diff --git a/doc/README.md b/doc/README.md index 8d4a44e6..adeb969e 100644 --- a/doc/README.md +++ b/doc/README.md @@ -178,22 +178,14 @@ and [here](https://content-security-policy.com/) how to change you CSP settings but be aware that **Changing CSP settings will potentially expose you Jenkins instance for security vulnerabilities**. -### Robot Framework 4.x compatibility +### Robot Framework 4.x+ compatibility -The plugin supports both Robot Framework 3.x and 4.x output files. However, in order to support both, the plugin -shows some extra information for both. [In Robot Framework 4.0 test criticality was removed and "SKIP" status was added](https://github.com/robotframework/robotframework/blob/master/doc/releasenotes/rf-4.0.rst). So for 3.x the -results overview will show a `Skipped` column, which will always be 0 and for Robot Framework 4.x output files -the `Critical tests` row will always be 0. +:heavy_exclamation_mark: As we're preparing to drop support for RF 3.x, the `onlyCritical` flag has been deprecated and no +longer has any effect. It will be removed in the future, but for now it's available for the transition period. +A new flag `countSkippedTests` has been added to the pipeline step to allow users to choose whether to count skipped +tests in the pass percentage calculation. -Skipped tests aren't taken into account when calculating pass percentage, but they are calculated to the total -amount of tests. - -Because criticality was removed in Robot Framework 4.0, having the `Use thresholds for critical tests only` checkbox -checked will always result in a passing step (because pass percentage is always considered to be 100% when there are -0 tests). In order to have set build status correctly, you **must** uncheck the checkbox or use `onlyCritical: false` -in your pipeline when you call `robot`. - -We are planning of dropping support for Robot Framework 3.x and earlier some time in distant future. +The plugin still supports RF 3.x, but no longer takes criticality into account. If you want to use RF 3.x with criticality, please downgrade to the last major version (4.0.0) ### Overall Screenshots diff --git a/src/main/java/hudson/plugins/robot/AggregatedRobotAction.java b/src/main/java/hudson/plugins/robot/AggregatedRobotAction.java index 5302ba91..cc7321ff 100644 --- a/src/main/java/hudson/plugins/robot/AggregatedRobotAction.java +++ b/src/main/java/hudson/plugins/robot/AggregatedRobotAction.java @@ -109,33 +109,34 @@ public static class AggregatedRobotResult extends RobotResult { private static final long serialVersionUID = 1L; private final transient AggregatedRobotAction parent; - private int passed, failed, skipped, criticalPassed, criticalFailed; + private int passed, failed, skipped; public AggregatedRobotResult(AggregatedRobotAction parent) { this.parent = parent; } public void addResult(RobotResult result) { - criticalFailed += result.getCriticalFailed(); - criticalPassed += result.getCriticalPassed(); failed += result.getOverallFailed(); passed += result.getOverallPassed(); skipped += result.getOverallSkipped(); } + @Deprecated @Override public long getCriticalPassed() { - return criticalPassed; + return this.getOverallPassed(); } + @Deprecated @Override public long getCriticalFailed() { - return criticalFailed; + return this.getOverallFailed(); } + @Deprecated @Override public long getCriticalTotal() { - return criticalFailed + criticalPassed; + return this.getOverallTotal(); } @Override diff --git a/src/main/java/hudson/plugins/robot/RobotBuildAction.java b/src/main/java/hudson/plugins/robot/RobotBuildAction.java index 056440d4..dd7a044a 100755 --- a/src/main/java/hudson/plugins/robot/RobotBuildAction.java +++ b/src/main/java/hudson/plugins/robot/RobotBuildAction.java @@ -65,6 +65,8 @@ public class RobotBuildAction extends AbstractTestResultAction private RobotResult result; private String xAxisLabel; + private boolean countSkippedTests; + static { XSTREAM.alias("result",RobotResult.class); XSTREAM.alias("suite",RobotSuiteResult.class); @@ -83,7 +85,8 @@ public class RobotBuildAction extends AbstractTestResultAction * @param enableCache Whether we want to enable caching or not */ public RobotBuildAction(Run build, RobotResult result, - String outputPath, TaskListener listener, String logFileLink, String logHtmlLink, boolean enableCache, String xAxisLabel) { + String outputPath, TaskListener listener, String logFileLink, String logHtmlLink, boolean enableCache, String xAxisLabel, + boolean countSkippedTests) { super(); super.onAttached(build); this.build = build; @@ -92,6 +95,7 @@ public RobotBuildAction(Run build, RobotResult result, this.logHtmlLink = logHtmlLink; this.enableCache = enableCache; this.xAxisLabel = xAxisLabel; + this.countSkippedTests = countSkippedTests; setResult(result, listener); } @@ -333,4 +337,12 @@ public String getUrlName() { public List getAllTests() { return getResult().getAllCases(); } + + public boolean isCountSkippedTests() { + return countSkippedTests; + } + + public void setCountSkippedTests(boolean countSkippedTests) { + this.countSkippedTests = countSkippedTests; + } } diff --git a/src/main/java/hudson/plugins/robot/RobotParser.java b/src/main/java/hudson/plugins/robot/RobotParser.java index a969752d..fdfe1d6a 100644 --- a/src/main/java/hudson/plugins/robot/RobotParser.java +++ b/src/main/java/hudson/plugins/robot/RobotParser.java @@ -293,7 +293,7 @@ private RobotCaseResult processTest(XMLStreamReader reader, RobotSuiteResult res caseResult.setLogFile(this.logFileName); //parse attributes caseResult.setName(reader.getAttributeValue(null, "name")); - setCriticalityIfAvailable(reader, caseResult); + //setCriticalityIfAvailable(reader, caseResult); caseResult.setId(reader.getAttributeValue(null, "id")); //parse test tags caseResult.setDescription(""); @@ -346,7 +346,7 @@ private RobotCaseResult processTest(XMLStreamReader reader, RobotSuiteResult res if (schemaVersion >= 5) { caseResult.setElapsedTime(reader.getAttributeValue(null, elapsedLocalName)); } - setCriticalityIfAvailable(reader, caseResult); +// setCriticalityIfAvailable(reader, caseResult); while(reader.hasNext()){ reader.next(); if(reader.isCharacters()){ diff --git a/src/main/java/hudson/plugins/robot/RobotPublisher.java b/src/main/java/hudson/plugins/robot/RobotPublisher.java index 7ea6b63c..48ae90c6 100755 --- a/src/main/java/hudson/plugins/robot/RobotPublisher.java +++ b/src/main/java/hudson/plugins/robot/RobotPublisher.java @@ -47,450 +47,468 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; public class RobotPublisher extends Recorder implements Serializable, - MatrixAggregatable, SimpleBuildStep { - - private static final long serialVersionUID = 1L; - - protected static final String DEFAULT_REPORT_FILE = "report.html"; - protected static final String DEFAULT_ARCHIVE_DIR = "robot-plugin"; - protected static final String DEFAULT_JENKINS_ARCHIVE_DIR = "archive"; - - private static final String DEFAULT_OUTPUT_FILE = "output.xml"; - private static final String DEFAULT_LOG_FILE = "log.html"; - - final private String archiveDirName; - final private String outputPath; - final private String reportFileName; - final private String logFileName; - final private String outputFileName; - final private boolean disableArchiveOutput; - final private double passThreshold; - final private double unstableThreshold; - private String[] otherFiles; - final private String overwriteXAxisLabel; - final private boolean enableCache; - - //Default to true - private boolean onlyCritical = true; - - /** - * Create new publisher for Robot Framework results - * - * @param archiveDirName - * Name of Archive dir - * @param outputPath - * Path to Robot Framework's output files - * @param outputFileName - * Name of Robot output xml - * @param disableArchiveOutput - * Disable Archiving output xml file to server - * @param reportFileName - * Name of Robot report html - * @param logFileName - * Name of Robot log html - * @param passThreshold - * Threshold of test pass percentage for successful builds - * @param unstableThreshold - * Threshold of test pass percentage for unstable builds - * @param onlyCritical - * True if only critical tests are included in pass percentage - * @param otherFiles - * Other files to be saved - * @param enableCache - * True if caching is used - */ - @DataBoundConstructor - public RobotPublisher(String archiveDirName, String outputPath, String outputFileName, - boolean disableArchiveOutput, String reportFileName, String logFileName, - double passThreshold, double unstableThreshold, - boolean onlyCritical, String otherFiles, boolean enableCache, String overwriteXAxisLabel) { - this.archiveDirName = archiveDirName; - this.outputPath = outputPath; - this.outputFileName = outputFileName; - this.disableArchiveOutput = disableArchiveOutput; - this.reportFileName = reportFileName; - this.passThreshold = passThreshold; - this.unstableThreshold = unstableThreshold; - this.logFileName = logFileName; - this.onlyCritical = onlyCritical; - this.enableCache = enableCache; - this.overwriteXAxisLabel = overwriteXAxisLabel; - - if (otherFiles != null) { - String[] filemasks = otherFiles.split(","); - for (int i = 0; i < filemasks.length; i++){ - filemasks[i] = StringUtils.strip(filemasks[i]); - } - this.otherFiles = filemasks; - } - } - - /** - * Gets the name of archive dir. Reverts to default if empty or - * whitespace. - * @return the name of archive dir - */ - public String getArchiveDirName() { - if (StringUtils.isBlank(archiveDirName)) - return DEFAULT_ARCHIVE_DIR; - return archiveDirName; - } - - /** - * Gets the output path of Robot files - * @return the output path of Robot files - */ - public String getOutputPath() { - return outputPath; - } - - /** - * Gets the name of output xml file. Reverts to default if empty or - * whitespace. - * @return the name of output xml file - */ - public String getOutputFileName() { - if (StringUtils.isBlank(outputFileName)) - return DEFAULT_OUTPUT_FILE; - return outputFileName; - } - - /** - * Get the value of disable Archive of output xml checkbox - * @return the value of disable Archive of output xml checkbox - */ - public boolean getDisableArchiveOutput() { - return disableArchiveOutput; - } - - /** - * Gets the name of report html file. Reverts to default if empty or - * whitespace. - * @return the name of report html file - */ - public String getReportFileName() { - if (StringUtils.isBlank(reportFileName)) - return DEFAULT_REPORT_FILE; - return reportFileName; - } - - /** - * Gets the name of log html file. Reverts to default if empty or - * whitespace. - * @return the name of log html file - */ - public String getLogFileName() { - if (StringUtils.isBlank(logFileName)) - return DEFAULT_LOG_FILE; - return logFileName; - } - - /** - * Gets the test pass percentage threshold for successful builds. - * @return the test pass percentage threshold for successful builds - */ - public double getPassThreshold() { - return passThreshold; - } - - /** - * Gets the test pass percentage threshold for unstable builds. - * @return the test pass percentage threshold for unstable builds - */ - public double getUnstableThreshold() { - return unstableThreshold; - } - - /** - * Gets if only critical tests should be accounted for the thresholds. - * @return true if only critical tests should be accounted for the thresholds - */ - public boolean getOnlyCritical() { - return onlyCritical; - } - - /** - * Gets value of enableCache - * @return true if cache is enabled - */ - public boolean getEnableCache() { return enableCache; } - - /** - * Gets the comma separated list of other filemasks to copy into build dir - * @return List of files as string - */ - public String getOtherFiles() { - return StringUtils.join(otherFiles, ","); - } - - /** - * Gets the value of overwriteXAxisLabel - * @return X axis label for the trend - */ - public String getOverwriteXAxisLabel() { - return overwriteXAxisLabel; - } - - /** - * {@inheritDoc} - */ - @Override - public Collection getProjectActions(AbstractProject project) { - Collection actions = new ArrayList<>(); - RobotProjectAction roboAction = new RobotProjectAction(project); - actions.add(roboAction); - return actions; - } - - protected RobotResult parse(String expandedTestResults, String expandedLogFileName, String expandedReportFileName, String outputPath, Run build, FilePath workspace, - Launcher launcher, TaskListener listener) throws IOException, - InterruptedException { - return new RobotParser().parse(expandedTestResults, outputPath, build, workspace, expandedLogFileName, expandedReportFileName); - } - - /** - * {@inheritDoc} - */ - @Override - @SuppressFBWarnings(value = "DCN_NULLPOINTER_EXCEPTION", - justification = "Lower risk to suppress the warning than to stop catching the null pointer exception") - public void perform(Run build, @NonNull FilePath workspace, @NonNull EnvVars buildEnv, @NonNull Launcher launcher, @NonNull TaskListener listener) throws InterruptedException, IOException { - if (build.getResult() != Result.ABORTED) { - PrintStream logger = listener.getLogger(); - logger.println(Messages.robot_publisher_started()); - logger.println(Messages.robot_publisher_only_critical()); - logger.println(Messages.robot_publisher_parsing()); - RobotResult result; - - try { - String expandedOutputFileName = buildEnv.expand(getOutputFileName()); - String expandedOutputPath = buildEnv.expand(getOutputPath()); - String expandedReportFileName = buildEnv.expand(getReportFileName()); - String expandedLogFileName = buildEnv.expand(getLogFileName()); - String logFileJavascripts = trimSuffix(expandedLogFileName) + ".js"; - - result = parse(expandedOutputFileName, expandedLogFileName, expandedReportFileName, expandedOutputPath, build, workspace, launcher, listener); - logger.println(Messages.robot_publisher_done()); - - // Check if log and report files exist - FilePath outputDir = new FilePath(workspace, expandedOutputPath); - if (outputDir.list(expandedLogFileName).length == 0) { - logger.println(Messages.robot_publisher_file_not_found() + " " + expandedLogFileName); - } - if (outputDir.list(expandedReportFileName).length == 0) { - logger.println(Messages.robot_publisher_file_not_found() + " " + expandedReportFileName); - } - - if (!DEFAULT_JENKINS_ARCHIVE_DIR.equalsIgnoreCase(getArchiveDirName())) { - logger.println(Messages.robot_publisher_copying()); - //Save configured Robot files (including split output) to build dir - copyFilesToBuildDir(build, workspace, expandedOutputPath, StringUtils.join(modifyMasksforSplittedOutput(new String[]{expandedReportFileName, expandedLogFileName, logFileJavascripts}), ",")); - - if (!getDisableArchiveOutput()) { - copyFilesToBuildDir(build, workspace, expandedOutputPath, StringUtils.join(modifyMasksforSplittedOutput(new String[]{expandedOutputFileName}), ",")); - } - - //Save other configured files to build dir - if(StringUtils.isNotBlank(getOtherFiles())) { - String filemask = buildEnv.expand(getOtherFiles()); - copyFilesToBuildDir(build, workspace, expandedOutputPath, filemask); - } - logger.println(Messages.robot_publisher_done()); - } - - logger.println(Messages.robot_publisher_assigning()); - - String label = buildEnv.expand(overwriteXAxisLabel); - RobotBuildAction action = new RobotBuildAction(build, result, getArchiveDirName(), listener, expandedReportFileName, expandedLogFileName, enableCache, label); - build.addAction(action); - - // set RobotProjectAction as project action - Job job = build.getParent(); - if (job != null) { - RobotProjectAction projectAction = new RobotProjectAction(job); - try { - job.addOrReplaceAction(projectAction); - } catch (UnsupportedOperationException | NullPointerException e) { - // it is possible that the action collection is an unmodifiable collection - // NullPointerException is thrown if a freestyle job runs - } - - logger.println(Messages.robot_publisher_done()); - logger.println(Messages.robot_publisher_checking()); - - Result buildResult = getBuildResult(build, result); - build.setResult(buildResult); - - logger.println(Messages.robot_publisher_done()); - logger.println(Messages.robot_publisher_finished()); - } - - } catch (RuntimeException e) { - throw e; - } catch (Exception e) { - logger.println(Messages.robot_publisher_fail()); - e.printStackTrace(logger); - build.setResult(Result.FAILURE); - } - } - } - - /** - * Copy files with given filemasks from input path relative to build into specific build file archive dir - * @param build The Jenkins run - * @param inputPath Base path for copy. Relative to build workspace. - * @param filemaskToCopy List of Ant GLOB style filemasks to copy from dirs specified at inputPathMask - * @param workspace Build workspace - * @throws IOException thrown exception - * @throws InterruptedException thrown exception - */ - public void copyFilesToBuildDir(Run build, FilePath workspace, - String inputPath, String filemaskToCopy) throws IOException, InterruptedException { - FilePath srcDir = new FilePath(workspace, inputPath); - FilePath destDir = new FilePath(new FilePath(build.getRootDir()), - getArchiveDirName()); - srcDir.copyRecursiveTo(filemaskToCopy, destDir); - } - - /** - * Return filename without file suffix. - * @param filename Filename with suffix - * @return filename as string - */ - public static String trimSuffix(String filename) { - int index = filename.lastIndexOf('.'); - if (index > 0) { - filename = filename.substring(0, index); - } - return filename; - } - - /** - * Return file suffix from string. - * @param filename Filename with suffix - * @return file suffix as string - */ - public static String getSuffix(String filename) { - int index = filename.lastIndexOf('.'); - if (index > 0) { - return filename.substring(index); - } - return ""; - } - - /** - * Add wildcard to filemasks between name and file extension in order to copy split output - * e.g. output-001.xml, output-002.xml etc. - * @param filemasks Files to be masked with wildcards - * @return Updated array of filemasks - */ - private static String[] modifyMasksforSplittedOutput(String[] filemasks){ - for (int i = 0; i < filemasks.length; i++){ - filemasks[i] = trimSuffix(filemasks[i]) + "*" + getSuffix(filemasks[i]); - } - return filemasks; - } - - /** - * Determines the build result based on set thresholds. If build is already - * failed before the tests it won't be changed to successful. - * - * @param build - * Build to be evaluated - * @param result - * Results associated to build - * @return Result of build - */ - protected Result getBuildResult(Run build, - RobotResult result) { - if (build.getResult() != Result.FAILURE) { - double passPercentage = result.getPassPercentage(onlyCritical); - if (passPercentage < getUnstableThreshold()) { - return Result.FAILURE; - } else if (passPercentage < getPassThreshold()) { - return Result.UNSTABLE; - } - return Result.SUCCESS; - } - return Result.FAILURE; - } - - /** - * Descriptor for the publisher - */ - @Extension - public static final class DescriptorImpl extends - BuildStepDescriptor { - - /** - * {@inheritDoc} - */ - @SuppressWarnings("rawtypes") - @Override - public boolean isApplicable(Class aClass) { - return true; - } - - /** - * {@inheritDoc} - */ - @Override - public String getDisplayName() { - return Messages.robot_description(); - } - - /** - * Validates the unstable threshold input field. - * @param value Value to be checked for the threshold - * @return OK if value is within threshold - * @throws IOException thrown exception - * @throws ServletException thrown exception - */ - public FormValidation doCheckUnstableThreshold( - @QueryParameter String value) throws IOException, - ServletException { - if (isPercentageValue(value)) - return FormValidation.ok(); - else - return FormValidation.error(Messages - .robot_config_percentvalidation()); - } - - /** - * Validates the pass threshold input field. - * @param value Value to be checked for the threshold - * @return OK if value is within threshold - * @throws IOException thrown exception - * @throws ServletException thrown exception - */ - public FormValidation doCheckPassThreshold(@QueryParameter String value) - throws IOException, ServletException { - if (isPercentageValue(value)) - return FormValidation.ok(); - else - return FormValidation.error(Messages - .robot_config_percentvalidation()); - } - - private boolean isPercentageValue(String value) { - try { - double doubleValue = Double.parseDouble(value); - if (doubleValue <= 100 && doubleValue >= 0) - return true; - else - return false; - } catch (NumberFormatException e) { - return false; - } - } - } - - public BuildStepMonitor getRequiredMonitorService() { - return BuildStepMonitor.NONE; - } - - public MatrixAggregator createAggregator(MatrixBuild build, - Launcher launcher, BuildListener listener) { - return new RobotResultAggregator(build, launcher, listener); - } + MatrixAggregatable, SimpleBuildStep { + + private static final long serialVersionUID = 1L; + + protected static final String DEFAULT_REPORT_FILE = "report.html"; + protected static final String DEFAULT_ARCHIVE_DIR = "robot-plugin"; + protected static final String DEFAULT_JENKINS_ARCHIVE_DIR = "archive"; + + private static final String DEFAULT_OUTPUT_FILE = "output.xml"; + private static final String DEFAULT_LOG_FILE = "log.html"; + + final private String archiveDirName; + final private String outputPath; + final private String reportFileName; + final private String logFileName; + final private String outputFileName; + final private boolean disableArchiveOutput; + final private double passThreshold; + final private double unstableThreshold; + private String[] otherFiles; + final private String overwriteXAxisLabel; + final private boolean enableCache; + + //Default to true + private boolean onlyCritical = true; + private boolean countSkippedTests = false; + + /** + * Create new publisher for Robot Framework results + * + * @param archiveDirName Name of Archive dir + * @param outputPath Path to Robot Framework's output files + * @param outputFileName Name of Robot output xml + * @param disableArchiveOutput Disable Archiving output xml file to server + * @param reportFileName Name of Robot report html + * @param logFileName Name of Robot log html + * @param passThreshold Threshold of test pass percentage for successful builds + * @param unstableThreshold Threshold of test pass percentage for unstable builds + * @param onlyCritical True if only critical tests are included in pass percentage + * @param otherFiles Other files to be saved + * @param enableCache True if caching is used + */ + @DataBoundConstructor + public RobotPublisher(String archiveDirName, String outputPath, String outputFileName, + boolean disableArchiveOutput, String reportFileName, String logFileName, + double passThreshold, double unstableThreshold, + boolean onlyCritical, boolean countSkippedTests, String otherFiles, boolean enableCache, String overwriteXAxisLabel) { + this.archiveDirName = archiveDirName; + this.outputPath = outputPath; + this.outputFileName = outputFileName; + this.disableArchiveOutput = disableArchiveOutput; + this.reportFileName = reportFileName; + this.passThreshold = passThreshold; + this.unstableThreshold = unstableThreshold; + this.logFileName = logFileName; + this.onlyCritical = onlyCritical; + this.countSkippedTests = countSkippedTests; + this.enableCache = enableCache; + this.overwriteXAxisLabel = overwriteXAxisLabel; + + if (otherFiles != null) { + String[] filemasks = otherFiles.split(","); + for (int i = 0; i < filemasks.length; i++) { + filemasks[i] = StringUtils.strip(filemasks[i]); + } + this.otherFiles = filemasks; + } + } + + /** + * Gets the name of archive dir. Reverts to default if empty or + * whitespace. + * + * @return the name of archive dir + */ + public String getArchiveDirName() { + if (StringUtils.isBlank(archiveDirName)) + return DEFAULT_ARCHIVE_DIR; + return archiveDirName; + } + + /** + * Gets the output path of Robot files + * + * @return the output path of Robot files + */ + public String getOutputPath() { + return outputPath; + } + + /** + * Gets the name of output xml file. Reverts to default if empty or + * whitespace. + * + * @return the name of output xml file + */ + public String getOutputFileName() { + if (StringUtils.isBlank(outputFileName)) + return DEFAULT_OUTPUT_FILE; + return outputFileName; + } + + /** + * Get the value of disable Archive of output xml checkbox + * + * @return the value of disable Archive of output xml checkbox + */ + public boolean getDisableArchiveOutput() { + return disableArchiveOutput; + } + + /** + * Gets the name of report html file. Reverts to default if empty or + * whitespace. + * + * @return the name of report html file + */ + public String getReportFileName() { + if (StringUtils.isBlank(reportFileName)) + return DEFAULT_REPORT_FILE; + return reportFileName; + } + + /** + * Gets the name of log html file. Reverts to default if empty or + * whitespace. + * + * @return the name of log html file + */ + public String getLogFileName() { + if (StringUtils.isBlank(logFileName)) + return DEFAULT_LOG_FILE; + return logFileName; + } + + /** + * Gets the test pass percentage threshold for successful builds. + * + * @return the test pass percentage threshold for successful builds + */ + public double getPassThreshold() { + return passThreshold; + } + + /** + * Gets the test pass percentage threshold for unstable builds. + * + * @return the test pass percentage threshold for unstable builds + */ + public double getUnstableThreshold() { + return unstableThreshold; + } + + /** + * Gets if only critical tests should be accounted for the thresholds. + * + * @return true if only critical tests should be accounted for the thresholds + */ + public boolean getOnlyCritical() { + return onlyCritical; + } + + /** + * Gets if skipped tests should be counted in the thresholds. + * + * @return true if skipped tests should be counted in the thresholds + */ + public boolean getCountSkippedTests() { + return countSkippedTests; + } + + /** + * Gets value of enableCache + * + * @return true if cache is enabled + */ + public boolean getEnableCache() { + return enableCache; + } + + /** + * Gets the comma separated list of other filemasks to copy into build dir + * + * @return List of files as string + */ + public String getOtherFiles() { + return StringUtils.join(otherFiles, ","); + } + + /** + * Gets the value of overwriteXAxisLabel + * + * @return X axis label for the trend + */ + public String getOverwriteXAxisLabel() { + return overwriteXAxisLabel; + } + + /** + * {@inheritDoc} + */ + @Override + public Collection getProjectActions(AbstractProject project) { + Collection actions = new ArrayList<>(); + RobotProjectAction roboAction = new RobotProjectAction(project); + actions.add(roboAction); + return actions; + } + + protected RobotResult parse(String expandedTestResults, String expandedLogFileName, String expandedReportFileName, String outputPath, Run build, FilePath workspace, + Launcher launcher, TaskListener listener) throws IOException, + InterruptedException { + return new RobotParser().parse(expandedTestResults, outputPath, build, workspace, expandedLogFileName, expandedReportFileName); + } + + /** + * {@inheritDoc} + */ + @Override + @SuppressFBWarnings(value = "DCN_NULLPOINTER_EXCEPTION", + justification = "Lower risk to suppress the warning than to stop catching the null pointer exception") + public void perform(Run build, @NonNull FilePath workspace, @NonNull EnvVars buildEnv, @NonNull Launcher launcher, @NonNull TaskListener listener) throws InterruptedException, IOException { + if (build.getResult() != Result.ABORTED) { + PrintStream logger = listener.getLogger(); + logger.println(Messages.robot_publisher_started()); + logger.println(Messages.robot_publisher_only_critical()); + logger.println(Messages.robot_publisher_parsing()); + RobotResult result; + + try { + String expandedOutputFileName = buildEnv.expand(getOutputFileName()); + String expandedOutputPath = buildEnv.expand(getOutputPath()); + String expandedReportFileName = buildEnv.expand(getReportFileName()); + String expandedLogFileName = buildEnv.expand(getLogFileName()); + String logFileJavascripts = trimSuffix(expandedLogFileName) + ".js"; + + result = parse(expandedOutputFileName, expandedLogFileName, expandedReportFileName, expandedOutputPath, build, workspace, launcher, listener); + logger.println(Messages.robot_publisher_done()); + + // Check if log and report files exist + FilePath outputDir = new FilePath(workspace, expandedOutputPath); + if (outputDir.list(expandedLogFileName).length == 0) { + logger.println(Messages.robot_publisher_file_not_found() + " " + expandedLogFileName); + } + if (outputDir.list(expandedReportFileName).length == 0) { + logger.println(Messages.robot_publisher_file_not_found() + " " + expandedReportFileName); + } + + if (!DEFAULT_JENKINS_ARCHIVE_DIR.equalsIgnoreCase(getArchiveDirName())) { + logger.println(Messages.robot_publisher_copying()); + //Save configured Robot files (including split output) to build dir + copyFilesToBuildDir(build, workspace, expandedOutputPath, StringUtils.join(modifyMasksforSplittedOutput(new String[]{expandedReportFileName, expandedLogFileName, logFileJavascripts}), ",")); + + if (!getDisableArchiveOutput()) { + copyFilesToBuildDir(build, workspace, expandedOutputPath, StringUtils.join(modifyMasksforSplittedOutput(new String[]{expandedOutputFileName}), ",")); + } + + //Save other configured files to build dir + if (StringUtils.isNotBlank(getOtherFiles())) { + String filemask = buildEnv.expand(getOtherFiles()); + copyFilesToBuildDir(build, workspace, expandedOutputPath, filemask); + } + logger.println(Messages.robot_publisher_done()); + } + + logger.println(Messages.robot_publisher_assigning()); + + String label = buildEnv.expand(overwriteXAxisLabel); + RobotBuildAction action = new RobotBuildAction(build, result, getArchiveDirName(), listener, expandedReportFileName, expandedLogFileName, enableCache, label, countSkippedTests); + build.addAction(action); + + // set RobotProjectAction as project action + Job job = build.getParent(); + if (job != null) { + RobotProjectAction projectAction = new RobotProjectAction(job); + try { + job.addOrReplaceAction(projectAction); + } catch (UnsupportedOperationException | NullPointerException e) { + // it is possible that the action collection is an unmodifiable collection + // NullPointerException is thrown if a freestyle job runs + } + + logger.println(Messages.robot_publisher_done()); + logger.println(Messages.robot_publisher_checking()); + + Result buildResult = getBuildResult(build, result); + build.setResult(buildResult); + + logger.println(Messages.robot_publisher_done()); + logger.println(Messages.robot_publisher_finished()); + } + + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + logger.println(Messages.robot_publisher_fail()); + e.printStackTrace(logger); + build.setResult(Result.FAILURE); + } + } + } + + /** + * Copy files with given filemasks from input path relative to build into specific build file archive dir + * + * @param build The Jenkins run + * @param inputPath Base path for copy. Relative to build workspace. + * @param filemaskToCopy List of Ant GLOB style filemasks to copy from dirs specified at inputPathMask + * @param workspace Build workspace + * @throws IOException thrown exception + * @throws InterruptedException thrown exception + */ + public void copyFilesToBuildDir(Run build, FilePath workspace, + String inputPath, String filemaskToCopy) throws IOException, InterruptedException { + FilePath srcDir = new FilePath(workspace, inputPath); + FilePath destDir = new FilePath(new FilePath(build.getRootDir()), + getArchiveDirName()); + srcDir.copyRecursiveTo(filemaskToCopy, destDir); + } + + /** + * Return filename without file suffix. + * + * @param filename Filename with suffix + * @return filename as string + */ + public static String trimSuffix(String filename) { + int index = filename.lastIndexOf('.'); + if (index > 0) { + filename = filename.substring(0, index); + } + return filename; + } + + /** + * Return file suffix from string. + * + * @param filename Filename with suffix + * @return file suffix as string + */ + public static String getSuffix(String filename) { + int index = filename.lastIndexOf('.'); + if (index > 0) { + return filename.substring(index); + } + return ""; + } + + /** + * Add wildcard to filemasks between name and file extension in order to copy split output + * e.g. output-001.xml, output-002.xml etc. + * + * @param filemasks Files to be masked with wildcards + * @return Updated array of filemasks + */ + private static String[] modifyMasksforSplittedOutput(String[] filemasks) { + for (int i = 0; i < filemasks.length; i++) { + filemasks[i] = trimSuffix(filemasks[i]) + "*" + getSuffix(filemasks[i]); + } + return filemasks; + } + + /** + * Determines the build result based on set thresholds. If build is already + * failed before the tests it won't be changed to successful. + * + * @param build Build to be evaluated + * @param result Results associated to build + * @return Result of build + */ + protected Result getBuildResult(Run build, + RobotResult result) { + if (build.getResult() != Result.FAILURE) { + double passPercentage = result.getPassPercentage(countSkippedTests); + if (passPercentage < getUnstableThreshold()) { + return Result.FAILURE; + } else if (passPercentage < getPassThreshold()) { + return Result.UNSTABLE; + } + return Result.SUCCESS; + } + return Result.FAILURE; + } + + /** + * Descriptor for the publisher + */ + @Extension + public static final class DescriptorImpl extends + BuildStepDescriptor { + + /** + * {@inheritDoc} + */ + @SuppressWarnings("rawtypes") + @Override + public boolean isApplicable(Class aClass) { + return true; + } + + /** + * {@inheritDoc} + */ + @Override + public String getDisplayName() { + return Messages.robot_description(); + } + + /** + * Validates the unstable threshold input field. + * + * @param value Value to be checked for the threshold + * @return OK if value is within threshold + * @throws IOException thrown exception + * @throws ServletException thrown exception + */ + public FormValidation doCheckUnstableThreshold( + @QueryParameter String value) throws IOException, + ServletException { + if (isPercentageValue(value)) + return FormValidation.ok(); + else + return FormValidation.error(Messages + .robot_config_percentvalidation()); + } + + /** + * Validates the pass threshold input field. + * + * @param value Value to be checked for the threshold + * @return OK if value is within threshold + * @throws IOException thrown exception + * @throws ServletException thrown exception + */ + public FormValidation doCheckPassThreshold(@QueryParameter String value) + throws IOException, ServletException { + if (isPercentageValue(value)) + return FormValidation.ok(); + else + return FormValidation.error(Messages + .robot_config_percentvalidation()); + } + + private boolean isPercentageValue(String value) { + try { + double doubleValue = Double.parseDouble(value); + if (doubleValue <= 100 && doubleValue >= 0) + return true; + else + return false; + } catch (NumberFormatException e) { + return false; + } + } + } + + public BuildStepMonitor getRequiredMonitorService() { + return BuildStepMonitor.NONE; + } + + public MatrixAggregator createAggregator(MatrixBuild build, + Launcher launcher, BuildListener listener) { + return new RobotResultAggregator(build, launcher, listener); + } } diff --git a/src/main/java/hudson/plugins/robot/RobotStep.java b/src/main/java/hudson/plugins/robot/RobotStep.java index a3a9d914..1383ac56 100644 --- a/src/main/java/hudson/plugins/robot/RobotStep.java +++ b/src/main/java/hudson/plugins/robot/RobotStep.java @@ -38,6 +38,7 @@ public class RobotStep extends Step { private @CheckForNull String[] otherFiles; private boolean enableCache = true; private boolean onlyCritical = true; + private boolean countSkippedTests = false; private @CheckForNull String overwriteXAxisLabel; @@ -95,6 +96,8 @@ public boolean getOnlyCritical() { return this.onlyCritical; } + public boolean getCountSkippedTests() { return this.countSkippedTests; } + public String getOverwriteXAxisLabel() { return this.overwriteXAxisLabel; } @@ -143,7 +146,12 @@ public void setEnableCache(boolean enableCache) { public void setOnlyCritical(boolean onlyCritical) { this.onlyCritical = onlyCritical; } - + + @DataBoundSetter + public void setCountSkippedTests(boolean countSkippedTests) { + this.countSkippedTests = countSkippedTests; + } + @DataBoundSetter public void setOtherFiles(String otherFiles) { otherFiles = Util.fixEmpty(otherFiles); diff --git a/src/main/java/hudson/plugins/robot/RobotStepExecution.java b/src/main/java/hudson/plugins/robot/RobotStepExecution.java index 8553507c..1872a3aa 100644 --- a/src/main/java/hudson/plugins/robot/RobotStepExecution.java +++ b/src/main/java/hudson/plugins/robot/RobotStepExecution.java @@ -27,7 +27,7 @@ public class RobotStepExecution extends SynchronousNonBlockingStepExecution context, FilePath workspace, TaskListener liste RobotBuildAction action = context.getAction(RobotBuildAction.class); if(action!=null){ RobotResult result = action.getResult(); - if(onlyCritical) - return Long.toString(result.getCriticalFailed()); - else - return Long.toString(result.getOverallFailed()); + return Long.toString(result.getOverallFailed()); } return ""; } diff --git a/src/main/java/hudson/plugins/robot/tokens/RobotPassPercentageTokenMacro.java b/src/main/java/hudson/plugins/robot/tokens/RobotPassPercentageTokenMacro.java index 6a82e6de..cbb6bd99 100644 --- a/src/main/java/hudson/plugins/robot/tokens/RobotPassPercentageTokenMacro.java +++ b/src/main/java/hudson/plugins/robot/tokens/RobotPassPercentageTokenMacro.java @@ -16,11 +16,8 @@ @Extension(optional = true) public class RobotPassPercentageTokenMacro extends DataBoundTokenMacro { - /** - * If true return only pass percentage of critical tests - */ @Parameter - public boolean onlyCritical; + public boolean countSkippedTests; @Override public String evaluate(AbstractBuild context, TaskListener listener, @@ -37,7 +34,7 @@ public String evaluate(Run context, FilePath workspace, TaskListener liste if (action!=null){ RobotResult result = action.getResult(); - return String.valueOf(result.getPassPercentage(onlyCritical)); + return String.valueOf(result.getPassPercentage(countSkippedTests)); } return ""; } diff --git a/src/main/java/hudson/plugins/robot/tokens/RobotPassRatioTokenMacro.java b/src/main/java/hudson/plugins/robot/tokens/RobotPassRatioTokenMacro.java index 64540199..55421d92 100644 --- a/src/main/java/hudson/plugins/robot/tokens/RobotPassRatioTokenMacro.java +++ b/src/main/java/hudson/plugins/robot/tokens/RobotPassRatioTokenMacro.java @@ -33,10 +33,7 @@ public String evaluate(Run context, FilePath workspace, TaskListener liste RobotBuildAction action = context.getAction(RobotBuildAction.class); if(action!=null){ RobotResult result = action.getResult(); - if(onlyCritical) - return result.getCriticalPassed() + " / " + result.getCriticalTotal(); - else - return result.getOverallPassed() + " / " + result.getOverallTotal(); + return result.getOverallPassed() + " / " + result.getOverallTotal(); } return ""; } diff --git a/src/main/java/hudson/plugins/robot/tokens/RobotPassTokenMacro.java b/src/main/java/hudson/plugins/robot/tokens/RobotPassTokenMacro.java index 5a3b636d..49f5dcb1 100644 --- a/src/main/java/hudson/plugins/robot/tokens/RobotPassTokenMacro.java +++ b/src/main/java/hudson/plugins/robot/tokens/RobotPassTokenMacro.java @@ -32,10 +32,7 @@ public String evaluate(Run context, FilePath workspace, TaskListener liste RobotBuildAction action = context.getAction(RobotBuildAction.class); if(action!=null){ RobotResult result = action.getResult(); - if(onlyCritical) - return Long.toString(result.getCriticalPassed()); - else - return Long.toString(result.getOverallPassed()); + return Long.toString(result.getOverallPassed()); } return ""; } diff --git a/src/main/java/hudson/plugins/robot/tokens/RobotTotalTokenMacro.java b/src/main/java/hudson/plugins/robot/tokens/RobotTotalTokenMacro.java index 1e6ea29a..96712566 100644 --- a/src/main/java/hudson/plugins/robot/tokens/RobotTotalTokenMacro.java +++ b/src/main/java/hudson/plugins/robot/tokens/RobotTotalTokenMacro.java @@ -32,10 +32,7 @@ public String evaluate(Run context, FilePath workspace, TaskListener liste RobotBuildAction action = context.getAction(RobotBuildAction.class); if(action!=null){ RobotResult result = action.getResult(); - if(onlyCritical) - return Long.toString(result.getCriticalTotal()); - else - return Long.toString(result.getOverallTotal()); + return Long.toString(result.getOverallTotal()); } return ""; } diff --git a/src/main/resources/hudson/plugins/robot/RobotPublisher/config.jelly b/src/main/resources/hudson/plugins/robot/RobotPublisher/config.jelly index 120cfb69..a13d9238 100755 --- a/src/main/resources/hudson/plugins/robot/RobotPublisher/config.jelly +++ b/src/main/resources/hudson/plugins/robot/RobotPublisher/config.jelly @@ -54,7 +54,10 @@ limitations under the License. - ${%thresholds.onlycritical} + ${%thresholds.onlyCritical} + + + ${%thresholds.countSkippedTests} diff --git a/src/main/resources/hudson/plugins/robot/RobotPublisher/config.properties b/src/main/resources/hudson/plugins/robot/RobotPublisher/config.properties index cacfda48..153e9c98 100644 --- a/src/main/resources/hudson/plugins/robot/RobotPublisher/config.properties +++ b/src/main/resources/hudson/plugins/robot/RobotPublisher/config.properties @@ -21,4 +21,5 @@ advanced.overwriteXAxisLabel.description=Overwrite default x-axis label for publ thresholds.label=Thresholds for build result -thresholds.onlycritical=Use thresholds for critical tests only \ No newline at end of file +thresholds.onlyCritical=DEPRECATED! THIS FLAG DOES NOTHING! - Use thresholds for critical tests only +thresholds.countSkippedTests=Include skipped tests in total count for thresholds \ No newline at end of file diff --git a/src/main/resources/hudson/plugins/robot/RobotStep/config.jelly b/src/main/resources/hudson/plugins/robot/RobotStep/config.jelly index 120cfb69..a13d9238 100755 --- a/src/main/resources/hudson/plugins/robot/RobotStep/config.jelly +++ b/src/main/resources/hudson/plugins/robot/RobotStep/config.jelly @@ -54,7 +54,10 @@ limitations under the License. - ${%thresholds.onlycritical} + ${%thresholds.onlyCritical} + + + ${%thresholds.countSkippedTests} diff --git a/src/main/resources/hudson/plugins/robot/RobotStep/config.properties b/src/main/resources/hudson/plugins/robot/RobotStep/config.properties index a324d676..b050debd 100644 --- a/src/main/resources/hudson/plugins/robot/RobotStep/config.properties +++ b/src/main/resources/hudson/plugins/robot/RobotStep/config.properties @@ -20,4 +20,5 @@ advanced.overwriteXAxisLabel=X-axis label advanced.overwriteXAxisLabel.description=Overwrite default x-axis label for publish trend thresholds.label=Thresholds for build result -thresholds.onlycritical=Use thresholds for critical tests only \ No newline at end of file +thresholds.onlyCritical=DEPRECATED! THIS FLAG DOES NOTHING! - Use thresholds for critical tests only +thresholds.countSkippedTests=Include skipped tests in total count for thresholds \ No newline at end of file diff --git a/src/main/resources/hudson/plugins/robot/util/robotsummary.jelly b/src/main/resources/hudson/plugins/robot/util/robotsummary.jelly index c0439ca2..55a0ccf0 100644 --- a/src/main/resources/hudson/plugins/robot/util/robotsummary.jelly +++ b/src/main/resources/hudson/plugins/robot/util/robotsummary.jelly @@ -1,4 +1,4 @@ - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
TotalFailedPassedSkippedPass %
Critical tests${attrs.action.result.criticalTotal} - - - - - ${attrs.action.result.criticalFailed} - ${attrs.action.result.criticalPassed}0${attrs.action.criticalPassPercentage}
All tests${attrs.action.result.overallTotal} - - - - - ${attrs.action.result.overallFailed} - ${attrs.action.result.overallPassed}${attrs.action.result.overallSkipped}${attrs.action.overallPassPercentage}
-

-

-

+ + + + + + + + + + + + + + + + + + + + +
TotalFailedPassedSkippedPass %
All tests${attrs.action.result.overallTotal} + + + + + ${attrs.action.result.overallFailed} + ${attrs.action.result.overallPassed}${attrs.action.result.overallSkipped} + + + + + ${totalPassPercentage} +
+

+

+

diff --git a/src/test/java/hudson/plugins/robot/RobotProjectActionTest.java b/src/test/java/hudson/plugins/robot/RobotProjectActionTest.java index ab98c7f1..1a3a02d5 100644 --- a/src/test/java/hudson/plugins/robot/RobotProjectActionTest.java +++ b/src/test/java/hudson/plugins/robot/RobotProjectActionTest.java @@ -59,7 +59,7 @@ public void testShouldDisplayGraph() throws IOException { when(build.getProject()).thenReturn(p); when(build.getRootDir()).thenReturn(tmpDir); RobotResult result = mock(RobotResult.class); - RobotBuildAction buildAction = new RobotBuildAction(build, result, "", null, null, null, false, ""); + RobotBuildAction buildAction = new RobotBuildAction(build, result, "", null, null, null, false, "", false); when(build.getAction(RobotBuildAction.class)).thenReturn(buildAction); when(p.getLastBuild()).thenReturn(build); @@ -75,7 +75,7 @@ public void testShouldGetLastBuildAction() throws IOException{ when(buildWithAction.getProject()).thenReturn(p); when(buildWithAction.getRootDir()).thenReturn(tmpDir); RobotResult result = mock(RobotResult.class); - RobotBuildAction buildAction = new RobotBuildAction(buildWithAction, result, "", null, null, null, false, ""); + RobotBuildAction buildAction = new RobotBuildAction(buildWithAction, result, "", null, null, null, false, "", false); when(buildWithAction.getAction(RobotBuildAction.class)).thenReturn(buildAction); when(p.getLastBuild()).thenReturn(lastBuild); diff --git a/src/test/java/hudson/plugins/robot/RobotPublisherSystemTest.java b/src/test/java/hudson/plugins/robot/RobotPublisherSystemTest.java index d7d671eb..fe3677cd 100644 --- a/src/test/java/hudson/plugins/robot/RobotPublisherSystemTest.java +++ b/src/test/java/hudson/plugins/robot/RobotPublisherSystemTest.java @@ -56,20 +56,20 @@ public class RobotPublisherSystemTest { @Test public void testRoundTripConfig() throws Exception { FreeStyleProject p = j.jenkins.createProject(FreeStyleProject.class, "testRoundTripConfig"); - RobotPublisher before = new RobotPublisher(null, "a", "b", false, "c", "d", 11, 27, true, "dir1/*.jpg, dir2/*.png", + RobotPublisher before = new RobotPublisher(null, "a", "b", false, "c", "d", 11, 27, true, false,"dir1/*.jpg, dir2/*.png", false, ""); p.getPublishersList().add(before); j.configRoundtrip(p); RobotPublisher after = p.getPublishersList().get(RobotPublisher.class); assertThat( - "outputPath,outputFileName,reportFileName,logFileName,passThreshold,unstableThreshold,onlyCritical,otherFiles", + "outputPath,outputFileName,reportFileName,logFileName,passThreshold,unstableThreshold,onlyCritical,countSkippedTests,otherFiles", before, samePropertyValuesAs(after)); } @Test public void testConfigView() throws Exception { FreeStyleProject p = j.jenkins.createProject(FreeStyleProject.class, "testConfigView"); - RobotPublisher before = new RobotPublisher(null, "a", "b", false, "c", "d", 11, 27, true, "dir1/*.jpg, dir2/*.png", + RobotPublisher before = new RobotPublisher(null, "a", "b", false, "c", "d", 11, 27, true, false,"dir1/*.jpg, dir2/*.png", false, ""); p.getPublishersList().add(before); HtmlPage page = j.createWebClient().getPage(p, "configure"); @@ -86,8 +86,8 @@ public void testConfigView() throws Exception { WebAssert.assertInputContainsValue(page, "_.unstableThreshold", "27.0"); WebAssert.assertInputPresent(page, "_.passThreshold"); WebAssert.assertInputContainsValue(page, "_.passThreshold", "11.0"); - WebAssert.assertInputPresent(page, "_.onlyCritical"); - WebAssert.assertInputContainsValue(page, "_.onlyCritical", "on"); + WebAssert.assertInputPresent(page, "_.countSkippedTests"); + WebAssert.assertInputContainsValue(page, "_.countSkippedTests", "on"); WebAssert.assertInputPresent(page, "_.otherFiles"); WebAssert.assertInputContainsValue(page, "_.otherFiles", "dir1/*.jpg,dir2/*.png"); } @@ -475,12 +475,7 @@ private Project getProject(String projectName) { private void verifyTotalsTable(HtmlPage page, int totalTests, int totalFailed, int totalSkipped, String totalPercents, int totalCritical, int criticalFailed, String criticalPercents) { HtmlTable table = page.getHtmlElementById("robot-summary-table"); - String expectedTable = "TotalFailedPassedSkippedPass%Criticaltests" - + "" + totalCritical + "" - + "" + criticalFailed + "" - + "" + (totalCritical - totalFailed) + "" - + "0" - + "" + criticalPercents + "" + String expectedTable = "TotalFailedPassedSkippedPass%" + "Alltests" + totalTests + "" + "" + totalFailed + "" + "" + (totalTests - totalFailed) + "" diff --git a/src/test/java/hudson/plugins/robot/RobotPublisherTest.java b/src/test/java/hudson/plugins/robot/RobotPublisherTest.java index 5053f3eb..3f0ab846 100644 --- a/src/test/java/hudson/plugins/robot/RobotPublisherTest.java +++ b/src/test/java/hudson/plugins/robot/RobotPublisherTest.java @@ -32,13 +32,14 @@ public class RobotPublisherTest { private final boolean onlyCritical = false; + private final boolean countSkipped = false; @Before public void setUp() throws Exception { } private RobotPublisher getRobotPublisher(double passThreshold, double unstableThreshold) { - return new RobotPublisher(null, "", "", false, "", "", passThreshold, unstableThreshold, onlyCritical, "", false, ""); + return new RobotPublisher(null, "", "", false, "", "", passThreshold, unstableThreshold, onlyCritical, countSkipped, "", false, ""); } @Test @@ -141,5 +142,5 @@ public void testOutputFileWithXXEShouldThrowException() throws URISyntaxExceptio RobotParser.RobotParserCallable remoteOperation = new RobotParser.RobotParserCallable("xxe_output.xml", null, null); RobotResult result = remoteOperation.invoke(new File(RobotPublisherTest.class.getResource("xxe_output.xml").toURI()).getParentFile(), null); } - + } diff --git a/src/test/java/hudson/plugins/robot/model/RobotResultTest.java b/src/test/java/hudson/plugins/robot/model/RobotResultTest.java index b8787f7a..925a447f 100644 --- a/src/test/java/hudson/plugins/robot/model/RobotResultTest.java +++ b/src/test/java/hudson/plugins/robot/model/RobotResultTest.java @@ -23,6 +23,7 @@ import hudson.plugins.robot.RobotParserTest; import org.apache.commons.lang.StringUtils; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import static org.junit.Assert.*; @@ -92,8 +93,10 @@ public void testShouldParseMultipleSameNamedSuites(){ assertNotNull(result.getSuite("Somecases_1")); } + @Deprecated @Test - //TODO; should add tests for all parsed fields? Refactor name to parsertest + @Ignore + // TODO: remove test when criticality is removed public void testShouldParseCriticalCases(){ assertEquals(19, result.getCriticalTotal()); } @@ -107,7 +110,10 @@ public void testShouldParseFailMessages(){ assertEquals("Test failed miserably!", errorMsg.trim()); } + @Deprecated @Test + @Ignore + // TODO: remove test when criticality is removed public void testShouldParseNewCriticalCases() throws Exception{ RobotParser.RobotParserCallable remoteOperation = new RobotParser.RobotParserCallable("new_critical_output.xml", null, null); @@ -127,12 +133,17 @@ public void testShouldParseFailedCases(){ assertEquals(10, result.getOverallFailed()); } + @Deprecated @Test + @Ignore public void testShouldParseFailedCriticalCases(){ assertEquals(9, result.getCriticalFailed()); } + @Deprecated @Test + @Ignore + // TODO: remove test when criticality is removed public void testShouldParseFailedNewCriticalCases() throws Exception{ RobotParser.RobotParserCallable remoteOperation = new RobotParser.RobotParserCallable("new_critical_output.xml", null, null); result = remoteOperation.invoke(new File(RobotSuiteResultTest.class.getResource("new_critical_output.xml").toURI()).getParentFile(), null); @@ -361,4 +372,22 @@ public void testCaseResultsShouldBeCorrectlySet() throws Exception { assertEquals(0.001748, caseResult.getElapsedtime(), 0.01); assertNull(caseResult.getEndtime()); } + + @Test + public void testGetPassPercentageWithoutSkippedTests() throws Exception { + RobotParser.RobotParserCallable remoteOperation = new RobotParser.RobotParserCallable("robot4_skip.xml", null, null); + result = remoteOperation.invoke(new File(RobotSuiteResultTest.class.getResource("robot4_skip.xml").toURI()).getParentFile(), null); + result.tally(null); + + assertEquals(66.6, result.getPassPercentage(false), 0); + } + + @Test + public void testGetPassPercentageWithSkippedTests() throws Exception { + RobotParser.RobotParserCallable remoteOperation = new RobotParser.RobotParserCallable("robot4_skip.xml", null, null); + result = remoteOperation.invoke(new File(RobotSuiteResultTest.class.getResource("robot4_skip.xml").toURI()).getParentFile(), null); + result.tally(null); + + assertEquals(33.3, result.getPassPercentage(true), 0); + } } diff --git a/src/test/java/hudson/plugins/robot/tokens/RobotFailTokenMacroTest.java b/src/test/java/hudson/plugins/robot/tokens/RobotFailTokenMacroTest.java index 3036fcf9..623a760c 100644 --- a/src/test/java/hudson/plugins/robot/tokens/RobotFailTokenMacroTest.java +++ b/src/test/java/hudson/plugins/robot/tokens/RobotFailTokenMacroTest.java @@ -26,7 +26,6 @@ public void setUp(){ RobotResult result = Mockito.mock(RobotResult.class); Mockito.when(result.getOverallFailed()).thenReturn(6l); - Mockito.when(result.getCriticalFailed()).thenReturn(5l); Mockito.when(action.getResult()).thenReturn(result); Mockito.when(build.getAction(RobotBuildAction.class)).thenReturn(action); } @@ -37,7 +36,7 @@ public void testAcceptsName(){ public void testTokenConversionWithCritical() throws MacroEvaluationException, IOException, InterruptedException{ token.onlyCritical = true; - assertEquals("5",token.evaluate(build, listener, macroName)); + assertEquals("6",token.evaluate(build, listener, macroName)); } public void testTokenConversionWithAll() throws MacroEvaluationException, IOException, InterruptedException{ diff --git a/src/test/java/hudson/plugins/robot/tokens/RobotPassPercentageTokenMacroTest.java b/src/test/java/hudson/plugins/robot/tokens/RobotPassPercentageTokenMacroTest.java index d148fa53..b938dc21 100644 --- a/src/test/java/hudson/plugins/robot/tokens/RobotPassPercentageTokenMacroTest.java +++ b/src/test/java/hudson/plugins/robot/tokens/RobotPassPercentageTokenMacroTest.java @@ -39,13 +39,14 @@ public void testAcceptsName(){ assertTrue(new RobotPassPercentageTokenMacro().acceptsMacroName(macroName)); } + // TODO: remove test when criticality is removed public void testTokenConversionWithOnlyCritical() throws MacroEvaluationException, IOException, InterruptedException{ - token.onlyCritical = true; + token.countSkippedTests = true; assertEquals("55.0",token.evaluate(build, listener, macroName)); } public void testTokenConversionWithAll() throws MacroEvaluationException, IOException, InterruptedException{ - token.onlyCritical = false; + token.countSkippedTests = false; assertEquals("41.0",token.evaluate(build, listener, macroName)); } } diff --git a/src/test/java/hudson/plugins/robot/tokens/RobotPassRatioTokenMacroTest.java b/src/test/java/hudson/plugins/robot/tokens/RobotPassRatioTokenMacroTest.java index 540c4a7d..3fdfdc73 100644 --- a/src/test/java/hudson/plugins/robot/tokens/RobotPassRatioTokenMacroTest.java +++ b/src/test/java/hudson/plugins/robot/tokens/RobotPassRatioTokenMacroTest.java @@ -29,8 +29,6 @@ public void setUp(){ Mockito.when(result.getOverallPassed()).thenReturn(6l); Mockito.when(result.getOverallTotal()).thenReturn(13l); - Mockito.when(result.getCriticalPassed()).thenReturn(5l); - Mockito.when(result.getCriticalTotal()).thenReturn(5l); Mockito.when(action.getResult()).thenReturn(result); Mockito.when(build.getAction(RobotBuildAction.class)).thenReturn(action); } @@ -39,9 +37,10 @@ public void testAcceptsName(){ assertTrue(new RobotPassRatioTokenMacro().acceptsMacroName(macroName)); } + // TODO: remove test when criticality is removed public void testTokenConversionWithCritical() throws MacroEvaluationException, IOException, InterruptedException{ token.onlyCritical = true; - assertEquals("5 / 5",token.evaluate(build, listener, macroName)); + assertEquals("6 / 13",token.evaluate(build, listener, macroName)); } public void testTokenConversionWithAll() throws MacroEvaluationException, IOException, InterruptedException{ diff --git a/src/test/java/hudson/plugins/robot/tokens/RobotPassTokenMacroTest.java b/src/test/java/hudson/plugins/robot/tokens/RobotPassTokenMacroTest.java index 08d9763e..f45df0de 100644 --- a/src/test/java/hudson/plugins/robot/tokens/RobotPassTokenMacroTest.java +++ b/src/test/java/hudson/plugins/robot/tokens/RobotPassTokenMacroTest.java @@ -26,7 +26,6 @@ public void setUp(){ RobotResult result = Mockito.mock(RobotResult.class); Mockito.when(result.getOverallPassed()).thenReturn(6l); - Mockito.when(result.getCriticalPassed()).thenReturn(5l); Mockito.when(action.getResult()).thenReturn(result); Mockito.when(build.getAction(RobotBuildAction.class)).thenReturn(action); } @@ -34,10 +33,11 @@ public void setUp(){ public void testAcceptsName(){ assertTrue(token.acceptsMacroName(macroName)); } - + +// TODO: remove test when criticality is removed public void testTokenConversionWithCritical() throws MacroEvaluationException, IOException, InterruptedException{ token.onlyCritical = true; - assertEquals("5",token.evaluate(build, listener, macroName)); + assertEquals("6",token.evaluate(build, listener, macroName)); } public void testTokenConversionWithAll() throws MacroEvaluationException, IOException, InterruptedException{ diff --git a/src/test/java/hudson/plugins/robot/tokens/RobotTotalTokenMacroTest.java b/src/test/java/hudson/plugins/robot/tokens/RobotTotalTokenMacroTest.java index 3a6e6ce3..1b77aa0f 100644 --- a/src/test/java/hudson/plugins/robot/tokens/RobotTotalTokenMacroTest.java +++ b/src/test/java/hudson/plugins/robot/tokens/RobotTotalTokenMacroTest.java @@ -26,7 +26,6 @@ public void setUp(){ RobotResult result = Mockito.mock(RobotResult.class); Mockito.when(result.getOverallTotal()).thenReturn(6l); - Mockito.when(result.getCriticalTotal()).thenReturn(5l); Mockito.when(action.getResult()).thenReturn(result); Mockito.when(build.getAction(RobotBuildAction.class)).thenReturn(action); } @@ -37,7 +36,7 @@ public void testAcceptsName(){ public void testTokenConversionWithCritical() throws MacroEvaluationException, IOException, InterruptedException{ token.onlyCritical = true; - assertEquals("5",token.evaluate(build, listener, macroName)); + assertEquals("6",token.evaluate(build, listener, macroName)); } public void testTokenConversionWithAll() throws MacroEvaluationException, IOException, InterruptedException{