diff --git a/build.gradle b/build.gradle index c99048e6ae..2e63741870 100644 --- a/build.gradle +++ b/build.gradle @@ -257,6 +257,18 @@ configure(subprojects.findAll { !it.name.contains("android") }) { classpath = configurations.testRuntimeClasspath } + tasks.register("jgivenPlainTextReport",JavaExec) { + //noinspection GroovyAccessibility + mainClass = 'com.tngtech.jgiven.report.ReportGenerator' + args '--sourceDir=build/reports/jgiven/json', + '--targetDir=build/reports/jgiven/text', + '--format=text', + '--exclude-empty-scenarios=true', + '--title=JGiven Report' + + classpath = configurations.testRuntimeClasspath + } + asciidoctor { sourceDir = new File('build/reports/jgiven/asciidoc') outputDir = new File('build/reports/jgiven/htmladoc') diff --git a/jgiven-core/src/main/java/com/tngtech/jgiven/impl/ScenarioBase.java b/jgiven-core/src/main/java/com/tngtech/jgiven/impl/ScenarioBase.java index 69244c33ed..921661a79c 100644 --- a/jgiven-core/src/main/java/com/tngtech/jgiven/impl/ScenarioBase.java +++ b/jgiven-core/src/main/java/com/tngtech/jgiven/impl/ScenarioBase.java @@ -57,6 +57,10 @@ public void finished() throws Throwable { executor.finished(); } + public void aborted(Throwable e){ + executor.aborted(e); + } + public ScenarioExecutor getExecutor() { return executor; } diff --git a/jgiven-core/src/main/java/com/tngtech/jgiven/impl/ScenarioExecutor.java b/jgiven-core/src/main/java/com/tngtech/jgiven/impl/ScenarioExecutor.java index d8bab431ec..cb7b4cca51 100644 --- a/jgiven-core/src/main/java/com/tngtech/jgiven/impl/ScenarioExecutor.java +++ b/jgiven-core/src/main/java/com/tngtech/jgiven/impl/ScenarioExecutor.java @@ -74,6 +74,12 @@ enum State { */ private Throwable failedException; + /** + * Set if an exception was thrown during the execution of the scenario and + * suppressStepExceptions is true. + */ + private Throwable abortedException; + private boolean failIfPass; /** @@ -356,6 +362,7 @@ public void finished() throws Throwable { } } + private void callFinishLifeCycleMethods() throws Throwable { Throwable firstThrownException = failedException; if (beforeScenarioMethodsExecuted) { @@ -415,6 +422,10 @@ public boolean hasFailed() { return failedException != null; } + public boolean hasAborted() { + return abortedException!= null; + } + public Throwable getFailedException() { return failedException; } @@ -423,6 +434,14 @@ public void setFailedException(Exception e) { failedException = e; } + public Throwable getAbortedException() { + return abortedException; + } + + public void setAbortedException(Exception e) { + abortedException= e; + } + /** * Handle ocurred exception and continue. */ @@ -438,6 +457,16 @@ public void failed(Throwable e) { } } + public void aborted(Throwable e) { + if (hasAborted()){ + log.error(e.getMessage(), e); + }else { + listener.scenarioAborted(e); + methodInterceptor.disableMethodExecution(); + abortedException = e; + } + } + /** * Starts a scenario with the given description. * diff --git a/jgiven-core/src/main/java/com/tngtech/jgiven/impl/ScenarioModelBuilder.java b/jgiven-core/src/main/java/com/tngtech/jgiven/impl/ScenarioModelBuilder.java index d82e3cd8d9..33c8f6fb4b 100644 --- a/jgiven-core/src/main/java/com/tngtech/jgiven/impl/ScenarioModelBuilder.java +++ b/jgiven-core/src/main/java/com/tngtech/jgiven/impl/ScenarioModelBuilder.java @@ -314,6 +314,13 @@ public void stepMethodFailed(Throwable t) { } } + @Override + public void stepMethodAborted(Throwable t) { + if (currentStep != null) { + currentStep.setStatus(StepStatus.ABORTED); + } + } + @Override public void stepMethodFinished(long durationInNanos, boolean hasNestedSteps) { if (hasNestedSteps && !parentSteps.isEmpty()) { @@ -364,6 +371,12 @@ public void scenarioFailed(Throwable e) { setException(e); } + @Override + public void scenarioAborted(Throwable e){ + setStatus(ExecutionStatus.ABORTED); + setException(e); + } + private void setCaseDescription(Class testClass, Method method, List namedArguments) { CaseAs annotation = null; diff --git a/jgiven-core/src/main/java/com/tngtech/jgiven/impl/intercept/NoOpScenarioListener.java b/jgiven-core/src/main/java/com/tngtech/jgiven/impl/intercept/NoOpScenarioListener.java index 93af7e2b19..5870d950a2 100644 --- a/jgiven-core/src/main/java/com/tngtech/jgiven/impl/intercept/NoOpScenarioListener.java +++ b/jgiven-core/src/main/java/com/tngtech/jgiven/impl/intercept/NoOpScenarioListener.java @@ -13,6 +13,9 @@ public class NoOpScenarioListener implements ScenarioListener { @Override public void scenarioFailed( Throwable e ) {} + @Override + public void scenarioAborted(Throwable e) {} + @Override public void scenarioStarted( String string ) {} @@ -31,6 +34,9 @@ public void stepCommentUpdated( String comment ) {} @Override public void stepMethodFailed( Throwable t ) {} + @Override + public void stepMethodAborted( Throwable t ) {} + @Override public void stepMethodFinished( long durationInNanos, boolean hasNestedSteps ) {} diff --git a/jgiven-core/src/main/java/com/tngtech/jgiven/impl/intercept/ScenarioListener.java b/jgiven-core/src/main/java/com/tngtech/jgiven/impl/intercept/ScenarioListener.java index 2bee06eee7..98d3ac9360 100644 --- a/jgiven-core/src/main/java/com/tngtech/jgiven/impl/intercept/ScenarioListener.java +++ b/jgiven-core/src/main/java/com/tngtech/jgiven/impl/intercept/ScenarioListener.java @@ -12,6 +12,8 @@ public interface ScenarioListener { void scenarioFailed( Throwable e ); + void scenarioAborted(Throwable e); + void scenarioStarted( String string ); void scenarioStarted( Class testClass, Method method, List arguments ); @@ -24,6 +26,8 @@ public interface ScenarioListener { void stepMethodFailed( Throwable t ); + void stepMethodAborted( Throwable t ); + void stepMethodFinished( long durationInNanos, boolean hasNestedSteps ); void scenarioFinished(); diff --git a/jgiven-core/src/main/java/com/tngtech/jgiven/impl/intercept/StepInterceptorImpl.java b/jgiven-core/src/main/java/com/tngtech/jgiven/impl/intercept/StepInterceptorImpl.java index 24399af4bd..af500f33e1 100644 --- a/jgiven-core/src/main/java/com/tngtech/jgiven/impl/intercept/StepInterceptorImpl.java +++ b/jgiven-core/src/main/java/com/tngtech/jgiven/impl/intercept/StepInterceptorImpl.java @@ -109,9 +109,7 @@ private Object doIntercept(Object receiver, Method method, Object[] parameters, try { return invoker.proceed(); - } catch( Exception e ) { - return handleThrowable( receiver, method, e, System.nanoTime() - started, handleMethod ); - } catch( AssertionError e ) { + } catch(Exception | AssertionError e ) { return handleThrowable( receiver, method, e, System.nanoTime() - started, handleMethod ); } finally { if( hasNestedSteps ) { @@ -217,13 +215,13 @@ private void handleMethod(Object stageInstance, Method paramMethod, Object[] arg private void handleThrowable( Throwable t ) throws Throwable { if( ThrowableUtil.isAssumptionException(t) ) { - throw t; + listener.stepMethodAborted(t); + scenarioExecutor.aborted(t); + }else { + listener.stepMethodFailed(t); + scenarioExecutor.failed( t ); } - listener.stepMethodFailed( t ); - - scenarioExecutor.failed( t ); - if (!suppressExceptions) { throw t; } diff --git a/jgiven-core/src/main/java/com/tngtech/jgiven/report/asciidoc/AsciiDocBlockConverter.java b/jgiven-core/src/main/java/com/tngtech/jgiven/report/asciidoc/AsciiDocBlockConverter.java index 6b9bcac8f1..9b8a9bb9f1 100644 --- a/jgiven-core/src/main/java/com/tngtech/jgiven/report/asciidoc/AsciiDocBlockConverter.java +++ b/jgiven-core/src/main/java/com/tngtech/jgiven/report/asciidoc/AsciiDocBlockConverter.java @@ -39,6 +39,7 @@ public String convertStatisticsBlock(final ListMultimap featureFiles = new ArrayList<>(); private final List failedScenarioFiles = new ArrayList<>(); private final List pendingScenarioFiles = new ArrayList<>(); + private final List abortedScenarioFiles = new ArrayList<>(); private File targetDir; private File featuresDir; @@ -73,6 +74,8 @@ public void generate() { writeIndexFileForPendingScenarios(); + writeIndexFileForAbortedScenarios(); + writeTotalStatisticsFile(); writeIndexFileForFullReport(config.getTitle()); @@ -115,6 +118,9 @@ private List collectReportBlocks(final ReportModelFile reportModelFile, if (statistics.numPendingScenarios > 0) { pendingScenarioFiles.add(featureFileName); } + if (statistics.numAbortedScenarios >0){ + abortedScenarioFiles.add(featureFileName); + } final AsciiDocReportModelVisitor visitor = new AsciiDocReportModelVisitor(blockConverter, statistics); reportModelFile.model().accept(visitor); @@ -151,6 +157,16 @@ private void writeIndexFileForPendingScenarios() { snippetGenerator.generateIndexSnippet()); } + private void writeIndexFileForAbortedScenarios() { + final String scenarioKind = "aborted"; + final AsciiDocSnippetGenerator snippetGenerator = new AsciiDocSnippetGenerator( + "Aborted Scenarios", "aborted scenarios", this.abortedScenarioFiles, scenarioKind, + this.completeReportModel.getTotalStatistics().numAbortedScenarios); + + writeAsciiDocBlocksToFile(new File(targetDir, scenarioKind + "Scenarios.asciidoc"), + snippetGenerator.generateIndexSnippet()); + } + private void writeTotalStatisticsFile() { final ListMultimap featureStatistics = completeReportModel.getAllReportModels() diff --git a/jgiven-core/src/main/java/com/tngtech/jgiven/report/asciidoc/MetadataMapper.java b/jgiven-core/src/main/java/com/tngtech/jgiven/report/asciidoc/MetadataMapper.java index 2ce1091791..7221f11ad2 100644 --- a/jgiven-core/src/main/java/com/tngtech/jgiven/report/asciidoc/MetadataMapper.java +++ b/jgiven-core/src/main/java/com/tngtech/jgiven/report/asciidoc/MetadataMapper.java @@ -8,6 +8,7 @@ final class MetadataMapper { private static final String ICON_CHECK_MARK = "icon:check-square[role=green]"; private static final String ICON_EXCLAMATION_MARK = "icon:exclamation-circle[role=red]"; private static final String ICON_BANNED = "icon:ban[role=silver]"; + private static final String ICON_TIMES_CIRCLE = "icon:times-circle[role=gray]"; private static final String ICON_STEP_FORWARD = "icon:step-forward[role=silver]"; private static final int NANOSECONDS_PER_MILLISECOND = 1000000; @@ -46,6 +47,8 @@ static String toHumanReadableStatus(final ExecutionStatus executionStatus) { return ICON_CHECK_MARK; case FAILED: return ICON_EXCLAMATION_MARK; + case ABORTED: + return ICON_TIMES_CIRCLE; default: return executionStatus.toString(); } diff --git a/jgiven-core/src/main/java/com/tngtech/jgiven/report/model/ExecutionStatus.java b/jgiven-core/src/main/java/com/tngtech/jgiven/report/model/ExecutionStatus.java index 98def5817a..c1054ee442 100644 --- a/jgiven-core/src/main/java/com/tngtech/jgiven/report/model/ExecutionStatus.java +++ b/jgiven-core/src/main/java/com/tngtech/jgiven/report/model/ExecutionStatus.java @@ -4,5 +4,6 @@ public enum ExecutionStatus { SCENARIO_PENDING, SUCCESS, FAILED, - SOME_STEPS_PENDING; + ABORTED, + SOME_STEPS_PENDING } \ No newline at end of file diff --git a/jgiven-core/src/main/java/com/tngtech/jgiven/report/model/ReportStatistics.java b/jgiven-core/src/main/java/com/tngtech/jgiven/report/model/ReportStatistics.java index 34ba93d330..bfeb2ddfa2 100644 --- a/jgiven-core/src/main/java/com/tngtech/jgiven/report/model/ReportStatistics.java +++ b/jgiven-core/src/main/java/com/tngtech/jgiven/report/model/ReportStatistics.java @@ -5,6 +5,7 @@ public class ReportStatistics { public int numScenarios; public int numFailedScenarios; public int numPendingScenarios; + public int numAbortedScenarios; public int numSuccessfulScenarios; public int numCases; public int numFailedCases; @@ -23,6 +24,7 @@ private void addModifying( ReportStatistics statistics ) { this.numScenarios += statistics.numScenarios; this.numFailedScenarios += statistics.numFailedScenarios; this.numPendingScenarios += statistics.numPendingScenarios; + this.numAbortedScenarios += statistics.numAbortedScenarios; this.numSuccessfulScenarios += statistics.numSuccessfulScenarios; this.numCases += statistics.numCases; this.numFailedCases += statistics.numFailedCases; diff --git a/jgiven-core/src/main/java/com/tngtech/jgiven/report/model/StatisticsCalculator.java b/jgiven-core/src/main/java/com/tngtech/jgiven/report/model/StatisticsCalculator.java index 8a2454f46e..e3f3d89426 100644 --- a/jgiven-core/src/main/java/com/tngtech/jgiven/report/model/StatisticsCalculator.java +++ b/jgiven-core/src/main/java/com/tngtech/jgiven/report/model/StatisticsCalculator.java @@ -29,6 +29,8 @@ public void visit( ScenarioModel scenarioModel ) { statistics.numFailedScenarios += 1; } else if( executionStatus == ExecutionStatus.SCENARIO_PENDING || executionStatus == ExecutionStatus.SOME_STEPS_PENDING) { statistics.numPendingScenarios += 1; + } else if (executionStatus == ExecutionStatus.ABORTED){ + statistics.numAbortedScenarios += 1; } else { statistics.numSuccessfulScenarios += 1; } diff --git a/jgiven-core/src/main/java/com/tngtech/jgiven/report/model/StepStatus.java b/jgiven-core/src/main/java/com/tngtech/jgiven/report/model/StepStatus.java index d9d7d6808a..698a7477d3 100644 --- a/jgiven-core/src/main/java/com/tngtech/jgiven/report/model/StepStatus.java +++ b/jgiven-core/src/main/java/com/tngtech/jgiven/report/model/StepStatus.java @@ -4,5 +4,6 @@ public enum StepStatus { PASSED, FAILED, SKIPPED, - PENDING; + PENDING, + ABORTED } diff --git a/jgiven-core/src/main/java/com/tngtech/jgiven/report/text/PlainTextReportGenerator.java b/jgiven-core/src/main/java/com/tngtech/jgiven/report/text/PlainTextReportGenerator.java index d09a545f5b..47a81134a0 100644 --- a/jgiven-core/src/main/java/com/tngtech/jgiven/report/text/PlainTextReportGenerator.java +++ b/jgiven-core/src/main/java/com/tngtech/jgiven/report/text/PlainTextReportGenerator.java @@ -10,19 +10,32 @@ import com.tngtech.jgiven.report.AbstractReportGenerator; import com.tngtech.jgiven.report.model.ReportModel; import com.tngtech.jgiven.report.model.ReportModelFile; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class PlainTextReportGenerator extends AbstractReportGenerator { + private static final Logger log = LoggerFactory.getLogger( PlainTextReportGenerator.class ); + public AbstractReportConfig createReportConfig( String... args ) { return new PlainTextReportConfig(args); } public void generate() { + generateOutputDirectory(); for( ReportModelFile reportModelFile : completeReportModel.getAllReportModels() ) { handleReportModel(reportModelFile.model(), reportModelFile.file()); } } + private void generateOutputDirectory() { + var outputDir = config.getTargetDir(); + if( !outputDir.exists() && !outputDir.mkdirs()) { + log.error( "Could not create target directory " + outputDir); + return; + } + } + public void handleReportModel( ReportModel model, File file ) { String targetFileName = Files.getNameWithoutExtension( file.getName() ) + ".feature"; PrintWriter printWriter = PrintWriterUtil.getPrintWriter( new File( config.getTargetDir(), targetFileName ) ); @@ -33,4 +46,5 @@ public void handleReportModel( ReportModel model, File file ) { ResourceUtil.close( printWriter ); } } + } diff --git a/jgiven-core/src/test/java/com/tngtech/jgiven/report/asciidoc/AsciiDocFeatureBlockConverterTest.java b/jgiven-core/src/test/java/com/tngtech/jgiven/report/asciidoc/AsciiDocFeatureBlockConverterTest.java index b5f64bb273..3609317de1 100644 --- a/jgiven-core/src/test/java/com/tngtech/jgiven/report/asciidoc/AsciiDocFeatureBlockConverterTest.java +++ b/jgiven-core/src/test/java/com/tngtech/jgiven/report/asciidoc/AsciiDocFeatureBlockConverterTest.java @@ -17,9 +17,10 @@ public class AsciiDocFeatureBlockConverterTest { public void convert_feature_header_without_description() { // given final ReportStatistics statistics = new ReportStatistics(); - statistics.numScenarios = 42; + statistics.numScenarios = 47; statistics.numFailedScenarios = 21; statistics.numPendingScenarios = 13; + statistics.numAbortedScenarios = 5; statistics.numSuccessfulScenarios = 8; // when @@ -27,33 +28,34 @@ public void convert_feature_header_without_description() { // then assertThatBlockContainsLines(block, - "=== My first feature", - "", - "icon:check-square[role=green] 8 Successful, icon:exclamation-circle[role=red] 21 Failed, " - + "icon:ban[role=silver] 13 Pending, 42 Total (0ms)"); + "=== My first feature", + "", + "icon:check-square[role=green] 8 Successful, icon:exclamation-circle[role=red] 21 Failed, " + + "icon:ban[role=silver] 13 Pending, icon:times-circle[role=gray] 5 Aborted, 47 Total (0ms)"); } @Test public void convert_feature_header_with_description() { // given final ReportStatistics statistics = new ReportStatistics(); - statistics.numScenarios = 42; + statistics.numScenarios = 47; statistics.numFailedScenarios = 21; statistics.numPendingScenarios = 13; + statistics.numAbortedScenarios = 5; statistics.numSuccessfulScenarios = 8; // when final String block = - converter.convertFeatureHeaderBlock("My first feature", statistics, "A very nice feature."); + converter.convertFeatureHeaderBlock("My first feature", statistics, "A very nice feature."); // then assertThatBlockContainsLines(block, - "=== My first feature", - "", - "icon:check-square[role=green] 8 Successful, icon:exclamation-circle[role=red] 21 Failed, " - + "icon:ban[role=silver] 13 Pending, 42 Total (0ms)", - "", - "+++A very nice feature.+++"); + "=== My first feature", + "", + "icon:check-square[role=green] 8 Successful, icon:exclamation-circle[role=red] 21 Failed, " + + "icon:ban[role=silver] 13 Pending, icon:times-circle[role=gray] 5 Aborted, 47 Total (0ms)", + "", + "+++A very nice feature.+++"); } private static void assertThatBlockContainsLines(final String block, final String... expectedLines) { diff --git a/jgiven-core/src/test/java/com/tngtech/jgiven/report/asciidoc/AsciiDocTableBlockConverterTest.java b/jgiven-core/src/test/java/com/tngtech/jgiven/report/asciidoc/AsciiDocTableBlockConverterTest.java index df13815ea8..1750284e4b 100644 --- a/jgiven-core/src/test/java/com/tngtech/jgiven/report/asciidoc/AsciiDocTableBlockConverterTest.java +++ b/jgiven-core/src/test/java/com/tngtech/jgiven/report/asciidoc/AsciiDocTableBlockConverterTest.java @@ -130,6 +130,7 @@ public void convert_statistics() { statisticsOne.numScenarios = 3; statisticsOne.numSuccessfulScenarios = 2; statisticsOne.numFailedScenarios = 1; + statisticsOne.numAbortedScenarios = 3; statisticsOne.numCases = 3; statisticsOne.numFailedCases = 1; statisticsOne.numSteps = 13; @@ -149,6 +150,7 @@ public void convert_statistics() { totalStatistics.numSuccessfulScenarios = 3; totalStatistics.numPendingScenarios = 1; totalStatistics.numFailedScenarios = 1; + totalStatistics.numAbortedScenarios = 3; totalStatistics.numCases = 5; totalStatistics.numFailedCases = 1; totalStatistics.numSteps = 21; @@ -166,11 +168,11 @@ public void convert_statistics() { ".Total Statistics", "[.jg-statisticsTable%autowidth%header%footer]", "|===", - "| feature | total classes | successful scenarios | failed scenarios | pending scenarios | " - + "total scenarios | failed cases | total cases | total steps | duration", - "| Feature One | 1 | 2 | 1 | 0 | 3 | 1 | 3 | 13 | 0ms", - "| Feature Two | 1 | 1 | 0 | 1 | 2 | 1 | 2 | 8 | 0ms", - "| sum | 2 | 3 | 1 | 1 | 5 | 1 | 5 | 21 | 0ms", + "| feature | total classes | successful scenarios | failed scenarios | pending scenarios | aborted scenarios " + + "| total scenarios | failed cases | total cases | total steps | duration", + "| Feature One | 1 | 2 | 1 | 0 | 3 | 3 | 1 | 3 | 13 | 0ms", + "| Feature Two | 1 | 1 | 0 | 1 | 0 | 2 | 1 | 2 | 8 | 0ms", + "| sum | 2 | 3 | 1 | 1 | 3 | 5 | 1 | 5 | 21 | 0ms", "|==="); } diff --git a/jgiven-core/src/test/java/com/tngtech/jgiven/report/text/PlainTextReportGeneratorTest.java b/jgiven-core/src/test/java/com/tngtech/jgiven/report/text/PlainTextReportGeneratorTest.java new file mode 100644 index 0000000000..3b798332c3 --- /dev/null +++ b/jgiven-core/src/test/java/com/tngtech/jgiven/report/text/PlainTextReportGeneratorTest.java @@ -0,0 +1,35 @@ +package com.tngtech.jgiven.report.text; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; + +import static org.assertj.core.api.Assertions.assertThat; + +public class PlainTextReportGeneratorTest { + @Rule + public final TemporaryFolder tmpFolder = new TemporaryFolder(); + + @Test + public void can_create_an_output_folder() throws IOException { + var outputDir = Path.of(tmpFolder.getRoot().getAbsolutePath(), "report", "jgiven", "text").toFile(); + var sourceFile = getClass().getClassLoader() + .getResource("com.tngtech.jgiven.report.text.ResourceForPlainTextReportGeneratorTest.json") + .getFile(); + var outputConfig = "--targetDir=" + outputDir.getAbsolutePath(); + var sourceConfig = "--sourceDir=" + new File(sourceFile).getParent(); + var underTest = new PlainTextReportGenerator(); + + underTest.generateWithConfig(underTest.createReportConfig(sourceConfig, outputConfig)); + + + assertThat(outputDir).exists(); + assertThat(outputDir).isDirectory(); + assertThat(outputDir.list()) + .contains("com.tngtech.jgiven.report.text.ResourceForPlainTextReportGeneratorTest.feature"); + } +} \ No newline at end of file diff --git a/jgiven-core/src/test/resources/com.tngtech.jgiven.report.text.ResourceForPlainTextReportGeneratorTest.json b/jgiven-core/src/test/resources/com.tngtech.jgiven.report.text.ResourceForPlainTextReportGeneratorTest.json new file mode 100644 index 0000000000..5f91045598 --- /dev/null +++ b/jgiven-core/src/test/resources/com.tngtech.jgiven.report.text.ResourceForPlainTextReportGeneratorTest.json @@ -0,0 +1,27 @@ +{ + "className": "com.tngtech.jgiven.report.text.ResourceForPlainTextReportGeneratorTest", + "name": "Empty", + "scenarios": [ + { + "className": "com.tngtech.jgiven.report.text.ResourceForPlainTextReportGeneratorTest", + "testMethodName": "someNonScenario", + "description": "some non scenario", + "tagIds": [], + "explicitParameters": [], + "derivedParameters": [], + "scenarioCases": [ + { + "caseNr": 1, + "steps": [], + "explicitArguments": [], + "derivedArguments": [], + "status": "SUCCESS", + "durationInNanos": 352000 + } + ], + "casesAsTable": false, + "durationInNanos": 352000 + } + ], + "tagMap": {} +} \ No newline at end of file diff --git a/jgiven-examples/build.gradle b/jgiven-examples/build.gradle index 2daed68304..c14db27059 100644 --- a/jgiven-examples/build.gradle +++ b/jgiven-examples/build.gradle @@ -12,8 +12,9 @@ dependencies { testImplementation libs.testng } +test.finalizedBy(jgivenPlainTextReport) test.finalizedBy jgivenHtml5Report test.finalizedBy jgivenAsciiDocReport -// there is a test that fails on purpose to show a failing test in the report +// there are tests that fail on purpose to show a failing test in the report test.ignoreFailures = true diff --git a/jgiven-examples/src/test/java/com/tngtech/jgiven/examples/FailingScenarioTest.java b/jgiven-examples/src/test/java/com/tngtech/jgiven/examples/FailingScenarioTest.java index b87bb2e2ad..b7a1da7bce 100644 --- a/jgiven-examples/src/test/java/com/tngtech/jgiven/examples/FailingScenarioTest.java +++ b/jgiven-examples/src/test/java/com/tngtech/jgiven/examples/FailingScenarioTest.java @@ -3,9 +3,11 @@ import com.tngtech.jgiven.examples.tags.FailingOnPurpose; import com.tngtech.jgiven.junit.SimpleScenarioTest; import com.tngtech.jgiven.tags.Issue; +import org.junit.Assume; import org.junit.Test; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assumptions.assumeThat; @FailingOnPurpose public class FailingScenarioTest extends SimpleScenarioTest { @@ -18,10 +20,24 @@ public void a_scenario_with_a_multi_line_error_message() { given().multi_line_error_message(); } + + + @Test + @Issue("#1625") + @FailingOnPurpose + public void failing_assumption(){ + given().false_assumption(); + } + public static class Steps { public Steps multi_line_error_message() { assertThat(true).as("This\nmessage\nhas\nmultiple lines").isFalse(); return this; } + + public Steps false_assumption(){ + assumeThat(true).as("This assumption fails on purpose").isFalse(); + return this; + } } } diff --git a/jgiven-examples/src/test/java/com/tngtech/jgiven/examples/assumptions/AssumptionFailureTestScenarios.java b/jgiven-examples/src/test/java/com/tngtech/jgiven/examples/assumptions/AssumptionFailureTestScenarios.java new file mode 100644 index 0000000000..f714e16ae5 --- /dev/null +++ b/jgiven-examples/src/test/java/com/tngtech/jgiven/examples/assumptions/AssumptionFailureTestScenarios.java @@ -0,0 +1,46 @@ +package com.tngtech.jgiven.examples.assumptions; + +import com.tngtech.jgiven.annotation.ScenarioStage; +import com.tngtech.jgiven.junit.JGivenClassRule; +import com.tngtech.jgiven.junit.JGivenMethodRule; +import org.junit.Assume; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; + +@SuppressWarnings("NewClassNamingConvention") +public class AssumptionFailureTestScenarios { + + @ClassRule + public static final JGivenClassRule writerRule = new JGivenClassRule(); + + @Rule + public final JGivenMethodRule scenarioRule = new JGivenMethodRule(); + + @ScenarioStage + AssumptionFailureTestStage givenTestStage; + + @SuppressWarnings("DataFlowIssue") + @Test + public void test_with_failing_assumption() { + Assume.assumeFalse(true); + } + + @Test + public void test_with_failing_assumption_in_stage() { + givenTestStage.given().a_failed_junit_assumption().and() + .nothing(); + + } + + @Test + public void test_with_failing_assumption_in_second_stage(){ + givenTestStage.given().nothing().and() + .a_failed_junit_assumption(); + } + + @Test + public void test_with_failing_assumption_in_a_nested_stage(){ + givenTestStage.given().nothing().a_failed_nested_step().and().nothing(); + } +} diff --git a/jgiven-examples/src/test/java/com/tngtech/jgiven/examples/assumptions/AssumptionFailureTestStage.java b/jgiven-examples/src/test/java/com/tngtech/jgiven/examples/assumptions/AssumptionFailureTestStage.java new file mode 100644 index 0000000000..c7461a3b0d --- /dev/null +++ b/jgiven-examples/src/test/java/com/tngtech/jgiven/examples/assumptions/AssumptionFailureTestStage.java @@ -0,0 +1,24 @@ +package com.tngtech.jgiven.examples.assumptions; + +import com.tngtech.jgiven.Stage; +import com.tngtech.jgiven.annotation.NestedSteps; +import org.junit.Assume; + +public class AssumptionFailureTestStage extends Stage { + + public AssumptionFailureTestStage nothing() { + return self(); + } + + @SuppressWarnings("DataFlowIssue") //fail on purpose + public AssumptionFailureTestStage a_failed_junit_assumption() { + Assume.assumeFalse(true); + return self(); + } + + @NestedSteps + public AssumptionFailureTestStage a_failed_nested_step() { + self().a_failed_junit_assumption(); + return self(); + } +} diff --git a/jgiven-junit/src/main/java/com/tngtech/jgiven/junit/JGivenMethodRule.java b/jgiven-junit/src/main/java/com/tngtech/jgiven/junit/JGivenMethodRule.java index e83793f399..e4b6eac3c5 100644 --- a/jgiven-junit/src/main/java/com/tngtech/jgiven/junit/JGivenMethodRule.java +++ b/jgiven-junit/src/main/java/com/tngtech/jgiven/junit/JGivenMethodRule.java @@ -1,16 +1,14 @@ package com.tngtech.jgiven.junit; -import static com.tngtech.jgiven.report.model.ExecutionStatus.FAILED; -import static com.tngtech.jgiven.report.model.ExecutionStatus.SUCCESS; -import static java.lang.String.format; -import static org.junit.Assume.assumeTrue; - -import java.lang.reflect.AccessibleObject; -import java.lang.reflect.Constructor; -import java.lang.reflect.Field; -import java.util.*; - -import org.junit.internal.AssumptionViolatedException; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Lists; +import com.google.common.primitives.Primitives; +import com.tngtech.jgiven.impl.ScenarioBase; +import com.tngtech.jgiven.impl.util.ParameterNameUtil; +import com.tngtech.jgiven.impl.util.ReflectionUtil; +import com.tngtech.jgiven.impl.util.ThrowableUtil; +import com.tngtech.jgiven.report.model.NamedArgument; +import com.tngtech.jgiven.report.model.ReportModel; import org.junit.rules.MethodRule; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -20,14 +18,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.Lists; -import com.google.common.primitives.Primitives; -import com.tngtech.jgiven.impl.ScenarioBase; -import com.tngtech.jgiven.impl.util.ParameterNameUtil; -import com.tngtech.jgiven.impl.util.ReflectionUtil; -import com.tngtech.jgiven.report.model.NamedArgument; -import com.tngtech.jgiven.report.model.ReportModel; +import java.lang.reflect.AccessibleObject; +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; +import java.util.*; + +import static com.tngtech.jgiven.report.model.ExecutionStatus.*; +import static java.lang.String.format; +import static org.junit.Assume.assumeTrue; /** * JUnit Rule to enable JGiven in a JUnit test @@ -38,7 +36,7 @@ public class JGivenMethodRule implements MethodRule { private static final String DATAPROVIDER_FRAMEWORK_METHOD = "com.tngtech.java.junit.dataprovider.DataProviderFrameworkMethod"; private static final String JUNITPARAMS_STATEMENT = "junitparams.internal.InvokeParameterisedMethod"; - private static final Logger log = LoggerFactory.getLogger( JGivenMethodRule.class ); + private static final Logger log = LoggerFactory.getLogger(JGivenMethodRule.class); protected final ScenarioBase scenario; @@ -52,7 +50,7 @@ public JGivenMethodRule() { /** * @since 0.8.1 */ - public JGivenMethodRule(ScenarioBase scenario ) { + public JGivenMethodRule(ScenarioBase scenario) { this.scenario = scenario; } @@ -64,18 +62,28 @@ public ScenarioBase getScenario() { } @Override - public Statement apply( final Statement base, final FrameworkMethod method, final Object target ) { + public Statement apply(final Statement base, final FrameworkMethod method, final Object target) { return new Statement() { @Override public void evaluate() throws Throwable { - starting( base, method, target ); + //TODO: make this nice! + starting(base, method, target); try { base.evaluate(); + } catch (Throwable t) { + if (ThrowableUtil.isAssumptionException(t)) { + aborted(t); + } else { + failed(t); + } + throw t; + } + try { succeeded(); - } catch( AssumptionViolatedException e ) { - throw e; - } catch( Throwable t ) { - failed( t ); + } catch (Throwable t) { + if (!ThrowableUtil.isAssumptionException(t)) { + failed(t); + } throw t; } } @@ -86,84 +94,95 @@ protected void succeeded() throws Throwable { scenario.finished(); // ignore test when scenario is not implemented - assumeTrue( EnumSet.of( SUCCESS, FAILED ).contains( scenario.getScenarioModel().getExecutionStatus() ) ); + assumeTrue(EnumSet.of(SUCCESS, FAILED, ABORTED).contains(scenario.getScenarioModel().getExecutionStatus())); } - protected void failed( Throwable e ) throws Throwable { - if( scenario.getExecutor().hasFailed() ) { + protected void failed(Throwable e) throws Throwable { + if (scenario.getExecutor().hasFailed()) { Throwable failedException = scenario.getExecutor().getFailedException(); - List errors = Lists.newArrayList( failedException, e ); - scenario.getExecutor().setFailedException( new MultipleFailureException( errors ) ); + List errors = Lists.newArrayList(failedException, e); + scenario.getExecutor().setFailedException(new MultipleFailureException(errors)); } else { - scenario.getExecutor().failed( e ); + scenario.getExecutor().failed(e); } scenario.finished(); } - protected void starting( Statement base, FrameworkMethod testMethod, Object target ) { - ReportModel reportModel = ScenarioModelHolder.getInstance().getReportModel( target.getClass() ); - scenario.setModel( reportModel ); - scenario.getExecutor().injectStages( target ); + protected void aborted(Throwable e) throws Throwable { + if (scenario.getExecutor().hasAborted()) { + Throwable failedException = scenario.getExecutor().getAbortedException(); + List errors = Lists.newArrayList(failedException, e); + scenario.getExecutor().setAbortedException(new MultipleFailureException(errors)); + } else { + scenario.getExecutor().aborted(e); + } + scenario.finished(); + } + + protected void starting(Statement base, FrameworkMethod testMethod, Object target) { + ReportModel reportModel = ScenarioModelHolder.getInstance().getReportModel(target.getClass()); + scenario.setModel(reportModel); + scenario.getExecutor().injectStages(target); - scenario.startScenario( target.getClass(), testMethod.getMethod(), getNamedArguments( base, testMethod, target ) ); + scenario.startScenario(target.getClass(), testMethod.getMethod(), getNamedArguments(base, testMethod, target)); // inject state from the test itself - scenario.getExecutor().readScenarioState( target ); + scenario.getExecutor().readScenarioState(target); } @VisibleForTesting - static List getNamedArguments( Statement base, FrameworkMethod method, Object target ) { + static List getNamedArguments(Statement base, FrameworkMethod method, Object target) { AccessibleObject constructorOrMethod = method.getMethod(); List arguments = Collections.emptyList(); - if( DATAPROVIDER_FRAMEWORK_METHOD.equals( method.getClass().getCanonicalName() ) ) { - arguments = getArgumentsFrom( method, "parameters" ); + if (DATAPROVIDER_FRAMEWORK_METHOD.equals(method.getClass().getCanonicalName())) { + arguments = getArgumentsFrom(method, "parameters"); } - if( JUNITPARAMS_STATEMENT.equals( base.getClass().getCanonicalName() ) ) { - arguments = getArgumentsFrom( base, "params" ); + if (JUNITPARAMS_STATEMENT.equals(base.getClass().getCanonicalName())) { + arguments = getArgumentsFrom(base, "params"); } - if( isParameterizedTest( target ) ) { - Constructor constructor = getOnlyConstructor( target.getClass() ); + if (isParameterizedTest(target)) { + Constructor constructor = getOnlyConstructor(target.getClass()); constructorOrMethod = constructor; - arguments = getArgumentsFrom( constructor, target ); + arguments = getArgumentsFrom(constructor, target); } - return ParameterNameUtil.mapArgumentsWithParameterNames( constructorOrMethod, arguments ); + return ParameterNameUtil.mapArgumentsWithParameterNames(constructorOrMethod, arguments); } - private static List getArgumentsFrom( Object object, String fieldName ) { + private static List getArgumentsFrom(Object object, String fieldName) { Class methodClass = object.getClass(); try { - Field field = methodClass.getDeclaredField( fieldName ); - field.setAccessible( true ); - return Arrays.asList( (Object[]) field.get( object ) ); - - } catch( NoSuchFieldException e ) { - log.warn( format( "Could not find field containing test method arguments in '%s'. " - + "Probably the internal representation has changed. Consider writing a bug report.", - methodClass.getSimpleName() ), e ); - } catch( IllegalAccessException e ) { - log.warn( format( "Not able to access field containing test method arguments in '%s'. " - + "Probably the internal representation has changed. Consider writing a bug report.", - methodClass.getSimpleName() ), e ); + Field field = methodClass.getDeclaredField(fieldName); + field.setAccessible(true); + return Arrays.asList((Object[]) field.get(object)); + + } catch (NoSuchFieldException e) { + log.warn(format("Could not find field containing test method arguments in '%s'. " + + "Probably the internal representation has changed. Consider writing a bug report.", + methodClass.getSimpleName()), e); + } catch (IllegalAccessException e) { + log.warn(format("Not able to access field containing test method arguments in '%s'. " + + "Probably the internal representation has changed. Consider writing a bug report.", + methodClass.getSimpleName()), e); } return Collections.emptyList(); } - private static boolean isParameterizedTest( Object target ) { - RunWith runWith = target.getClass().getAnnotation( RunWith.class ); - return runWith != null && Parameterized.class.equals( runWith.value() ); + private static boolean isParameterizedTest(Object target) { + RunWith runWith = target.getClass().getAnnotation(RunWith.class); + return runWith != null && Parameterized.class.equals(runWith.value()); } - private static Constructor getOnlyConstructor( Class testClass ) { + private static Constructor getOnlyConstructor(Class testClass) { Constructor[] constructors = testClass.getConstructors(); - if( constructors.length != 1 ) { - log.warn( "Test class can only have one public constructor, " - + "see org.junit.runners.Parameterized.TestClassRunnerForParameters.validateConstructor(List)" ); + if (constructors.length != 1) { + log.warn("Test class can only have one public constructor, " + + "see org.junit.runners.Parameterized.TestClassRunnerForParameters.validateConstructor(List)"); } return constructors[0]; } @@ -176,35 +195,35 @@ private static Constructor getOnlyConstructor( Class testClass ) { * type, the order of {@link ReflectionUtil#getAllNonStaticFieldValuesFrom(Class, Object, String)} is used. * * @param constructor {@link Constructor} from which argument types should be retrieved - * @param target {@link Parameterized} test instance from which arguments tried to be retrieved + * @param target {@link Parameterized} test instance from which arguments tried to be retrieved * @return the determined arguments, never {@code null} */ - private static List getArgumentsFrom( Constructor constructor, Object target ) { + private static List getArgumentsFrom(Constructor constructor, Object target) { Class testClass = target.getClass(); Class[] constructorParamClasses = constructor.getParameterTypes(); - List fieldValues = ReflectionUtil.getAllNonStaticFieldValuesFrom( testClass, target, - " Consider writing a bug report." ); + List fieldValues = ReflectionUtil.getAllNonStaticFieldValuesFrom(testClass, target, + " Consider writing a bug report."); - return getTypeMatchingValuesInOrderOf( constructorParamClasses, fieldValues ); + return getTypeMatchingValuesInOrderOf(constructorParamClasses, fieldValues); } - private static List getTypeMatchingValuesInOrderOf( Class[] expectedClasses, List values ) { - List valuesCopy = Lists.newArrayList( values ); - List result = new ArrayList(); - for( Class argumentClass : expectedClasses ) { - for( Iterator iterator = valuesCopy.iterator(); iterator.hasNext(); ) { + private static List getTypeMatchingValuesInOrderOf(Class[] expectedClasses, List values) { + List valuesCopy = Lists.newArrayList(values); + List result = new ArrayList<>(); + for (Class argumentClass : expectedClasses) { + for (Iterator iterator = valuesCopy.iterator(); iterator.hasNext(); ) { Object value = iterator.next(); - if( Primitives.wrap( argumentClass ).isInstance( value ) ) { - result.add( value ); + if (Primitives.wrap(argumentClass).isInstance(value)) { + result.add(value); iterator.remove(); break; } } } - if( result.size() < expectedClasses.length ) { - log.warn( format( "Couldn't find matching values in '%s' for expected classes '%s',", valuesCopy, - Arrays.toString( expectedClasses ) ) ); + if (result.size() < expectedClasses.length) { + log.warn(format("Couldn't find matching values in '%s' for expected classes '%s',", valuesCopy, + Arrays.toString(expectedClasses))); } return result; } diff --git a/jgiven-junit/src/test/java/com/tngtech/jgiven/junit/AssumptionTest.java b/jgiven-junit/src/test/java/com/tngtech/jgiven/junit/AssumptionTest.java index df506bfbab..91248797d0 100644 --- a/jgiven-junit/src/test/java/com/tngtech/jgiven/junit/AssumptionTest.java +++ b/jgiven-junit/src/test/java/com/tngtech/jgiven/junit/AssumptionTest.java @@ -4,32 +4,36 @@ import com.tngtech.jgiven.report.model.ScenarioCaseModel; import com.tngtech.jgiven.report.model.StepStatus; import org.junit.Assume; +import org.junit.AssumptionViolatedException; import org.junit.Test; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.assertj.core.api.Assumptions.assumeThat; -@Description("Scenarios can have sections") +@Description("Scenarios can have assumptions") public class AssumptionTest extends SimpleScenarioTest { @Test - public void should_pass_on_assertJ_assumptions() throws Throwable { - assertThatThrownBy(() -> when().I_assume_something_using_assertJ()) - .isInstanceOf(catchException(AssumptionTest::assertJAssumptionFailure)); + public void should_report_aborted_on_assertJ_assumptions() throws Throwable { + when().I_assume_something_using_assertJ(); getScenario().finished(); + + assertThat(getScenario().getExecutor().hasAborted()).isTrue(); + assertThat(getScenario().getExecutor().getAbortedException()).isInstanceOf(AssumptionViolatedException.class); ScenarioCaseModel aCase = getScenario().getModel().getLastScenarioModel().getCase(0); - assertThat(aCase.getStep(0).getStatus()).isEqualTo(StepStatus.PASSED); + assertThat(aCase.getStep(0).getStatus()).isEqualTo(StepStatus.ABORTED); } @Test - public void should_pass_on_junit5_assumptions() throws Throwable { - assertThatThrownBy(() -> when().I_assume_something_using_junit5()) - .isInstanceOf(catchException(AssumptionTest::junitAssumptionFailure)); + public void should_report_aborted_on_junit_assumptions() throws Throwable { + when().I_assume_something_using_junit5(); getScenario().finished(); + + assertThat(getScenario().getExecutor().hasAborted()).isTrue(); + assertThat(getScenario().getExecutor().getAbortedException()).isInstanceOf(org.junit.AssumptionViolatedException.class); ScenarioCaseModel aCase = getScenario().getModel().getLastScenarioModel().getCase(0); - assertThat(aCase.getStep(0).getStatus()).isEqualTo(StepStatus.PASSED); + assertThat(aCase.getStep(0).getStatus()).isEqualTo(StepStatus.ABORTED); } static class TestStage { @@ -50,13 +54,4 @@ private static void assertJAssumptionFailure() { private static void junitAssumptionFailure() { Assume.assumeTrue(false); } - - private Class catchException(Runnable runnable) { - try { - runnable.run(); - return null; - } catch (Exception e) { - return e.getClass(); - } - } } \ No newline at end of file diff --git a/jgiven-junit5/src/main/java/com/tngtech/jgiven/junit5/JGivenExtension.java b/jgiven-junit5/src/main/java/com/tngtech/jgiven/junit5/JGivenExtension.java index 27c5cb0856..fddf7e411a 100644 --- a/jgiven-junit5/src/main/java/com/tngtech/jgiven/junit5/JGivenExtension.java +++ b/jgiven-junit5/src/main/java/com/tngtech/jgiven/junit5/JGivenExtension.java @@ -1,14 +1,12 @@ package com.tngtech.jgiven.junit5; -import static com.tngtech.jgiven.report.model.ExecutionStatus.FAILED; -import static com.tngtech.jgiven.report.model.ExecutionStatus.SUCCESS; - import com.tngtech.jgiven.base.ScenarioTestBase; import com.tngtech.jgiven.config.AbstractJGivenConfiguration; import com.tngtech.jgiven.config.ConfigurationUtil; import com.tngtech.jgiven.exception.JGivenWrongUsageException; import com.tngtech.jgiven.impl.ScenarioBase; import com.tngtech.jgiven.impl.ScenarioHolder; +import com.tngtech.jgiven.impl.util.ThrowableUtil; import com.tngtech.jgiven.report.impl.CommonReportHelper; import com.tngtech.jgiven.report.model.ReportModel; import java.util.EnumSet; @@ -23,6 +21,8 @@ import org.junit.jupiter.api.extension.ExtensionContext.Namespace; import org.junit.jupiter.api.extension.TestInstancePostProcessor; +import static com.tngtech.jgiven.report.model.ExecutionStatus.*; + /** * This extension enables JGiven for JUnit 5 Tests. *

@@ -85,14 +85,14 @@ public void beforeEach(ExtensionContext context) { public void afterTestExecution(ExtensionContext context) throws Exception { ScenarioBase scenario = getScenario(); try { - if (context.getExecutionException().isPresent()) { + if (context.getExecutionException().isPresent() && !ThrowableUtil.isAssumptionException(context.getExecutionException().get())) { scenario.getExecutor().failed(context.getExecutionException().get()); } scenario.finished(); // ignore test when scenario is not implemented Assumptions.assumeTrue( - EnumSet.of(SUCCESS, FAILED).contains(scenario.getScenarioModel().getExecutionStatus())); + EnumSet.of(SUCCESS, FAILED, ABORTED).contains(scenario.getScenarioModel().getExecutionStatus())); } catch (Exception e) { throw e; diff --git a/jgiven-junit5/src/test/java/com/tngtech/jgiven/junit5/test/AssumptionsTest.java b/jgiven-junit5/src/test/java/com/tngtech/jgiven/junit5/test/AssumptionsTest.java index d50904a23a..ffbec84bc6 100644 --- a/jgiven-junit5/src/test/java/com/tngtech/jgiven/junit5/test/AssumptionsTest.java +++ b/jgiven-junit5/src/test/java/com/tngtech/jgiven/junit5/test/AssumptionsTest.java @@ -7,7 +7,6 @@ import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.assertj.core.api.Assumptions.assumeThat; class AssumptionsTest extends SimpleScenarioTest { @@ -15,20 +14,24 @@ class AssumptionsTest extends SimpleScenarioTest { @Test void should_pass_on_assertJ_assumptions() throws Throwable { - assertThatThrownBy(() -> when().I_assume_something_using_assertJ()) - .isInstanceOf(catchException(AssumptionsTest::assertJAssumptionFailure)); + when().I_assume_something_using_assertJ(); getScenario().finished(); - ScenarioCaseModel aCase = getScenario().getModel().getLastScenarioModel().getCase( 0 ); - assertThat( aCase.getStep( 0 ).getStatus() ).isEqualTo( StepStatus.PASSED ); + + assertThat(getScenario().getExecutor().hasAborted()).isTrue(); + assertThat(getScenario().getExecutor().getAbortedException()).isInstanceOf(org.junit.AssumptionViolatedException.class); + ScenarioCaseModel aCase = getScenario().getModel().getLastScenarioModel().getCase(0); + assertThat(aCase.getStep(0).getStatus()).isEqualTo(StepStatus.ABORTED); } @Test void should_pass_on_junit5_assumptions() throws Throwable { - assertThatThrownBy(() -> when().I_assume_something_using_junit5()) - .isInstanceOf(catchException(AssumptionsTest::junitAssumptionFailure)); + when().I_assume_something_using_junit5(); getScenario().finished(); - ScenarioCaseModel aCase = getScenario().getModel().getLastScenarioModel().getCase( 0 ); - assertThat( aCase.getStep( 0 ).getStatus() ).isEqualTo( StepStatus.PASSED ); + + assertThat(getScenario().getExecutor().hasAborted()).isTrue(); + assertThat(getScenario().getExecutor().getAbortedException()).isInstanceOf(org.opentest4j.TestAbortedException.class); + ScenarioCaseModel aCase = getScenario().getModel().getLastScenarioModel().getCase(0); + assertThat(aCase.getStep(0).getStatus()).isEqualTo(StepStatus.ABORTED); } static class TestStage { @@ -48,12 +51,4 @@ private static void assertJAssumptionFailure(){ private static void junitAssumptionFailure(){ Assumptions.abort(); } - private Class catchException(Runnable runnable) { - try { - runnable.run(); - return null; - } catch (Exception e) { - return e.getClass(); - } - } } diff --git a/jgiven-junit5/src/test/java/com/tngtech/jgiven/junit5/test/JUnit5NoScenarioTest.java b/jgiven-junit5/src/test/java/com/tngtech/jgiven/junit5/test/JUnit5NoScenarioTest.java new file mode 100644 index 0000000000..f29ec43cc9 --- /dev/null +++ b/jgiven-junit5/src/test/java/com/tngtech/jgiven/junit5/test/JUnit5NoScenarioTest.java @@ -0,0 +1,42 @@ +package com.tngtech.jgiven.junit5.test; + +import com.tngtech.jgiven.Stage; +import com.tngtech.jgiven.annotation.ScenarioStage; +import com.tngtech.jgiven.impl.ScenarioHolder; +import com.tngtech.jgiven.junit5.JGivenExtension; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +import static org.assertj.core.api.Assertions.assertThat; + +@ExtendWith(JGivenExtension.class) +class JUnit5NoScenarioTest { + + @ScenarioStage + TestStage testStage; + + @Test + void running_jgiven_without_scenario(){ + testStage.given().a_class_with_only_an_extension(); + testStage.when().i_run_the_test(); + testStage.then().i_get_a_report(); + + } + + + @SuppressWarnings("UnusedReturnValue") + static class TestStage extends Stage{ + TestStage a_class_with_only_an_extension(){ + return self(); + } + TestStage i_run_the_test(){ + return self(); + } + + TestStage i_get_a_report(){ + var myScenario = ScenarioHolder.get().getScenarioOfCurrentThread(); + assertThat(myScenario.getModel().getClassName()).isEqualTo(JUnit5NoScenarioTest.class.getName()); + return self(); + } + } +} diff --git a/jgiven-testng/src/main/java/com/tngtech/jgiven/testng/ScenarioTestListener.java b/jgiven-testng/src/main/java/com/tngtech/jgiven/testng/ScenarioTestListener.java index 9d19e91155..0229a4665b 100644 --- a/jgiven-testng/src/main/java/com/tngtech/jgiven/testng/ScenarioTestListener.java +++ b/jgiven-testng/src/main/java/com/tngtech/jgiven/testng/ScenarioTestListener.java @@ -102,6 +102,7 @@ public void onTestFailure(ITestResult paramITestResult) { @Override public void onTestSkipped(ITestResult testResult) { + testFinished(testResult); } private void testFinished(ITestResult testResult) { diff --git a/jgiven-testng/src/test/java/com/tngtech/jgiven/testng/AssumptionsTest.java b/jgiven-testng/src/test/java/com/tngtech/jgiven/testng/AssumptionsTest.java index 70ad33d979..0646f2c8b4 100644 --- a/jgiven-testng/src/test/java/com/tngtech/jgiven/testng/AssumptionsTest.java +++ b/jgiven-testng/src/test/java/com/tngtech/jgiven/testng/AssumptionsTest.java @@ -19,16 +19,16 @@ public void should_pass_on_assertJ_assumptions() throws Throwable { .isInstanceOf(catchException(AssumptionsTest::assertJAssumptionFailure)); getScenario().finished(); ScenarioCaseModel aCase = getScenario().getModel().getLastScenarioModel().getCase( 0 ); - assertThat( aCase.getStep( 0 ).getStatus() ).isEqualTo( StepStatus.PASSED ); + assertThat( aCase.getStep( 0 ).getStatus() ).isEqualTo( StepStatus.ABORTED); } @Test - public void should_pass_on_junit5_assumptions() throws Throwable { + public void should_pass_on_testng_assumptions() throws Throwable { assertThatThrownBy(() -> when().I_assume_something_using_junit5()) .isInstanceOf(catchException(AssumptionsTest::testNgAssumptionFailure)); getScenario().finished(); ScenarioCaseModel aCase = getScenario().getModel().getLastScenarioModel().getCase( 0 ); - assertThat( aCase.getStep( 0 ).getStatus() ).isEqualTo( StepStatus.PASSED ); + assertThat( aCase.getStep( 0 ).getStatus() ).isEqualTo( StepStatus.ABORTED); } static class TestStage { diff --git a/jgiven-testng/src/test/java/com/tngtech/jgiven/testng/SkipExceptionTest.java b/jgiven-testng/src/test/java/com/tngtech/jgiven/testng/SkipExceptionTest.java deleted file mode 100644 index 4f2926cbf3..0000000000 --- a/jgiven-testng/src/test/java/com/tngtech/jgiven/testng/SkipExceptionTest.java +++ /dev/null @@ -1,29 +0,0 @@ -package com.tngtech.jgiven.testng; - -import static org.assertj.core.api.Assertions.assertThat; - -import org.assertj.core.api.Assertions; - -import com.tngtech.jgiven.annotation.Description; -import com.tngtech.jgiven.report.model.ScenarioCaseModel; -import com.tngtech.jgiven.report.model.StepStatus; -import org.testng.SkipException; -import org.testng.annotations.Test; - -@Description( "SkipException are handled correctly" ) -public class SkipExceptionTest extends SimpleScenarioTest { - - @Test - public void TestNG_skipped_exceptions_should_be_treated_correctly() throws Throwable { - try { - given().skipped_exception_is_thrown(); - Assertions.fail( "SkipException should have been thrown" ); - } catch( SkipException e ) {} - - getScenario().finished(); - - ScenarioCaseModel aCase = getScenario().getModel().getLastScenarioModel().getCase( 0 ); - assertThat( aCase.getStep( 0 ).getStatus() ).isEqualTo( StepStatus.PASSED ); - } - -} diff --git a/jgiven-tests/build.gradle b/jgiven-tests/build.gradle index 0b5d8156ae..98b09a7cd7 100644 --- a/jgiven-tests/build.gradle +++ b/jgiven-tests/build.gradle @@ -9,7 +9,9 @@ dependencies { implementation platform(libs.junit.bom) implementation 'org.junit.jupiter:junit-jupiter-api' implementation 'org.junit.jupiter:junit-jupiter-engine' + implementation(libs.slf4j.java.util.logging) implementation 'org.junit.platform:junit-platform-runner' + implementation 'org.junit.jupiter:junit-jupiter-params' implementation libs.testng implementation 'com.beust:jcommander:1.82' implementation libs.assertj @@ -18,5 +20,7 @@ dependencies { testImplementation 'io.github.bonigarcia:webdrivermanager:5.9.2' testImplementation 'org.apache.commons:commons-io:1.3.2' } -test.finalizedBy(jgivenHtml5Report) -test.finalizedBy(jgivenAsciiDocReport) +test { + finalizedBy(jgivenHtml5Report) + finalizedBy(jgivenAsciiDocReport) +} diff --git a/jgiven-tests/src/main/java/com/tngtech/jgiven/tests/GivenTestStage.java b/jgiven-tests/src/main/java/com/tngtech/jgiven/tests/GivenTestStage.java index 198ee9d1bf..02ab6d277d 100644 --- a/jgiven-tests/src/main/java/com/tngtech/jgiven/tests/GivenTestStage.java +++ b/jgiven-tests/src/main/java/com/tngtech/jgiven/tests/GivenTestStage.java @@ -1,6 +1,12 @@ package com.tngtech.jgiven.tests; import com.tngtech.jgiven.Stage; +import com.tngtech.jgiven.annotation.NestedSteps; +import org.junit.Assume; +import org.junit.jupiter.api.Assumptions; +import org.testng.SkipException; + +import static org.assertj.core.api.Assumptions.assumeThat; public class GivenTestStage extends Stage { public GivenTestStage an_exception_is_thrown() { @@ -8,15 +14,31 @@ public GivenTestStage an_exception_is_thrown() { } public GivenTestStage nothing() { - return this; + return self(); } public GivenTestStage a_failed_step(boolean fail) { if (fail) { throw new IllegalArgumentException(); } - return this; + return self(); + } + + @SuppressWarnings("DataFlowIssue") //fail on purpose + public GivenTestStage a_failed_junit5_assumption() { + Assumptions.assumeFalse(true); + return self(); + } + + @SuppressWarnings("DataFlowIssue") //fail on purpose + public GivenTestStage a_failed_testng_assumption(){ + throw new SkipException("Fail on purpose"); } -} \ No newline at end of file + @SuppressWarnings("DataFlowIssue") //fail on purpose + public GivenTestStage a_failed_junit_assumption() { + Assume.assumeFalse(true); + return self(); + } +} diff --git a/jgiven-tests/src/main/java/com/tngtech/jgiven/tests/TestScenarioRepository.java b/jgiven-tests/src/main/java/com/tngtech/jgiven/tests/TestScenarioRepository.java index 741939c1f4..8f6e906f09 100644 --- a/jgiven-tests/src/main/java/com/tngtech/jgiven/tests/TestScenarioRepository.java +++ b/jgiven-tests/src/main/java/com/tngtech/jgiven/tests/TestScenarioRepository.java @@ -4,6 +4,7 @@ import com.tngtech.jgiven.annotation.Description; import java.util.List; +import java.util.stream.Collectors; public class TestScenarioRepository { @@ -14,6 +15,7 @@ public static class SearchCriteria { public Integer numberOfSteps; public Integer failingStep; public Integer numberOfFailingStages; + public Boolean assumptionFailed; public Boolean failIfPassed; public Boolean executeSteps; public Boolean tagAnnotation; @@ -23,6 +25,9 @@ public static class SearchCriteria { public String testClassDescription; public boolean matches(ScenarioCriteria criteria) { + if(assumptionFailed != null && assumptionFailed != criteria.assumptionFailed){ + return false; + } if (pending != criteria.pending) { return false; } @@ -82,6 +87,7 @@ public static class ScenarioCriteria { public boolean executeSteps; public boolean failing; public Integer failingStep; + public boolean assumptionFailed; public int numberOfSteps = 1; public boolean tagAnnotation; private int numberOfFailingStages; @@ -95,6 +101,11 @@ public ScenarioCriteria pending() { return this; } + public ScenarioCriteria assumptionFailed() { + assumptionFailed = true; + return this; + } + public ScenarioCriteria failIfPassed() { failIfPassed = true; return this; @@ -174,13 +185,10 @@ public TestScenario(Class testClass, String testMethod) { final static List testScenarios = setupTestScenarios(); - public static TestScenario findScenario(SearchCriteria searchCriteria) { - for (TestScenario scenario : testScenarios) { - if (searchCriteria.matches(scenario.criteria)) { - return scenario; - } - } - throw new IllegalArgumentException("No matching scenario found"); + public static List findScenario(SearchCriteria searchCriteria) { + return testScenarios.stream() + .filter(s -> searchCriteria.matches(s.criteria)) + .collect(Collectors.toList()); } private static ScenarioCriteria addTestScenario(List list, Class testClass) { diff --git a/jgiven-tests/src/main/java/com/tngtech/jgiven/tests/TestScenarios.java b/jgiven-tests/src/main/java/com/tngtech/jgiven/tests/TestScenarios.java index c200cc6673..c31fc42450 100644 --- a/jgiven-tests/src/main/java/com/tngtech/jgiven/tests/TestScenarios.java +++ b/jgiven-tests/src/main/java/com/tngtech/jgiven/tests/TestScenarios.java @@ -3,13 +3,17 @@ import com.tngtech.jgiven.annotation.Pending; import com.tngtech.jgiven.testng.ScenarioTestListener; import org.junit.Test; +import org.junit.jupiter.api.Assumptions; +import org.junit.Assume; import org.junit.jupiter.api.extension.ExtendWith; +import org.testng.SkipException; import org.testng.annotations.Listeners; +import static org.assertj.core.api.Assumptions.assumeThat; + @Listeners( ScenarioTestListener.class ) @ExtendWith(JGivenReportExtractingExtension.class) public class TestScenarios extends ScenarioTestForTesting { - @Test @org.junit.jupiter.api.Test @org.testng.annotations.Test diff --git a/jgiven-tests/src/main/java/com/tngtech/jgiven/tests/assumptions/JUnit5AssumptionTestScenarios.java b/jgiven-tests/src/main/java/com/tngtech/jgiven/tests/assumptions/JUnit5AssumptionTestScenarios.java new file mode 100644 index 0000000000..2c0e2e4e64 --- /dev/null +++ b/jgiven-tests/src/main/java/com/tngtech/jgiven/tests/assumptions/JUnit5AssumptionTestScenarios.java @@ -0,0 +1,34 @@ +package com.tngtech.jgiven.tests.assumptions; + +import com.tngtech.jgiven.annotation.ScenarioStage; +import com.tngtech.jgiven.tests.GivenTestStage; +import com.tngtech.jgiven.tests.JGivenReportExtractingExtension; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +@SuppressWarnings("NewClassNamingConvention") +@ExtendWith(JGivenReportExtractingExtension.class) +public class JUnit5AssumptionTestScenarios { + + @ScenarioStage + GivenTestStage givenTestStage; + + @SuppressWarnings("DataFlowIssue") + @Test + void test_with_failing_assumption() { + Assumptions.assumeFalse(true); + } + + @Test + void test_with_failing_assumption_in_stage() { + givenTestStage.given().a_failed_junit5_assumption() + .and().nothing(); + } + + @Test + void test_with_failing_assumption_in_second_stage(){ + givenTestStage.given().nothing() + .and().a_failed_junit5_assumption(); + } +} diff --git a/jgiven-tests/src/main/java/com/tngtech/jgiven/tests/assumptions/JUnitAssumptionTestScenarios.java b/jgiven-tests/src/main/java/com/tngtech/jgiven/tests/assumptions/JUnitAssumptionTestScenarios.java new file mode 100644 index 0000000000..fa52f23baa --- /dev/null +++ b/jgiven-tests/src/main/java/com/tngtech/jgiven/tests/assumptions/JUnitAssumptionTestScenarios.java @@ -0,0 +1,42 @@ +package com.tngtech.jgiven.tests.assumptions; + +import com.tngtech.jgiven.annotation.ScenarioStage; +import com.tngtech.jgiven.junit.JGivenClassRule; +import com.tngtech.jgiven.junit.JGivenMethodRule; +import com.tngtech.jgiven.tests.GivenTestStage; +import org.junit.Assume; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; + +@SuppressWarnings("NewClassNamingConvention") +public class JUnitAssumptionTestScenarios { + + @ClassRule + public static final JGivenClassRule writerRule = new JGivenClassRule(); + + @Rule + public final JGivenMethodRule scenarioRule = new JGivenMethodRule(); + + @ScenarioStage + GivenTestStage givenTestStage; + + @SuppressWarnings("DataFlowIssue") + @Test + public void test_with_failing_assumption() { + Assume.assumeFalse(true); + } + + @Test + public void test_with_failing_assumption_in_stage() { + givenTestStage.given().a_failed_junit_assumption().and() + .nothing(); + + } + + @Test + public void test_with_failing_assumption_in_second_stage(){ + givenTestStage.given().nothing().and() + .a_failed_junit_assumption(); + } +} diff --git a/jgiven-tests/src/main/java/com/tngtech/jgiven/tests/assumptions/TestNgAssumptionTestScenarios.java b/jgiven-tests/src/main/java/com/tngtech/jgiven/tests/assumptions/TestNgAssumptionTestScenarios.java new file mode 100644 index 0000000000..da0f93a66f --- /dev/null +++ b/jgiven-tests/src/main/java/com/tngtech/jgiven/tests/assumptions/TestNgAssumptionTestScenarios.java @@ -0,0 +1,35 @@ +package com.tngtech.jgiven.tests.assumptions; + +import com.tngtech.jgiven.annotation.ScenarioStage; +import com.tngtech.jgiven.testng.ScenarioTestListener; +import com.tngtech.jgiven.tests.GivenTestStage; +import com.tngtech.jgiven.tests.ScenarioTestForTesting; +import com.tngtech.jgiven.tests.ThenTestStage; +import com.tngtech.jgiven.tests.WhenTestStage; +import org.testng.SkipException; +import org.testng.annotations.Listeners; +import org.testng.annotations.Test; + +@Listeners(ScenarioTestListener.class) +public class TestNgAssumptionTestScenarios { + + @ScenarioStage + GivenTestStage givenTestStage; + + @Test + public void test_with_failing_assumption() { + throw new SkipException("Fail on purpose"); + } + + @Test + public void test_with_failing_assumption_in_second_stage() { + givenTestStage.given().nothing().and() + .a_failed_testng_assumption(); + } + + @Test + public void test_with_failing_assumption_in_stage() { + givenTestStage.given().a_failed_testng_assumption().and() + .nothing(); + } +} diff --git a/jgiven-tests/src/test/java/com/tngtech/jgiven/AssumptionsTest.java b/jgiven-tests/src/test/java/com/tngtech/jgiven/AssumptionsTest.java new file mode 100644 index 0000000000..901bfd85f7 --- /dev/null +++ b/jgiven-tests/src/test/java/com/tngtech/jgiven/AssumptionsTest.java @@ -0,0 +1,79 @@ +package com.tngtech.jgiven; + + +import com.tngtech.jgiven.testframework.ThenTestFramework; +import com.tngtech.jgiven.testframework.WhenTestFramework; +import com.tngtech.jgiven.tests.assumptions.JUnit5AssumptionTestScenarios; +import com.tngtech.jgiven.tests.assumptions.JUnitAssumptionTestScenarios; +import com.tngtech.jgiven.tests.assumptions.TestNgAssumptionTestScenarios; +import org.junit.Test; + +public class AssumptionsTest extends JGivenScenarioTest, WhenTestFramework, ThenTestFramework> { + + @Test + public void testng_does_not_fail_for_failing_assumption() { + testNg_handles_assumptions_correctly("test_with_failing_assumption"); + } + + @Test + public void testng_does_not_fail_for_failing_assumption_in_stage() { + testNg_handles_assumptions_correctly("test_with_failing_assumption_in_stage"); + } + @Test + public void testng_does_not_fail_for_failing_assumption_in_second_stage() { + testNg_handles_assumptions_correctly("test_with_failing_assumption_in_second_stage"); + } + + private void testNg_handles_assumptions_correctly(String testName) { + given().a_test_named_$(testName, TestNgAssumptionTestScenarios.class); + when().the_test_is_executed_with_TestNG(); + then().the_test_is_ignored() + .the_report_model_contains_$_scenarios(1); + } + + @Test + public void junit_does_not_fail_for_failing_assumption() { + junit_handles_assumptions_correctly("test_with_failing_assumption"); + } + + @Test + public void junit_does_not_fail_for_failing_assumption_in_stage() { + junit_handles_assumptions_correctly("test_with_failing_assumption_in_stage"); + } + + @Test + public void junit_does_not_fail_for_failing_assumption_in_second_stage() { + junit_handles_assumptions_correctly("test_with_failing_assumption_in_second_stage"); + } + + private void junit_handles_assumptions_correctly(String testName) { + given().a_test_named_$(testName, JUnitAssumptionTestScenarios.class); + when().the_test_is_executed_with_JUnit(); + then().the_test_is_ignored() + .the_report_model_contains_$_scenarios(1); + } + + @Test + public void junit5_does_not_fail_for_failing_assumption() { + junit5_handles_assumptions_correctly("test_with_failing_assumption"); + } + + @Test + public void junit5_does_not_fail_for_failing_assumption_in_stage() { + junit5_handles_assumptions_correctly("test_with_failing_assumption_in_stage"); + } + + @Test + public void junit5_does_not_fail_for_failing_assumption_in_second_stage() { + junit5_handles_assumptions_correctly("test_with_failing_assumption_in_second_stage"); + } + + private void junit5_handles_assumptions_correctly(String testName) { + given().a_test_named_$(testName, JUnit5AssumptionTestScenarios.class); + when().the_test_is_executed_with_JUnit5(); + then().the_test_is_ignored() + .the_report_model_contains_$_scenarios(1); + } +} + + diff --git a/jgiven-tests/src/test/java/com/tngtech/jgiven/GivenScenarioTest.java b/jgiven-tests/src/test/java/com/tngtech/jgiven/GivenScenarioTest.java index ae92a1cf73..f381ba1d2f 100644 --- a/jgiven-tests/src/test/java/com/tngtech/jgiven/GivenScenarioTest.java +++ b/jgiven-tests/src/test/java/com/tngtech/jgiven/GivenScenarioTest.java @@ -40,11 +40,23 @@ public SELF a_failing_test() { return self(); } + + public SELF a_test_named_$(String testName, Class testClass){ + testScenario = new TestScenario(testClass, testName); + return self(); + } + + public SELF the_test_has_$_failing_stages(int n) { criteria.numberOfFailingStages = n; return self(); } + public SELF the_test_has_a_failed_assumption(){ + criteria.assumptionFailed = true; + return self(); + } + public SELF stage_$_has_a_failing_after_stage_method(int i) { criteria.stageWithFailingAfterStageMethod = i; return self(); @@ -74,7 +86,7 @@ public SELF the_test_has_a_tag_annotation_named(String name) { @AfterStage public void findScenario() { if (testScenario == null) { - testScenario = TestScenarioRepository.findScenario(criteria); + testScenario = TestScenarioRepository.findScenario(criteria).stream().findFirst().orElseThrow(); } } diff --git a/jgiven-tests/src/test/java/com/tngtech/jgiven/junit/JUnitExecutor.java b/jgiven-tests/src/test/java/com/tngtech/jgiven/junit/JUnitExecutor.java index 3213d36e74..2ded59d02c 100644 --- a/jgiven-tests/src/test/java/com/tngtech/jgiven/junit/JUnitExecutor.java +++ b/jgiven-tests/src/test/java/com/tngtech/jgiven/junit/JUnitExecutor.java @@ -32,22 +32,12 @@ interface RequestSupplier { @Override public TestExecutionResult execute( final Class testClass, final String testMethod ) { - return execute( new RequestSupplier() { - @Override - public Request supply() { - return Request.method( testClass, testMethod ); - } - } ); + return execute(() -> Request.method( testClass, testMethod )); } @Override public TestExecutionResult execute(final Class testClass ) { - return execute( new RequestSupplier() { - @Override - public Request supply() { - return Request.aClass( testClass ); - } - } ); + return execute(() -> Request.aClass( testClass )); } static class TestRunListener extends RunListener { diff --git a/jgiven-tests/src/test/java/com/tngtech/jgiven/testframework/WhenTestFramework.java b/jgiven-tests/src/test/java/com/tngtech/jgiven/testframework/WhenTestFramework.java index 9d608d7624..6aadb27774 100644 --- a/jgiven-tests/src/test/java/com/tngtech/jgiven/testframework/WhenTestFramework.java +++ b/jgiven-tests/src/test/java/com/tngtech/jgiven/testframework/WhenTestFramework.java @@ -1,12 +1,11 @@ package com.tngtech.jgiven.testframework; -import org.assertj.core.api.Assertions; - import com.tngtech.jgiven.Stage; import com.tngtech.jgiven.annotation.ExpectedScenarioState; import com.tngtech.jgiven.annotation.ProvidedScenarioState; import com.tngtech.jgiven.report.model.ReportModel; import com.tngtech.jgiven.tests.TestScenarioRepository.TestScenario; +import org.assertj.core.api.Assertions; public class WhenTestFramework> extends Stage {