Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix assumption errors #1645

Draft
wants to merge 30 commits into
base: master
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
00c905c
chore(Test framework): switch to using junit5 by default for jgiven-t…
l-1squared Jun 5, 2024
549ce68
fix(Code Quality): Correct testNG test name
l-1squared Jun 5, 2024
f5a0131
fix(Issue-1625): Pass multiple test cases to JGiven test
l-1squared Jun 5, 2024
1eb7609
fix(Issue-1625): Implement test cases with failing assumptions
l-1squared Jun 5, 2024
4baee85
fix(Issue-1625): Implement test cases with failing assumptions
l-1squared Jun 6, 2024
38837c6
chore(Code Quality): replace anonymous class by lambda
l-1squared Jun 7, 2024
1710b37
fix(Issue-1625): use dedicated JUnit4 errors
l-1squared Jun 7, 2024
484127c
fix(Issue-1625): switch back to JUnit4 test execution for now
l-1squared Jun 7, 2024
a946f25
fix(Issue-1625): JUnit5 Scenarios run without inheritance
l-1squared Jun 10, 2024
040e414
fix(Issue-1625): remove list test scenarios
l-1squared Jun 12, 2024
ba4c869
fix(Issue-1625): cleanup scenario tests
l-1squared Jun 12, 2024
d5a7a2a
fix(Issue-1625): switch back to Junit5
l-1squared Jun 12, 2024
73e246e
fix(Issue-1625): throw out assertJ tests
l-1squared Jun 12, 2024
cce1171
fix(Issue-1625): WIP
l-1squared Jun 12, 2024
1750cb9
fix(Issue-1625): Logging for tests in main
l-1squared Jun 13, 2024
627585f
fix(Issue-1625): Throw out junit 5 junit4 converter
l-1squared Jun 13, 2024
2286a42
fix(Issue-1625): Refactor assumptions tests
l-1squared Jun 13, 2024
ccec987
fix(Issue-1625): add an example for a failing assumption
l-1squared Jun 14, 2024
9145f2d
fix(Issue-1625): add a method in the vicinity of a failing assumption
l-1squared Jun 14, 2024
3371571
fix(Issue-1625): junit and testng should report a scenario with a fai…
l-1squared Jun 14, 2024
12b9fa6
fix(Issue-1625): Tentatively implement aborted exception for JUnit
l-1squared Jun 14, 2024
df87519
feat(Issue-1625): Introduce new ABORTED status
l-1squared Jun 17, 2024
69c24ed
feat(Issue-1625): remove duplicated skip exception test
l-1squared Jun 20, 2024
5b6e6b7
feat(Issue-1625): fix catching pending scenarios as aborted
l-1squared Jun 20, 2024
8e1da95
feat(Issue-1625): asciidoc generator recognizes aborted status
l-1squared Aug 5, 2024
57be682
feat(Issue-1625): write example tests for aborted scenarios
l-1squared Aug 5, 2024
15e42bf
feat(Issue-1625): process aborted status in asciidoc report
l-1squared Aug 6, 2024
550ec65
fix(Issue-1625): Automatically produce a plain text report for examples
l-1squared Aug 9, 2024
17a3ddd
fix(Issue-1625): Plain text report generator can create the output di…
l-1squared Aug 9, 2024
9855275
fix(Issue-1625): Usage of Java 21 features in java 11
l-1squared Aug 9, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -257,6 +257,18 @@ configure(subprojects.findAll { !it.name.contains("android") }) {
classpath = configurations.testRuntimeClasspath
}

tasks.register("jgivenPlainTextReport",JavaExec) {
//noinspection GroovyAccessibility
mainClass = 'com.tngtech.jgiven.report.ReportGenerator'
args '--sourceDir=build/reports/jgiven/json',
'--targetDir=build/reports/jgiven/text',
'--format=text',
'--exclude-empty-scenarios=true',
'--title=JGiven Report'

classpath = configurations.testRuntimeClasspath
}

asciidoctor {
sourceDir = new File('build/reports/jgiven/asciidoc')
outputDir = new File('build/reports/jgiven/htmladoc')
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,10 @@ public void finished() throws Throwable {
executor.finished();
}

public void aborted(Throwable e){
executor.aborted(e);
}

public ScenarioExecutor getExecutor() {
return executor;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,12 @@ enum State {
*/
private Throwable failedException;

/**
* Set if an exception was thrown during the execution of the scenario and
* suppressStepExceptions is true.
*/
private Throwable abortedException;

private boolean failIfPass;

/**
Expand Down Expand Up @@ -356,6 +362,7 @@ public void finished() throws Throwable {
}
}


private void callFinishLifeCycleMethods() throws Throwable {
Throwable firstThrownException = failedException;
if (beforeScenarioMethodsExecuted) {
Expand Down Expand Up @@ -415,6 +422,10 @@ public boolean hasFailed() {
return failedException != null;
}

public boolean hasAborted() {
return abortedException!= null;
}

public Throwable getFailedException() {
return failedException;
}
Expand All @@ -423,6 +434,14 @@ public void setFailedException(Exception e) {
failedException = e;
}

public Throwable getAbortedException() {
return abortedException;
}

public void setAbortedException(Exception e) {
abortedException= e;
}

/**
* Handle ocurred exception and continue.
*/
Expand All @@ -438,6 +457,16 @@ public void failed(Throwable e) {
}
}

public void aborted(Throwable e) {
if (hasAborted()){
log.error(e.getMessage(), e);
}else {
listener.scenarioAborted(e);
methodInterceptor.disableMethodExecution();
abortedException = e;
}
}

/**
* Starts a scenario with the given description.
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -314,6 +314,13 @@ public void stepMethodFailed(Throwable t) {
}
}

@Override
public void stepMethodAborted(Throwable t) {
if (currentStep != null) {
currentStep.setStatus(StepStatus.ABORTED);
}
}

@Override
public void stepMethodFinished(long durationInNanos, boolean hasNestedSteps) {
if (hasNestedSteps && !parentSteps.isEmpty()) {
Expand Down Expand Up @@ -364,6 +371,12 @@ public void scenarioFailed(Throwable e) {
setException(e);
}

@Override
public void scenarioAborted(Throwable e){
setStatus(ExecutionStatus.ABORTED);
setException(e);
}

private void setCaseDescription(Class<?> testClass, Method method, List<NamedArgument> namedArguments) {

CaseAs annotation = null;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,9 @@ public class NoOpScenarioListener implements ScenarioListener {
@Override
public void scenarioFailed( Throwable e ) {}

@Override
public void scenarioAborted(Throwable e) {}

@Override
public void scenarioStarted( String string ) {}

Expand All @@ -31,6 +34,9 @@ public void stepCommentUpdated( String comment ) {}
@Override
public void stepMethodFailed( Throwable t ) {}

@Override
public void stepMethodAborted( Throwable t ) {}

@Override
public void stepMethodFinished( long durationInNanos, boolean hasNestedSteps ) {}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ public interface ScenarioListener {

void scenarioFailed( Throwable e );

void scenarioAborted(Throwable e);

void scenarioStarted( String string );

void scenarioStarted( Class<?> testClass, Method method, List<NamedArgument> arguments );
Expand All @@ -24,6 +26,8 @@ public interface ScenarioListener {

void stepMethodFailed( Throwable t );

void stepMethodAborted( Throwable t );

void stepMethodFinished( long durationInNanos, boolean hasNestedSteps );

void scenarioFinished();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -109,9 +109,7 @@ private Object doIntercept(Object receiver, Method method, Object[] parameters,

try {
return invoker.proceed();
} catch( Exception e ) {
return handleThrowable( receiver, method, e, System.nanoTime() - started, handleMethod );
} catch( AssertionError e ) {
} catch(Exception | AssertionError e ) {
return handleThrowable( receiver, method, e, System.nanoTime() - started, handleMethod );
} finally {
if( hasNestedSteps ) {
Expand Down Expand Up @@ -217,13 +215,13 @@ private void handleMethod(Object stageInstance, Method paramMethod, Object[] arg

private void handleThrowable( Throwable t ) throws Throwable {
if( ThrowableUtil.isAssumptionException(t) ) {
throw t;
listener.stepMethodAborted(t);
scenarioExecutor.aborted(t);
}else {
listener.stepMethodFailed(t);
scenarioExecutor.failed( t );
}

listener.stepMethodFailed( t );

scenarioExecutor.failed( t );

if (!suppressExceptions) {
throw t;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ public String convertStatisticsBlock(final ListMultimap<String, ReportStatistics
statisticsTable.append("| successful scenarios ");
statisticsTable.append("| failed scenarios ");
statisticsTable.append("| pending scenarios ");
statisticsTable.append("| aborted scenarios ");
statisticsTable.append("| total scenarios ");
statisticsTable.append("| failed cases ");
statisticsTable.append("| total cases ");
Expand Down Expand Up @@ -68,6 +69,8 @@ public String convertFeatureHeaderBlock(final String featureName, final ReportSt
.append(statistics.numFailedScenarios).append(" Failed, ");
blockContent.append(MetadataMapper.toHumanReadableStatus(ExecutionStatus.SCENARIO_PENDING)).append(" ")
.append(statistics.numPendingScenarios).append(" Pending, ");
blockContent.append(MetadataMapper.toHumanReadableStatus(ExecutionStatus.ABORTED)).append(" ")
.append(statistics.numAbortedScenarios).append(" Aborted, ");
blockContent.append(statistics.numScenarios).append(" Total");
blockContent.append(" (").append(MetadataMapper.toHumanReadableScenarioDuration(statistics.durationInNanos))
.append(")");
Expand Down Expand Up @@ -326,6 +329,7 @@ private static void appendStatisticsRowFragment(final StringBuilder builder, fin
builder.append(" | ").append(statistics.numSuccessfulScenarios);
builder.append(" | ").append(statistics.numFailedScenarios);
builder.append(" | ").append(statistics.numPendingScenarios);
builder.append(" | ").append(statistics.numAbortedScenarios);
builder.append(" | ").append(statistics.numScenarios);
builder.append(" | ").append(statistics.numFailedCases);
builder.append(" | ").append(statistics.numCases);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ public class AsciiDocReportGenerator extends AbstractReportGenerator {
private final List<String> featureFiles = new ArrayList<>();
private final List<String> failedScenarioFiles = new ArrayList<>();
private final List<String> pendingScenarioFiles = new ArrayList<>();
private final List<String> abortedScenarioFiles = new ArrayList<>();
private File targetDir;
private File featuresDir;

Expand Down Expand Up @@ -73,6 +74,8 @@ public void generate() {

writeIndexFileForPendingScenarios();

writeIndexFileForAbortedScenarios();

writeTotalStatisticsFile();

writeIndexFileForFullReport(config.getTitle());
Expand Down Expand Up @@ -115,6 +118,9 @@ private List<String> collectReportBlocks(final ReportModelFile reportModelFile,
if (statistics.numPendingScenarios > 0) {
pendingScenarioFiles.add(featureFileName);
}
if (statistics.numAbortedScenarios >0){
abortedScenarioFiles.add(featureFileName);
}

final AsciiDocReportModelVisitor visitor = new AsciiDocReportModelVisitor(blockConverter, statistics);
reportModelFile.model().accept(visitor);
Expand Down Expand Up @@ -151,6 +157,16 @@ private void writeIndexFileForPendingScenarios() {
snippetGenerator.generateIndexSnippet());
}

private void writeIndexFileForAbortedScenarios() {
final String scenarioKind = "aborted";
final AsciiDocSnippetGenerator snippetGenerator = new AsciiDocSnippetGenerator(
"Aborted Scenarios", "aborted scenarios", this.abortedScenarioFiles, scenarioKind,
this.completeReportModel.getTotalStatistics().numAbortedScenarios);

writeAsciiDocBlocksToFile(new File(targetDir, scenarioKind + "Scenarios.asciidoc"),
snippetGenerator.generateIndexSnippet());
}

private void writeTotalStatisticsFile() {

final ListMultimap<String, ReportStatistics> featureStatistics = completeReportModel.getAllReportModels()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ final class MetadataMapper {
private static final String ICON_CHECK_MARK = "icon:check-square[role=green]";
private static final String ICON_EXCLAMATION_MARK = "icon:exclamation-circle[role=red]";
private static final String ICON_BANNED = "icon:ban[role=silver]";
private static final String ICON_TIMES_CIRCLE = "icon:times-circle[role=gray]";
private static final String ICON_STEP_FORWARD = "icon:step-forward[role=silver]";
private static final int NANOSECONDS_PER_MILLISECOND = 1000000;

Expand Down Expand Up @@ -46,6 +47,8 @@ static String toHumanReadableStatus(final ExecutionStatus executionStatus) {
return ICON_CHECK_MARK;
case FAILED:
return ICON_EXCLAMATION_MARK;
case ABORTED:
return ICON_TIMES_CIRCLE;
default:
return executionStatus.toString();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,5 +4,6 @@ public enum ExecutionStatus {
SCENARIO_PENDING,
SUCCESS,
FAILED,
SOME_STEPS_PENDING;
ABORTED,
SOME_STEPS_PENDING
}
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ public class ReportStatistics {
public int numScenarios;
public int numFailedScenarios;
public int numPendingScenarios;
public int numAbortedScenarios;
public int numSuccessfulScenarios;
public int numCases;
public int numFailedCases;
Expand All @@ -23,6 +24,7 @@ private void addModifying( ReportStatistics statistics ) {
this.numScenarios += statistics.numScenarios;
this.numFailedScenarios += statistics.numFailedScenarios;
this.numPendingScenarios += statistics.numPendingScenarios;
this.numAbortedScenarios += statistics.numAbortedScenarios;
this.numSuccessfulScenarios += statistics.numSuccessfulScenarios;
this.numCases += statistics.numCases;
this.numFailedCases += statistics.numFailedCases;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ public void visit( ScenarioModel scenarioModel ) {
statistics.numFailedScenarios += 1;
} else if( executionStatus == ExecutionStatus.SCENARIO_PENDING || executionStatus == ExecutionStatus.SOME_STEPS_PENDING) {
statistics.numPendingScenarios += 1;
} else if (executionStatus == ExecutionStatus.ABORTED){
statistics.numAbortedScenarios += 1;
} else {
statistics.numSuccessfulScenarios += 1;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,5 +4,6 @@ public enum StepStatus {
PASSED,
FAILED,
SKIPPED,
PENDING;
PENDING,
ABORTED
}
Original file line number Diff line number Diff line change
Expand Up @@ -10,19 +10,32 @@
import com.tngtech.jgiven.report.AbstractReportGenerator;
import com.tngtech.jgiven.report.model.ReportModel;
import com.tngtech.jgiven.report.model.ReportModelFile;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class PlainTextReportGenerator extends AbstractReportGenerator {

private static final Logger log = LoggerFactory.getLogger( PlainTextReportGenerator.class );

public AbstractReportConfig createReportConfig( String... args ) {
return new PlainTextReportConfig(args);
}

public void generate() {
generateOutputDirectory();
for( ReportModelFile reportModelFile : completeReportModel.getAllReportModels() ) {
handleReportModel(reportModelFile.model(), reportModelFile.file());
}
}

private void generateOutputDirectory() {
var outputDir = config.getTargetDir();
if( !outputDir.exists() && !outputDir.mkdirs()) {
log.error( "Could not create target directory " + outputDir);
return;
}
}

public void handleReportModel( ReportModel model, File file ) {
String targetFileName = Files.getNameWithoutExtension( file.getName() ) + ".feature";
PrintWriter printWriter = PrintWriterUtil.getPrintWriter( new File( config.getTargetDir(), targetFileName ) );
Expand All @@ -33,4 +46,5 @@ public void handleReportModel( ReportModel model, File file ) {
ResourceUtil.close( printWriter );
}
}

}
Loading
Loading