diff --git a/pitest-entry/src/main/java/org/pitest/mutationtest/MutationResult.java b/pitest-entry/src/main/java/org/pitest/mutationtest/MutationResult.java index c41c0d94a..afa9cea17 100644 --- a/pitest-entry/src/main/java/org/pitest/mutationtest/MutationResult.java +++ b/pitest-entry/src/main/java/org/pitest/mutationtest/MutationResult.java @@ -45,6 +45,18 @@ public List getSucceedingTests() { return this.status.getSucceedingTests(); } + public List getTimeoutTests() { + return this.status.getTimeOutTests(); + } + + public List getMemoryErrorTests() { + return this.status.getMemoryErrorTests(); + } + + public List getRunErrorTests() { + return this.status.getRunErrorTests(); + } + public DetectionStatus getStatus() { return this.status.getStatus(); } diff --git a/pitest-entry/src/main/java/org/pitest/mutationtest/MutationStatusMap.java b/pitest-entry/src/main/java/org/pitest/mutationtest/MutationStatusMap.java index 90c1507f1..07fed632d 100644 --- a/pitest-entry/src/main/java/org/pitest/mutationtest/MutationStatusMap.java +++ b/pitest-entry/src/main/java/org/pitest/mutationtest/MutationStatusMap.java @@ -14,6 +14,7 @@ */ package org.pitest.mutationtest; +import static org.pitest.functional.prelude.Prelude.or; import static org.pitest.functional.prelude.Prelude.putToMap; import java.util.Collection; @@ -74,6 +75,15 @@ public Collection getUnfinishedRuns() { .collect(Collectors.toList()); } + public Collection getCrashed() { + return this.mutationMap.entrySet().stream() + .filter(or(hasStatus(DetectionStatus.RUN_ERROR), + or(hasStatus(DetectionStatus.MEMORY_ERROR), + hasStatus(DetectionStatus.TIMED_OUT)))) + .map(toMutationDetails()) + .collect(Collectors.toList()); + } + public Set allMutations() { return this.mutationMap.keySet(); } diff --git a/pitest-entry/src/main/java/org/pitest/mutationtest/build/MutationTestUnit.java b/pitest-entry/src/main/java/org/pitest/mutationtest/build/MutationTestUnit.java index cea1968ed..72ec5a97d 100644 --- a/pitest-entry/src/main/java/org/pitest/mutationtest/build/MutationTestUnit.java +++ b/pitest-entry/src/main/java/org/pitest/mutationtest/build/MutationTestUnit.java @@ -16,13 +16,17 @@ import java.io.IOException; import java.util.Collection; +import java.util.Collections; import java.util.logging.Logger; import org.pitest.classinfo.ClassName; +import org.pitest.coverage.TestInfo; import org.pitest.mutationtest.DetectionStatus; import org.pitest.mutationtest.MutationMetaData; import org.pitest.mutationtest.MutationStatusMap; +import org.pitest.mutationtest.MutationStatusTestPair; import org.pitest.mutationtest.engine.MutationDetails; +import org.pitest.mutationtest.engine.MutationIdentifier; import org.pitest.mutationtest.execute.MutationTestProcess; import org.pitest.util.ExitCode; import org.pitest.util.Log; @@ -75,17 +79,55 @@ private void runTestInSeperateProcessForMutationRange( final Collection remainingMutations = mutations .getUnrunMutations(); - final MutationTestProcess worker = this.workerFactory.createWorker( + + //First run mutants normally + MutationTestProcess worker = this.workerFactory.createWorker( remainingMutations, this.testClasses); worker.start(); setFirstMutationToStatusOfStartedInCaseMinionFailsAtBoot(mutations, remainingMutations); - final ExitCode exitCode = waitForMinionToDie(worker); + ExitCode exitCode = waitForMinionToDie(worker); worker.results(mutations); correctResultForProcessExitCode(mutations, exitCode); + + //rerun crashing mutants with isolation + if (this.workerFactory.isFullMutationMatrix() && !exitCode.isOk()) { + Collection crashedRuns = mutations.getCrashed(); + //not rerun crashed from previous range + crashedRuns.retainAll(remainingMutations); + + LOG.info("Rerunning " + crashedRuns.size() + " mutant(s) because of minion crash"); + for (MutationDetails d : crashedRuns) { + MutationStatusTestPair result = null; + for (TestInfo t : d.getTestsInOrder()) { + MutationDetails singleTest = new MutationDetails(new MutationIdentifier(d.getId().getLocation(), + d.getId().getIndexes(),d.getMutator()), d.getFilename(), d.getDescription(), + d.getLineNumber(), d.getBlock()); + singleTest.addTestsInOrder(Collections.singleton(t)); + worker = this.workerFactory.createWorker(Collections.singleton(singleTest), this.testClasses); + worker.start(); + exitCode = waitForMinionToDie(worker); + MutationStatusTestPair r = worker.results(singleTest); + + if (exitCode != ExitCode.OK) { + r.setErrorStatusAndName(DetectionStatus.getForErrorExitCode(exitCode), t.getName()); + } + + if (result == null) { + result = r; + } else { + result.accumulate(r, t.getName()); + } + } + + if (result != null) { + mutations.setStatusForMutation(d, result); + } + } + } } private static ExitCode waitForMinionToDie(final MutationTestProcess worker) { @@ -123,7 +165,4 @@ private static void correctResultForProcessExitCode( private static MutationMetaData reportResults(final MutationStatusMap mutationsMap) { return new MutationMetaData(mutationsMap.createMutationResults()); } - - - -} +} \ No newline at end of file diff --git a/pitest-entry/src/main/java/org/pitest/mutationtest/build/WorkerFactory.java b/pitest-entry/src/main/java/org/pitest/mutationtest/build/WorkerFactory.java index bc3eed399..26ac1d9f6 100644 --- a/pitest-entry/src/main/java/org/pitest/mutationtest/build/WorkerFactory.java +++ b/pitest-entry/src/main/java/org/pitest/mutationtest/build/WorkerFactory.java @@ -75,4 +75,8 @@ private SideEffect1 captureStdOutIfVerbose() { } + public boolean isFullMutationMatrix() { + return this.fullMutationMatrix; + } + } diff --git a/pitest-entry/src/main/java/org/pitest/mutationtest/execute/MutationTestProcess.java b/pitest-entry/src/main/java/org/pitest/mutationtest/execute/MutationTestProcess.java index f4f047753..b4255e977 100644 --- a/pitest-entry/src/main/java/org/pitest/mutationtest/execute/MutationTestProcess.java +++ b/pitest-entry/src/main/java/org/pitest/mutationtest/execute/MutationTestProcess.java @@ -31,6 +31,10 @@ public void start() throws IOException, InterruptedException { this.process.start(); } + public MutationStatusTestPair results(MutationDetails mutant) { + return this.thread.getStatus(mutant.getId()); + } + public void results(final MutationStatusMap allmutations) throws IOException { for (final MutationDetails each : allmutations.allMutations()) { diff --git a/pitest-entry/src/main/java/org/pitest/mutationtest/report/xml/XMLReportListener.java b/pitest-entry/src/main/java/org/pitest/mutationtest/report/xml/XMLReportListener.java index 102f0099d..b0fc665da 100644 --- a/pitest-entry/src/main/java/org/pitest/mutationtest/report/xml/XMLReportListener.java +++ b/pitest-entry/src/main/java/org/pitest/mutationtest/report/xml/XMLReportListener.java @@ -20,13 +20,16 @@ import static org.pitest.mutationtest.report.xml.Tag.killingTest; import static org.pitest.mutationtest.report.xml.Tag.killingTests; import static org.pitest.mutationtest.report.xml.Tag.lineNumber; +import static org.pitest.mutationtest.report.xml.Tag.memoryErrorTests; import static org.pitest.mutationtest.report.xml.Tag.methodDescription; import static org.pitest.mutationtest.report.xml.Tag.mutatedClass; import static org.pitest.mutationtest.report.xml.Tag.mutatedMethod; import static org.pitest.mutationtest.report.xml.Tag.mutation; import static org.pitest.mutationtest.report.xml.Tag.mutator; +import static org.pitest.mutationtest.report.xml.Tag.runErrorTests; import static org.pitest.mutationtest.report.xml.Tag.sourceFile; import static org.pitest.mutationtest.report.xml.Tag.succeedingTests; +import static org.pitest.mutationtest.report.xml.Tag.timeoutTests; import java.io.IOException; import java.io.Writer; @@ -42,7 +45,8 @@ import org.pitest.util.Unchecked; enum Tag { - mutation, sourceFile, mutatedClass, mutatedMethod, methodDescription, lineNumber, mutator, index, killingTest, killingTests, succeedingTests, description, block; + mutation, sourceFile, mutatedClass, mutatedMethod, methodDescription, lineNumber, mutator, index, killingTest, + killingTests, succeedingTests, description, block, timeoutTests, runErrorTests, memoryErrorTests; } public class XMLReportListener implements MutationResultListener { @@ -95,6 +99,12 @@ private String makeMutationNode(final MutationResult mutation) { createTestDesc(mutation.getKillingTests()), killingTests) + makeNodeWhenConditionSatisfied(fullMutationMatrix, createTestDesc(mutation.getSucceedingTests()), succeedingTests) + + makeNodeWhenConditionSatisfied(fullMutationMatrix, + createTestDesc(mutation.getTimeoutTests()), timeoutTests) + + makeNodeWhenConditionSatisfied(fullMutationMatrix, + createTestDesc(mutation.getRunErrorTests()), runErrorTests) + + makeNodeWhenConditionSatisfied(fullMutationMatrix, + createTestDesc(mutation.getMemoryErrorTests()), memoryErrorTests) + makeNode(clean(details.getDescription()), description); } diff --git a/pitest-entry/src/test/java/com/example/coverage/execute/samples/mutationMatrix/SimpleCalculator.java b/pitest-entry/src/test/java/com/example/coverage/execute/samples/mutationMatrix/SimpleCalculator.java index fb38b658f..404f9b08e 100644 --- a/pitest-entry/src/test/java/com/example/coverage/execute/samples/mutationMatrix/SimpleCalculator.java +++ b/pitest-entry/src/test/java/com/example/coverage/execute/samples/mutationMatrix/SimpleCalculator.java @@ -1,5 +1,6 @@ package com.example.coverage.execute.samples.mutationMatrix; + public class SimpleCalculator { public static int sum(int x, int y) { diff --git a/pitest-entry/src/test/java/com/example/coverage/execute/samples/mutationMatrix/TestsForSimpleCalculator.java b/pitest-entry/src/test/java/com/example/coverage/execute/samples/mutationMatrix/TestsForSimpleCalculator.java index 883dfd89e..3d0c88d5a 100644 --- a/pitest-entry/src/test/java/com/example/coverage/execute/samples/mutationMatrix/TestsForSimpleCalculator.java +++ b/pitest-entry/src/test/java/com/example/coverage/execute/samples/mutationMatrix/TestsForSimpleCalculator.java @@ -4,6 +4,9 @@ import org.junit.Test; +import java.util.ArrayList; +import java.util.List; + public class TestsForSimpleCalculator { @Test @@ -20,4 +23,24 @@ public void testSumWithNegativeNumber() { public void pseudoTestSum() { SimpleCalculator.sum(2, 1); } + + @Test + public void unknownErrorOnMutant() { + if (SimpleCalculator.sum(2,1) != 3) { + System.exit(13); + } + } + + @Test + public void timeoutOnMutant() { + if (SimpleCalculator.sum(2,1) != 3) { +// System.exit(14); + try { + Thread.sleep(5000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + } diff --git a/pitest-entry/src/test/java/org/pitest/mutationtest/MutationCoverageReportSystemTest.java b/pitest-entry/src/test/java/org/pitest/mutationtest/MutationCoverageReportSystemTest.java index eb2862b18..1d5dec517 100644 --- a/pitest-entry/src/test/java/org/pitest/mutationtest/MutationCoverageReportSystemTest.java +++ b/pitest-entry/src/test/java/org/pitest/mutationtest/MutationCoverageReportSystemTest.java @@ -39,6 +39,7 @@ import org.pitest.SystemTest; import org.pitest.classpath.ClassPath; import org.pitest.help.PitHelpError; +import org.pitest.mutationtest.build.PercentAndConstantTimeoutStrategy; import org.pitest.mutationtest.engine.gregor.Generated; import org.pitest.util.FileUtil; import org.pitest.util.IsolationUtils; @@ -212,18 +213,24 @@ public void computesFullMutationMatrix() { .setTargetTests(predicateFor("com.example.coverage.execute.samples.mutationMatrix.*")); this.data.setTargetClasses(asList("com.example.coverage.execute.samples.mutationMatrix.*")); this.data.setExcludedClasses(asGlobs(TestsForSimpleCalculator.class)); + this.data.setExcludedMethods(asList("crash")); this.data.setFullMutationMatrix(true); this.data.addOutputFormats(Arrays.asList("XML")); this.data.setMutators(Arrays.asList("MATH")); + this.data.setTimeoutConstant(PercentAndConstantTimeoutStrategy.DEFAULT_CONSTANT); + this.data.setTimeoutFactor(PercentAndConstantTimeoutStrategy.DEFAULT_FACTOR); createAndRun(); List resultData = this.metaDataExtractor.getData(); assertEquals(1, resultData.size()); MutationResult mutation = resultData.get(0); assertEquals(KILLED, mutation.getStatus()); - assertEquals(3, mutation.getNumberOfTestsRun()); + assertEquals(5, mutation.getNumberOfTestsRun()); assertEquals(2, mutation.getKillingTests().size()); assertEquals(1, mutation.getSucceedingTests().size()); + assertEquals(0, mutation.getMemoryErrorTests().size()); + assertEquals(1, mutation.getTimeoutTests().size()); + assertEquals(1, mutation.getRunErrorTests().size()); } @Test(expected = PitHelpError.class) diff --git a/pitest-entry/src/test/java/org/pitest/mutationtest/report/xml/XMLReportListenerTest.java b/pitest-entry/src/test/java/org/pitest/mutationtest/report/xml/XMLReportListenerTest.java index fcc18e567..1279de29d 100644 --- a/pitest-entry/src/test/java/org/pitest/mutationtest/report/xml/XMLReportListenerTest.java +++ b/pitest-entry/src/test/java/org/pitest/mutationtest/report/xml/XMLReportListenerTest.java @@ -66,7 +66,7 @@ public void shouldOutputFullMutationMatrixWhenEnabled() throws IOException { new MutationStatusTestPair(3, DetectionStatus.KILLED, Arrays.asList("foo", "foo2"), Arrays.asList("bar"))); this.testee .handleMutationResult(MutationTestResultMother.createClassResults(mr)); - final String expected = "fileclazzmethod()I42mutator10foo|foo2bardesc\n"; + final String expected = "fileclazzmethod()I42mutator10foo|foo2bardesc\n"; assertEquals(expected, this.out.toString()); } diff --git a/pitest/src/main/java/org/pitest/mutationtest/MutationStatusTestPair.java b/pitest/src/main/java/org/pitest/mutationtest/MutationStatusTestPair.java index 69d94f91a..ac895e1c0 100644 --- a/pitest/src/main/java/org/pitest/mutationtest/MutationStatusTestPair.java +++ b/pitest/src/main/java/org/pitest/mutationtest/MutationStatusTestPair.java @@ -16,6 +16,7 @@ import java.io.Serializable; import java.util.Collections; +import java.util.LinkedList; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -24,28 +25,43 @@ public final class MutationStatusTestPair implements Serializable { private static final long serialVersionUID = 1L; - private final int numberOfTestsRun; - private final DetectionStatus status; + private int numberOfTestsRun; + private DetectionStatus status; private final List killingTests; private final List succeedingTests; + private final List timeOutTests; + private final List runErrorTests; + private final List memoryErrorTests; public static MutationStatusTestPair notAnalysed(int testsRun, DetectionStatus status) { - return new MutationStatusTestPair(testsRun, status, Collections.emptyList(), Collections.emptyList()); + return new MutationStatusTestPair(testsRun, status, Collections.emptyList(), Collections.emptyList(), + Collections.emptyList(), Collections.emptyList(), Collections.emptyList()); } public MutationStatusTestPair(final int numberOfTestsRun, final DetectionStatus status, final String killingTest) { this(numberOfTestsRun, status, killingTestToList(killingTest), - Collections.emptyList()); + Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), Collections.emptyList()); + } + + public MutationStatusTestPair(final int numberOfTestsRun, + final DetectionStatus status, final List killingTests, + final List succeedingTests) { + this(numberOfTestsRun, status, killingTests, succeedingTests, Collections.emptyList(), + Collections.emptyList(), Collections.emptyList()); } public MutationStatusTestPair(final int numberOfTestsRun, final DetectionStatus status, final List killingTests, - final List succeedingTests) { + final List succeedingTests, final List timeOutTests, + final List runErrorTests, final List memoryErrorTests) { this.status = status; - this.killingTests = killingTests; - this.succeedingTests = succeedingTests; + this.killingTests = new LinkedList<>(killingTests); + this.succeedingTests = new LinkedList<>(succeedingTests); this.numberOfTestsRun = numberOfTestsRun; + this. timeOutTests = new LinkedList<>(timeOutTests); + this.runErrorTests = new LinkedList<>(runErrorTests); + this.memoryErrorTests = new LinkedList<>(memoryErrorTests); } private static List killingTestToList(String killingTest) { @@ -107,9 +123,12 @@ public int hashCode() { + ((this.killingTests == null) ? 0 : this.killingTests.hashCode()); result = (prime * result) + ((this.succeedingTests == null) ? 0 : this.succeedingTests.hashCode()); - result = (prime * result) + this.numberOfTestsRun; result = (prime * result) - + ((this.status == null) ? 0 : this.status.hashCode()); + + ((this.memoryErrorTests == null) ? 0 : this.memoryErrorTests.hashCode()); + result = (prime * result) + + ((this.runErrorTests == null) ? 0 : this.runErrorTests.hashCode()); + result = (prime * result) + + ((this.timeOutTests == null) ? 0 : this.timeOutTests.hashCode()); return result; } @@ -131,13 +150,81 @@ public boolean equals(final Object obj) { if (!Objects.equals(this.succeedingTests, other.succeedingTests)) { return false; } - if (this.numberOfTestsRun != other.numberOfTestsRun) { + if (!Objects.equals(this.memoryErrorTests, other.memoryErrorTests)) { return false; } - if (this.status != other.status) { + if (!Objects.equals(this.runErrorTests, other.runErrorTests)) { + return false; + } + if (!Objects.equals(this.timeOutTests, other.timeOutTests)) { return false; } return true; } + public void accumulate(MutationStatusTestPair status, String testName) { + this.numberOfTestsRun += status.numberOfTestsRun; + + if (status.status.equals(DetectionStatus.KILLED)) { + if (!this.killingTests.contains(testName)) { + this.killingTests.add(testName); + } + this.succeedingTests.remove(testName); + } else if (status.status.equals(DetectionStatus.SURVIVED) && !this.killingTests.contains(testName) + && !this.succeedingTests.contains(testName)) { + this.succeedingTests.add(testName); + } else if (status.status.equals(DetectionStatus.MEMORY_ERROR)) { + if (!this.memoryErrorTests.contains(testName)) { + this.memoryErrorTests.add(testName); + } + this.succeedingTests.remove(testName); + } else if (status.status.equals(DetectionStatus.RUN_ERROR)) { + if (!this.runErrorTests.contains(testName)) { + this.runErrorTests.add(testName); + } + this.succeedingTests.remove(testName); + } else if (status.status.equals(DetectionStatus.TIMED_OUT)) { + if (!this.timeOutTests.contains(testName)) { + this.timeOutTests.add(testName); + } + this.succeedingTests.remove(testName); + } + + if (!this.killingTests.isEmpty()) { + this.status = DetectionStatus.KILLED; + } else if (!this.runErrorTests.isEmpty()) { + this.status = DetectionStatus.RUN_ERROR; + } else if (!this.memoryErrorTests.isEmpty()) { + this.status = DetectionStatus.MEMORY_ERROR; + } else if (!this.timeOutTests.isEmpty()) { + this.status = DetectionStatus.TIMED_OUT; + } else if (!this.succeedingTests.isEmpty()) { + this.status = DetectionStatus.SURVIVED; + } else { + this.status = status.status; + } + } + + public List getTimeOutTests() { + return timeOutTests; + } + + public List getRunErrorTests() { + return runErrorTests; + } + + public List getMemoryErrorTests() { + return memoryErrorTests; + } + + public void setErrorStatusAndName (DetectionStatus status, String testName) { + this.status = status; + if (this.status == DetectionStatus.RUN_ERROR) { + this.runErrorTests.add(testName); + } else if (this.status == DetectionStatus.MEMORY_ERROR) { + this.memoryErrorTests.add(testName); + } else if (this.status == DetectionStatus.TIMED_OUT) { + this.timeOutTests.add(testName); + } + } }