Runner code checks for problems against flattened testsets (#119)

* Simplify tests for `any_problems`

* Support `any_problems` on vectors

* Test that `flatten_results!` returns a vector

* Runner code checks for problems against flattened testsets

* Set project version to 1.1.1

* warn about flatten_results! behavior in docs

---------

Co-authored-by: Philip Bittihn <[email protected]>
Co-authored-by: Frames White <[email protected]>

Author: 3 people

Diff for: Project.toml

@@ -1,6 +1,6 @@
 name = "TestReports"
 uuid = "dcd651b4-b50a-5b6b-8f22-87e9f253a252"
-version = "1.1.0"
+version = "1.1.1"
 
 [deps]
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"

Diff for: src/runner.jl

@@ -151,10 +151,11 @@ function gen_runner_code(testfilename, logfilename, test_args)
         end
 
         # Flatten before calling `report` to avoid a `deepcopy`.
+        flattened_testsets = TestReports.flatten_results!(ts)
         open($(repr(logfilename)), "w") do io
-            prettyprint(io, report(TestReports.flatten_results!(ts)))
+            prettyprint(io, report(flattened_testsets))
         end
-        any_problems(ts) && exit(TestReports.TESTS_FAILED)
+        any_problems(flattened_testsets) && exit(TestReports.TESTS_FAILED)
         """
     return runner_code
 end

Diff for: src/testsets.jl

@@ -238,6 +238,7 @@ does not throw an exception on a failure. Thus to set the exit code from
 the runner code, we check it using `exit(any_problems(top_level_testset))`.
 """
 any_problems(ts::AbstractTestSet) = any(any_problems.(ts.results))
+any_problems(v::AbstractVector{<:AbstractTestSet}) = any(any_problems.(v))
 any_problems(rs::ReportingResult) = any_problems(rs.result)
 any_problems(::Pass) = false
 any_problems(::Fail) = true
@@ -262,6 +263,9 @@ end
 
 Returns a flat vector of `TestSet`s which only contain `Result`s. This is necessary for
 writing a JUnit XML report the schema does not allow nested XML `testsuite` elements.
+
+Warning: Alters the original hierarchy inside `ts` and potentially removes testsets
+         from it. Be sure to rely only on the returned vector after usage.
 """
 flatten_results!(ts::AbstractTestSet) = _flatten_results!(ts, 0)
 

Diff for: test/reportgeneration.jl

@@ -90,6 +90,7 @@ end
 @testset "Test packages" begin
     # Errors
     test_package_expected_fail("FailedTest")
+    test_package_expected_fail("FailedNestedTest")
     test_package_expected_fail("ErroredTest")
     test_package_expected_fail("NoTestFile")
 

Diff for: test/test_packages/FailedNestedTest/Project.toml

@@ -0,0 +1,9 @@
+name = "FailedNestedTest"
+uuid = "ab7034c0-1419-4241-b1b9-b226c1793904"
+version = "0.1.0"
+
+[extras]
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+
+[targets]
+test = ["Test"]

Diff for: test/test_packages/FailedNestedTest/src/FailedNestedTest.jl

@@ -0,0 +1,3 @@
+module FailedNestedTest
+
+end # module

Diff for: test/test_packages/FailedNestedTest/test/runtests.jl

@@ -0,0 +1,9 @@
+using FailedNestedTest, Test
+
+@testset "FailedNestedTest" begin
+    @test true
+
+    @testset "nested" begin
+        @test false
+    end
+end

Diff for: test/testsets.jl

@@ -11,6 +11,7 @@ end
         @test 2 == 2
     end
     flattened_testsets = TestReports.flatten_results!(ts)
+    @test flattened_testsets isa Vector
     @test length(flattened_testsets) == 1
     @test all(ts -> ts isa AbstractTestSet, flattened_testsets)
     @test flattened_testsets[1].description == "Top-Level"
@@ -135,32 +136,38 @@ end
 end
 
 @testset "any_problems" begin
-    @test any_problems(Pass(Symbol(), nothing, nothing, nothing)) == false
-    @test any_problems(Fail(Symbol(), nothing, nothing, nothing, LineNumberNode(1))) == true
-    @test any_problems(Broken(Symbol(1), nothing)) == false
-    @test any_problems(Error(Symbol(), nothing, nothing, nothing, LineNumberNode(1))) == true
-
-    fail_code = """
-    using Test
-    using TestReports
-    ts = @testset ReportingTestSet "eg" begin
-        @test false == true
-    end;
-    exit(any_problems(ts))
-    """
-
-    @test_throws Exception run(`$(Base.julia_cmd()) -e $(fail_code)`)
-
-    pass_code = """
-    using Test
-    using TestReports
-    ts = @testset ReportingTestSet "eg" begin
-        @test true == true
-    end;
-    exit(any_problems(ts))
-    """
-
-    @test run(`$(Base.julia_cmd()) -e $(pass_code)`) isa Any #this line would error if fail
+    pass = Pass(Symbol(), nothing, nothing, nothing)
+    fail = Fail(Symbol(), "false", nothing, nothing, LineNumberNode(1))
+
+    @testset "results" begin
+        @test any_problems(pass) === false
+        @test any_problems(fail) === true
+        @test any_problems(Broken(Symbol(1), nothing)) === false
+        @test any_problems(Error(Symbol(), nothing, nothing, nothing, LineNumberNode(1))) === true
+    end
+
+    @testset "testsets" begin
+        ts = DefaultTestSet("")
+        Test.record(ts, pass)
+        @test any_problems(ts) === false
+        Test.record(ts, fail)
+        @test any_problems(ts) === true
+
+        ts = ReportingTestSet("")
+        Test.record(ts, pass)
+        @test any_problems(ts) === false
+        Test.record(ts, fail)
+        @test any_problems(ts) === true
+    end
+
+    @testset "vector" begin
+        ts = [ReportingTestSet("first"), ReportingTestSet("second")]
+        Test.record(ts[1], pass)
+        Test.record(ts[2], pass)
+        @test any_problems(ts) === false
+        Test.record(ts[2], fail)
+        @test any_problems(ts) === true
+    end
 end
 
 @testset "Timing" begin