Store timeout limit as elapsed time when recording error (#94)

* Stored timeout limit as elapsed time when recording error

* Bump version

* Add comment

Author: nickrobinson251

Project.toml

@@ -1,6 +1,6 @@
 name = "ReTestItems"
 uuid = "817f1d60-ba6b-4fd5-9520-3cf149f6a823"
-version = "1.12.2"
+version = "1.13.0"
 
 [deps]
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"

src/ReTestItems.jl

@@ -366,15 +366,15 @@ function record_timeout!(testitem, run_number::Int, timeout_limit::Real)
         end
     end
     msg = "Timed out after $time_str evaluating test item $(repr(testitem.name)) (run=$run_number)"
-    record_test_error!(testitem, msg)
+    record_test_error!(testitem, msg, timeout_limit)
 end
 
 function record_worker_terminated!(testitem, run_number::Int)
     msg = "Worker aborted evaluating test item $(repr(testitem.name)) (run=$run_number)"
     record_test_error!(testitem, msg)
 end
 
-function record_test_error!(testitem, msg)
+function record_test_error!(testitem, msg, elapsed_seconds::Real=0.0)
     Test.TESTSET_PRINT_ENABLE[] = false
     ts = DefaultTestSet(testitem.name)
     err = ErrorException(msg)
@@ -386,6 +386,9 @@ function record_test_error!(testitem, msg)
     catch e2
         e2 isa TestSetException || rethrow()
     end
+    # Since we're manually constructing a TestSet here to report tests that already ran and
+    # were killed, we need to manually set how long those tests were running (if known).
+    ts.time_end = ts.time_start + elapsed_seconds
     Test.TESTSET_PRINT_ENABLE[] = true
     push!(testitem.testsets, ts)
     push!(testitem.stats, PerfStats())  # No data since testitem didn't complete

test/integrationtests.jl

@@ -738,4 +738,18 @@ end
     @test_throws expected runtests(file; nworkers=1)
 end
 
+@testset "Timeout failures accurately record elapsed time" begin
+    timeout = 3
+    results = encased_testset() do
+        runtests(joinpath(TEST_FILES_DIR, "_timeout_tests.jl"); nworkers=1, testitem_timeout=timeout)
+    end
+    # unwrap results down to the testset for the timed-out testitem to check its elapsed time
+    results = only(results.results) # test
+    results = only(results.results) # test/testfiles
+    results = only(results.results) # test/testfiles/_timeout_tests.jl
+    ts = results.results[1]
+    @assert ts.description == "Test item takes 60 seconds"
+    @test ts.time_end - ts.time_start ≈ timeout
+end
+
 end # integrationtests.jl testset