Add ability to pass an expression to run after every testitem (#108)

* WIP: Pass expression to run after every testitem

* don't nest expr

* Add tests

* fixup! Add tests

* Add docs

* fixup! fixup! Add tests

* Update test/integrationtests.jl

* Run test_end_expr in softscope, same as testitem

* Bump version

* fixup! Run test_end_expr in softscope, same as testitem

* Add TestEndExpr.jl to TEST_PKGS

* Don't import testsetups in test_end_expr

Author: nickrobinson251

Project.toml

@@ -1,6 +1,6 @@
 name = "ReTestItems"
 uuid = "817f1d60-ba6b-4fd5-9520-3cf149f6a823"
-version = "1.16.0"
+version = "1.17.0"
 
 [deps]
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"

README.md

@@ -107,6 +107,8 @@ By default, `Test` and the package being tested will be imported into the `@test
 
 Since a `@testitem` is the block of code that will be executed, `@testitem`s cannot be nested.
 
+#### Test setup
+
 If some test-specific code needs to be shared by multiple `@testitem`s, this code can be placed in a `module` and marked as `@testsetup`,
 and the `@testitem`s can depend on it via the `setup` keyword.
 
@@ -124,7 +126,38 @@ end
 end
 ```
 
-### Summary
+The `setup` is run once on each worker process that requires it;
+it is not run before every `@testitem` that depends on the setup.
+
+#### Post-testitem hook
+
+If there is something that should be checked after every single `@testitem`, then it's possible to pass an expression to `runtests` using the `test_end_expr` keyword.
+This expression will be run immediately after each `@testitem`.
+
+```julia
+test_end_expr = quote
+    @testset "global Foo unchanged" begin
+        foo = get_global_foo()
+        @test foo.changes == 0
+    end
+end
+runtests("frozzle_tests.jl"; test_end_expr)
+```
+
+#### Worker process start-up
+
+If there is some set-up that should be done on each worker process before it is used to evaluated test-items, then it is possible to pass an expression to `runtests` via the `worker_init_expr` keyword.
+This expression will be run on each worker process as soon as it is started.
+
+```julia
+nworkers = 3
+worker_init_expr = quote
+    set_global_foo_memory_limit!(Sys.total_memory()/nworkers)
+end
+runtests("frobble_tests.jl"; nworkers, worker_init_expr)
+```
+
+## Summary
 
 1. Write tests inside of an `@testitem` block.
     - These are like an `@testset`, except that they must contain all the code they need to run;
@@ -157,6 +190,8 @@ end
       using ReTestItems, MyPackage
       runtests(MyPackage)
       ```
+    - Pass to `runtests` any configuration you want your tests to run with, such as `retries`, `testitem_timeout`, `nworkers`, `nworker_threads`, `worker_init_expr`, `test_end_expr`.
+      See the `runtests` docstring for details.
 
 ---
 

src/ReTestItems.jl

@@ -42,6 +42,16 @@ function softscope(@nospecialize ex)
     return ex
 end
 
+# Call softscope on each top-level body expr
+# which has the effect of the body acting like you're at the REPL or
+# inside a testset, except imports/using/etc all still work as expected
+# more info: https://docs.julialang.org/en/v1.10-dev/manual/variables-and-scoping/#on-soft-scope
+function softscope_all!(@nospecialize ex)
+    for i = 1:length(ex.args)
+        ex.args[i] = softscope(ex.args[i])
+    end
+end
+
 include("workers.jl")
 using .Workers
 include("macros.jl")
@@ -127,6 +137,8 @@ will be run.
   supported through a string (e.g. "auto,2").
 - `worker_init_expr::Expr`: an expression that will be evaluated on each worker process before any tests are run.
   Can be used to load packages or set up the environment. Must be a `:block` expression.
+- `test_end_expr::Expr`: an expression that will be evaluated after each testitem is run.
+  Can be used to verify that global state is unchanged after running a test. Must be a `:block` expression.
 - `report::Bool=false`: If `true`, write a JUnit-format XML file summarising the test results.
   Can also be set using the `RETESTITEMS_REPORT` environment variable. The location at which
   the XML report is saved can be set using the `RETESTITEMS_REPORT_LOCATION` environment variable.
@@ -182,7 +194,8 @@ function runtests(
     tags::Union{Symbol,AbstractVector{Symbol},Nothing}=nothing,
     report::Bool=parse(Bool, get(ENV, "RETESTITEMS_REPORT", "false")),
     logs::Symbol=default_log_display_mode(report, nworkers),
-    verbose_results::Bool=(logs !== :issues && isinteractive())
+    verbose_results::Bool=(logs !== :issues && isinteractive()),
+    test_end_expr::Expr=Expr(:block),
 )
     nworker_threads = _validated_nworker_threads(nworker_threads)
     paths′ = filter(paths) do p
@@ -208,10 +221,10 @@ function runtests(
     debuglvl = Int(debug)
     if debuglvl > 0
         LoggingExtras.withlevel(LoggingExtras.Debug; verbosity=debuglvl) do
-            _runtests(shouldrun_combined, paths′, nworkers, nworker_threads, worker_init_expr, testitem_timeout, retries, verbose_results, debuglvl, report, logs)
+            _runtests(shouldrun_combined, paths′, nworkers, nworker_threads, worker_init_expr, test_end_expr, testitem_timeout, retries, verbose_results, debuglvl, report, logs)
         end
     else
-        return _runtests(shouldrun_combined, paths′, nworkers, nworker_threads, worker_init_expr, testitem_timeout, retries, verbose_results, debuglvl, report, logs)
+        return _runtests(shouldrun_combined, paths′, nworkers, nworker_threads, worker_init_expr, test_end_expr, testitem_timeout, retries, verbose_results, debuglvl, report, logs)
     end
 end
 
@@ -225,7 +238,7 @@ end
 # By tracking and reusing test environments, we can avoid this issue.
 const TEST_ENVS = Dict{String, String}()
 
-function _runtests(shouldrun, paths, nworkers::Int, nworker_threads::String, worker_init_expr::Expr, testitem_timeout::Real, retries::Int, verbose_results::Bool, debug::Int, report::Bool, logs::Symbol)
+function _runtests(shouldrun, paths, nworkers::Int, nworker_threads::String, worker_init_expr::Expr, test_end_expr::Expr, testitem_timeout::Real, retries::Int, verbose_results::Bool, debug::Int, report::Bool, logs::Symbol)
     # Don't recursively call `runtests` e.g. if we `include` a file which calls it.
     # So we ignore the `runtests(...)` call in `test/runtests.jl` when `runtests(...)`
     # was called from the command line.
@@ -245,7 +258,7 @@ function _runtests(shouldrun, paths, nworkers::Int, nworker_threads::String, wor
         if is_running_test_runtests_jl(proj_file)
             # Assume this is `Pkg.test`, so test env already active.
             @debugv 2 "Running in current environment `$(Base.active_project())`"
-            return _runtests_in_current_env(shouldrun, paths, proj_file, nworkers, nworker_threads, worker_init_expr, testitem_timeout, retries, verbose_results, debug, report, logs)
+            return _runtests_in_current_env(shouldrun, paths, proj_file, nworkers, nworker_threads, worker_init_expr, test_end_expr, testitem_timeout, retries, verbose_results, debug, report, logs)
         else
             @debugv 1 "Activating test environment for `$proj_file`"
             orig_proj = Base.active_project()
@@ -258,7 +271,7 @@ function _runtests(shouldrun, paths, nworkers::Int, nworker_threads::String, wor
                     testenv = TestEnv.activate()
                     TEST_ENVS[proj_file] = testenv
                 end
-                _runtests_in_current_env(shouldrun, paths, proj_file, nworkers, nworker_threads, worker_init_expr, testitem_timeout, retries, verbose_results, debug, report, logs)
+                _runtests_in_current_env(shouldrun, paths, proj_file, nworkers, nworker_threads, worker_init_expr, test_end_expr, testitem_timeout, retries, verbose_results, debug, report, logs)
             finally
                 Base.set_active_project(orig_proj)
             end
@@ -267,7 +280,7 @@ function _runtests(shouldrun, paths, nworkers::Int, nworker_threads::String, wor
 end
 
 function _runtests_in_current_env(
-    shouldrun, paths, projectfile::String, nworkers::Int, nworker_threads, worker_init_expr::Expr,
+    shouldrun, paths, projectfile::String, nworkers::Int, nworker_threads, worker_init_expr::Expr, test_end_expr::Expr,
     testitem_timeout::Real, retries::Int, verbose_results::Bool, debug::Int, report::Bool, logs::Symbol,
 )
     start_time = time()
@@ -294,7 +307,7 @@ function _runtests_in_current_env(
                 run_number = 1
                 max_runs = 1 + max(retries, testitem.retries)
                 while run_number ≤ max_runs
-                    res = runtestitem(testitem, ctx; verbose_results, logs)
+                    res = runtestitem(testitem, ctx; test_end_expr, verbose_results, logs)
                     ts = res.testset
                     print_errors_and_captured_logs(testitem, run_number; logs)
                     report_empty_testsets(testitem, ts)
@@ -333,7 +346,7 @@ function _runtests_in_current_env(
                 ti = starting[i]
                 @spawn begin
                     with_logger(original_logger) do
-                        manage_worker($w, $proj_name, $testitems, $ti, $nworker_threads, $worker_init_expr, $testitem_timeout, $retries, $verbose_results, $debug, $report, $logs)
+                        manage_worker($w, $proj_name, $testitems, $ti, $nworker_threads, $worker_init_expr, $test_end_expr, $testitem_timeout, $retries, $verbose_results, $debug, $report, $logs)
                     end
                 end
             end
@@ -441,15 +454,15 @@ function record_test_error!(testitem, msg, elapsed_seconds::Real=0.0)
 end
 
 function manage_worker(
-    worker::Worker, proj_name, testitems, testitem, nworker_threads, worker_init_expr,
+    worker::Worker, proj_name, testitems, testitem, nworker_threads, worker_init_expr::Expr, test_end_expr::Expr,
     timeout::Real, retries::Int, verbose_results::Bool, debug::Int, report::Bool, logs::Symbol
 )
     ntestitems = length(testitems.testitems)
     run_number = 1
     while testitem !== nothing
         ch = Channel{TestItemResult}(1)
         testitem.workerid[] = worker.pid
-        fut = remote_eval(worker, :(ReTestItems.runtestitem($testitem, GLOBAL_TEST_CONTEXT; verbose_results=$verbose_results, logs=$(QuoteNode(logs)))))
+        fut = remote_eval(worker, :(ReTestItems.runtestitem($testitem, GLOBAL_TEST_CONTEXT; test_end_expr=$(QuoteNode(test_end_expr)), verbose_results=$verbose_results, logs=$(QuoteNode(logs)))))
         max_runs = 1 + max(retries, testitem.retries)
         try
             timer = Timer(timeout) do tm
@@ -823,19 +836,22 @@ end
 # when `runtestitem` called directly or `@testitem` called outside of `runtests`.
 function runtestitem(
     ti::TestItem, ctx::TestContext;
-    logs::Symbol=:eager, verbose_results::Bool=true, finish_test::Bool=true,
+    test_end_expr::Expr=Expr(:block), logs::Symbol=:eager, verbose_results::Bool=true, finish_test::Bool=true,
 )
     name = ti.name
     log_testitem_start(ti, ctx.ntestitems)
     ts = DefaultTestSet(name; verbose=verbose_results)
     stats = PerfStats()
-    # start with empty block expr and build up our @testitem module body
+    # start with empty block expr and build up our `@testitem` and `test_end_expr` module bodies
     body = Expr(:block)
+    test_end_body = Expr(:block)
     if ti.default_imports
         push!(body.args, :(using Test))
+        push!(test_end_body.args, :(using Test))
         if !isempty(ctx.projectname)
             # this obviously assumes we're in an environment where projectname is reachable
             push!(body.args, :(using $(Symbol(ctx.projectname))))
+            push!(test_end_body.args, :(using $(Symbol(ctx.projectname))))
         end
     end
     Test.push_testset(ts)
@@ -865,27 +881,29 @@ function runtestitem(
             push!(body.args, :(const $setup = $ts_mod))
         end
         @debugv 1 "Setup for test item $(repr(name)) done$(_on_worker())."
-        # add our @testitem quoted code to module body expr
+
+        # add our `@testitem` quoted code to module body expr
         append!(body.args, ti.code.args)
         mod_expr = :(module $(gensym(name)) end)
-        # replace the module body with our built up expr
-        # we're being a bit sneaky here by calling softscope on each top-level body expr
-        # which has the effect of test item body acting like you're at the REPL or
-        # inside a testset, except imports/using/etc all still work as expected
-        # more info: https://docs.julialang.org/en/v1.10-dev/manual/variables-and-scoping/#on-soft-scope
-        for i = 1:length(body.args)
-            body.args[i] = softscope(body.args[i])
-        end
+        softscope_all!(body)
         mod_expr.args[3] = body
+
+        # add the `test_end_expr` to a module to be run after the test item
+        append!(test_end_body.args, test_end_expr.args)
+        softscope_all!(test_end_body)
+        test_end_mod_expr = :(module $(gensym(name * " test_end")) end)
+        test_end_mod_expr.args[3] = test_end_body
+
         # eval the testitem into a temporary module, so that all results can be GC'd
         # once the test is done and sent over the wire. (However, note that anonymous modules
         # aren't always GC'd right now: https://github.com/JuliaLang/julia/issues/48711)
-        @debugv 1 "Evaluating test item $(repr(name))$(_on_worker())."
         # disabled for now since there were issues when tests tried serialize/deserialize
         # with things defined in an anonymous module
         # environment = Module()
+        @debugv 1 "Evaluating test item $(repr(name))$(_on_worker())."
         _, stats = @timed_with_compilation _redirect_logs(logs == :eager ? DEFAULT_STDOUT[] : logpath(ti)) do
             with_source_path(() -> Core.eval(Main, mod_expr), ti.file)
+            with_source_path(() -> Core.eval(Main, test_end_mod_expr), ti.file)
             nothing # return nothing as the first return value of @timed_with_compilation
         end
         @debugv 1 "Done evaluating test item $(repr(name))$(_on_worker())."

test/_integration_test_tools.jl

@@ -6,6 +6,7 @@
 # (ii) do _not_ want expected fail/errors to cause ReTestItems' tests to fail/error
 # This is not `@test_throws` etc, because we're not testing that the code fails/errors
 # we're testing that _the tests themselves_ fail/error.
+using Test
 
 """
     EncasedTestSet(desc, results) <: AbstractTestset