Skip to content

Commit 0b72753

Browse files
Clearer terminology 'evaluate' -> 'run' (#116)
* Clearer terminology 'evaluate' -> 'run' * Bump version
1 parent 7532642 commit 0b72753

File tree

5 files changed

+27
-27
lines changed

5 files changed

+27
-27
lines changed

Project.toml

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
name = "ReTestItems"
22
uuid = "817f1d60-ba6b-4fd5-9520-3cf149f6a823"
3-
version = "1.19.0"
3+
version = "1.20.0"
44

55
[deps]
66
Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"

src/ReTestItems.jl

+12-12
Original file line numberDiff line numberDiff line change
@@ -142,16 +142,16 @@ will be run.
142142
- `nworker_threads::Union{String,Int}`: The number of threads to use for each worker process. Defaults to 2.
143143
Can also be set using the `RETESTITEMS_NWORKER_THREADS` environment variable. Interactive threads are
144144
supported through a string (e.g. "auto,2").
145-
- `worker_init_expr::Expr`: an expression that will be evaluated on each worker process before any tests are run.
145+
- `worker_init_expr::Expr`: an expression that will be run on each worker process before any tests are run.
146146
Can be used to load packages or set up the environment. Must be a `:block` expression.
147-
- `test_end_expr::Expr`: an expression that will be evaluated after each testitem is run.
147+
- `test_end_expr::Expr`: an expression that will be run after each testitem is run.
148148
Can be used to verify that global state is unchanged after running a test. Must be a `:block` expression.
149149
- `memory_threshold::Real`: Sets the fraction of memory that can be in use before a worker processes are
150150
restarted to free memory. Defaults to $(DEFAULT_MEMORY_THRESHOLD[]). Only supported with `nworkers > 0`.
151151
For example, if set to 0.8, then when >80% of the available memory is in use, a worker process will be killed and
152-
replaced with a new worker before the next testitem is evaluated. The testitem will then be run on the new worker
152+
replaced with a new worker before the next testitem is run. The testitem will then be run on the new worker
153153
process, regardless of if memory pressure dropped below the threshold. If the memory pressure remains above the
154-
threshold, then a worker process will again be replaced before the next testitem is evaluated.
154+
threshold, then a worker process will again be replaced before the next testitem is run.
155155
Can also be set using the `RETESTITEMS_MEMORY_THRESHOLD` environment variable.
156156
**Note**: the `memory_threshold` keyword is experimental and may be removed in future versions.
157157
- `report::Bool=false`: If `true`, write a JUnit-format XML file summarising the test results.
@@ -357,7 +357,7 @@ function _runtests_in_current_env(
357357
end
358358
end
359359
# Now all workers are started, we can begin processing test items.
360-
@info "Starting evaluating test items"
360+
@info "Starting running test items"
361361
starting = get_starting_testitems(testitems, nworkers)
362362
@sync for (i, w) in enumerate(workers)
363363
ti = starting[i]
@@ -380,7 +380,7 @@ function _runtests_in_current_env(
380380
return nothing
381381
end
382382

383-
# Start a new `Worker` with `nworker_threads` threads and evaluate `worker_init_expr` on it.
383+
# Start a new `Worker` with `nworker_threads` threads and run `worker_init_expr` on it.
384384
# The provided `worker_num` is only for logging purposes, and not persisted as part of the worker.
385385
function start_worker(proj_name, nworker_threads, worker_init_expr, ntestitems; worker_num=nothing)
386386
w = Worker(; threads="$nworker_threads")
@@ -439,13 +439,13 @@ function record_timeout!(testitem, run_number::Int, timeout_limit::Real)
439439
string(mins, "m", lpad(secs, 2, "0"), "s")
440440
end
441441
end
442-
msg = "Timed out after $time_str evaluating test item $(repr(testitem.name)) (run=$run_number)"
442+
msg = "Timed out after $time_str running test item $(repr(testitem.name)) (run=$run_number)"
443443
record_test_error!(testitem, msg, timeout_limit)
444444
end
445445

446446
function record_worker_terminated!(testitem, worker::Worker, run_number::Int)
447447
termsignal = worker.process.termsignal
448-
msg = "Worker process aborted (signal=$termsignal) evaluating test item $(repr(testitem.name)) (run=$run_number)"
448+
msg = "Worker process aborted (signal=$termsignal) running test item $(repr(testitem.name)) (run=$run_number)"
449449
record_test_error!(testitem, msg)
450450
end
451451

@@ -537,11 +537,11 @@ function manage_worker(
537537
@debugv 1 "Test item $(repr(testitem.name)) timed out. Terminating worker $worker"
538538
terminate!(worker)
539539
wait(worker)
540-
@error "$worker timed out evaluating test item $(repr(testitem.name)) after $timeout seconds. \
540+
@error "$worker timed out running test item $(repr(testitem.name)) after $timeout seconds. \
541541
Recording test error."
542542
record_timeout!(testitem, run_number, timeout)
543543
elseif e isa WorkerTerminatedException
544-
@error "$worker died evaluating test item $(repr(testitem.name)). \
544+
@error "$worker died running test item $(repr(testitem.name)). \
545545
Recording test error."
546546
record_worker_terminated!(testitem, worker, run_number)
547547
else
@@ -924,13 +924,13 @@ function runtestitem(
924924
# disabled for now since there were issues when tests tried serialize/deserialize
925925
# with things defined in an anonymous module
926926
# environment = Module()
927-
@debugv 1 "Evaluating test item $(repr(name))$(_on_worker())."
927+
@debugv 1 "Running test item $(repr(name))$(_on_worker())."
928928
_, stats = @timed_with_compilation _redirect_logs(logs == :eager ? DEFAULT_STDOUT[] : logpath(ti)) do
929929
with_source_path(() -> Core.eval(Main, mod_expr), ti.file)
930930
with_source_path(() -> Core.eval(Main, test_end_mod_expr), ti.file)
931931
nothing # return nothing as the first return value of @timed_with_compilation
932932
end
933-
@debugv 1 "Done evaluating test item $(repr(name))$(_on_worker())."
933+
@debugv 1 "Done running test item $(repr(name))$(_on_worker())."
934934
catch err
935935
err isa InterruptException && rethrow()
936936
# Handle exceptions thrown outside a `@test` in the body of the @testitem:

src/macros.jl

+6-6
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ gettls(k, d) = get(task_local_storage(), k, d)
77
"""
88
TestSetup(name, code)
99
10-
A module that a `TestItem` can require to be evaluated before that `TestItem` is run.
10+
A module that a `TestItem` can require to be run before that `TestItem` is run.
1111
Used for declaring code that multiple `TestItem`s rely on.
1212
Should only be created via the `@testsetup` macro.
1313
"""
@@ -32,7 +32,7 @@ A module only used for tests, and which `@testitem`s can depend on.
3232
3333
Useful for setup logic that is used across multiple test items.
3434
The setup will run once, before any `@testitem` that requires it is executed.
35-
If running with multiple processes, each test-setup with be evaluated once on each process.
35+
If running with multiple processes, each test-setup with be run once on each process.
3636
3737
Each test-setup module will live for the lifetime of the tests.
3838
Mutable state should be avoided, since the order in which test items run is non-deterministic,
@@ -125,9 +125,9 @@ struct TestItem
125125
code::Any
126126
testsetups::Vector{TestSetup} # populated by runtests coordinator
127127
workerid::Base.RefValue{Int} # populated when the test item is scheduled
128-
testsets::Vector{DefaultTestSet} # populated when the test item is finished evaluating
129-
eval_number::Base.RefValue{Int} # to keep track of how many items have been evaluated so far
130-
stats::Vector{PerfStats} # populated when the test item is finished evaluating
128+
testsets::Vector{DefaultTestSet} # populated when the test item is finished running
129+
eval_number::Base.RefValue{Int} # to keep track of how many items have been run so far
130+
stats::Vector{PerfStats} # populated when the test item is finished running
131131
scheduled_for_evaluation::ScheduledForEvaluation # to keep track of whether the test item has been scheduled for evaluation
132132
end
133133
function TestItem(number, name, id, tags, default_imports, setups, retries, file, line, project_root, code)
@@ -182,7 +182,7 @@ A `@testitem` is wrapped into a module when run, so must import any additional p
182182
end
183183
end
184184
185-
The test item's code is evaluated as top-level code in a new module, so it can include imports, define new structs or helper functions, and declare tests and testsets.
185+
The test item's code is run as top-level code in a new module, so it can include imports, define new structs or helper functions, and declare tests and testsets.
186186
187187
@testitem "DoCoolStuff" begin
188188
function do_really_cool_stuff()

test/integrationtests.jl

+4-4
Original file line numberDiff line numberDiff line change
@@ -573,7 +573,7 @@ end
573573
""",
574574
replace(c.output, r" on worker \d+" => ""))
575575

576-
# Since the test setup never succeeds it will be evaluated mutliple times. Here we test
576+
# Since the test setup never succeeds it will be run mutliple times. Here we test
577577
# that we don't accumulate logs from all previous failed attempts (which would get
578578
# really spammy if the test setup is used by 100 test items).
579579
@test !occursin("""
@@ -610,7 +610,7 @@ end
610610
# Test the error is as expected
611611
err = only(non_passes(results))
612612
@test err.test_type == :nontest_error
613-
@test err.value == string(ErrorException("Worker process aborted (signal=6) evaluating test item \"Abort\" (run=1)"))
613+
@test err.value == string(ErrorException("Worker process aborted (signal=6) running test item \"Abort\" (run=1)"))
614614
end
615615

616616
@testset "test retrying failing testitem" begin
@@ -665,7 +665,7 @@ end
665665
# Test the error is as expected
666666
err = only(non_passes(results))
667667
@test err.test_type == :nontest_error
668-
@test err.value == string(ErrorException("Timed out after 4s evaluating test item \"Test item takes 60 seconds\" (run=1)"))
668+
@test err.value == string(ErrorException("Timed out after 4s running test item \"Test item takes 60 seconds\" (run=1)"))
669669
end
670670

671671
@testset "testitem timeout set via env variable" begin
@@ -682,7 +682,7 @@ end
682682
# Test the error is as expected
683683
err = only(non_passes(results))
684684
@test err.test_type == :nontest_error
685-
@test err.value == string(ErrorException("Timed out after 4s evaluating test item \"Test item takes 60 seconds\" (run=1)"))
685+
@test err.value == string(ErrorException("Timed out after 4s running test item \"Test item takes 60 seconds\" (run=1)"))
686686
end
687687

688688
@testset "Error outside `@testitem`" begin

test/references/retry_tests_report_1worker.xml

+4-4
Original file line numberDiff line numberDiff line change
@@ -244,7 +244,7 @@ No Captured Logs for test item "Has retries=1 and always fails" at test/testfile
244244
<error message="Error during test at test/testfiles/_retry_tests.jl:54">Error in testset "Timeout always" on worker 72554:
245245
Error During Test at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:54
246246
Got exception outside of a @test
247-
Timed out after 5s evaluating test item "Timeout always" (run=1)
247+
Timed out after 5s running test item "Timeout always" (run=1)
248248
Captured Logs for test item "Timeout always" at test/testfiles/_retry_tests.jl:54 on worker 72554
249249

250250
signal (15): Terminated: 15
@@ -269,7 +269,7 @@ Allocations: 5231943 (Pool: 5229611; Big: 2332); GC: 30
269269
<error message="Error during test at test/testfiles/_retry_tests.jl:54">Error in testset "Timeout always" on worker 72554:
270270
Error During Test at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:54
271271
Got exception outside of a @test
272-
Timed out after 5s evaluating test item "Timeout always" (run=2)
272+
Timed out after 5s running test item "Timeout always" (run=2)
273273
Captured Logs for test item "Timeout always" at test/testfiles/_retry_tests.jl:54 on worker 72554
274274

275275
signal (15): Terminated: 15
@@ -294,7 +294,7 @@ Allocations: 3068888 (Pool: 3067775; Big: 1113); GC: 3
294294
<error message="Error during test at test/testfiles/_retry_tests.jl:54">Error in testset "Timeout always" on worker 72554:
295295
Error During Test at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:54
296296
Got exception outside of a @test
297-
Timed out after 5s evaluating test item "Timeout always" (run=3)
297+
Timed out after 5s running test item "Timeout always" (run=3)
298298
Captured Logs for test item "Timeout always" at test/testfiles/_retry_tests.jl:54 on worker 72554
299299

300300
signal (15): Terminated: 15
@@ -319,7 +319,7 @@ Allocations: 3068845 (Pool: 3067734; Big: 1111); GC: 3
319319
<error message="Error during test at test/testfiles/_retry_tests.jl:64">Error in testset "Timeout first, pass after" on worker 72859:
320320
Error During Test at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:64
321321
Got exception outside of a @test
322-
Timed out after 5s evaluating test item "Timeout first, pass after" (run=1)
322+
Timed out after 5s running test item "Timeout first, pass after" (run=1)
323323
Captured Logs for test item "Timeout first, pass after" at test/testfiles/_retry_tests.jl:64 on worker 72859
324324

325325
signal (15): Terminated: 15

0 commit comments

Comments
 (0)