Skip to content

Commit 341515b

Browse files
charleskawczynskicharleskawczynski
committed
Add unit tests for some reproducibility infrastructure
1 parent 3c2e640 commit 341515b

File tree

4 files changed

+149
-64
lines changed

4 files changed

+149
-64
lines changed

‎reproducibility_tests/compute_mse.jl

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ function reproducibility_test(;
6464
paths = String[] # initialize for later handling
6565

6666
if haskey(ENV, "BUILDKITE_COMMIT")
67-
paths = latest_comparable_paths(10)
67+
paths = latest_comparable_paths(; n = 10)
6868
isempty(paths) && return (reference_mse, paths)
6969
@info "`ds_filename_computed`: `$ds_filename_computed`"
7070
ds_filename_references =

‎reproducibility_tests/latest_comparable_paths.jl

Lines changed: 58 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ are found.
1212
"""
1313
function sorted_dataset_folder(; dir = pwd())
1414
matching_paths = filter(ispath, readdir(dir; join = true))
15-
isempty(matching_paths) && return ""
15+
isempty(matching_paths) && return String[]
1616
# sort by timestamp
1717
sorted_paths =
1818
sort(matching_paths; by = f -> Dates.unix2datetime(stat(f).mtime))
@@ -40,85 +40,80 @@ function ref_counters_per_path(paths)
4040
end
4141

4242
"""
43-
latest_comparable_paths(n::Integer)
43+
paths = latest_comparable_paths(;
44+
n = 5,
45+
root_path = "/central/scratch/esm/slurm-buildkite/climaatmos-main",
46+
ref_counter_PR = read_ref_counter(joinpath(@__DIR__, "ref_counter.jl"))
47+
)
4448
4549
Returns a vector of strings, containing the `n`
46-
latest comparable paths based on
47-
`reproducibility_tests/ref_counter.jl`.
48-
"""
49-
function latest_comparable_paths(n = 5)
50-
if get(ENV, "BUILDKITE_PIPELINE_SLUG", nothing) != "climaatmos-ci"
51-
@warn "Not using climaatmos-ci pipeline slug, assuming no comparable references"
52-
@info "Please review output results before merging."
53-
return String[]
54-
end
50+
latest comparable paths. The assumed folder structure
51+
is:
52+
53+
```
54+
root_path/some_folder_1/ref_counter.jl
55+
root_path/some_folder_2/ref_counter.jl
56+
root_path/some_folder_3/ref_counter.jl
57+
```
5558
56-
# Note: cluster_data_prefix is also defined in move_output.jl
57-
cluster_data_prefix = "/central/scratch/esm/slurm-buildkite/climaatmos-main"
59+
If a subfolder does not contain a `ref_counter.jl` file
60+
then it is filtered out as not-comparable. The `ref_counter.jl`
61+
files are assumed to start with a single integer,
62+
which is read. If that integer matches `ref_counter_PR`,
63+
then that path is considered comparable.
64+
65+
`paths[1]` is the most recent comparable path, and
66+
`paths[end]` is the oldest comparable path.
67+
"""
68+
function latest_comparable_paths(;
69+
n = 5,
70+
root_path = "/central/scratch/esm/slurm-buildkite/climaatmos-main",
71+
ref_counter_PR = read_ref_counter(joinpath(@__DIR__, "ref_counter.jl")),
72+
)
73+
@info "---Finding the latest comparable paths"
74+
# Note: root_path is also defined in move_output.jl
5875
# Get (sorted) array of paths, `pop!(sorted_paths)`
5976
# is the most recent merged folder.
60-
sorted_paths = sorted_dataset_folder(; dir = cluster_data_prefix)
77+
sorted_paths = sorted_dataset_folder(; dir = root_path)
6178
if isempty(sorted_paths)
62-
@warn "No paths on main found, assuming no comparable references"
63-
@info "Please review output results before merging."
79+
@warn "No paths found in $root_path"
6480
return String[]
6581
end
6682
# Find oldest path in main with the same reference
6783
# counter as the one in the PR. If none exists,
6884
# then assume no comparable references.
6985

70-
ref_counter_file_PR = joinpath(@__DIR__, "ref_counter.jl")
71-
@assert isfile(ref_counter_file_PR)
72-
ref_counter_PR = read_ref_counter(ref_counter_file_PR)
73-
74-
ref_counters_main = ref_counters_per_path(sorted_paths)
75-
i_comparable_references = findall(ref_counters_main) do ref_counter_main
76-
ref_counter_main == ref_counter_PR
77-
end
78-
if isnothing(i_comparable_references)
79-
@warn "`ref_counter.jl` not found on main, assuming no comparable references"
80-
@info "Please review output results before merging."
86+
# Short circuit if we don't find anything:
87+
found_ref_counters =
88+
filter(p -> isfile(joinpath(p, "ref_counter.jl")), sorted_paths)
89+
if isempty(found_ref_counters)
90+
@warn "No reference counters found in paths: $sorted_paths"
8191
return String[]
8292
end
83-
@info "Found $(length(i_comparable_references)) comparable references:$i_comparable_references"
84-
# Largest ref-counter reference path:
85-
paths = map(i -> sorted_paths[i], i_comparable_references)
86-
@info "$(length(paths)) paths found:"
87-
for p in paths
88-
@info " $p, $(Dates.unix2datetime(stat(p).mtime))"
93+
94+
# Find comparable paths
95+
comparable_paths = String[]
96+
@info "Reference counters found:"
97+
for (i, path) in enumerate(sorted_paths)
98+
ref_counter_file = joinpath(path, "ref_counter.jl")
99+
!isfile(ref_counter_file) && continue
100+
rc = read_ref_counter(ref_counter_file)
101+
comparable = ref_counter_PR == rc
102+
suffix = comparable ? ", comparable" : ""
103+
@info " $path: $rc$suffix"
104+
comparable && push!(comparable_paths, path)
89105
end
90-
ref_counter_files_main = map(p -> joinpath(p, "ref_counter.jl"), paths)
91-
@info "$(length(ref_counter_files_main)) reference counter paths on central"
92-
filter!(isfile, ref_counter_files_main)
93-
@info "$(length(ref_counter_files_main)) reference counter paths on central after filtering isfile"
94-
95-
# for p in paths
96-
# @info "Files in $p:" # for debugging
97-
# for file_on_main in readdir(p)
98-
# @info " File:`$file_on_main`"
99-
# end
100-
# end
101-
@assert all(isfile, ref_counter_files_main)
102-
ref_counters_main = map(read_ref_counter, ref_counter_files_main)
103-
if all(rc -> ref_counter_PR == rc + 1, ref_counters_main) # new reference
104-
@warn "`ref_counter.jl` incremented, assuming no comparable references"
105-
@info "Ref counters main: $ref_counters_main."
106-
@info "Please review output results before merging."
106+
107+
if isempty(comparable_paths)
108+
@warn "No comparable paths found in any of the paths:$sorted_paths"
107109
return String[]
108-
elseif all(rc -> ref_counter_PR == rc, ref_counters_main) # unchanged reference
109-
@info "Ref counters main: $ref_counters_main."
110-
@info "Comparing results against main path:$paths"
111-
else
112-
error(
113-
"Unexpected reference. Please open an issue pointing to this build.",
114-
)
115110
end
116111

117-
paths = reverse(paths)[1:min(n, length(paths))]
118-
@info "Limiting comparable paths to $n:"
119-
for p in paths
120-
@info " $p, $(Dates.unix2datetime(stat(p).mtime))"
112+
comparable_paths = reverse(comparable_paths) # sort so that
113+
114+
if length(comparable_paths) > n # limit to n comparable paths
115+
comparable_paths = comparable_paths[1:min(n, length(comparable_paths))]
121116
end
122-
# Get the top 10 most recent paths to compare against:
123-
return paths
117+
118+
return comparable_paths
124119
end

‎test/runtests.jl

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ using Test
2222
@safetestset "Model getters" begin @time include("solver/model_getters.jl") end
2323
@safetestset "Topography tests" begin @time include("topography.jl") end
2424
@safetestset "Restarts" begin @time include("restart.jl") end
25+
@safetestset "Reproducibility infra" begin @time include("unit_reproducibility_infra.jl") end
2526

2627
#! format: on
2728

‎test/unit_reproducibility_infra.jl

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
#=
2+
using Revise; include("test/unit_reproducibility_infra.jl")
3+
=#
4+
using Test
5+
using Dates
6+
7+
include(joinpath("..", "reproducibility_tests/latest_comparable_paths.jl"))
8+
9+
function make_ref_file_counter(dir, pathname, i)
10+
d = mkdir(pathname)
11+
open(io -> println(io, i), joinpath(d, "ref_counter.jl"), "w")
12+
return joinpath(dir, d)
13+
end
14+
15+
@testset "Reproducibility infrastructure: latest_comparable_paths" begin
16+
# No paths at all
17+
mktempdir() do path
18+
cd(path) do
19+
paths =
20+
latest_comparable_paths(; root_path = path, ref_counter_PR = 2)
21+
@test paths == []
22+
end
23+
end
24+
25+
# No paths with ref counters
26+
mktempdir() do path
27+
cd(path) do
28+
p1 = mkdir("d1")
29+
paths =
30+
latest_comparable_paths(; root_path = path, ref_counter_PR = 2)
31+
@test paths == []
32+
end
33+
end
34+
35+
# No paths with matching ref counters
36+
mktempdir() do path
37+
cd(path) do
38+
p1 = make_ref_file_counter(path, "d1", 1)
39+
paths =
40+
latest_comparable_paths(; root_path = path, ref_counter_PR = 2)
41+
@test paths == []
42+
end
43+
end
44+
45+
# 1 matching ref counter
46+
mktempdir() do path
47+
cd(path) do
48+
p1 = make_ref_file_counter(path, "d1", 1)
49+
p2 = make_ref_file_counter(path, "d2", 2)
50+
p3 = make_ref_file_counter(path, "d3", 3)
51+
paths =
52+
latest_comparable_paths(; root_path = path, ref_counter_PR = 2)
53+
@test paths == [p2]
54+
end
55+
end
56+
57+
# multiple matching ref counters
58+
mktempdir() do path
59+
cd(path) do
60+
p1 = make_ref_file_counter(path, "d1", 1)
61+
p2 = make_ref_file_counter(path, "d2", 2)
62+
p3 = make_ref_file_counter(path, "d3", 3)
63+
p4 = make_ref_file_counter(path, "d4", 3)
64+
p5 = make_ref_file_counter(path, "d5", 3)
65+
p6 = make_ref_file_counter(path, "d6", 3)
66+
paths =
67+
latest_comparable_paths(; root_path = path, ref_counter_PR = 3)
68+
@test paths == [p6, p5, p4, p3] # p6 is most recent
69+
end
70+
end
71+
72+
# matching ref counters that exceed n
73+
mktempdir() do path
74+
cd(path) do
75+
p1 = make_ref_file_counter(path, "d1", 1)
76+
p2 = make_ref_file_counter(path, "d2", 2)
77+
p3 = make_ref_file_counter(path, "d3", 3)
78+
p4 = make_ref_file_counter(path, "d4", 3)
79+
p5 = make_ref_file_counter(path, "d5", 3)
80+
p6 = make_ref_file_counter(path, "d6", 3)
81+
paths = latest_comparable_paths(;
82+
n = 2,
83+
root_path = path,
84+
ref_counter_PR = 3,
85+
)
86+
@test paths == [p6, p5] # p6 is most recent
87+
end
88+
end
89+
end

0 commit comments

Comments
 (0)