From f802f8b09ec4194a1205b306b7707ef7aa84cc55 Mon Sep 17 00:00:00 2001 From: Charles Kawczynski Date: Sat, 9 Nov 2024 11:31:28 -0500 Subject: [PATCH] Use env for repro tests, improve repro names Fixes for gpu repro tests, auto-compare all state variables Improve error message, increment ref counter Fix zero_dict calls Fix dict init Fixes to zero_dict Improve debug info --- .buildkite/pipeline.yml | 117 +++- config/default_configs/default_config.yml | 3 - ...sphere_baroclinic_wave_rhoe_equilmoist.yml | 1 - .../diagnostic_edmfx_aquaplanet.yml | 1 - .../single_column_precipitation_test.yml | 1 - ...lmoist_allsky_gw_raw_zonallyasymmetric.yml | 1 - ...sphere_baroclinic_wave_rhoe_equilmoist.yml | 1 - ..._suarez_rhoe_equilmoist_hightop_sponge.yml | 1 - examples/hybrid/driver.jl | 15 +- reproducibility_tests/README.md | 17 +- reproducibility_tests/compute_mse.jl | 184 ++---- reproducibility_tests/move_output.jl | 10 +- reproducibility_tests/mse_tables.jl | 554 ++---------------- reproducibility_tests/print_new_mse.jl | 40 +- .../print_new_ref_counter.jl | 26 +- reproducibility_tests/ref_counter.jl | 8 +- .../reproducibility_tests.jl | 37 +- reproducibility_tests/test_mse.jl | 8 - reproducibility_tests/test_reset.jl | 14 - 19 files changed, 292 insertions(+), 747 deletions(-) delete mode 100644 reproducibility_tests/test_reset.jl diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 3f60677ee2..d5e12d7326 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -56,12 +56,6 @@ steps: - wait - - group: "Reproducibility tests" - steps: - - - label: ":computer: Ensure mse tables are reset when necessary" - command: "julia --color=yes --project=examples reproducibility_tests/test_reset.jl" - - group: "Radiation" steps: @@ -149,6 +143,8 @@ steps: --job_id single_column_hydrostatic_balance_ft64 --out_dir single_column_hydrostatic_balance_ft64/output_active artifact_paths: "single_column_hydrostatic_balance_ft64/output_active/*" + env: + test_reproducibility: "true" - group: "Box Examples" steps: @@ -163,6 +159,8 @@ steps: --job_id box_hydrostatic_balance_rhoe --out_dir box_hydrostatic_balance_rhoe/output_active artifact_paths: "box_hydrostatic_balance_rhoe/output_active/*" + env: + test_reproducibility: "true" - label: ":computer: 3D density current" command: > @@ -174,6 +172,8 @@ steps: --job_id box_density_current_test --out_dir box_density_current_test/output_active artifact_paths: "box_density_current_test/output_active/*" + env: + test_reproducibility: "true" - label: ":computer: Box (ρe_tot) rcemipii with diagnostic edmf" command: > @@ -185,6 +185,8 @@ steps: --job_id rcemipii_box_diagnostic_edmfx --out_dir rcemipii_box_diagnostic_edmfx/output_active artifact_paths: "rcemipii_box_diagnostic_edmfx/output_active/*" + env: + test_reproducibility: "true" agents: slurm_mem: 20GB @@ -199,6 +201,7 @@ steps: --out_dir les_isdac_box/output_active artifact_paths: "les_isdac_box/*" env: + test_reproducibility: "true" CLIMACOMMS_DEVICE: "CUDA" agents: slurm_mem: 16G @@ -217,6 +220,8 @@ steps: --job_id plane_agnesi_mountain_test_uniform --out_dir plane_agnesi_mountain_test_uniform/output_active artifact_paths: "plane_agnesi_mountain_test_uniform/output_active/*" + env: + test_reproducibility: "true" - label: ":computer: Agnesi linear hydrostatic mountain experiment (stretched)" command: > @@ -228,6 +233,8 @@ steps: --job_id plane_agnesi_mountain_test_stretched --out_dir plane_agnesi_mountain_test_stretched/output_active artifact_paths: "plane_agnesi_mountain_test_stretched/output_active/*" + env: + test_reproducibility: "true" - label: ":computer: Density current experiment" command: > @@ -239,6 +246,8 @@ steps: --job_id plane_density_current_test --out_dir plane_density_current_test/output_active artifact_paths: "plane_density_current_test/output_active/*" + env: + test_reproducibility: "true" - group: "Conservation check" @@ -277,6 +286,8 @@ steps: artifact_paths: "sphere_hydrostatic_balance_rhoe_ft64/output_active/*" agents: slurm_mem: 20GB + env: + test_reproducibility: "true" - label: ":computer: baroclinic wave (ρe)" key: sphere_baroclinic_wave_rhoe @@ -289,6 +300,8 @@ steps: --job_id sphere_baroclinic_wave_rhoe --out_dir sphere_baroclinic_wave_rhoe/output_active artifact_paths: "sphere_baroclinic_wave_rhoe/output_active/*" + env: + test_reproducibility: "true" - label: ":computer: no lim baroclinic wave (ρe) equilmoist" command: > @@ -302,6 +315,8 @@ steps: artifact_paths: "sphere_baroclinic_wave_rhoe_equilmoist/output_active/*" agents: slurm_constraint: icelake|cascadelake|skylake|epyc + env: + test_reproducibility: "true" - label: ":computer: no lim baroclinic wave (ρe) equilmoist (deep sphere)" command: > @@ -315,6 +330,8 @@ steps: artifact_paths: "deep_sphere_baroclinic_wave_rhoe_equilmoist/output_active/*" agents: slurm_constraint: icelake|cascadelake|skylake|epyc + env: + test_reproducibility: "true" # Add this back when we figure out what to do with SSP and zalesak # - label: ":computer: SSP zalesak tracer & energy upwind baroclinic wave (ρe_tot) equilmoist" @@ -339,6 +356,8 @@ steps: --job_id sphere_held_suarez_rhoe_hightop --out_dir sphere_held_suarez_rhoe_hightop/output_active artifact_paths: "sphere_held_suarez_rhoe_hightop/output_active/*" + env: + test_reproducibility: "true" - label: ":computer: no lim held suarez (ρe) equilmoist hightop sponge" command: > @@ -352,6 +371,8 @@ steps: artifact_paths: "sphere_held_suarez_rhoe_equilmoist_hightop_sponge/output_active/*" agents: slurm_constraint: icelake|cascadelake|skylake|epyc + env: + test_reproducibility: "true" - group: "Sphere Examples (Aquaplanet)" steps: @@ -368,6 +389,8 @@ steps: artifact_paths: "sphere_aquaplanet_rhoe_equilmoist_allsky_gw_res/output_active/*" agents: slurm_mem: 20GB + env: + test_reproducibility: "true" - label: ":computer: aquaplanet (ρe_tot) equil allsky monin_obukhov varying insol gravity wave (raw_topo) high top zonally asymmetric" command: > @@ -379,6 +402,8 @@ steps: --job_id sphere_aquaplanet_rhoe_equilmoist_allsky_gw_raw_zonallyasymmetric --out_dir sphere_aquaplanet_rhoe_equilmoist_allsky_gw_raw_zonallyasymmetric/output_active artifact_paths: "sphere_aquaplanet_rhoe_equilmoist_allsky_gw_raw_zonallyasymmetric/output_active/*" + env: + test_reproducibility: "true" agents: slurm_mem: 20GB slurm_constraint: icelake|cascadelake|skylake|epyc @@ -393,6 +418,8 @@ steps: --job_id sphere_aquaplanet_rhoe_nonequilmoist_allsky --out_dir sphere_aquaplanet_rhoe_nonequilmoist_allsky/output_active artifact_paths: "sphere_aquaplanet_rhoe_nonequilmoist_allsky/output_active/*" + env: + test_reproducibility: "true" agents: slurm_mem: 20GB @@ -406,6 +433,8 @@ steps: --job_id aquaplanet_rhoe_equil_clearsky_tvinsol_0M_slabocean --out_dir aquaplanet_rhoe_equil_clearsky_tvinsol_0M_slabocean/output_active artifact_paths: "aquaplanet_rhoe_equil_clearsky_tvinsol_0M_slabocean/output_active/*" + env: + test_reproducibility: "true" agents: slurm_mem: 20GB @@ -419,6 +448,8 @@ steps: --job_id aquaplanet_rhoe_equil_clearsky_tvinsol_0M_slabocean_ft64 --out_dir aquaplanet_rhoe_equil_clearsky_tvinsol_0M_slabocean_ft64/output_active artifact_paths: "aquaplanet_rhoe_equil_clearsky_tvinsol_0M_slabocean_ft64/output_active/*" + env: + test_reproducibility: "true" agents: slurm_mem: 20GB @@ -432,6 +463,8 @@ steps: --job_id rcemipii_sphere_diagnostic_edmfx --out_dir rcemipii_sphere_diagnostic_edmfx/output_active artifact_paths: "rcemipii_sphere_diagnostic_edmfx/output_active/*" + env: + test_reproducibility: "true" agents: slurm_mem: 20GB @@ -448,6 +481,8 @@ steps: --job_id sphere_baroclinic_wave_rhoe_topography_dcmip_rs --out_dir sphere_baroclinic_wave_rhoe_topography_dcmip_rs/output_active artifact_paths: "sphere_baroclinic_wave_rhoe_topography_dcmip_rs/output_active/*" + env: + test_reproducibility: "true" - label: ":computer: held suarez (ρe) topography (dcmip)" command: > @@ -459,6 +494,8 @@ steps: --job_id sphere_held_suarez_rhoe_topography_dcmip --out_dir sphere_held_suarez_rhoe_topography_dcmip/output_active artifact_paths: "sphere_held_suarez_rhoe_topography_dcmip/output_active/*" + env: + test_reproducibility: "true" - label: ":computer: held suarez (ρe) equilmoist topography (dcmip)" command: > @@ -470,6 +507,8 @@ steps: --job_id sphere_held_suarez_rhoe_equilmoist_topography_dcmip --out_dir sphere_held_suarez_rhoe_equilmoist_topography_dcmip/output_active artifact_paths: "sphere_held_suarez_rhoe_equilmoist_topography_dcmip/output_active/*" + env: + test_reproducibility: "true" agents: slurm_mem: 20GB @@ -483,6 +522,8 @@ steps: --job_id sphere_ssp_baroclinic_wave_rhoe_equilmoist_dcmip200 --out_dir sphere_ssp_baroclinic_wave_rhoe_equilmoist_dcmip200/output_active artifact_paths: "sphere_ssp_baroclinic_wave_rhoe_equilmoist_dcmip200/output_active/*" + env: + test_reproducibility: "true" - label: ":computer: Diagnostic Earth surface elevation spectra" command: > @@ -494,6 +535,8 @@ steps: --job_id sphere_ssp_baroclinic_wave_rhoe_equilmoist_earth --out_dir sphere_ssp_baroclinic_wave_rhoe_equilmoist_earth/output_active artifact_paths: "sphere_ssp_baroclinic_wave_rhoe_equilmoist_earth/output_active/*" + env: + test_reproducibility: "true" - group: "Restarting" steps: @@ -607,6 +650,7 @@ steps: --out_dir mpi_sphere_aquaplanet_rhoe_equilmoist_clearsky/output_active artifact_paths: "mpi_sphere_aquaplanet_rhoe_equilmoist_clearsky/output_active/*" env: + test_reproducibility: "true" CLIMACOMMS_CONTEXT: "MPI" agents: slurm_ntasks: 2 @@ -669,6 +713,8 @@ steps: --job_id diagnostic_edmfx_test_box --out_dir diagnostic_edmfx_test_box/output_active artifact_paths: "diagnostic_edmfx_test_box/output_active/*" + env: + test_reproducibility: "true" agents: slurm_mem: 20GB @@ -682,6 +728,8 @@ steps: --job_id diagnostic_edmfx_gabls_box --out_dir diagnostic_edmfx_gabls_box/output_active artifact_paths: "diagnostic_edmfx_gabls_box/output_active/*" + env: + test_reproducibility: "true" agents: slurm_mem: 20GB @@ -695,6 +743,8 @@ steps: --job_id diagnostic_edmfx_bomex_box --out_dir diagnostic_edmfx_bomex_box/output_active artifact_paths: "diagnostic_edmfx_bomex_box/output_active/*" + env: + test_reproducibility: "true" agents: slurm_mem: 20GB @@ -708,6 +758,8 @@ steps: --job_id diagnostic_edmfx_bomex_stretched_box --out_dir diagnostic_edmfx_bomex_stretched_box/output_active artifact_paths: "diagnostic_edmfx_bomex_stretched_box/output_active/*" + env: + test_reproducibility: "true" agents: slurm_mem: 20GB @@ -721,6 +773,8 @@ steps: --job_id diagnostic_edmfx_dycoms_rf01_explicit_box --out_dir diagnostic_edmfx_dycoms_rf01_explicit_box/output_active artifact_paths: "diagnostic_edmfx_dycoms_rf01_explicit_box/output_active/*" + env: + test_reproducibility: "true" agents: slurm_mem: 20GB @@ -734,6 +788,8 @@ steps: --job_id diagnostic_edmfx_dycoms_rf01_box --out_dir diagnostic_edmfx_dycoms_rf01_box/output_active artifact_paths: "diagnostic_edmfx_dycoms_rf01_box/output_active/*" + env: + test_reproducibility: "true" agents: slurm_mem: 20GB @@ -747,6 +803,8 @@ steps: --job_id diagnostic_edmfx_dycoms_rf02_box --out_dir diagnostic_edmfx_dycoms_rf02_box/output_active artifact_paths: "diagnostic_edmfx_dycoms_rf02_box/output_active/*" + env: + test_reproducibility: "true" agents: slurm_mem: 20GB @@ -760,6 +818,8 @@ steps: --job_id diagnostic_edmfx_rico_box --out_dir diagnostic_edmfx_rico_box/output_active artifact_paths: "diagnostic_edmfx_rico_box/output_active/*" + env: + test_reproducibility: "true" agents: slurm_mem: 20GB @@ -773,6 +833,8 @@ steps: --job_id diagnostic_edmfx_trmm_box --out_dir diagnostic_edmfx_trmm_box/output_active artifact_paths: "diagnostic_edmfx_trmm_box/output_active/*" + env: + test_reproducibility: "true" agents: slurm_mem: 20GB @@ -786,6 +848,8 @@ steps: --job_id diagnostic_edmfx_trmm_stretched_box --out_dir diagnostic_edmfx_trmm_stretched_box/output_active artifact_paths: "diagnostic_edmfx_trmm_stretched_box/output_active/*" + env: + test_reproducibility: "true" agents: slurm_mem: 20GB @@ -799,6 +863,8 @@ steps: --job_id diagnostic_edmfx_trmm_box_0M --out_dir diagnostic_edmfx_trmm_box_0M/output_active artifact_paths: "diagnostic_edmfx_trmm_box_0M/output_active/*" + env: + test_reproducibility: "true" agents: slurm_mem: 20GB @@ -812,6 +878,8 @@ steps: --job_id diagnostic_edmfx_aquaplanet --out_dir diagnostic_edmfx_aquaplanet/output_active artifact_paths: "diagnostic_edmfx_aquaplanet/output_active/*" + env: + test_reproducibility: "true" agents: slurm_mem: 20GB slurm_constraint: icelake|cascadelake|skylake|epyc @@ -829,6 +897,8 @@ steps: --job_id prognostic_edmfx_adv_test_column --out_dir prognostic_edmfx_adv_test_column/output_active artifact_paths: "prognostic_edmfx_adv_test_column/output_active/*" + env: + test_reproducibility: "true" agents: slurm_mem: 20GB @@ -843,6 +913,8 @@ steps: --job_id prognostic_edmfx_simpleplume_column --out_dir prognostic_edmfx_simpleplume_column/output_active artifact_paths: "prognostic_edmfx_simpleplume_column/output_active/*" + env: + test_reproducibility: "true" agents: slurm_mem: 20GB @@ -856,6 +928,8 @@ steps: --job_id prognostic_edmfx_gabls_column --out_dir prognostic_edmfx_gabls_column/output_active artifact_paths: "prognostic_edmfx_gabls_column/output_active/*" + env: + test_reproducibility: "true" agents: slurm_mem: 20GB @@ -869,6 +943,8 @@ steps: --job_id prognostic_edmfx_bomex_pigroup_column --out_dir prognostic_edmfx_bomex_pigroup_column/output_active artifact_paths: "prognostic_edmfx_bomex_pigroup_column/output_active/*" + env: + test_reproducibility: "true" agents: slurm_mem: 20GB @@ -882,6 +958,8 @@ steps: --job_id prognostic_edmfx_bomex_fixtke_column --out_dir prognostic_edmfx_bomex_fixtke_column/output_active artifact_paths: "prognostic_edmfx_bomex_fixtke_column/output_active/*" + env: + test_reproducibility: "true" agents: slurm_mem: 20GB @@ -895,6 +973,8 @@ steps: --job_id prognostic_edmfx_bomex_stretched_column --out_dir prognostic_edmfx_bomex_stretched_column/output_active artifact_paths: "prognostic_edmfx_bomex_stretched_column/output_active/*" + env: + test_reproducibility: "true" agents: slurm_mem: 20GB @@ -908,6 +988,8 @@ steps: --job_id prognostic_edmfx_bomex_column --out_dir prognostic_edmfx_bomex_column/output_active artifact_paths: "prognostic_edmfx_bomex_column/output_active/*" + env: + test_reproducibility: "true" agents: slurm_mem: 20GB @@ -921,6 +1003,8 @@ steps: --job_id prognostic_edmfx_bomex_column_implicit --out_dir prognostic_edmfx_bomex_column_implicit/output_active artifact_paths: "prognostic_edmfx_bomex_column_implicit/output_active/*" + env: + test_reproducibility: "true" agents: slurm_mem: 20GB @@ -934,6 +1018,8 @@ steps: --job_id prognostic_edmfx_dycoms_rf01_column --out_dir prognostic_edmfx_dycoms_rf01_column/output_active artifact_paths: "prognostic_edmfx_dycoms_rf01_column/output_active/*" + env: + test_reproducibility: "true" agents: slurm_mem: 20GB @@ -947,6 +1033,8 @@ steps: --job_id prognostic_edmfx_rico_column --out_dir prognostic_edmfx_rico_column/output_active artifact_paths: "prognostic_edmfx_rico_column/output_active/*" + env: + test_reproducibility: "true" agents: slurm_mem: 20GB @@ -960,6 +1048,8 @@ steps: --job_id prognostic_edmfx_trmm_column --out_dir prognostic_edmfx_trmm_column/output_active artifact_paths: "prognostic_edmfx_trmm_column/output_active/*" + env: + test_reproducibility: "true" agents: slurm_mem: 20GB @@ -973,6 +1063,8 @@ steps: --job_id prognostic_edmfx_trmm_column_0M --out_dir prognostic_edmfx_trmm_column_0M/output_active artifact_paths: "prognostic_edmfx_trmm_column_0M/output_active/*" + env: + test_reproducibility: "true" agents: slurm_mem: 20GB @@ -986,6 +1078,8 @@ steps: --job_id prognostic_edmfx_gcmdriven_column --out_dir prognostic_edmfx_gcmdriven_column/output_active artifact_paths: "prognostic_edmfx_gcmdriven_column/output_active/*" + env: + test_reproducibility: "true" agents: slurm_mem: 20GB soft_fail: true @@ -1000,6 +1094,8 @@ steps: --job_id prognostic_edmfx_bomex_box --out_dir prognostic_edmfx_bomex_box/output_active artifact_paths: "prognostic_edmfx_bomex_box/*" + env: + test_reproducibility: "true" agents: slurm_mem: 20GB @@ -1013,6 +1109,8 @@ steps: --job_id prognostic_edmfx_aquaplanet --out_dir prognostic_edmfx_aquaplanet/output_active artifact_paths: "prognostic_edmfx_aquaplanet/output_active/*" + env: + test_reproducibility: "true" agents: slurm_mem: 20GB @@ -1043,6 +1141,7 @@ steps: --out_dir sphere_baroclinic_wave_rhoe_gpu/output_active artifact_paths: "sphere_baroclinic_wave_rhoe_gpu/output_active/*" env: + test_reproducibility: "true" CLIMACOMMS_DEVICE: "CUDA" agents: slurm_mem: 16G @@ -1099,6 +1198,7 @@ steps: --out_dir target_gpu_implicit_baroclinic_wave_4process/output_active artifact_paths: "target_gpu_implicit_baroclinic_wave_4process/output_active/*" env: + test_reproducibility: "true" CLIMACOMMS_CONTEXT: "MPI" CLIMACOMMS_DEVICE: "CUDA" agents: @@ -1120,6 +1220,7 @@ steps: --out_dir central_gpu_hs_rhoe_equil_55km_nz63_0M/output_active artifact_paths: "central_gpu_hs_rhoe_equil_55km_nz63_0M/output_active/*" env: + test_reproducibility: "true" CLIMACOMMS_DEVICE: "CUDA" agents: slurm_gpus: 1 @@ -1138,6 +1239,7 @@ steps: --out_dir central_cloud_diag_gpu_hs_rhoe_equil_55km_nz63_0M/output_active artifact_paths: "central_cloud_diag_gpu_hs_rhoe_equil_55km_nz63_0M/output_active/*" env: + test_reproducibility: "true" CLIMACOMMS_DEVICE: "CUDA" agents: slurm_gpus: 1 @@ -1156,6 +1258,7 @@ steps: --out_dir gpu_aquaplanet_dyamond/output_active artifact_paths: "gpu_aquaplanet_dyamond/output_active/*" env: + test_reproducibility: "true" CLIMACOMMS_DEVICE: "CUDA" agents: slurm_gpus: 1 @@ -1173,6 +1276,7 @@ steps: --out_dir diagnostic_edmfx_aquaplanet_gpu/output_active artifact_paths: "diagnostic_edmfx_aquaplanet_gpu/output_active/*" env: + test_reproducibility: "true" CLIMACOMMS_DEVICE: "CUDA" agents: slurm_gpus: 1 @@ -1189,6 +1293,7 @@ steps: --out_dir prognostic_edmfx_aquaplanet_gpu/output_active artifact_paths: "prognostic_edmfx_aquaplanet_gpu/output_active/*" env: + test_reproducibility: "true" CLIMACOMMS_DEVICE: "CUDA" agents: slurm_gpus: 1 diff --git a/config/default_configs/default_config.yml b/config/default_configs/default_config.yml index 708cdf0801..61b050b068 100644 --- a/config/default_configs/default_config.yml +++ b/config/default_configs/default_config.yml @@ -217,9 +217,6 @@ non_orographic_gravity_wave: nh_poly: help: "Horizontal polynomial degree. Note: The number of quadrature points in 1D within each horizontal element is then Nq = <--nh_poly> + 1" value: 3 -reproducibility_test: - help: "(Bool) perform reproducibility test" - value: false check_conservation: help: "Check conservation of mass and energy [`false` (default), `true`]" value: false diff --git a/config/model_configs/deep_sphere_baroclinic_wave_rhoe_equilmoist.yml b/config/model_configs/deep_sphere_baroclinic_wave_rhoe_equilmoist.yml index c3f9430f68..f3c6bbd6e9 100644 --- a/config/model_configs/deep_sphere_baroclinic_wave_rhoe_equilmoist.yml +++ b/config/model_configs/deep_sphere_baroclinic_wave_rhoe_equilmoist.yml @@ -1,6 +1,5 @@ precip_model: "0M" dt_save_state_to_disk: "2days" -reproducibility_test: true initial_condition: "MoistBaroclinicWave" dt: "450secs" t_end: "10days" diff --git a/config/model_configs/diagnostic_edmfx_aquaplanet.yml b/config/model_configs/diagnostic_edmfx_aquaplanet.yml index cdcc207fba..973ad33c95 100644 --- a/config/model_configs/diagnostic_edmfx_aquaplanet.yml +++ b/config/model_configs/diagnostic_edmfx_aquaplanet.yml @@ -19,6 +19,5 @@ cloud_model: "quadrature_sgs" precip_model: 1M dt: 120secs t_end: 3hours -reproducibility_test: true toml: [toml/diagnostic_edmfx.toml] ode_algo: ARS343 diff --git a/config/model_configs/single_column_precipitation_test.yml b/config/model_configs/single_column_precipitation_test.yml index 88a2728150..8d9f2999b9 100644 --- a/config/model_configs/single_column_precipitation_test.yml +++ b/config/model_configs/single_column_precipitation_test.yml @@ -14,7 +14,6 @@ precip_model: "1M" vert_diff: "FriersonDiffusion" implicit_diffusion: true approximate_linear_solve_iters: 2 -reproducibility_test: false toml: [toml/single_column_precipitation_test.toml] diagnostics: - short_name: [hus, clw, cli, husra, hussn, ta, wa] diff --git a/config/model_configs/sphere_aquaplanet_rhoe_equilmoist_allsky_gw_raw_zonallyasymmetric.yml b/config/model_configs/sphere_aquaplanet_rhoe_equilmoist_allsky_gw_raw_zonallyasymmetric.yml index 5a810a84c7..9b8af6a0d4 100644 --- a/config/model_configs/sphere_aquaplanet_rhoe_equilmoist_allsky_gw_raw_zonallyasymmetric.yml +++ b/config/model_configs/sphere_aquaplanet_rhoe_equilmoist_allsky_gw_raw_zonallyasymmetric.yml @@ -16,7 +16,6 @@ cloud_model: "grid_scale" surface_temperature: "ZonallyAsymmetric" moist: "equil" albedo_model: "RegressionFunctionAlbedo" -reproducibility_test: true aerosol_radiation: true prescribed_aerosols: ["CB1", "CB2", "DST01", "DST02", "DST03", "DST04", "OC1", "OC2", "SO4"] toml: [toml/sphere_aquaplanet.toml] diff --git a/config/model_configs/sphere_baroclinic_wave_rhoe_equilmoist.yml b/config/model_configs/sphere_baroclinic_wave_rhoe_equilmoist.yml index 5e5079aa77..dfdc588796 100644 --- a/config/model_configs/sphere_baroclinic_wave_rhoe_equilmoist.yml +++ b/config/model_configs/sphere_baroclinic_wave_rhoe_equilmoist.yml @@ -1,6 +1,5 @@ precip_model: "0M" dt_save_state_to_disk: "2days" -reproducibility_test: true initial_condition: "MoistBaroclinicWave" dt: "450secs" t_end: "10days" diff --git a/config/model_configs/sphere_held_suarez_rhoe_equilmoist_hightop_sponge.yml b/config/model_configs/sphere_held_suarez_rhoe_equilmoist_hightop_sponge.yml index 5fba8640ee..ba7d0f983b 100644 --- a/config/model_configs/sphere_held_suarez_rhoe_equilmoist_hightop_sponge.yml +++ b/config/model_configs/sphere_held_suarez_rhoe_equilmoist_hightop_sponge.yml @@ -9,6 +9,5 @@ t_end: "4days" vert_diff: true forcing: "held_suarez" precip_model: "0M" -reproducibility_test: true moist: "equil" toml: [toml/sphere_held_suarez.toml] diff --git a/examples/hybrid/driver.jl b/examples/hybrid/driver.jl index 796f2b86d4..c95c48f407 100644 --- a/examples/hybrid/driver.jl +++ b/examples/hybrid/driver.jl @@ -87,8 +87,8 @@ end include( joinpath(@__DIR__, "..", "..", "reproducibility_tests", "mse_tables.jl"), ) -if config.parsed_args["reproducibility_test"] - # Test results against main branch +if get(ENV, "test_reproducibility", "false") == "true" + # Export reproducibility results, to later test against the main branch include( joinpath( @__DIR__, @@ -98,17 +98,10 @@ if config.parsed_args["reproducibility_test"] "reproducibility_tests.jl", ), ) - @testset "Test reproducibility table entries" begin - mse_keys = sort(collect(keys(all_best_mse[simulation.job_id]))) - pcs = collect(Fields.property_chains(sol.u[end])) - for prop_chain in mse_keys - @test prop_chain in pcs - end - end - perform_reproducibility_tests( + export_reproducibility_results( + config.comms_ctx, simulation.job_id, sol.u[end], - all_best_mse, simulation.output_dir, ) end diff --git a/reproducibility_tests/README.md b/reproducibility_tests/README.md index 40f4d65ed2..b683eb458a 100644 --- a/reproducibility_tests/README.md +++ b/reproducibility_tests/README.md @@ -64,9 +64,8 @@ To update the mse tables: To add a new reproducibility test: - - Set the command-line `reproducibility_test` to true, and add `julia --color=yes --project=examples reproducibility_tests/test_mse.jl --job_id [job_id] --out_dir [job_id]` as a separate command for the new (or existing) job - - Copy the `all_best_mse` dict template from the job's log - - Paste the `all_best_mse` dict template into `reproducibility_test/mse_tables.jl` + - Add `julia --color=yes --project=examples reproducibility_tests/test_mse.jl --job_id [job_id] --out_dir [job_id]` as a separate command for the new (or existing) job, and set the `test_reproducibility` environment flag. For example: `test_reproducibility: "true"`. + - Add the job's `job_id` into the `reproducibility_test_job_ids` vector in `reproducibility_test/mse_tables.jl`. @@ -90,19 +89,17 @@ We cannot (easily) compare the output with a reference if we change the spatial ## A detailed procedure of how reproducibility tests are performed -Reprodicibility tests are performed at the end of `examples/hybrid/driver.jl`, after a simulation completes, and relies on a unique job id (`job_id`). Here is an outline of the reproducibility test procedure: +Reprodicibility results are computed at the end of the `examples/hybrid/driver.jl` script, and tested in the `reproducibility_tests/test_mse.jl`. This separation helps us delay. Here is an outline of the reproducibility test procedure: 0) Run a simulation, with a particular `job_id`, to the final time. - 1) Load a dictionary, `all_best_mse`, of previous "best" mean-squared errors from `mse_tables.jl` and extract the mean squared errors for the given `job_id` (store in job-specific dictionary, `best_mse`). - 2) Export the solution (a `FieldVector`) at the final simulation time to an `NCDataset` file. - 3) Compute the errors between the exported solution and the exported solution from the reference `NCDataset` files (which are saved in a dedicated folders on the Caltech Central cluster) and save into a dictionary, called `computed_mse`. - 4) Export this dictionary (`computed_mse`) to the output folder - 5) Test that `computed_mse` is no worse than `best_mse` (determines if reproducibility test passes or not). + 1) Export the solution (a `FieldVector`) at the final simulation time to an HDF5 file. + 2) Compute the mean squared errors (MSE) against all other comparable references (which are saved in a dedicated folders on the Caltech Central cluster) for all fieldvector variables in the prognostic state. + 3) Convert this set of MSEs to a dictionary (called `computed_mse`), and export it to a file in the output folder. After these steps are performed at the end of the driver, additional jobs are run: 1) Print `computed_mse` for all jobs to make updating `reproducibility_tests/mse_tables.jl` easy - 2) If we're on the github queue merging branch (all tests have passed, and the PR is effectively merging), move the `NCDataset`s from the scratch directory onto the dedicated folder on the Caltech Central cluster. + 2) If we're on the github queue merging branch (all tests have passed, and the PR is effectively merging), move the HDF5 files from the scratch directory onto the dedicated folder on the Caltech Central cluster. ## How we track which dataset to compare against diff --git a/reproducibility_tests/compute_mse.jl b/reproducibility_tests/compute_mse.jl index f5bfce3266..3629b6fb41 100644 --- a/reproducibility_tests/compute_mse.jl +++ b/reproducibility_tests/compute_mse.jl @@ -5,67 +5,67 @@ import ClimaCoreTempestRemap as CCTR include("latest_comparable_paths.jl") -function get_nc_data(ds, var::String) - if haskey(ds, var) - return ds[var] - else - for key in keys(ds.group) - if haskey(ds.group[key], var) - return ds.group[key][var] - end - end +""" + to_dict(filename::String, comms_ctx) + +Convert the HDF5 file containing the +prognostic field `Y` into a `Dict` +using ClimaCore's `property_chains` and +`single_field` functions. +""" +function to_dict(filename::String, comms_ctx) + dict = Dict{Any, AbstractArray}() + reader = InputOutput.HDF5Reader(filename, comms_ctx) + Y = InputOutput.read_field(reader, "Y") + Base.close(reader) + for prop_chain in Fields.property_chains(Y) + dict[prop_chain] = + vec(Array(parent(Fields.single_field(Y, prop_chain)))) end - error("No key $var for mse computation.") - return nothing + return dict end """ - to_dict(nc_filename::String, reference_keys::Vector{String}) + zero_dict(filename::String, comms_ctx) -Convert an NCDatasets file to a `Dict`. +Return a dict of zeros for all `ClimaCore.Fields.property_chains` +in the fieldvector `Y` contained in the HDF5 file `filename`. """ -function to_dict(nc_filename::String, reference_keys::Vector{String}) - dict = Dict{String, AbstractArray}() - NCDatasets.Dataset(nc_filename, "r") do ds - for key in reference_keys - dict[key] = vec(Array(get_nc_data(ds, key))) - end +function zero_dict(filename::String, comms_ctx) + dict = Dict{Any, AbstractArray}() + reader = InputOutput.HDF5Reader(filename, comms_ctx) + Y = InputOutput.read_field(reader, "Y") + Base.close(reader) + for prop_chain in Fields.property_chains(Y) + dict[prop_chain] = + vec(Array(parent(Fields.single_field(Y, prop_chain)))) .* 0 end return dict end """ - reproducibility_test(; + reproducibility_results( + comms_ctx; job_id, - reference_mse, ds_filename_computed, ds_filename_reference = nothing, - varname, ) Returns a `Dict` of mean-squared errors between -`NCDataset`s `ds_filename_computed` and -`ds_filename_reference` for all keys in `reference_mse`. -Keys in `reference_mse` may directly map to keys in -the `NCDataset`s, or they may be mapped to the keys -via `varname`. +datasets `ds_filename_computed` and +`ds_filename_reference` for all variables. If running on buildkite, we get `ds_filename_reference` from the latest merged dataset on Caltech central. """ -function reproducibility_test(; - job_id, - reference_mse, - ds_filename_computed, - varname, -) +function reproducibility_results(comms_ctx; job_id, ds_filename_computed) local ds_filename_reference - reference_keys = map(k -> varname(k), collect(keys(reference_mse))) paths = String[] # initialize for later handling if haskey(ENV, "BUILDKITE_COMMIT") paths = latest_comparable_paths(10) - isempty(paths) && return (reference_mse, paths) + isempty(paths) && + return (zero_dict(ds_filename_computed, comms_ctx), paths) @info "`ds_filename_computed`: `$ds_filename_computed`" ds_filename_references = map(p -> joinpath(p, ds_filename_computed), paths) @@ -94,40 +94,41 @@ function reproducibility_test(; @warn "There is no reference dataset, and no NC tar file." end end - if !isfile(ds_filename_reference) - msg = "\n\n" - msg *= "Pull request author:\n" - msg *= " It seems that a new dataset,\n" - msg *= "\n" - msg *= "dataset file:`$(ds_filename_computed)`," - msg *= "\n" - msg *= " was created, or the name of the dataset\n" - msg *= " has changed. Please increment the reference\n" - msg *= " counter in `reproducibility_tests/ref_counter.jl`.\n" - msg *= "\n" - msg *= " If this is not the case, then please\n" - msg *= " open an issue with a link pointing to this\n" - msg *= " PR and build.\n" - msg *= "\n" - msg *= "For more information, please find\n" - msg *= "`reproducibility_tests/README.md` and read the section\n\n" - msg *= " `How to merge pull requests (PR) that get approved\n" - msg *= " but *break* reproducibility tests`\n\n" - msg *= "for how to merge this PR." - error(msg) - end + end + non_existent_files = filter(x -> !isfile(x), ds_filename_references) + if !isempty(non_existent_files) + msg = "\n\n" + msg *= "Pull request author:\n" + msg *= " It seems that a new dataset,\n" + msg *= "\n" + msg *= "dataset file(s):`$(non_existent_files)`," + msg *= "\n" + msg *= " was created, or the name of the dataset\n" + msg *= " has changed. Please increment the reference\n" + msg *= " counter in `reproducibility_tests/ref_counter.jl`.\n" + msg *= "\n" + msg *= " If this is not the case, then please\n" + msg *= " open an issue with a link pointing to this\n" + msg *= " PR and build.\n" + msg *= "\n" + msg *= "For more information, please find\n" + msg *= "`reproducibility_tests/README.md` and read the section\n\n" + msg *= " `How to merge pull requests (PR) that get approved\n" + msg *= " but *break* reproducibility tests`\n\n" + msg *= "for how to merge this PR." + error(msg) end else @warn "Buildkite not detected. Skipping reproducibility tests." @info "Please review output results before merging." - return (reference_mse, paths) + return (zero_dict(ds_filename_computed, comms_ctx), paths) end local computed_mse - @info "Prescribed reference keys $reference_keys" - dict_computed = to_dict(ds_filename_computed, reference_keys) - dict_references = - map(ds -> to_dict(ds, reference_keys), ds_filename_references) + dict_computed = to_dict(ds_filename_computed, comms_ctx) + dict_references = map(ds -> to_dict(ds, comms_ctx), ds_filename_references) + reference_keys = keys(first(dict_references)) + @info "Reference keys $reference_keys" @info "Computed keys $(collect(keys(dict_computed)))" @info "Reference keys $(collect(keys(first(dict_references))))" if all(dr -> keys(dict_computed) == keys(dr), dict_references) && all( @@ -153,62 +154,3 @@ function reproducibility_test(; return (computed_mses, paths) end - - -##### TODO: move below functions to ClimaCore - -function first_center_space(fv::Fields.FieldVector) - for prop_chain in Fields.property_chains(fv) - f = Fields.single_field(fv, prop_chain) - space = axes(f) - if space isa Spaces.CenterExtrudedFiniteDifferenceSpace - return space - end - end - error("Unfound space") -end - -function first_face_space(fv::Fields.FieldVector) - for prop_chain in Fields.property_chains(fv) - f = Fields.single_field(fv, prop_chain) - space = axes(f) - if space isa Spaces.FaceExtrudedFiniteDifferenceSpace - return space - end - end - error("Unfound space") -end - -function export_nc( - Y::Fields.FieldVector; - nc_filename, - t_now = 0.0, - center_space = first_center_space, - face_space = first_face_space, - filter_prop_chain = pn -> true, # use all fields - varname::Function, -) - prop_chains = Fields.property_chains(Y) - filter!(filter_prop_chain, prop_chains) - cspace = center_space(Y) - fspace = face_space(Y) - # create a temporary dir for intermediate data - FT = eltype(Y) - NCDatasets.NCDataset(nc_filename, "c") do nc - # defines the appropriate dimensions and variables for a space coordinate - # defines the appropriate dimensions and variables for a time coordinate (by default, unlimited size) - nc_time = CCTR.def_time_coord(nc) - CCTR.def_space_coord(nc, cspace, type = "cgll") - CCTR.def_space_coord(nc, fspace, type = "cgll") - # define variables for the prognostic states - for prop_chain in Fields.property_chains(Y) - f = Fields.single_field(Y, prop_chain) - space = axes(f) - nc_var = CCTR.defVar(nc, varname(prop_chain), FT, space, ("time",)) - nc_var[:, 1] = f - end - # TODO: interpolate w onto center space and save it the same way as the other vars - nc_time[1] = t_now - end - return nothing -end diff --git a/reproducibility_tests/move_output.jl b/reproducibility_tests/move_output.jl index 157251285e..2c9ce5346d 100644 --- a/reproducibility_tests/move_output.jl +++ b/reproducibility_tests/move_output.jl @@ -2,12 +2,8 @@ include(joinpath(@__DIR__, "latest_comparable_paths.jl")) paths = latest_comparable_paths() -all_lines = readlines(joinpath(@__DIR__, "mse_tables.jl")) -lines = deepcopy(all_lines) -filter!(x -> occursin("] = OrderedCollections", x), lines) -job_ids = getindex.(split.(lines, "\""), 2) -@assert count(x -> occursin("OrderedDict", x), all_lines) == length(job_ids) + 1 -@assert length(job_ids) ≠ 0 # safety net +include(joinpath(@__DIR__, "mse_tables.jl")) +job_ids = reproducibility_test_job_ids # Note: cluster_data_prefix is also defined in compute_mse.jl cluster_data_prefix = "/central/scratch/esm/slurm-buildkite/climaatmos-main" @@ -21,7 +17,7 @@ if buildkite_ci @info "commit = $(commit)" using Glob - @show readdir(joinpath(@__DIR__, "..")) + # @show readdir(joinpath(@__DIR__, "..")) # if a contributor manually merged, we still want to move data # from scratch to `cluster_data_prefix`. So, let's also try moving # data if this is running on the main branch. diff --git a/reproducibility_tests/mse_tables.jl b/reproducibility_tests/mse_tables.jl index be9e9078ba..80f9acc81f 100644 --- a/reproducibility_tests/mse_tables.jl +++ b/reproducibility_tests/mse_tables.jl @@ -1,492 +1,62 @@ -################################# -################################# MSE tables -################################# -#! format: off -# -all_best_mse = OrderedCollections.OrderedDict() -# -all_best_mse["sphere_baroclinic_wave_rhoe_equilmoist"] = OrderedCollections.OrderedDict() -all_best_mse["sphere_baroclinic_wave_rhoe_equilmoist"][(:c, :ρ)] = 0 -all_best_mse["sphere_baroclinic_wave_rhoe_equilmoist"][(:c, :ρe_tot)] = 0 -all_best_mse["sphere_baroclinic_wave_rhoe_equilmoist"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["sphere_baroclinic_wave_rhoe_equilmoist"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["sphere_baroclinic_wave_rhoe_equilmoist"][(:c, :ρq_tot)] = 0 -all_best_mse["sphere_baroclinic_wave_rhoe_equilmoist"][(:f, :u₃, :components, :data, 1)] = 0 -# -all_best_mse["sphere_held_suarez_rhoe_equilmoist_hightop_sponge"] = OrderedCollections.OrderedDict() -all_best_mse["sphere_held_suarez_rhoe_equilmoist_hightop_sponge"][(:c, :ρ)] = 0 -all_best_mse["sphere_held_suarez_rhoe_equilmoist_hightop_sponge"][(:c, :ρe_tot)] = 0 -all_best_mse["sphere_held_suarez_rhoe_equilmoist_hightop_sponge"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["sphere_held_suarez_rhoe_equilmoist_hightop_sponge"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["sphere_held_suarez_rhoe_equilmoist_hightop_sponge"][(:c, :ρq_tot)] = 0 -all_best_mse["sphere_held_suarez_rhoe_equilmoist_hightop_sponge"][(:f, :u₃, :components, :data, 1)] = 0 -# -all_best_mse["sphere_aquaplanet_rhoe_equilmoist_allsky_gw_res"] = OrderedCollections.OrderedDict() -all_best_mse["sphere_aquaplanet_rhoe_equilmoist_allsky_gw_res"][(:c, :ρ)] = 0 -all_best_mse["sphere_aquaplanet_rhoe_equilmoist_allsky_gw_res"][(:c, :ρe_tot)] = 0 -all_best_mse["sphere_aquaplanet_rhoe_equilmoist_allsky_gw_res"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["sphere_aquaplanet_rhoe_equilmoist_allsky_gw_res"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["sphere_aquaplanet_rhoe_equilmoist_allsky_gw_res"][(:c, :ρq_tot)] = 0 -all_best_mse["sphere_aquaplanet_rhoe_equilmoist_allsky_gw_res"][(:f, :u₃, :components, :data, 1)] = 0 -# -all_best_mse["sphere_aquaplanet_rhoe_equilmoist_allsky_gw_raw_zonallyasymmetric"] = OrderedCollections.OrderedDict() -all_best_mse["sphere_aquaplanet_rhoe_equilmoist_allsky_gw_raw_zonallyasymmetric"][(:c, :ρ)] = 0 -all_best_mse["sphere_aquaplanet_rhoe_equilmoist_allsky_gw_raw_zonallyasymmetric"][(:c, :ρe_tot)] = 0 -all_best_mse["sphere_aquaplanet_rhoe_equilmoist_allsky_gw_raw_zonallyasymmetric"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["sphere_aquaplanet_rhoe_equilmoist_allsky_gw_raw_zonallyasymmetric"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["sphere_aquaplanet_rhoe_equilmoist_allsky_gw_raw_zonallyasymmetric"][(:c, :ρq_tot)] = 0 -all_best_mse["sphere_aquaplanet_rhoe_equilmoist_allsky_gw_raw_zonallyasymmetric"][(:f, :u₃, :components, :data, 1)] = 0 -# -all_best_mse["deep_sphere_baroclinic_wave_rhoe_equilmoist"] = OrderedCollections.OrderedDict() -all_best_mse["deep_sphere_baroclinic_wave_rhoe_equilmoist"][(:c, :ρ)] = 0 -all_best_mse["deep_sphere_baroclinic_wave_rhoe_equilmoist"][(:c, :ρe_tot)] = 0 -all_best_mse["deep_sphere_baroclinic_wave_rhoe_equilmoist"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["deep_sphere_baroclinic_wave_rhoe_equilmoist"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["deep_sphere_baroclinic_wave_rhoe_equilmoist"][(:c, :ρq_tot)] = 0 -all_best_mse["deep_sphere_baroclinic_wave_rhoe_equilmoist"][(:f, :u₃, :components, :data, 1)] = 0 -# -all_best_mse["diagnostic_edmfx_aquaplanet"] = OrderedCollections.OrderedDict() -all_best_mse["diagnostic_edmfx_aquaplanet"][(:c, :ρ)] = 0 -all_best_mse["diagnostic_edmfx_aquaplanet"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["diagnostic_edmfx_aquaplanet"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["diagnostic_edmfx_aquaplanet"][(:c, :ρe_tot)] = 0 -all_best_mse["diagnostic_edmfx_aquaplanet"][(:c, :ρq_tot)] = 0 -all_best_mse["diagnostic_edmfx_aquaplanet"][(:c, :sgs⁰, :ρatke)] = 0 -all_best_mse["diagnostic_edmfx_aquaplanet"][(:f, :u₃, :components, :data, 1)] = 0 -# -all_best_mse["single_column_hydrostatic_balance_ft64"] = OrderedCollections.OrderedDict() -all_best_mse["single_column_hydrostatic_balance_ft64"][(:c, :ρ)] = 0 -all_best_mse["single_column_hydrostatic_balance_ft64"][(:c, :ρe_tot)] = 0 -all_best_mse["single_column_hydrostatic_balance_ft64"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["single_column_hydrostatic_balance_ft64"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["single_column_hydrostatic_balance_ft64"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["single_column_hydrostatic_balance_ft64"][(:c, :ρq_tot)] = 0 -# -all_best_mse["box_hydrostatic_balance_rhoe"] = OrderedCollections.OrderedDict() -all_best_mse["box_hydrostatic_balance_rhoe"][(:c, :ρ)] = 0 -all_best_mse["box_hydrostatic_balance_rhoe"][(:c, :ρe_tot)] = 0 -all_best_mse["box_hydrostatic_balance_rhoe"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["box_hydrostatic_balance_rhoe"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["box_hydrostatic_balance_rhoe"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["box_hydrostatic_balance_rhoe"][(:c, :ρq_tot)] = 0 -# -all_best_mse["box_density_current_test"] = OrderedCollections.OrderedDict() -all_best_mse["box_density_current_test"][(:c, :ρ)] = 0 -all_best_mse["box_density_current_test"][(:c, :ρe_tot)] = 0 -all_best_mse["box_density_current_test"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["box_density_current_test"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["box_density_current_test"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["box_density_current_test"][(:c, :ρq_tot)] = 0 -# -all_best_mse["rcemipii_box_diagnostic_edmfx"] = OrderedCollections.OrderedDict() -all_best_mse["rcemipii_box_diagnostic_edmfx"][(:c, :ρ)] = 0 -all_best_mse["rcemipii_box_diagnostic_edmfx"][(:c, :ρe_tot)] = 0 -all_best_mse["rcemipii_box_diagnostic_edmfx"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["rcemipii_box_diagnostic_edmfx"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["rcemipii_box_diagnostic_edmfx"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["rcemipii_box_diagnostic_edmfx"][(:c, :ρq_tot)] = 0 -# -all_best_mse["les_isdac_box"] = OrderedCollections.OrderedDict() -all_best_mse["les_isdac_box"][(:c, :ρ)] = 0 -all_best_mse["les_isdac_box"][(:c, :ρe_tot)] = 0 -all_best_mse["les_isdac_box"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["les_isdac_box"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["les_isdac_box"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["les_isdac_box"][(:c, :ρq_tot)] = 0 -# -all_best_mse["plane_agnesi_mountain_test_uniform"] = OrderedCollections.OrderedDict() -all_best_mse["plane_agnesi_mountain_test_uniform"][(:c, :ρ)] = 0 -all_best_mse["plane_agnesi_mountain_test_uniform"][(:c, :ρe_tot)] = 0 -all_best_mse["plane_agnesi_mountain_test_uniform"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["plane_agnesi_mountain_test_uniform"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["plane_agnesi_mountain_test_uniform"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["plane_agnesi_mountain_test_uniform"][(:c, :ρq_tot)] = 0 -# -all_best_mse["plane_agnesi_mountain_test_stretched"] = OrderedCollections.OrderedDict() -all_best_mse["plane_agnesi_mountain_test_stretched"][(:c, :ρ)] = 0 -all_best_mse["plane_agnesi_mountain_test_stretched"][(:c, :ρe_tot)] = 0 -all_best_mse["plane_agnesi_mountain_test_stretched"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["plane_agnesi_mountain_test_stretched"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["plane_agnesi_mountain_test_stretched"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["plane_agnesi_mountain_test_stretched"][(:c, :ρq_tot)] = 0 -# -all_best_mse["plane_density_current_test"] = OrderedCollections.OrderedDict() -all_best_mse["plane_density_current_test"][(:c, :ρ)] = 0 -all_best_mse["plane_density_current_test"][(:c, :ρe_tot)] = 0 -all_best_mse["plane_density_current_test"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["plane_density_current_test"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["plane_density_current_test"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["plane_density_current_test"][(:c, :ρq_tot)] = 0 -# -all_best_mse["sphere_hydrostatic_balance_rhoe_ft64"] = OrderedCollections.OrderedDict() -all_best_mse["sphere_hydrostatic_balance_rhoe_ft64"][(:c, :ρ)] = 0 -all_best_mse["sphere_hydrostatic_balance_rhoe_ft64"][(:c, :ρe_tot)] = 0 -all_best_mse["sphere_hydrostatic_balance_rhoe_ft64"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["sphere_hydrostatic_balance_rhoe_ft64"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["sphere_hydrostatic_balance_rhoe_ft64"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["sphere_hydrostatic_balance_rhoe_ft64"][(:c, :ρq_tot)] = 0 -# -all_best_mse["sphere_baroclinic_wave_rhoe"] = OrderedCollections.OrderedDict() -all_best_mse["sphere_baroclinic_wave_rhoe"][(:c, :ρ)] = 0 -all_best_mse["sphere_baroclinic_wave_rhoe"][(:c, :ρe_tot)] = 0 -all_best_mse["sphere_baroclinic_wave_rhoe"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["sphere_baroclinic_wave_rhoe"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["sphere_baroclinic_wave_rhoe"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["sphere_baroclinic_wave_rhoe"][(:c, :ρq_tot)] = 0 -# -all_best_mse["sphere_held_suarez_rhoe_hightop"] = OrderedCollections.OrderedDict() -all_best_mse["sphere_held_suarez_rhoe_hightop"][(:c, :ρ)] = 0 -all_best_mse["sphere_held_suarez_rhoe_hightop"][(:c, :ρe_tot)] = 0 -all_best_mse["sphere_held_suarez_rhoe_hightop"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["sphere_held_suarez_rhoe_hightop"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["sphere_held_suarez_rhoe_hightop"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["sphere_held_suarez_rhoe_hightop"][(:c, :ρq_tot)] = 0 -# -all_best_mse["sphere_aquaplanet_rhoe_nonequilmoist_allsky"] = OrderedCollections.OrderedDict() -all_best_mse["sphere_aquaplanet_rhoe_nonequilmoist_allsky"][(:c, :ρ)] = 0 -all_best_mse["sphere_aquaplanet_rhoe_nonequilmoist_allsky"][(:c, :ρe_tot)] = 0 -all_best_mse["sphere_aquaplanet_rhoe_nonequilmoist_allsky"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["sphere_aquaplanet_rhoe_nonequilmoist_allsky"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["sphere_aquaplanet_rhoe_nonequilmoist_allsky"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["sphere_aquaplanet_rhoe_nonequilmoist_allsky"][(:c, :ρq_tot)] = 0 -# -all_best_mse["aquaplanet_rhoe_equil_clearsky_tvinsol_0M_slabocean"] = OrderedCollections.OrderedDict() -all_best_mse["aquaplanet_rhoe_equil_clearsky_tvinsol_0M_slabocean"][(:c, :ρ)] = 0 -all_best_mse["aquaplanet_rhoe_equil_clearsky_tvinsol_0M_slabocean"][(:c, :ρe_tot)] = 0 -all_best_mse["aquaplanet_rhoe_equil_clearsky_tvinsol_0M_slabocean"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["aquaplanet_rhoe_equil_clearsky_tvinsol_0M_slabocean"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["aquaplanet_rhoe_equil_clearsky_tvinsol_0M_slabocean"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["aquaplanet_rhoe_equil_clearsky_tvinsol_0M_slabocean"][(:c, :ρq_tot)] = 0 -# -all_best_mse["aquaplanet_rhoe_equil_clearsky_tvinsol_0M_slabocean_ft64"] = OrderedCollections.OrderedDict() -all_best_mse["aquaplanet_rhoe_equil_clearsky_tvinsol_0M_slabocean_ft64"][(:c, :ρ)] = 0 -all_best_mse["aquaplanet_rhoe_equil_clearsky_tvinsol_0M_slabocean_ft64"][(:c, :ρe_tot)] = 0 -all_best_mse["aquaplanet_rhoe_equil_clearsky_tvinsol_0M_slabocean_ft64"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["aquaplanet_rhoe_equil_clearsky_tvinsol_0M_slabocean_ft64"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["aquaplanet_rhoe_equil_clearsky_tvinsol_0M_slabocean_ft64"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["aquaplanet_rhoe_equil_clearsky_tvinsol_0M_slabocean_ft64"][(:c, :ρq_tot)] = 0 -# -all_best_mse["rcemipii_sphere_diagnostic_edmfx"] = OrderedCollections.OrderedDict() -all_best_mse["rcemipii_sphere_diagnostic_edmfx"][(:c, :ρ)] = 0 -all_best_mse["rcemipii_sphere_diagnostic_edmfx"][(:c, :ρe_tot)] = 0 -all_best_mse["rcemipii_sphere_diagnostic_edmfx"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["rcemipii_sphere_diagnostic_edmfx"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["rcemipii_sphere_diagnostic_edmfx"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["rcemipii_sphere_diagnostic_edmfx"][(:c, :ρq_tot)] = 0 -# -all_best_mse["sphere_baroclinic_wave_rhoe_topography_dcmip_rs"] = OrderedCollections.OrderedDict() -all_best_mse["sphere_baroclinic_wave_rhoe_topography_dcmip_rs"][(:c, :ρ)] = 0 -all_best_mse["sphere_baroclinic_wave_rhoe_topography_dcmip_rs"][(:c, :ρe_tot)] = 0 -all_best_mse["sphere_baroclinic_wave_rhoe_topography_dcmip_rs"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["sphere_baroclinic_wave_rhoe_topography_dcmip_rs"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["sphere_baroclinic_wave_rhoe_topography_dcmip_rs"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["sphere_baroclinic_wave_rhoe_topography_dcmip_rs"][(:c, :ρq_tot)] = 0 -# -all_best_mse["sphere_held_suarez_rhoe_topography_dcmip"] = OrderedCollections.OrderedDict() -all_best_mse["sphere_held_suarez_rhoe_topography_dcmip"][(:c, :ρ)] = 0 -all_best_mse["sphere_held_suarez_rhoe_topography_dcmip"][(:c, :ρe_tot)] = 0 -all_best_mse["sphere_held_suarez_rhoe_topography_dcmip"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["sphere_held_suarez_rhoe_topography_dcmip"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["sphere_held_suarez_rhoe_topography_dcmip"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["sphere_held_suarez_rhoe_topography_dcmip"][(:c, :ρq_tot)] = 0 -# -all_best_mse["sphere_held_suarez_rhoe_equilmoist_topography_dcmip"] = OrderedCollections.OrderedDict() -all_best_mse["sphere_held_suarez_rhoe_equilmoist_topography_dcmip"][(:c, :ρ)] = 0 -all_best_mse["sphere_held_suarez_rhoe_equilmoist_topography_dcmip"][(:c, :ρe_tot)] = 0 -all_best_mse["sphere_held_suarez_rhoe_equilmoist_topography_dcmip"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["sphere_held_suarez_rhoe_equilmoist_topography_dcmip"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["sphere_held_suarez_rhoe_equilmoist_topography_dcmip"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["sphere_held_suarez_rhoe_equilmoist_topography_dcmip"][(:c, :ρq_tot)] = 0 -# -all_best_mse["sphere_ssp_baroclinic_wave_rhoe_equilmoist_dcmip200"] = OrderedCollections.OrderedDict() -all_best_mse["sphere_ssp_baroclinic_wave_rhoe_equilmoist_dcmip200"][(:c, :ρ)] = 0 -all_best_mse["sphere_ssp_baroclinic_wave_rhoe_equilmoist_dcmip200"][(:c, :ρe_tot)] = 0 -all_best_mse["sphere_ssp_baroclinic_wave_rhoe_equilmoist_dcmip200"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["sphere_ssp_baroclinic_wave_rhoe_equilmoist_dcmip200"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["sphere_ssp_baroclinic_wave_rhoe_equilmoist_dcmip200"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["sphere_ssp_baroclinic_wave_rhoe_equilmoist_dcmip200"][(:c, :ρq_tot)] = 0 -# -all_best_mse["sphere_ssp_baroclinic_wave_rhoe_equilmoist_earth"] = OrderedCollections.OrderedDict() -all_best_mse["sphere_ssp_baroclinic_wave_rhoe_equilmoist_earth"][(:c, :ρ)] = 0 -all_best_mse["sphere_ssp_baroclinic_wave_rhoe_equilmoist_earth"][(:c, :ρe_tot)] = 0 -all_best_mse["sphere_ssp_baroclinic_wave_rhoe_equilmoist_earth"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["sphere_ssp_baroclinic_wave_rhoe_equilmoist_earth"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["sphere_ssp_baroclinic_wave_rhoe_equilmoist_earth"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["sphere_ssp_baroclinic_wave_rhoe_equilmoist_earth"][(:c, :ρq_tot)] = 0 -# -all_best_mse["mpi_sphere_aquaplanet_rhoe_equilmoist_clearsky"] = OrderedCollections.OrderedDict() -all_best_mse["mpi_sphere_aquaplanet_rhoe_equilmoist_clearsky"][(:c, :ρ)] = 0 -all_best_mse["mpi_sphere_aquaplanet_rhoe_equilmoist_clearsky"][(:c, :ρe_tot)] = 0 -all_best_mse["mpi_sphere_aquaplanet_rhoe_equilmoist_clearsky"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["mpi_sphere_aquaplanet_rhoe_equilmoist_clearsky"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["mpi_sphere_aquaplanet_rhoe_equilmoist_clearsky"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["mpi_sphere_aquaplanet_rhoe_equilmoist_clearsky"][(:c, :ρq_tot)] = 0 -# -all_best_mse["diagnostic_edmfx_test_box"] = OrderedCollections.OrderedDict() -all_best_mse["diagnostic_edmfx_test_box"][(:c, :ρ)] = 0 -all_best_mse["diagnostic_edmfx_test_box"][(:c, :ρe_tot)] = 0 -all_best_mse["diagnostic_edmfx_test_box"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["diagnostic_edmfx_test_box"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["diagnostic_edmfx_test_box"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["diagnostic_edmfx_test_box"][(:c, :ρq_tot)] = 0 -# -all_best_mse["diagnostic_edmfx_gabls_box"] = OrderedCollections.OrderedDict() -all_best_mse["diagnostic_edmfx_gabls_box"][(:c, :ρ)] = 0 -all_best_mse["diagnostic_edmfx_gabls_box"][(:c, :ρe_tot)] = 0 -all_best_mse["diagnostic_edmfx_gabls_box"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["diagnostic_edmfx_gabls_box"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["diagnostic_edmfx_gabls_box"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["diagnostic_edmfx_gabls_box"][(:c, :ρq_tot)] = 0 -# -all_best_mse["diagnostic_edmfx_bomex_box"] = OrderedCollections.OrderedDict() -all_best_mse["diagnostic_edmfx_bomex_box"][(:c, :ρ)] = 0 -all_best_mse["diagnostic_edmfx_bomex_box"][(:c, :ρe_tot)] = 0 -all_best_mse["diagnostic_edmfx_bomex_box"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["diagnostic_edmfx_bomex_box"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["diagnostic_edmfx_bomex_box"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["diagnostic_edmfx_bomex_box"][(:c, :ρq_tot)] = 0 -# -all_best_mse["diagnostic_edmfx_bomex_stretched_box"] = OrderedCollections.OrderedDict() -all_best_mse["diagnostic_edmfx_bomex_stretched_box"][(:c, :ρ)] = 0 -all_best_mse["diagnostic_edmfx_bomex_stretched_box"][(:c, :ρe_tot)] = 0 -all_best_mse["diagnostic_edmfx_bomex_stretched_box"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["diagnostic_edmfx_bomex_stretched_box"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["diagnostic_edmfx_bomex_stretched_box"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["diagnostic_edmfx_bomex_stretched_box"][(:c, :ρq_tot)] = 0 -# -all_best_mse["diagnostic_edmfx_dycoms_rf01_explicit_box"] = OrderedCollections.OrderedDict() -all_best_mse["diagnostic_edmfx_dycoms_rf01_explicit_box"][(:c, :ρ)] = 0 -all_best_mse["diagnostic_edmfx_dycoms_rf01_explicit_box"][(:c, :ρe_tot)] = 0 -all_best_mse["diagnostic_edmfx_dycoms_rf01_explicit_box"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["diagnostic_edmfx_dycoms_rf01_explicit_box"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["diagnostic_edmfx_dycoms_rf01_explicit_box"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["diagnostic_edmfx_dycoms_rf01_explicit_box"][(:c, :ρq_tot)] = 0 -# -all_best_mse["diagnostic_edmfx_dycoms_rf01_box"] = OrderedCollections.OrderedDict() -all_best_mse["diagnostic_edmfx_dycoms_rf01_box"][(:c, :ρ)] = 0 -all_best_mse["diagnostic_edmfx_dycoms_rf01_box"][(:c, :ρe_tot)] = 0 -all_best_mse["diagnostic_edmfx_dycoms_rf01_box"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["diagnostic_edmfx_dycoms_rf01_box"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["diagnostic_edmfx_dycoms_rf01_box"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["diagnostic_edmfx_dycoms_rf01_box"][(:c, :ρq_tot)] = 0 -# -all_best_mse["diagnostic_edmfx_dycoms_rf02_box"] = OrderedCollections.OrderedDict() -all_best_mse["diagnostic_edmfx_dycoms_rf02_box"][(:c, :ρ)] = 0 -all_best_mse["diagnostic_edmfx_dycoms_rf02_box"][(:c, :ρe_tot)] = 0 -all_best_mse["diagnostic_edmfx_dycoms_rf02_box"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["diagnostic_edmfx_dycoms_rf02_box"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["diagnostic_edmfx_dycoms_rf02_box"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["diagnostic_edmfx_dycoms_rf02_box"][(:c, :ρq_tot)] = 0 -# -all_best_mse["diagnostic_edmfx_rico_box"] = OrderedCollections.OrderedDict() -all_best_mse["diagnostic_edmfx_rico_box"][(:c, :ρ)] = 0 -all_best_mse["diagnostic_edmfx_rico_box"][(:c, :ρe_tot)] = 0 -all_best_mse["diagnostic_edmfx_rico_box"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["diagnostic_edmfx_rico_box"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["diagnostic_edmfx_rico_box"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["diagnostic_edmfx_rico_box"][(:c, :ρq_tot)] = 0 -# -all_best_mse["diagnostic_edmfx_trmm_box"] = OrderedCollections.OrderedDict() -all_best_mse["diagnostic_edmfx_trmm_box"][(:c, :ρ)] = 0 -all_best_mse["diagnostic_edmfx_trmm_box"][(:c, :ρe_tot)] = 0 -all_best_mse["diagnostic_edmfx_trmm_box"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["diagnostic_edmfx_trmm_box"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["diagnostic_edmfx_trmm_box"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["diagnostic_edmfx_trmm_box"][(:c, :ρq_tot)] = 0 -# -all_best_mse["diagnostic_edmfx_trmm_stretched_box"] = OrderedCollections.OrderedDict() -all_best_mse["diagnostic_edmfx_trmm_stretched_box"][(:c, :ρ)] = 0 -all_best_mse["diagnostic_edmfx_trmm_stretched_box"][(:c, :ρe_tot)] = 0 -all_best_mse["diagnostic_edmfx_trmm_stretched_box"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["diagnostic_edmfx_trmm_stretched_box"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["diagnostic_edmfx_trmm_stretched_box"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["diagnostic_edmfx_trmm_stretched_box"][(:c, :ρq_tot)] = 0 -# -all_best_mse["diagnostic_edmfx_trmm_box_0M"] = OrderedCollections.OrderedDict() -all_best_mse["diagnostic_edmfx_trmm_box_0M"][(:c, :ρ)] = 0 -all_best_mse["diagnostic_edmfx_trmm_box_0M"][(:c, :ρe_tot)] = 0 -all_best_mse["diagnostic_edmfx_trmm_box_0M"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["diagnostic_edmfx_trmm_box_0M"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["diagnostic_edmfx_trmm_box_0M"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["diagnostic_edmfx_trmm_box_0M"][(:c, :ρq_tot)] = 0 -# -all_best_mse["prognostic_edmfx_adv_test_column"] = OrderedCollections.OrderedDict() -all_best_mse["prognostic_edmfx_adv_test_column"][(:c, :ρ)] = 0 -all_best_mse["prognostic_edmfx_adv_test_column"][(:c, :ρe_tot)] = 0 -all_best_mse["prognostic_edmfx_adv_test_column"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["prognostic_edmfx_adv_test_column"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["prognostic_edmfx_adv_test_column"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["prognostic_edmfx_adv_test_column"][(:c, :ρq_tot)] = 0 -# -all_best_mse["prognostic_edmfx_simpleplume_column"] = OrderedCollections.OrderedDict() -all_best_mse["prognostic_edmfx_simpleplume_column"][(:c, :ρ)] = 0 -all_best_mse["prognostic_edmfx_simpleplume_column"][(:c, :ρe_tot)] = 0 -all_best_mse["prognostic_edmfx_simpleplume_column"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["prognostic_edmfx_simpleplume_column"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["prognostic_edmfx_simpleplume_column"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["prognostic_edmfx_simpleplume_column"][(:c, :ρq_tot)] = 0 -# -all_best_mse["prognostic_edmfx_gabls_column"] = OrderedCollections.OrderedDict() -all_best_mse["prognostic_edmfx_gabls_column"][(:c, :ρ)] = 0 -all_best_mse["prognostic_edmfx_gabls_column"][(:c, :ρe_tot)] = 0 -all_best_mse["prognostic_edmfx_gabls_column"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["prognostic_edmfx_gabls_column"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["prognostic_edmfx_gabls_column"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["prognostic_edmfx_gabls_column"][(:c, :ρq_tot)] = 0 -# -all_best_mse["prognostic_edmfx_bomex_pigroup_column"] = OrderedCollections.OrderedDict() -all_best_mse["prognostic_edmfx_bomex_pigroup_column"][(:c, :ρ)] = 0 -all_best_mse["prognostic_edmfx_bomex_pigroup_column"][(:c, :ρe_tot)] = 0 -all_best_mse["prognostic_edmfx_bomex_pigroup_column"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["prognostic_edmfx_bomex_pigroup_column"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["prognostic_edmfx_bomex_pigroup_column"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["prognostic_edmfx_bomex_pigroup_column"][(:c, :ρq_tot)] = 0 -# -all_best_mse["prognostic_edmfx_bomex_fixtke_column"] = OrderedCollections.OrderedDict() -all_best_mse["prognostic_edmfx_bomex_fixtke_column"][(:c, :ρ)] = 0 -all_best_mse["prognostic_edmfx_bomex_fixtke_column"][(:c, :ρe_tot)] = 0 -all_best_mse["prognostic_edmfx_bomex_fixtke_column"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["prognostic_edmfx_bomex_fixtke_column"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["prognostic_edmfx_bomex_fixtke_column"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["prognostic_edmfx_bomex_fixtke_column"][(:c, :ρq_tot)] = 0 -# -all_best_mse["prognostic_edmfx_bomex_stretched_column"] = OrderedCollections.OrderedDict() -all_best_mse["prognostic_edmfx_bomex_stretched_column"][(:c, :ρ)] = 0 -all_best_mse["prognostic_edmfx_bomex_stretched_column"][(:c, :ρe_tot)] = 0 -all_best_mse["prognostic_edmfx_bomex_stretched_column"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["prognostic_edmfx_bomex_stretched_column"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["prognostic_edmfx_bomex_stretched_column"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["prognostic_edmfx_bomex_stretched_column"][(:c, :ρq_tot)] = 0 -# -all_best_mse["prognostic_edmfx_bomex_column"] = OrderedCollections.OrderedDict() -all_best_mse["prognostic_edmfx_bomex_column"][(:c, :ρ)] = 0 -all_best_mse["prognostic_edmfx_bomex_column"][(:c, :ρe_tot)] = 0 -all_best_mse["prognostic_edmfx_bomex_column"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["prognostic_edmfx_bomex_column"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["prognostic_edmfx_bomex_column"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["prognostic_edmfx_bomex_column"][(:c, :ρq_tot)] = 0 -# -all_best_mse["prognostic_edmfx_bomex_column_implicit"] = OrderedCollections.OrderedDict() -all_best_mse["prognostic_edmfx_bomex_column_implicit"][(:c, :ρ)] = 0 -all_best_mse["prognostic_edmfx_bomex_column_implicit"][(:c, :ρe_tot)] = 0 -all_best_mse["prognostic_edmfx_bomex_column_implicit"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["prognostic_edmfx_bomex_column_implicit"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["prognostic_edmfx_bomex_column_implicit"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["prognostic_edmfx_bomex_column_implicit"][(:c, :ρq_tot)] = 0 -# -all_best_mse["prognostic_edmfx_dycoms_rf01_column"] = OrderedCollections.OrderedDict() -all_best_mse["prognostic_edmfx_dycoms_rf01_column"][(:c, :ρ)] = 0 -all_best_mse["prognostic_edmfx_dycoms_rf01_column"][(:c, :ρe_tot)] = 0 -all_best_mse["prognostic_edmfx_dycoms_rf01_column"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["prognostic_edmfx_dycoms_rf01_column"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["prognostic_edmfx_dycoms_rf01_column"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["prognostic_edmfx_dycoms_rf01_column"][(:c, :ρq_tot)] = 0 -# -all_best_mse["prognostic_edmfx_rico_column"] = OrderedCollections.OrderedDict() -all_best_mse["prognostic_edmfx_rico_column"][(:c, :ρ)] = 0 -all_best_mse["prognostic_edmfx_rico_column"][(:c, :ρe_tot)] = 0 -all_best_mse["prognostic_edmfx_rico_column"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["prognostic_edmfx_rico_column"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["prognostic_edmfx_rico_column"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["prognostic_edmfx_rico_column"][(:c, :ρq_tot)] = 0 -# -all_best_mse["prognostic_edmfx_trmm_column"] = OrderedCollections.OrderedDict() -all_best_mse["prognostic_edmfx_trmm_column"][(:c, :ρ)] = 0 -all_best_mse["prognostic_edmfx_trmm_column"][(:c, :ρe_tot)] = 0 -all_best_mse["prognostic_edmfx_trmm_column"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["prognostic_edmfx_trmm_column"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["prognostic_edmfx_trmm_column"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["prognostic_edmfx_trmm_column"][(:c, :ρq_tot)] = 0 -# -all_best_mse["prognostic_edmfx_trmm_column_0M"] = OrderedCollections.OrderedDict() -all_best_mse["prognostic_edmfx_trmm_column_0M"][(:c, :ρ)] = 0 -all_best_mse["prognostic_edmfx_trmm_column_0M"][(:c, :ρe_tot)] = 0 -all_best_mse["prognostic_edmfx_trmm_column_0M"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["prognostic_edmfx_trmm_column_0M"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["prognostic_edmfx_trmm_column_0M"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["prognostic_edmfx_trmm_column_0M"][(:c, :ρq_tot)] = 0 -# -all_best_mse["prognostic_edmfx_gcmdriven_column"] = OrderedCollections.OrderedDict() -all_best_mse["prognostic_edmfx_gcmdriven_column"][(:c, :ρ)] = 0 -all_best_mse["prognostic_edmfx_gcmdriven_column"][(:c, :ρe_tot)] = 0 -all_best_mse["prognostic_edmfx_gcmdriven_column"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["prognostic_edmfx_gcmdriven_column"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["prognostic_edmfx_gcmdriven_column"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["prognostic_edmfx_gcmdriven_column"][(:c, :ρq_tot)] = 0 -# -all_best_mse["prognostic_edmfx_bomex_box"] = OrderedCollections.OrderedDict() -all_best_mse["prognostic_edmfx_bomex_box"][(:c, :ρ)] = 0 -all_best_mse["prognostic_edmfx_bomex_box"][(:c, :ρe_tot)] = 0 -all_best_mse["prognostic_edmfx_bomex_box"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["prognostic_edmfx_bomex_box"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["prognostic_edmfx_bomex_box"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["prognostic_edmfx_bomex_box"][(:c, :ρq_tot)] = 0 -# -all_best_mse["prognostic_edmfx_aquaplanet"] = OrderedCollections.OrderedDict() -all_best_mse["prognostic_edmfx_aquaplanet"][(:c, :ρ)] = 0 -all_best_mse["prognostic_edmfx_aquaplanet"][(:c, :ρe_tot)] = 0 -all_best_mse["prognostic_edmfx_aquaplanet"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["prognostic_edmfx_aquaplanet"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["prognostic_edmfx_aquaplanet"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["prognostic_edmfx_aquaplanet"][(:c, :ρq_tot)] = 0 -# -all_best_mse["sphere_baroclinic_wave_rhoe_gpu"] = OrderedCollections.OrderedDict() -all_best_mse["sphere_baroclinic_wave_rhoe_gpu"][(:c, :ρ)] = 0 -all_best_mse["sphere_baroclinic_wave_rhoe_gpu"][(:c, :ρe_tot)] = 0 -all_best_mse["sphere_baroclinic_wave_rhoe_gpu"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["sphere_baroclinic_wave_rhoe_gpu"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["sphere_baroclinic_wave_rhoe_gpu"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["sphere_baroclinic_wave_rhoe_gpu"][(:c, :ρq_tot)] = 0 -# -all_best_mse["diagnostic_edmfx_aquaplanet_gpu"] = OrderedCollections.OrderedDict() -all_best_mse["diagnostic_edmfx_aquaplanet_gpu"][(:c, :ρ)] = 0 -all_best_mse["diagnostic_edmfx_aquaplanet_gpu"][(:c, :ρe_tot)] = 0 -all_best_mse["diagnostic_edmfx_aquaplanet_gpu"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["diagnostic_edmfx_aquaplanet_gpu"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["diagnostic_edmfx_aquaplanet_gpu"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["diagnostic_edmfx_aquaplanet_gpu"][(:c, :ρq_tot)] = 0 -# -all_best_mse["prognostic_edmfx_aquaplanet_gpu"] = OrderedCollections.OrderedDict() -all_best_mse["prognostic_edmfx_aquaplanet_gpu"][(:c, :ρ)] = 0 -all_best_mse["prognostic_edmfx_aquaplanet_gpu"][(:c, :ρe_tot)] = 0 -all_best_mse["prognostic_edmfx_aquaplanet_gpu"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["prognostic_edmfx_aquaplanet_gpu"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["prognostic_edmfx_aquaplanet_gpu"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["prognostic_edmfx_aquaplanet_gpu"][(:c, :ρq_tot)] = 0 -# -all_best_mse["central_gpu_hs_rhoe_equil_55km_nz63_0M"] = OrderedCollections.OrderedDict() -all_best_mse["central_gpu_hs_rhoe_equil_55km_nz63_0M"][(:c, :ρ)] = 0 -all_best_mse["central_gpu_hs_rhoe_equil_55km_nz63_0M"][(:c, :ρe_tot)] = 0 -all_best_mse["central_gpu_hs_rhoe_equil_55km_nz63_0M"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["central_gpu_hs_rhoe_equil_55km_nz63_0M"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["central_gpu_hs_rhoe_equil_55km_nz63_0M"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["central_gpu_hs_rhoe_equil_55km_nz63_0M"][(:c, :ρq_tot)] = 0 -# -all_best_mse["central_cloud_diag_gpu_hs_rhoe_equil_55km_nz63_0M"] = OrderedCollections.OrderedDict() -all_best_mse["central_cloud_diag_gpu_hs_rhoe_equil_55km_nz63_0M"][(:c, :ρ)] = 0 -all_best_mse["central_cloud_diag_gpu_hs_rhoe_equil_55km_nz63_0M"][(:c, :ρe_tot)] = 0 -all_best_mse["central_cloud_diag_gpu_hs_rhoe_equil_55km_nz63_0M"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["central_cloud_diag_gpu_hs_rhoe_equil_55km_nz63_0M"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["central_cloud_diag_gpu_hs_rhoe_equil_55km_nz63_0M"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["central_cloud_diag_gpu_hs_rhoe_equil_55km_nz63_0M"][(:c, :ρq_tot)] = 0 -# -all_best_mse["gpu_aquaplanet_dyamond"] = OrderedCollections.OrderedDict() -all_best_mse["gpu_aquaplanet_dyamond"][(:c, :ρ)] = 0 -all_best_mse["gpu_aquaplanet_dyamond"][(:c, :ρe_tot)] = 0 -all_best_mse["gpu_aquaplanet_dyamond"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["gpu_aquaplanet_dyamond"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["gpu_aquaplanet_dyamond"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["gpu_aquaplanet_dyamond"][(:c, :ρq_tot)] = 0 -# -all_best_mse["target_gpu_implicit_baroclinic_wave_4process"] = OrderedCollections.OrderedDict() -all_best_mse["target_gpu_implicit_baroclinic_wave_4process"][(:c, :ρ)] = 0 -all_best_mse["target_gpu_implicit_baroclinic_wave_4process"][(:c, :ρe_tot)] = 0 -all_best_mse["target_gpu_implicit_baroclinic_wave_4process"][(:c, :uₕ, :components, :data, 1)] = 0 -all_best_mse["target_gpu_implicit_baroclinic_wave_4process"][(:c, :uₕ, :components, :data, 2)] = 0 -all_best_mse["target_gpu_implicit_baroclinic_wave_4process"][(:f, :u₃, :components, :data, 1)] = 0 -all_best_mse["target_gpu_implicit_baroclinic_wave_4process"][(:c, :ρq_tot)] = 0 -# -#! format: on -################################# -################################# -################################# +reproducibility_test_job_ids = [ # TODO: can we instead automatically grab this from the .buildkite file? + "sphere_baroclinic_wave_rhoe_equilmoist", + "sphere_held_suarez_rhoe_equilmoist_hightop_sponge", + "sphere_aquaplanet_rhoe_equilmoist_allsky_gw_res", + "sphere_aquaplanet_rhoe_equilmoist_allsky_gw_raw_zonallyasymmetric", + "deep_sphere_baroclinic_wave_rhoe_equilmoist", + "diagnostic_edmfx_aquaplanet", + "single_column_hydrostatic_balance_ft64", + "box_hydrostatic_balance_rhoe", + "box_density_current_test", + "rcemipii_box_diagnostic_edmfx", + "les_isdac_box", + "plane_agnesi_mountain_test_uniform", + "plane_agnesi_mountain_test_stretched", + "plane_density_current_test", + "sphere_hydrostatic_balance_rhoe_ft64", + "sphere_baroclinic_wave_rhoe", + "sphere_held_suarez_rhoe_hightop", + "sphere_aquaplanet_rhoe_nonequilmoist_allsky", + "aquaplanet_rhoe_equil_clearsky_tvinsol_0M_slabocean", + "aquaplanet_rhoe_equil_clearsky_tvinsol_0M_slabocean_ft64", + "rcemipii_sphere_diagnostic_edmfx", + "sphere_baroclinic_wave_rhoe_topography_dcmip_rs", + "sphere_held_suarez_rhoe_topography_dcmip", + "sphere_held_suarez_rhoe_equilmoist_topography_dcmip", + "sphere_ssp_baroclinic_wave_rhoe_equilmoist_dcmip200", + "sphere_ssp_baroclinic_wave_rhoe_equilmoist_earth", + "mpi_sphere_aquaplanet_rhoe_equilmoist_clearsky", + "diagnostic_edmfx_test_box", + "diagnostic_edmfx_gabls_box", + "diagnostic_edmfx_bomex_box", + "diagnostic_edmfx_bomex_stretched_box", + "diagnostic_edmfx_dycoms_rf01_explicit_box", + "diagnostic_edmfx_dycoms_rf01_box", + "diagnostic_edmfx_dycoms_rf02_box", + "diagnostic_edmfx_rico_box", + "diagnostic_edmfx_trmm_box", + "diagnostic_edmfx_trmm_stretched_box", + "diagnostic_edmfx_trmm_box_0M", + "prognostic_edmfx_adv_test_column", + "prognostic_edmfx_simpleplume_column", + "prognostic_edmfx_gabls_column", + "prognostic_edmfx_bomex_pigroup_column", + "prognostic_edmfx_bomex_fixtke_column", + "prognostic_edmfx_bomex_stretched_column", + "prognostic_edmfx_bomex_column", + "prognostic_edmfx_bomex_column_implicit", + "prognostic_edmfx_dycoms_rf01_column", + "prognostic_edmfx_rico_column", + "prognostic_edmfx_trmm_column", + "prognostic_edmfx_trmm_column_0M", + "prognostic_edmfx_gcmdriven_column", + "prognostic_edmfx_bomex_box", + "prognostic_edmfx_aquaplanet", + "sphere_baroclinic_wave_rhoe_gpu", + "diagnostic_edmfx_aquaplanet_gpu", + "prognostic_edmfx_aquaplanet_gpu", + "central_gpu_hs_rhoe_equil_55km_nz63_0M", + "central_cloud_diag_gpu_hs_rhoe_equil_55km_nz63_0M", + "gpu_aquaplanet_dyamond", + "target_gpu_implicit_baroclinic_wave_4process", +] diff --git a/reproducibility_tests/print_new_mse.jl b/reproducibility_tests/print_new_mse.jl index f7bb2f8a54..fcd7030748 100644 --- a/reproducibility_tests/print_new_mse.jl +++ b/reproducibility_tests/print_new_mse.jl @@ -5,14 +5,8 @@ import JSON include(joinpath(@__DIR__, "latest_comparable_paths.jl")) paths = latest_comparable_paths() -all_lines = readlines(joinpath(@__DIR__, "mse_tables.jl")) -lines = deepcopy(all_lines) -filter!(x -> occursin("] = OrderedCollections", x), lines) -job_ids = getindex.(split.(lines, "\""), 2) -@assert count(x -> occursin("OrderedDict", x), all_lines) == length(job_ids) + 1 -@assert length(job_ids) ≠ 0 # safety net - include(joinpath(@__DIR__, "mse_tables.jl")) +job_ids = reproducibility_test_job_ids computed_mse = OrderedCollections.OrderedDict() files_skipped = OrderedCollections.OrderedDict() @@ -21,10 +15,11 @@ for job_id in job_ids files_skipped[job_id] = false end +@info "length(job_ids) = $(length(job_ids))" for job_id in job_ids all_filenames = readdir(joinpath(job_id, "output_active"); join = true) mse_filenames = filter(is_mse_file, all_filenames) - @info "mse_filenames: $mse_filenames" + isempty(mse_filenames) || @info "mse_filenames: $mse_filenames" for filename in mse_filenames if !isfile(filename) @warn "File $filename skipped" @@ -41,26 +36,22 @@ for job_id in job_ids end end -println("#################################") -println("################################# MSE tables") -println("#################################") +println("################################# Computed MSEs") println("#! format: off") println("#") -println("all_best_mse = OrderedCollections.OrderedDict()\n#") for job_id in keys(computed_mse) - println("all_best_mse[\"$job_id\"] = OrderedCollections.OrderedDict()") for var in keys(computed_mse[job_id]) if computed_mse[job_id][var] == "NA" println( - "all_best_mse[\"$job_id\"][$(var)] = \"$(computed_mse[job_id][var])\"", + "mse_dict[\"$job_id\"][$(var)] = \"$(computed_mse[job_id][var])\"", ) else # It's easier to update the reference counter, rather than updating # the mse tables, so let's always print zeros: computed_mse[job_id][var] = 0 println( - "all_best_mse[\"$job_id\"][$(var)] = $(computed_mse[job_id][var])", + "mse_dict[\"$job_id\"][$(var)] = $(computed_mse[job_id][var])", ) end end @@ -68,17 +59,9 @@ for job_id in keys(computed_mse) end println("#! format: on") -println("#################################") -println("#################################") println("#################################") -if isempty(paths) - @warn string( - "The printed `all_best_mse` values have", - "been set to zero, due to no comparable references,", - "for copy-paste convenience.", - ) -end +isempty(paths) && @warn string("No comparable references.") # Cleanup for job_id in job_ids @@ -91,15 +74,6 @@ end println("-- DO NOT COPY --") -for job_id in keys(computed_mse) - for var in keys(computed_mse[job_id]) - if haskey(all_best_mse[job_id], var) - all_best_mse[job_id][var] isa Real || continue # skip if "NA" - computed_mse[job_id][var] isa Real || continue # skip if "NA" - end - end -end - if any(values(files_skipped)) @info "Skipped files:" for key in keys(files_skipped) diff --git a/reproducibility_tests/print_new_ref_counter.jl b/reproducibility_tests/print_new_ref_counter.jl index 89e6d25e56..de242145fd 100644 --- a/reproducibility_tests/print_new_ref_counter.jl +++ b/reproducibility_tests/print_new_ref_counter.jl @@ -1,25 +1,31 @@ import Dates -function find_latest_dataset_folder(; dir = pwd()) - matching_paths = String[] - for file in readdir(dir) - !ispath(joinpath(dir, file)) && continue - # Skip folders without the ref_counter - isfile(joinpath(dir, "ref_counter.jl")) || continue - push!(matching_paths, joinpath(dir, file)) - end +""" + sorted_dataset_folder(; dir=pwd()) + +Return a the subdirectory paths within the given `dir` (defaults +to the current working directory) sorted by modification time +(oldest to newest). Return an empty vector if no subdirectories +are found. +""" +function sorted_dataset_folder(; dir = pwd()) + matching_paths = filter(ispath, readdir(dir; join = true)) isempty(matching_paths) && return "" # sort by timestamp sorted_paths = sort(matching_paths; by = f -> Dates.unix2datetime(stat(f).mtime)) - return pop!(sorted_paths) + return sorted_paths end +find_latest_dataset_folder(; dir = pwd()) = pop!(sorted_dataset_folder(; dir)) + cluster_data_prefix = "/central/scratch/esm/slurm-buildkite/climaatmos-main" -path = find_latest_dataset_folder(; dir = cluster_data_prefix) +sorted_folders = sorted_dataset_folder(; dir = cluster_data_prefix) +path = sorted_folders[end] ref_counter = 0 # (error) if isempty(path) # no folders found ref_counter = 1 + @warn "sorted_folders = $sorted_folders" @warn "path: `$path` is empty, setting `ref_counter = 1`" elseif !isfile(joinpath(path, "ref_counter.jl")) # no file found @warn "file `$(joinpath(path, "ref_counter.jl"))` not found" diff --git a/reproducibility_tests/ref_counter.jl b/reproducibility_tests/ref_counter.jl index ba179e0e02..d7e38dc7fd 100644 --- a/reproducibility_tests/ref_counter.jl +++ b/reproducibility_tests/ref_counter.jl @@ -1,4 +1,4 @@ -186 +187 # **README** # @@ -21,6 +21,12 @@ #= +187 +- Data for reproducibility tests are now exported via HDF5 instead of NC files, + and incrementing the reference counter is far easier to update our system + than supporting the comparison of data exported (and some pre-processed) + from different formats. + 186 - Topography dataset has been modified to the 60 arc-second ETOPO2022 dataset. This is behaviour changing for the gravity-wave (raw-topo) parameterization diff --git a/reproducibility_tests/reproducibility_tests.jl b/reproducibility_tests/reproducibility_tests.jl index 5ab9493c33..cec655a04a 100644 --- a/reproducibility_tests/reproducibility_tests.jl +++ b/reproducibility_tests/reproducibility_tests.jl @@ -1,41 +1,28 @@ import JSON -import ClimaCore.Fields as Fields +import ClimaCore.Fields +import ClimaCore.InputOutput include(joinpath(@__DIR__, "compute_mse.jl")) -function perform_reproducibility_tests( +function export_reproducibility_results( + comms_ctx, job_id::String, Y_last::Fields.FieldVector, - all_best_mse::AbstractDict, output_dir::String, ) # This is helpful for starting up new tables - @info "Job-specific MSE table format:" - println("all_best_mse[\"$job_id\"] = OrderedCollections.OrderedDict()") + @info "Comparing variables:" for prop_chain in Fields.property_chains(Y_last) - println("all_best_mse[\"$job_id\"][$prop_chain] = 0.0") + println(" $prop_chain") end - # Extract best mse for this job: - best_mse = all_best_mse[job_id] - ds_filename_computed = joinpath(output_dir, "prog_state.nc") + ds_filename_computed = joinpath(output_dir, "prog_state.hdf5") - function process_name(s::AbstractString) - # "c_ρ", "c_ρe", "c_uₕ_1", "c_uₕ_2", "f_w_1", "c_sgs⁰_ρatke" - s = replace(s, "components_data_" => "") - s = replace(s, "ₕ" => "_h") - s = replace(s, "ρ" => "rho") - s = replace(s, "⁰" => "_0") - return s - end - varname(pc::Tuple) = process_name(join(pc, "_")) + hdfwriter = InputOutput.HDF5Writer(ds_filename_computed, comms_ctx) + InputOutput.write!(hdfwriter, Y_last, "Y") + Base.close(hdfwriter) - export_nc(Y_last; nc_filename = ds_filename_computed, varname) - (computed_mses, paths) = reproducibility_test(; - job_id, - reference_mse = best_mse, - ds_filename_computed, - varname, - ) + (computed_mses, paths) = + reproducibility_results(config.comms_ctx; job_id, ds_filename_computed) for (computed_mse, path) in zip(computed_mses, paths) commit_hash = basename(path) diff --git a/reproducibility_tests/test_mse.jl b/reproducibility_tests/test_mse.jl index 5ea4a5cd38..61c9ee2654 100644 --- a/reproducibility_tests/test_mse.jl +++ b/reproducibility_tests/test_mse.jl @@ -2,9 +2,7 @@ Please see ClimaAtmos.jl/reproducibility_tests/README.md for a more detailed information on how reproducibility tests work. =# -@info "##########################################" @info "########################################## Reproducibility tests" -@info "##########################################" import OrderedCollections import JSON @@ -37,10 +35,6 @@ function get_params() end (; job_id, out_dir, test_broken_report_flakiness) = get_params() -include(joinpath(@__DIR__, "mse_tables.jl")) -best_mse = all_best_mse[job_id] -best_mse_string = - Dict(map(x -> string(x) => best_mse[x], collect(keys(best_mse)))) import ClimaReproducibilityTests as CRT using Test @@ -98,5 +92,3 @@ test_reproducibility_results( ) @info "##########################################" -@info "##########################################" -@info "##########################################" diff --git a/reproducibility_tests/test_reset.jl b/reproducibility_tests/test_reset.jl deleted file mode 100644 index 783ece2412..0000000000 --- a/reproducibility_tests/test_reset.jl +++ /dev/null @@ -1,14 +0,0 @@ -import OrderedCollections - -# Get cases from JobIDs in mse_tables file: -include(joinpath(@__DIR__, "latest_comparable_paths.jl")) -paths = latest_comparable_paths() -include(joinpath(@__DIR__, "mse_tables.jl")) - -#### Test that mse values are all zero if ref counter is incremented -mse_vals = collect(Iterators.flatten(map(x -> values(x), values(all_best_mse)))) -if isempty(paths) && !all(mse_vals .== 0) - error( - "All mse values in `reproducibility_tests/mse_tables.jl` must be set to zero when the reference counter is incremented", - ) -end