Merge branch 'main' into 371-update-the-pr-checklist

Author: NParsonsMO

CMEW/app/add_datasets/bin/add_datasets_to_share.py

@@ -225,6 +225,40 @@ def dict_namelists_in_work_dir():
     return filepaths
 
 
+def use_facet_as_key(filepath, key_facet="suite_id"):
+    """
+    Edit a YAML file in place, from a list to a dictionary.
+
+    The keys of the new dictionary are the values of the specified facet,
+    wich must be present in each section of the list and be unique.
+
+    Parameters
+    ----------
+    filepath: str
+        The file path to the YAML file to be edited.
+    key_facet: str
+        The facet to use as the key in the new dictionary.
+        Defaults to 'suite_id', which is the unique identifier for now.
+    """
+    # Read the YAML file that has datasets as a list
+    with open(filepath, "r") as f:
+        data = yaml.safe_load(f)
+
+    # Create a new dictionary with the same sections as the list
+    new_dict = {}
+    for section in data:
+
+        # Use the facet as a unique key
+        unique = section[key_facet]
+
+        # The information in each section remains unchanged
+        new_dict[unique] = section
+
+    # Write the new dictionary back to the existing YAML filepath
+    with open(filepath, "w") as f:
+        yaml.dump(new_dict, f)
+
+
 if __name__ == "__main__":
     # Read the target (shared) directory from the environment
     target_dir = os.environ["DATASETS_LIST_DIR"]
@@ -240,3 +274,6 @@ def dict_namelists_in_work_dir():
 
         # Write the datasets to a YAML file in the target directory
         write_datasets_to_yaml(datasets, basename, target_dir)
+
+    # Reformat the model_runs YAML file to use suite_ids as keys
+    use_facet_as_key(f"{target_dir}/model_runs.yml")

CMEW/app/add_datasets/bin/test_add_datasets_to_share.py

@@ -8,10 +8,12 @@
     write_dict_to_yaml,
     write_datasets_to_yaml,
     dict_namelists_in_work_dir,
+    use_facet_as_key,
 )
 from pathlib import Path
 import pytest
 import yaml
+import shutil
 import tempfile
 from unittest.mock import patch
 
@@ -24,36 +26,47 @@ def mock_env_vars(monkeypatch):
 
 
 @pytest.fixture
-def path_to_kgo_yaml():
+def path_to_mock_nl():
     path = (
         Path(__file__).parent.parent.parent
         / "unittest"
-        / "kgo"
-        / "model_runs.yml"
+        / "mock_data"
+        / "model_runs.nl"
     )
     return str(path)
 
 
 @pytest.fixture
-def path_to_mock_nl():
+def path_to_kgo_dict():
+    path = (
+        Path(__file__).parent.parent.parent
+        / "unittest"
+        / "kgo"
+        / "basic_dict.yml"
+    )
+    return path
+
+
+@pytest.fixture
+def path_to_mock_yaml_list():
     path = (
         Path(__file__).parent.parent.parent
         / "unittest"
         / "mock_data"
-        / "model_runs.nl"
+        / "model_runs_as_list.yml"
     )
     return str(path)
 
 
 @pytest.fixture
-def path_to_kgo_dict():
+def path_to_kgo_yaml_dict():
     path = (
         Path(__file__).parent.parent.parent
         / "unittest"
         / "kgo"
-        / "basic_dict.yml"
+        / "model_runs_as_dict.yml"
     )
-    return path
+    return str(path)
 
 
 def test_extract_sections_from_naml(path_to_mock_nl):
@@ -152,7 +165,6 @@ def test_process_naml_file(path_to_mock_nl, mock_env_vars):
     assert actual == expected
 
 
-# This one was only *some* random copying of Google
 def test_write_dict_to_yaml(path_to_kgo_dict):
     # Note the keys are not alphabetical here but are in the output
     test_dict = {
@@ -204,3 +216,22 @@ def test_dict_namelists_in_work_dir(mock_dirname, mock_listdir, mock_env_vars):
     }
     actual = dict_namelists_in_work_dir()
     assert expected == actual
+
+
+def test_use_facet_as_key(path_to_mock_yaml_list, path_to_kgo_yaml_dict):
+    # Copy known input to a temp file
+    with tempfile.NamedTemporaryFile() as tmp:
+        shutil.copyfile(path_to_mock_yaml_list, tmp.name)
+
+        # The filepath is given by .name
+        use_facet_as_key(str(tmp.name))
+
+        # Read the result
+        tmp.seek(0)
+        actual = yaml.safe_load(tmp)
+
+    # Load the expected output
+    with open(path_to_kgo_yaml_dict, "r") as file_handle:
+        expected = yaml.safe_load(file_handle)
+
+    assert actual == expected

CMEW/app/configure_for/bin/test_update_recipe_file.py

@@ -12,16 +12,18 @@ def mock_env_vars(monkeypatch):
     # Time window
     monkeypatch.setenv("START_YEAR", "1993")
     monkeypatch.setenv("NUMBER_OF_YEARS", "1")
-    monkeypatch.setenv("LABEL_FOR_PLOTS", "Test Label")
-    monkeypatch.setenv("REF_LABEL_FOR_PLOTS", "Ref Test Label")
 
     # Reference run metadata
     monkeypatch.setenv("REF_MODEL_ID", "HadGEM3-GC31-LL")
     monkeypatch.setenv("REF_VARIANT_LABEL", "r1i1p1f3")
+    monkeypatch.setenv("REF_LABEL_FOR_PLOTS", "Ref Test Label")
+    monkeypatch.setenv("REF_EXPERIMENT_ID", "historical")
 
     # Evaluation run metadata
     monkeypatch.setenv("MODEL_ID", "UKESM1-0-LL")
     monkeypatch.setenv("VARIANT_LABEL", "r1i1p1f1")
+    monkeypatch.setenv("LABEL_FOR_PLOTS", "Test Label")
+    monkeypatch.setenv("EXPERIMENT_ID", "amip")
 
     # For adding CMIP6 datasets
     monkeypatch.setenv(

CMEW/app/configure_for/bin/update_recipe_file.py

@@ -34,11 +34,12 @@ def update_recipe(recipe_path):
 
     datasets:
     - {activity: <activity>, alias: <ref_alias>, dataset: <ref_model_id>,
-      end_year: <end_year>, ensemble: <ref_variant>, exp: <exp>, grid: <grid>,
-      project: <project>, start_year: <start_year>}
+      end_year: <end_year>, ensemble: <ref_variant>, exp: <ref_experiment_id>,
+      grid: <grid>, project: <project>, start_year: <start_year>}
     - {activity: <activity>, alias: <alias>, dataset: <eval_model_id>,
-      end_year: <end_year>, ensemble: <eval_variant>, exp: <exp>, grid: <grid>,
-      project: <project>, start_year: <start_year>}
+      end_year: <end_year>, ensemble: <eval_variant>,
+      exp: <eval_experiment_id>, grid: <grid>, project: <project>,
+      start_year: <start_year>}
 
     Notes
     -----
@@ -66,8 +67,10 @@ def update_recipe(recipe_path):
     # Model metadata from environment
     ref_model_id = os.environ["REF_MODEL_ID"]
     ref_variant = os.environ["REF_VARIANT_LABEL"]
+    ref_experiment_id = os.environ["REF_EXPERIMENT_ID"]
     eval_model_id = os.environ["MODEL_ID"]
     eval_variant = os.environ["VARIANT_LABEL"]
+    eval_experiment_id = os.environ["EXPERIMENT_ID"]
 
     # Read given reference alias or use the suite ID
     if os.environ.get("REF_LABEL_FOR_PLOTS"):
@@ -91,14 +94,13 @@ def update_recipe(recipe_path):
             "one for the reference and one for the evaluation run."
         )
 
-    # Reference dataset: treat as a GCModelDev / ESMVal / amip run,
-    # using REF_MODEL_ID & REF_VARIANT_LABEL, with the configured time window.
+    # Reference dataset
     ref_dataset = datasets[0]
     ref_dataset.update(
         {
             "dataset": ref_model_id,
             "project": "ESMVal",
-            "exp": "amip",
+            "exp": ref_experiment_id,
             "activity": "ESMVal",
             "institute": "MOHC",
             "ensemble": ref_variant,
@@ -108,13 +110,13 @@ def update_recipe(recipe_path):
         }
     )
 
-    # Evaluation dataset: ESMVal / amip run using MODEL_ID and VARIANT_LABEL
+    # Evaluation dataset
     eval_dataset = datasets[1]
     eval_dataset.update(
         {
             "dataset": eval_model_id,
             "project": "ESMVal",
-            "exp": "amip",
+            "exp": eval_experiment_id,
             "activity": "ESMVal",
             "institute": "MOHC",
             "ensemble": eval_variant,

CMEW/app/configure_standardise/bin/configure_standardise.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# (C) Crown Copyright 2024-2025, Met Office.
+# (C) Crown Copyright 2024-2026, Met Office.
 # The LICENSE.md file contains full licensing details.
 # Send the output from 'set -x' to 'stdout' rather than 'stderr'.
 BASH_XTRACEFD=1
@@ -20,9 +20,11 @@ echo "[INFO] Using REQUEST_PATH_EVAL=${REQUEST_PATH_EVAL}"
 : "${REF_MODEL_ID:?REF_MODEL_ID must be set}"
 : "${REF_SUITE_ID:?REF_SUITE_ID must be set}"
 : "${REF_CALENDAR:?REF_CALENDAR must be set}"
+: "${REF_EXPERIMENT_ID:??REF_EXPERIMENT_ID must be set}"
 : "${MODEL_ID:?MODEL_ID (evaluation) must be set}"
 : "${SUITE_ID:?SUITE_ID (evaluation) must be set}"
 : "${CALENDAR:?CALENDAR (evaluation) must be set}"
+: "${EXPERIMENT_ID:?EXPERIMENT_ID (evaluation) must be set}"
 
 # ---------------------------------------------------------------------------
 # 1. Create variables.txt once (shared by both runs)
@@ -40,6 +42,7 @@ create_for_run() {
     local run_suite_id=""
     local run_calendar=""
     local run_variant=""
+    local run_experiment_id=""
     local run_request=""
 
     case "${RUN_LABEL}" in
@@ -48,6 +51,7 @@ create_for_run() {
             run_suite_id="${REF_SUITE_ID}"
             run_calendar="${REF_CALENDAR}"
             run_variant="${REF_VARIANT_LABEL:-}"
+            run_experiment_id="${REF_EXPERIMENT_ID}"
             run_request="${REQUEST_PATH_REF}"
             ;;
         EVAL)
@@ -56,6 +60,7 @@ create_for_run() {
             run_suite_id="${SUITE_ID}"
             run_calendar="${CALENDAR}"
             run_variant="${VARIANT_LABEL:-}"
+            run_experiment_id="${EXPERIMENT_ID}"
             run_request="${REQUEST_PATH_EVAL}"
             ;;
         *)
@@ -70,6 +75,7 @@ create_for_run() {
         export SUITE_ID="${run_suite_id}"
         export CALENDAR="${run_calendar}"
         export VARIANT_LABEL="${run_variant}"
+        export EXPERIMENT_ID="${run_experiment_id}"
         export REQUEST_PATH="${run_request}"
 
         echo "[INFO] Creating request for ${RUN_LABEL} run at: ${REQUEST_PATH}"

CMEW/app/configure_standardise/bin/create_request_file.py

@@ -25,7 +25,7 @@ def create_request():
         "base_date": "1850-01-01T00:00:00",
         "branch_method": "no parent",
         "calendar": os.environ["CALENDAR"],
-        "experiment_id": "amip",
+        "experiment_id": os.environ["EXPERIMENT_ID"],
         "institution_id": os.environ["INSTITUTION_ID"],
         "license": "GCModelDev model data is licensed under the Open Government License v3 (https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/)",  # noqa: E501
         "mip": "ESMVal",

CMEW/app/configure_standardise/bin/test_create_request_file.py

@@ -10,6 +10,7 @@ def test_create_request(monkeypatch):
     monkeypatch.setenv("START_YEAR", "1993")
     monkeypatch.setenv("NUMBER_OF_YEARS", "1")
     monkeypatch.setenv("CALENDAR", "360_day")
+    monkeypatch.setenv("EXPERIMENT_ID", "amip")
     monkeypatch.setenv("INSTITUTION_ID", "MOHC")
     monkeypatch.setenv("MODEL_ID", "UKESM1-0-LL")
     monkeypatch.setenv("ROOT_PROC_DIR", "/path/to/proc/dir/")

CMEW/app/unittest/kgo/model_runs_as_dict.yml

@@ -0,0 +1,18 @@
+u-bv526:
+  calendar: 360_day
+  end_year: 2002
+  label_for_plots: HadGEM3-GC3.1 N96ORCA1
+  model_id: HadGEM3-GC31-LL
+  project: CMIP6
+  start_year: 1993
+  suite_id: u-bv526
+  variant_label: r5i1p1f3
+u-cw673:
+  calendar: gregorian
+  end_year: 2002
+  label_for_plots: HadGEM3-GC5E-LL N96ORCA1
+  model_id: HadGEM3-GC5E-LL
+  project: CMIP6
+  start_year: 1993
+  suite_id: u-cw673
+  variant_label: r1i1p1f1

CMEW/app/unittest/kgo/test_updated_radiation_budget_recipe.yml

@@ -10,7 +10,7 @@ datasets:
   dataset: HadGEM3-GC31-LL
   end_year: 1993
   ensemble: r1i1p1f3
-  exp: amip
+  exp: historical
   grid: gn
   institute: MOHC
   project: ESMVal