updates following PR reviews

Author: mo-nikosbaltas

CMEW/app/configure_standardise/bin/configure_standardise.sh

@@ -3,62 +3,15 @@
 # The LICENSE.md file contains full licensing details.
 # Send the output from 'set -x' to 'stdout' rather than 'stderr'.
 BASH_XTRACEFD=1
-set -xeuo pipefail
-
-# ---------------------------------------------------------------------------
-# 0. Defensive programming (multi-run)
-# ---------------------------------------------------------------------------
-: "${RUN_LABEL:?RUN_LABEL must be set (suite_id e.g. u-bv526)}"
-: "${REQUEST_PATH:?REQUEST_PATH must be set}"
-: "${VARIABLES_PATH:?VARIABLES_PATH must be set}"
-
-: "${START_YEAR:?START_YEAR must be set}"
-: "${NUMBER_OF_YEARS:?NUMBER_OF_YEARS must be set}"
-: "${INSTITUTION_ID:?INSTITUTION_ID must be set}"
-: "${ROOT_PROC_DIR:?ROOT_PROC_DIR must be set}"
-: "${ROOT_DATA_DIR:?ROOT_DATA_DIR must be set}"
+set -xeu
 
 # We support two-model metadata via REF_* + non-REF vars.
 # Decide which set must be present based on RUN_LABEL.
 REF_SUITE_ID="${REF_SUITE_ID:-}"
 SUITE_ID="${SUITE_ID:-}"
 
-echo "[INFO] RUN_LABEL=${RUN_LABEL}"
-echo "[INFO] REQUEST_PATH=${REQUEST_PATH}"
-echo "[INFO] VARIABLES_PATH=${VARIABLES_PATH}"
-echo "[INFO] START_YEAR=${START_YEAR}"
-echo "[INFO] NUMBER_OF_YEARS=${NUMBER_OF_YEARS}"
-echo "[INFO] INSTITUTION_ID=${INSTITUTION_ID}"
-echo "[INFO] ROOT_PROC_DIR=${ROOT_PROC_DIR}"
-echo "[INFO] ROOT_DATA_DIR=${ROOT_DATA_DIR}"
-
-if [[ -n "${REF_SUITE_ID}" && "${RUN_LABEL}" == "${REF_SUITE_ID}" ]]; then
-  : "${REF_MODEL_ID:?REF_MODEL_ID must be set (because RUN_LABEL == REF_SUITE_ID)}"
-  : "${REF_SUITE_ID:?REF_SUITE_ID must be set (because RUN_LABEL == REF_SUITE_ID)}"
-  : "${REF_CALENDAR:?REF_CALENDAR must be set (because RUN_LABEL == REF_SUITE_ID)}"
-  : "${REF_VARIANT_LABEL:?REF_VARIANT_LABEL must be set (because RUN_LABEL == REF_SUITE_ID)}"
-
-  echo "[INFO] Mode: REF (matched RUN_LABEL==REF_SUITE_ID)"
-  echo "[INFO] REF_MODEL_ID=${REF_MODEL_ID}"
-  echo "[INFO] REF_SUITE_ID=${REF_SUITE_ID}"
-  echo "[INFO] REF_CALENDAR=${REF_CALENDAR}"
-  echo "[INFO] REF_VARIANT_LABEL=${REF_VARIANT_LABEL}"
-else
-  # evaluation / default path
-  : "${MODEL_ID:?MODEL_ID must be set}"
-  : "${SUITE_ID:?SUITE_ID must be set}"
-  : "${CALENDAR:?CALENDAR must be set}"
-  : "${VARIANT_LABEL:?VARIANT_LABEL must be set}"
-
-  echo "[INFO] Mode: non-REF (legacy/eval/default)"
-  echo "[INFO] MODEL_ID=${MODEL_ID}"
-  echo "[INFO] SUITE_ID=${SUITE_ID}"
-  echo "[INFO] CALENDAR=${CALENDAR}"
-  echo "[INFO] VARIANT_LABEL=${VARIANT_LABEL}"
-fi
-
 # ---------------------------------------------------------------------------
-# 1. Create variables.txt once (shared by both runs)
+# 1. Create variables.txt
 # ---------------------------------------------------------------------------
 echo "[INFO] Creating variables file from ESMValTool recipe"
 cmew-esmvaltool-env create_variables_file.py

CMEW/app/configure_standardise/bin/create_request_file.py

@@ -29,14 +29,32 @@
 Environment variables are accessed directly via os.environ[...].
 """
 
-from __future__ import annotations
-
 import configparser
 import os
 from pathlib import Path
 
 
-def create_request() -> configparser.ConfigParser:
+def create_request():
+    """
+    Build a CDDS request configuration for the run identified by RUN_LABEL.
+
+    The function expects a two-run CMEW configuration to be present in the
+    environment: one reference run (REF_*) and
+                 one evaluation run (MODEL_ID/SUITE_ID/...).
+
+    Behaviour:
+    - Reads all required metadata directly from environment variables.
+    - Selects the reference or evaluation metadata according to RUN_LABEL.
+    - Raises KeyError if a required environment variable is missing.
+    - Raises KeyError if RUN_LABEL matches neither REF_SUITE_ID nor SUITE_ID.
+
+    Returns
+    -------
+    configparser.ConfigParser
+    A populated CDDS request configuration with sections:
+    metadata, common, data, misc, and conversion.
+    """
+
     # Required time window
     start_year = int(os.environ["START_YEAR"])
     number_of_years = int(os.environ["NUMBER_OF_YEARS"])
@@ -62,7 +80,6 @@ def create_request() -> configparser.ConfigParser:
     # Must be set in two-run mode
     run_label = os.environ["RUN_LABEL"].strip()
 
-    # Optional experiment IDs (default to "amip" if not provided)
     ref_experiment_id = (
         os.environ.get("REF_EXPERIMENT_ID", "amip").strip() or "amip"
     )
@@ -143,15 +160,35 @@ def create_request() -> configparser.ConfigParser:
     return request
 
 
-def write_request(
-    request: configparser.ConfigParser, target_path: Path
-) -> None:
+def write_request(request, target_path):
+    """
+    Write a CDDS request configuration to disk.
+
+    The parent directory of ``target_path`` is created if it does
+    not already exist.
+
+    Parameters
+    ----------
+    request : configparser.ConfigParser
+    The populated request configuration to write.
+    target_path : pathlib.Path
+    Destination file path for the generated request configuration.
+    """
+
     target_path.parent.mkdir(parents=True, exist_ok=True)
     with open(target_path, mode="w", encoding="utf-8") as fh:
         request.write(fh)
 
 
-def main() -> None:
+def main():
+    """
+    Generate and write the request file for the current task environment.
+
+    The output file location is taken from the REQUEST_PATH environment
+    variable. All other required inputs are read from the environment
+    by ``create_request()``.
+    """
+
     target_path = Path(os.environ["REQUEST_PATH"])
     request = create_request()
     write_request(request, target_path)

CMEW/app/restructure_dirs/bin/restructure_dirs.sh

@@ -1,18 +1,11 @@
 #!/bin/bash
-# (C) Crown Copyright 2024-2026, Met Office.
+# (C) Crown Copyright 2024-2025, Met Office.
 # The LICENSE.md file contains full licensing details.
-
 # Send the output from 'set -x' to 'stdout' rather than 'stderr'.
 BASH_XTRACEFD=1
-set -xeuo pipefail
-
-: "${ROOT_DATA_DIR:?ROOT_DATA_DIR must be set}"
-: "${CYLC_WORKFLOW_SHARE_DIR:?CYLC_WORKFLOW_SHARE_DIR must be set}"
-: "${CDDS_SOFTWARE_DIR:?CDDS_SOFTWARE_DIR must be set}"
+set -xeu
 
 RESTRUCTURE_COMMAND="${CDDS_SOFTWARE_DIR}/ceda-mip-tools/bin/restructure_for_cmip6"
 ROOT_RESTRUCTURED_DIR="${CYLC_WORKFLOW_SHARE_DIR}/work/"
 
-# This ensures restructuring occurs once after both conversions complete.
-echo "[INFO] Restructuring ${ROOT_DATA_DIR} -> ${ROOT_RESTRUCTURED_DIR}"
 "${RESTRUCTURE_COMMAND}" -d "${ROOT_RESTRUCTURED_DIR}" "${ROOT_DATA_DIR}"

CMEW/app/standardise_model_data/bin/standardise_model_data.sh

@@ -4,7 +4,7 @@
 # Standardise data for both REF and EVAL runs via CDDS.
 
 BASH_XTRACEFD=1
-set -xeuo pipefail
+set -xeu
 
 : "${RUN_LABEL:?RUN_LABEL must be set (ref/eval)}"
 : "${REQUEST_PATH:?REQUEST_PATH must be set}"

CMEW/app/standardise_model_data/rose-app.conf

@@ -1,4 +1,4 @@
-# (C) Crown Copyright 2024-2026, Met Office.
+# (C) Crown Copyright 2024-2025, Met Office.
 # The LICENSE.md file contains full licensing details.
 
 [command]

CMEW/flow.cylc

@@ -24,14 +24,11 @@
 {%- if not UNITTEST %}
             install_env_file => configure_recipe & add_datasets
             add_datasets => configure_for<recipe>
-
             configure_for<recipe> => configure_standardise<recipe,dataset>
+            => standardise_model_data<dataset> => restructure_dirs
+            configure_recipe & restructure_dirs
+            => run_recipe<recipe> & housekeeping
 
-            configure_standardise<recipe,dataset> => standardise_model_data<dataset>
-            standardise_model_data<dataset> => restructure_dirs
-
-            configure_recipe & restructure_dirs => run_recipe<recipe>
-            run_recipe<recipe> => housekeeping
     {%- if TEST %}
             run_recipe<recipe> => compare<recipe>
     {%- endif %}
@@ -66,6 +63,11 @@
             ROOT_SOFTWARE_DIR = ${CDDS_SOFTWARE_DIR}
             CDDS_VERSION = {{ CDDS_VERSION }}
 
+    [[PER_DATASET]]
+        [[[environment]]]
+            RUN_LABEL = ${CYLC_TASK_PARAM_dataset}
+            REQUEST_PATH = ${CYLC_WORKFLOW_SHARE_DIR}/etc/request_${CYLC_TASK_PARAM_dataset}.cfg
+
     # Multi-run metadata family: reference and evaluation runs
     [[MODEL_RUNS]]
         [[[environment]]]
@@ -123,21 +125,17 @@
             REF_SUITE_ID = {{ REF_SUITE_ID }}
 
     [[configure_standardise<recipe,dataset>]]
-        inherit = STANDARDISE, RECIPE, MODEL_RUNS
+        inherit = STANDARDISE, RECIPE, MODEL_RUNS, PER_DATASET
         [[[environment]]]
             ROSE_TASK_APP = configure_standardise
-            RUN_LABEL = %(dataset)s
-            REQUEST_PATH = ${CYLC_WORKFLOW_SHARE_DIR}/etc/request_%(dataset)s.cfg
             START_YEAR = {{ START_YEAR }}
             NUMBER_OF_YEARS = {{ NUMBER_OF_YEARS }}
 
     [[standardise_model_data<dataset>]]
-        inherit = STANDARDISE
+        inherit = STANDARDISE, PER_DATASET
         [[[environment]]]
             ROSE_TASK_APP = standardise_model_data
             SITE = {{ SITE }}
-            RUN_LABEL = %(dataset)s
-            REQUEST_PATH = ${CYLC_WORKFLOW_SHARE_DIR}/etc/request_%(dataset)s.cfg
 
     [[restructure_dirs]]
         inherit = STANDARDISE