Skip to content

Commit 274e9be

Browse files
mo-nikosbaltasmo-nikosbaltas
committed
changes strictly for parametrisation of two-model runs
1 parent 2e0f9e8 commit 274e9be

File tree

2 files changed

+80
-112
lines changed

2 files changed

+80
-112
lines changed

‎CMEW/app/configure_standardise/bin/create_request_file.py‎

Lines changed: 74 additions & 104 deletions
Original file line numberDiff line numberDiff line change
@@ -2,25 +2,31 @@
22
# (C) Crown Copyright 2024-2026, Met Office.
33
# The LICENSE.md file contains full licensing details.
44
"""
5-
Generate CDDS request configuration.
5+
Generate CDDS request configuration for CMEW.
66
7-
THIS VERSION USES ONLY environment variables populated from rose-suite.conf.
7+
Two-run only:
88
9-
Supported modes:
9+
Reference run (REF_*):
10+
- REF_MODEL_ID
11+
- REF_SUITE_ID
12+
- REF_CALENDAR
13+
- REF_VARIANT_LABEL
1014
11-
1) "Legacy" serial/unit-test mode (RUN_LABEL may be unset):
12-
- REQUIRES two-run configuration in the environment (REF_* and eval vars).
13-
- If RUN_LABEL is unset, defaults to generating the EVAL request
14-
(uses SUITE_ID / MODEL_ID / CALENDAR / VARIANT_LABEL).
15+
Evaluation run (non-REF):
16+
- MODEL_ID
17+
- SUITE_ID
18+
- CALENDAR
19+
- VARIANT_LABEL
1520
16-
2) Two-model "multi-run" mode via task parameterisation:
17-
- RUN_LABEL is set to a suite_id (e.g. u-bv526 or u-cw673).
18-
- If RUN_LABEL == REF_SUITE_ID -> use REF_* variables.
19-
- If RUN_LABEL == SUITE_ID -> use non-REF variables.
20-
- Any other RUN_LABEL is an error (no silent fallback).
21+
Selection rule:
22+
- RUN_LABEL must be set and must match either REF_SUITE_ID or SUITE_ID.
23+
- If RUN_LABEL == REF_SUITE_ID -> use REF_* metadata.
24+
- If RUN_LABEL == SUITE_ID -> use non-REF metadata.
2125
2226
Naming requirement:
23-
- ALWAYS set workflow_basename = suite_id, so CDDS paths are cdds_<suite_id>.
27+
- ALWAYS set workflow_basename = suite_id so CDDS paths are cdds_<suite_id>.
28+
29+
Environment variables are accessed directly via os.environ[...].
2430
"""
2531

2632
from __future__ import annotations
@@ -31,106 +37,75 @@
3137

3238

3339
def create_request() -> configparser.ConfigParser:
34-
# ---------------------------------------------------------------------
35-
# 0) Enforce "two-run legacy": require BOTH ref and eval env variables
36-
# ---------------------------------------------------------------------
37-
required = [
38-
"START_YEAR",
39-
"NUMBER_OF_YEARS",
40-
"INSTITUTION_ID",
41-
"ROOT_PROC_DIR",
42-
"ROOT_DATA_DIR",
43-
"VARIABLES_PATH",
44-
# Reference run
45-
"REF_MODEL_ID",
46-
"REF_SUITE_ID",
47-
"REF_CALENDAR",
48-
"REF_VARIANT_LABEL",
49-
# Evaluation run
50-
"MODEL_ID",
51-
"SUITE_ID",
52-
"CALENDAR",
53-
"VARIANT_LABEL",
54-
]
55-
missing = [
56-
k for k in required if not (os.environ.get(k, "") or "").strip()
57-
]
58-
if missing:
59-
raise KeyError(
60-
"Two-run legacy is required; missing environment variables: "
61-
+ ", ".join(missing)
62-
)
63-
64-
# ---------------------------------------------------------------------
65-
# 1) Time window
66-
# ---------------------------------------------------------------------
67-
start_year = int(os.environ.get("START_YEAR", "").strip())
68-
number_of_years = int(os.environ.get("NUMBER_OF_YEARS", "").strip())
40+
# Required time window
41+
start_year = int(os.environ["START_YEAR"])
42+
number_of_years = int(os.environ["NUMBER_OF_YEARS"])
6943
end_year = start_year + number_of_years
7044

71-
# ---------------------------------------------------------------------
72-
# 2) Resolve which run we're generating (Option B)
73-
# - RUN_LABEL may be unset -> default to SUITE_ID (eval)
74-
# ---------------------------------------------------------------------
75-
run_label = (os.environ.get("RUN_LABEL", "") or "").strip()
76-
ref_suite_id = (os.environ.get("REF_SUITE_ID", "") or "").strip()
77-
suite_id = (os.environ.get("SUITE_ID", "") or "").strip()
45+
# Required global metadata
46+
institution_id = os.environ["INSTITUTION_ID"]
47+
variables_path = os.environ["VARIABLES_PATH"]
48+
root_proc_dir = os.environ["ROOT_PROC_DIR"]
49+
root_data_dir = os.environ["ROOT_DATA_DIR"]
50+
51+
# Enforce two-run config exists (KeyError if missing)
52+
ref_model_id = os.environ["REF_MODEL_ID"]
53+
ref_suite_id = os.environ["REF_SUITE_ID"]
54+
ref_calendar = os.environ["REF_CALENDAR"]
55+
ref_variant_label = os.environ["REF_VARIANT_LABEL"]
56+
57+
model_id = os.environ["MODEL_ID"]
58+
suite_id = os.environ["SUITE_ID"]
59+
calendar = os.environ["CALENDAR"]
60+
variant_label = os.environ["VARIANT_LABEL"]
61+
62+
# Must be set in two-run mode
63+
run_label = os.environ["RUN_LABEL"].strip()
7864

79-
if not run_label:
80-
run_label = suite_id
65+
# Optional experiment IDs (default to "amip" if not provided)
66+
ref_experiment_id = (
67+
os.environ.get("REF_EXPERIMENT_ID", "amip").strip() or "amip"
68+
)
69+
experiment_id = os.environ.get("EXPERIMENT_ID", "amip").strip() or "amip"
8170

82-
# ---------------------------------------------------------------------
83-
# 3) Resolve per-run metadata from env only
84-
# ---------------------------------------------------------------------
8571
if run_label == ref_suite_id:
86-
meta_model_id = (os.environ.get("REF_MODEL_ID", "") or "").strip()
87-
meta_suite_id = ref_suite_id
88-
meta_calendar = (os.environ.get("REF_CALENDAR", "") or "").strip()
89-
meta_variant_label = (
90-
os.environ.get("REF_VARIANT_LABEL", "") or ""
91-
).strip()
92-
meta_experiment_id = (
93-
os.environ.get("REF_EXPERIMENT_ID", "amip") or "amip"
94-
).strip()
72+
chosen_model_id = ref_model_id
73+
chosen_suite_id = ref_suite_id
74+
chosen_calendar = ref_calendar
75+
chosen_variant_label = ref_variant_label
76+
chosen_experiment_id = ref_experiment_id
9577
elif run_label == suite_id:
96-
meta_model_id = (os.environ.get("MODEL_ID", "") or "").strip()
97-
meta_suite_id = suite_id
98-
meta_calendar = (os.environ.get("CALENDAR", "") or "").strip()
99-
meta_variant_label = (
100-
os.environ.get("VARIANT_LABEL", "") or ""
101-
).strip()
102-
meta_experiment_id = (
103-
os.environ.get("EXPERIMENT_ID", "amip") or "amip"
104-
).strip()
78+
chosen_model_id = model_id
79+
chosen_suite_id = suite_id
80+
chosen_calendar = calendar
81+
chosen_variant_label = variant_label
82+
chosen_experiment_id = experiment_id
10583
else:
10684
raise KeyError(
107-
"RUN_LABEL must match one of the configured suite IDs. "
108-
f"Got RUN_LABEL='{run_label}'. "
109-
f"Expected REF_SUITE_ID='{ref_suite_id}' or SUITE_ID='{suite_id}'."
85+
"RUN_LABEL must match REF_SUITE_ID or SUITE_ID. "
86+
f"Got RUN_LABEL='{run_label}', REF_SUITE_ID='{ref_suite_id}',"
87+
f"SUITE_ID='{suite_id}'."
11088
)
11189

112-
# Naming requirement: always suite_id
113-
workflow_basename = meta_suite_id
90+
# Requirement: ALWAYS use suite_id for basename (so cdds_<suite_id>)
91+
workflow_basename = chosen_suite_id
11492

115-
# Avoid ConfigParser interpolation problems (e.g. '%' in URLs)
93+
# Avoid ConfigParser interpolation issues (e.g. '%' in URLs)
11694
request = configparser.RawConfigParser()
11795

118-
# ---------------------------------------------------------------------
119-
# 4) Populate request sections
120-
# ---------------------------------------------------------------------
12196
request["metadata"] = {
12297
"base_date": "1850-01-01T00:00:00",
12398
"branch_method": "no parent",
124-
"calendar": meta_calendar,
125-
"experiment_id": meta_experiment_id or "amip",
126-
"institution_id": (os.environ.get("INSTITUTION_ID", "") or "").strip(),
99+
"calendar": chosen_calendar,
100+
"experiment_id": chosen_experiment_id,
101+
"institution_id": institution_id,
127102
"license": "GCModelDev model data is licensed under the Open Government License v3 (https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/)", # noqa: E501
128103
"mip": "ESMVal",
129104
"mip_era": "GCModelDev",
130-
"model_id": meta_model_id,
105+
"model_id": chosen_model_id,
131106
"model_type": "AGCM AER",
132107
"sub_experiment_id": "none",
133-
"variant_label": meta_variant_label,
108+
"variant_label": chosen_variant_label,
134109
}
135110

136111
request["common"] = {
@@ -141,22 +116,20 @@ def create_request() -> configparser.ConfigParser:
141116
),
142117
"mode": "relaxed",
143118
"package": "round-1",
144-
"root_proc_dir": (os.environ.get("ROOT_PROC_DIR", "") or "").strip(),
145-
"root_data_dir": (os.environ.get("ROOT_DATA_DIR", "") or "").strip(),
119+
"root_proc_dir": root_proc_dir,
120+
"root_data_dir": root_data_dir,
146121
"workflow_basename": workflow_basename,
147122
}
148123

149124
request["data"] = {
150125
"end_date": f"{end_year}-01-01T00:00:00",
151126
"mass_data_class": "crum",
152127
"model_workflow_branch": "trunk",
153-
"model_workflow_id": meta_suite_id,
128+
"model_workflow_id": chosen_suite_id,
154129
"model_workflow_revision": "not used except with data request",
155130
"start_date": f"{start_year}-01-01T00:00:00",
156131
"streams": "apm",
157-
"variable_list_file": (
158-
os.environ.get("VARIABLES_PATH", "") or ""
159-
).strip(),
132+
"variable_list_file": variables_path,
160133
}
161134

162135
request["misc"] = {"atmos_timestep": "1200"}
@@ -174,15 +147,12 @@ def write_request(
174147
request: configparser.ConfigParser, target_path: Path
175148
) -> None:
176149
target_path.parent.mkdir(parents=True, exist_ok=True)
177-
with open(target_path, mode="w", encoding="utf-8") as file_handle:
178-
request.write(file_handle)
150+
with open(target_path, mode="w", encoding="utf-8") as fh:
151+
request.write(fh)
179152

180153

181154
def main() -> None:
182-
request_path = (os.environ.get("REQUEST_PATH", "") or "").strip()
183-
if not request_path:
184-
raise KeyError("REQUEST_PATH must be set")
185-
target_path = Path(request_path)
155+
target_path = Path(os.environ["REQUEST_PATH"])
186156
request = create_request()
187157
write_request(request, target_path)
188158

‎CMEW/app/configure_standardise/bin/test_create_request_file.py‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,8 @@
66

77

88
def test_create_request(monkeypatch):
9-
# This create_request_file.py enforces "two-run legacy" even in unit tests,
10-
# so we must set BOTH eval and ref environment variables.
11-
#
12-
# RUN_LABEL is intentionally NOT set here; the code defaults to SUITE_ID
13-
# (i.e. generates the EVAL request).
9+
# Two-run only: must set BOTH eval and ref environment variables
10+
# and must set RUN_LABEL to select which request to generate.
1411

1512
# Shared / common env
1613
monkeypatch.setenv("START_YEAR", "1993")
@@ -32,6 +29,9 @@ def test_create_request(monkeypatch):
3229
monkeypatch.setenv("REF_SUITE_ID", "u-bv526")
3330
monkeypatch.setenv("REF_VARIANT_LABEL", "r5i1p1f3")
3431

32+
# Select EVAL branch explicitly
33+
monkeypatch.setenv("RUN_LABEL", "u-az513")
34+
3535
config = create_request()
3636
actual = {
3737
section: dict(config.items(section)) for section in config.sections()
@@ -74,9 +74,7 @@ def test_create_request(monkeypatch):
7474
"streams": "apm",
7575
"variable_list_file": "/path/to/variables.txt",
7676
},
77-
"misc": {
78-
"atmos_timestep": "1200",
79-
},
77+
"misc": {"atmos_timestep": "1200"},
8078
"conversion": {
8179
"mip_convert_plugin": "UKESM1",
8280
"skip_archive": "True",

0 commit comments

Comments
 (0)