From f335631778c537fbec2b2664d6c3154dbbbb2cae Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Fri, 24 Oct 2025 10:29:01 +0200 Subject: [PATCH 1/8] Update pyproject.toml for Python 3.14 - Update version classifiers. - Bump requires-python to 3.10. --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 55a10b0f0..8ab1dbc73 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,16 +20,16 @@ classifiers = [ "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Programming Language :: R", "Topic :: Scientific/Engineering", "Topic :: Scientific/Engineering :: Information Analysis", ] -requires-python = ">=3.9" +requires-python = ">=3.10" dependencies = [ "click", "ixmp >= 3.11", From 8f636fde71a7cd28e0cf50b7d1ed5833c5786954 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Fri, 24 Oct 2025 10:33:36 +0200 Subject: [PATCH 2/8] Update CI workflows for Python 3.14 - Add Python 3.14; use for single-version jobs. - Drop Python 3.9. Drop use of macos-13 for Python 3.9. --- .github/workflows/nightly.yaml | 2 +- .github/workflows/pytest.yaml | 21 +++++---------------- 2 files changed, 6 insertions(+), 17 deletions(-) diff --git a/.github/workflows/nightly.yaml b/.github/workflows/nightly.yaml index c60825bc0..cf224c368 100644 --- a/.github/workflows/nightly.yaml +++ b/.github/workflows/nightly.yaml @@ -24,7 +24,7 @@ env: # Oldest version that can reliably be downloaded gams-version: 48.6.1 os: ubuntu-latest - python-version: "3.13" + python-version: "3.14" permissions: {contents: read} diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index c65e72b04..66cbcfaec 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -24,7 +24,7 @@ env: GAMS_VERSION: 48.6.1 # Oldest version of GAMS that can reliably be downloaded depth: 100 # Must be large enough to include the most recent release label: "safe to test" # Label that must be applied to run on PRs from forks - python-version: "3.13" # For non-matrixed jobs + python-version: "3.14" # For non-matrixed jobs # Install: # - dask: to work around https://github.com/khaeru/genno/issues/171 # - ixmp: from its `main` branch. @@ -57,16 +57,15 @@ jobs: strategy: matrix: os: - - macos-13 - macos-latest - ubuntu-latest - windows-latest python-version: - - "3.9" # Earliest version supported by message_ix - - "3.10" + - "3.10" # Earliest version supported by message_ix - "3.11" - "3.12" - - "3.13" # Latest version supported by message_ix + - "3.13" + - "3.14" # Latest version supported by message_ix # Below this comment are newly released or development versions of # Python. For these versions, binary wheels are not available for some @@ -74,17 +73,7 @@ jobs: # these on the job runner requires a more elaborate build environment, # currently out of scope for the message_ix project. - # - "3.14.0-alpha.1" # Development version - - exclude: - # Specific version combinations that are invalid / not to be used - # No arm64 distributions of JPype for these Pythons - - { os: macos-latest, python-version: "3.9" } - # Redundant with macos-latest - - { os: macos-13, python-version: "3.10" } - - { os: macos-13, python-version: "3.11" } - - { os: macos-13, python-version: "3.12" } - - { os: macos-13, python-version: "3.13" } + # - "3.15.0-alpha.1" # Development version fail-fast: false From dcd96b9eea2b8f21420a5e1500e8b63e81551ff2 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Fri, 24 Oct 2025 10:47:49 +0200 Subject: [PATCH 3/8] Use type union operator in 26 files - Drop import & use of typing.{Optional,Union}. --- doc/conf.py | 3 +- message_ix/common.py | 8 ++-- message_ix/core.py | 41 +++++++++--------- message_ix/macro/calibrate.py | 10 ++--- message_ix/report/__init__.py | 6 +-- message_ix/report/pyam.py | 10 ++--- message_ix/testing/__init__.py | 12 +++--- .../tests/model/message/test_cap_comm.py | 4 +- message_ix/tests/test_core.py | 4 +- message_ix/tests/test_feature_addon.py | 4 +- .../test_feature_bound_activity_shares.py | 3 +- .../tests/test_feature_bound_emission.py | 10 ++--- .../tests/test_feature_duration_time.py | 3 +- .../tests/test_feature_price_commodity.py | 8 ++-- .../tests/test_feature_price_emission.py | 6 +-- message_ix/tests/test_feature_storage.py | 7 ++-- .../test_feature_vintage_and_active_years.py | 5 +-- message_ix/tests/test_macro.py | 4 +- message_ix/tests/test_tutorials.py | 10 ++--- message_ix/tests/tools/test_add_year.py | 6 +-- message_ix/tools/add_year/__init__.py | 42 +++++++++---------- message_ix/tools/lp_diag/cli.py | 3 +- message_ix/util/__init__.py | 4 +- message_ix/util/gams_io.py | 30 +++++++------ message_ix/util/scenario_data.py | 25 +++++------ message_ix/util/scenario_setup.py | 12 +++--- 26 files changed, 132 insertions(+), 148 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index 2fc56a29f..95e083c63 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -7,7 +7,6 @@ import os from importlib.metadata import version as get_version from pathlib import Path -from typing import Optional # -- Project information --------------------------------------------------------------- @@ -193,7 +192,7 @@ # -- Options for sphinx.ext.intersphinx ------------------------------------------------ -def local_inv(name: str, *parts: str) -> Optional[str]: +def local_inv(name: str, *parts: str) -> str | None: """Construct the path to a local intersphinx inventory.""" from importlib.util import find_spec diff --git a/message_ix/common.py b/message_ix/common.py index 3ec1abe3a..0f32c3ce3 100644 --- a/message_ix/common.py +++ b/message_ix/common.py @@ -6,7 +6,7 @@ from copy import copy from dataclasses import InitVar, dataclass, field from pathlib import Path -from typing import TYPE_CHECKING, Any, Optional +from typing import TYPE_CHECKING, Any import ixmp.model.gams from ixmp import config @@ -96,7 +96,7 @@ class Item: dims: tuple[str, ...] = field(default_factory=tuple) #: Text description of the item. - description: Optional[str] = None + description: str | None = None def __post_init__(self, expr): if expr == "": @@ -169,12 +169,12 @@ class GAMSModel(ixmp.model.gams.GAMSModel): model_dir: Path #: Optional minimum version of GAMS. - GAMS_min_version: Optional[str] = None + GAMS_min_version: str | None = None #: Keyword arguments to map to GAMS `solve_args`. keyword_to_solve_arg: list[tuple[str, type, str]] - def __init__(self, name: Optional[str] = None, **model_options) -> None: + def __init__(self, name: str | None = None, **model_options) -> None: if gmv := self.GAMS_min_version: # Check the minimum GAMS version. version = ixmp.model.gams.gams_version() or "" diff --git a/message_ix/core.py b/message_ix/core.py index 007c1d097..e3b269f6e 100644 --- a/message_ix/core.py +++ b/message_ix/core.py @@ -3,7 +3,7 @@ from collections.abc import Iterable, Mapping, Sequence from functools import lru_cache from itertools import chain, product, zip_longest -from typing import Optional, TypeVar, Union +from typing import TypeVar from warnings import warn import ixmp @@ -255,12 +255,15 @@ def cat(self, name, cat): def add_par( self, name: str, - key_or_data: Optional[ - Union[int, str, Sequence[Union[int, str]], dict, pd.DataFrame] - ] = None, - value: Union[float, Iterable[float], None] = None, - unit: Union[str, Iterable[str], None] = None, - comment: Union[str, Iterable[str], None] = None, + key_or_data: int + | str + | Sequence[int | str] + | dict + | pd.DataFrame + | None = None, + value: float | Iterable[float] | None = None, + unit: str | Iterable[str] | None = None, + comment: str | Iterable[str] | None = None, ) -> None: # ixmp.Scenario.add_par() is typed as accepting only str, but this method also # accepts int for "year"-like dimensions. Proxy the call to avoid type check @@ -295,14 +298,12 @@ def add_par( def add_set( self, name: str, - key: Union[ - int, - str, - Iterable[object], - dict[str, Union[Sequence[int], Sequence[str]]], - pd.DataFrame, - ], - comment: Union[str, Sequence[str], None] = None, + key: int + | str + | Iterable[object] + | dict[str, Sequence[int] | Sequence[str]] + | pd.DataFrame, + comment: str | Sequence[str] | None = None, ) -> None: # ixmp.Scenario.add_par() is typed as accepting only str, but this method also # accepts int for "year"-like dimensions. Proxy the call to avoid type check @@ -373,8 +374,8 @@ def recurse(k, v, parent="World"): def add_horizon( self, year: Iterable[int] = [], - firstmodelyear: Optional[int] = None, - data: Optional[dict] = None, + firstmodelyear: int | None = None, + data: dict | None = None, ) -> None: """Set the scenario time horizon via ``year`` and related categories. @@ -502,7 +503,7 @@ def add_horizon( def vintage_and_active_years( self, - ya_args: Union[tuple[str, str], tuple[str, str, Union[int, str]], None] = None, + ya_args: tuple[str, str] | tuple[str, str, int | str] | None = None, tl_only: bool = True, **kwargs, ) -> pd.DataFrame: @@ -653,7 +654,7 @@ def _valid(elem): #: Alias for :meth:`vintage_and_active_years`. yv_ya = vintage_and_active_years - def years_active(self, node: str, tec: str, yr_vtg: Union[int, str]) -> list[int]: + def years_active(self, node: str, tec: str, yr_vtg: int | str) -> list[int]: """Return periods in which `tec` hnology of `yr_vtg` can be active in `node`. The :ref:`parameters ` ``duration_period`` and @@ -762,7 +763,7 @@ def solve(self, model="MESSAGE", solve_options={}, **kwargs): def add_macro( self, - data: Union[Mapping, os.PathLike], + data: Mapping | os.PathLike, scenario=None, check_convergence=True, **kwargs, diff --git a/message_ix/macro/calibrate.py b/message_ix/macro/calibrate.py index 7b0a27ce9..edc476793 100644 --- a/message_ix/macro/calibrate.py +++ b/message_ix/macro/calibrate.py @@ -5,7 +5,7 @@ from functools import partial from operator import itemgetter, mul from pathlib import Path -from typing import TYPE_CHECKING, Optional, Union +from typing import TYPE_CHECKING import numpy as np import pandas as pd @@ -460,7 +460,7 @@ def validate_transform( return df.set_index(idx)["value"] -def _validate_data(name: Optional[str], df: "DataFrame", s: Structures) -> list: +def _validate_data(name: str | None, df: "DataFrame", s: Structures) -> list: """Validate input `df` against `s` for MACRO parameter `name` calibration . Parameters @@ -553,7 +553,7 @@ def ym1(df: "Series", macro_periods: Collection[int]) -> int: def add_model_data( - base: "Scenario", clone: "Scenario", data: Union[Mapping, os.PathLike] + base: "Scenario", clone: "Scenario", data: Mapping | os.PathLike ) -> None: """Calculate and add MACRO structure and data to `clone`. @@ -643,8 +643,8 @@ def calibrate(s, check_convergence: bool = True, **kwargs): def prepare_computer( base: "Scenario", - target: Optional["Scenario"] = None, - data: Union[Mapping, os.PathLike, None] = None, + target: "Scenario | None" = None, + data: Mapping | os.PathLike | None = None, ) -> "genno.Computer": """Prepare a :class:`.Reporter` to perform MACRO calibration calculations. diff --git a/message_ix/report/__init__.py b/message_ix/report/__init__.py index 931106fda..90816ea54 100644 --- a/message_ix/report/__init__.py +++ b/message_ix/report/__init__.py @@ -2,7 +2,7 @@ from collections.abc import Mapping from functools import lru_cache, partial from operator import itemgetter -from typing import TYPE_CHECKING, Union, cast +from typing import TYPE_CHECKING, cast from genno.operator import broadcast_map from ixmp.report import ( @@ -220,7 +220,7 @@ def from_scenario(cls, scenario, **kwargs) -> "Reporter": f'Scenario "{scenario.model}/{scenario.scenario}" has no solution' ) log.warning("Some reporting may not function as expected") - fail_action: Union[int, str] = logging.DEBUG + fail_action: int | str = logging.DEBUG else: fail_action = "raise" @@ -288,7 +288,7 @@ def add_sankey( # Generate the plotly.Figure object; return the key return str(self.add(f"sankey figure {unique}", sankey, k[1], k[2])) - def add_tasks(self, fail_action: Union[int, str] = "raise") -> None: + def add_tasks(self, fail_action: int | str = "raise") -> None: """Add the pre-defined MESSAGEix reporting tasks to the Reporter. Parameters diff --git a/message_ix/report/pyam.py b/message_ix/report/pyam.py index 716154671..91c148627 100644 --- a/message_ix/report/pyam.py +++ b/message_ix/report/pyam.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Optional, TypedDict +from typing import TYPE_CHECKING, TypedDict if TYPE_CHECKING: import pandas @@ -8,15 +8,15 @@ class CollapseMessageColsKw(TypedDict, total=False): """Type hint for :class:`dict` of keyword args to :func:`collapse_message_cols`.""" df: "pandas.DataFrame" - var: Optional[str] - kind: Optional[str] + var: str | None + kind: str | None var_cols: list[str] def collapse_message_cols( df: "pandas.DataFrame", - var: Optional[str] = None, - kind: Optional[str] = None, + var: str | None = None, + kind: str | None = None, var_cols=[], ) -> "pandas.DataFrame": """:mod:`genno.compat.pyam` `collapse=...` callback for MESSAGEix quantities. diff --git a/message_ix/testing/__init__.py b/message_ix/testing/__init__.py index aaba66bd4..73a36399f 100644 --- a/message_ix/testing/__init__.py +++ b/message_ix/testing/__init__.py @@ -3,7 +3,7 @@ from collections.abc import Callable, Generator from itertools import product from pathlib import Path -from typing import TYPE_CHECKING, Any, Optional, Union +from typing import TYPE_CHECKING, Any import ixmp import numpy as np @@ -90,7 +90,7 @@ def pytest_sessionstart() -> None: # Create and populate ixmp databases -_ms: list[Union[str, float]] = [ +_ms: list[str | float] = [ SCENARIO["dantzig"]["model"], SCENARIO["dantzig"]["scenario"], ] @@ -165,7 +165,7 @@ def make_austria( # noqa: C901 solve: bool = False, quiet: bool = True, *, - request: Optional["pytest.FixtureRequest"] = None, + request: "pytest.FixtureRequest | None" = None, ) -> Scenario: """Return an :class:`message_ix.Scenario` for the Austrian energy system. @@ -352,7 +352,7 @@ def make_dantzig( solve: bool = False, multi_year: bool = False, *, - request: Optional["pytest.FixtureRequest"] = None, + request: "pytest.FixtureRequest | None" = None, **solve_opts, ) -> Scenario: """Return an :class:`message_ix.Scenario` for Dantzig's canning problem. @@ -521,7 +521,7 @@ def make_westeros( quiet: bool = True, model_horizon: list[int] = [700, 710, 720], *, - request: Optional["pytest.FixtureRequest"] = None, + request: "pytest.FixtureRequest | None" = None, ) -> Scenario: """Return a new :class:`message_ix.Scenario` containing the ‘Westeros’ model. @@ -715,7 +715,7 @@ def make_subannual( var_cost={}, operation_factor={}, *, - request: Optional["pytest.FixtureRequest"] = None, + request: "pytest.FixtureRequest | None" = None, ) -> Scenario: """Return an :class:`message_ix.Scenario` with subannual time resolution. diff --git a/message_ix/tests/model/message/test_cap_comm.py b/message_ix/tests/model/message/test_cap_comm.py index 21b85278a..eb43db1b9 100644 --- a/message_ix/tests/model/message/test_cap_comm.py +++ b/message_ix/tests/model/message/test_cap_comm.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING import numpy as np import pandas as pd @@ -12,7 +12,7 @@ from ixmp import Platform -COMMON: dict[str, Union[int, str]] = dict( +COMMON: dict[str, int | str] = dict( commodity="coal", level="end_of_life", node_dest="World", diff --git a/message_ix/tests/test_core.py b/message_ix/tests/test_core.py index fcf03d72b..04cd4c418 100644 --- a/message_ix/tests/test_core.py +++ b/message_ix/tests/test_core.py @@ -3,7 +3,7 @@ from copy import deepcopy from pathlib import Path from subprocess import run -from typing import Any, Optional +from typing import Any import ixmp import numpy as np @@ -301,7 +301,7 @@ def test_add_spatial_hierarchy(test_mp: ixmp.Platform) -> None: ], ) def test_add_horizon( - test_mp: ixmp.Platform, args, kwargs, exp: Optional[dict[str, list[int]]] + test_mp: ixmp.Platform, args, kwargs, exp: dict[str, list[int]] | None ) -> None: scen = Scenario(test_mp, **SCENARIO["dantzig"], version="new") diff --git a/message_ix/tests/test_feature_addon.py b/message_ix/tests/test_feature_addon.py index 06763b527..f7a7451a8 100644 --- a/message_ix/tests/test_feature_addon.py +++ b/message_ix/tests/test_feature_addon.py @@ -1,5 +1,3 @@ -from typing import Union - import numpy as np import pandas as pd from ixmp import Platform @@ -29,7 +27,7 @@ def add_addon( - s: Scenario, costs: Union[bool, int] = False, zero_output: bool = False + s: Scenario, costs: bool | int = False, zero_output: bool = False ) -> None: s.check_out() s.add_set("technology", "canning_addon") diff --git a/message_ix/tests/test_feature_bound_activity_shares.py b/message_ix/tests/test_feature_bound_activity_shares.py index 49fce690a..4574981d6 100644 --- a/message_ix/tests/test_feature_bound_activity_shares.py +++ b/message_ix/tests/test_feature_bound_activity_shares.py @@ -1,5 +1,4 @@ from pathlib import Path -from typing import Union import numpy as np import numpy.testing as npt @@ -259,7 +258,7 @@ def calc_share(s): VALUE = 0.45 # common operations for both subtests - def add_data(s: Scenario, modes: Union[str, list[str]]) -> None: + def add_data(s: Scenario, modes: str | list[str]) -> None: with s.transact("Add share_commodity_up"): s.add_set("shares", SHARES) s.add_cat("technology", tt_share, "canning_plant") diff --git a/message_ix/tests/test_feature_bound_emission.py b/message_ix/tests/test_feature_bound_emission.py index 6a1554007..0a1f6f488 100644 --- a/message_ix/tests/test_feature_bound_emission.py +++ b/message_ix/tests/test_feature_bound_emission.py @@ -1,5 +1,3 @@ -from typing import Union - from ixmp import Platform from message_ix import Scenario @@ -18,14 +16,14 @@ def model_setup(scen: Scenario, years: list[int]) -> None: scen.add_set("technology", ["tec1", "tec2"]) scen.add_set("mode", "mode") - output_specs: list[Union[int, str]] = ["node", "comm", "level", "year", "year"] + output_specs: list[int | str] = ["node", "comm", "level", "year", "year"] dict_var_cost = {"tec1": 1, "tec2": 2} dict_em_factor = {"tec1": 1.5, "tec2": 1} for yr in years: scen.add_par("demand", ["node", "comm", "level", yr, "year"], 1, "GWa") for t in dict_var_cost.keys(): - tec_specs: list[Union[int, str]] = ["node", t, yr, yr, "mode"] + tec_specs: list[int | str] = ["node", t, yr, yr, "mode"] scen.add_par("output", tec_specs + output_specs, 1, "GWa") scen.add_par("var_cost", tec_specs + ["year"], dict_var_cost[t], "USD/GWa") scen.add_par( @@ -34,14 +32,14 @@ def model_setup(scen: Scenario, years: list[int]) -> None: def add_bound_emission( - scen: Scenario, bound: float, year: Union[int, str] = "cumulative" + scen: Scenario, bound: float, year: int | str = "cumulative" ) -> None: scen.check_out() scen.add_par("bound_emission", ["node", "emiss_type", "all", year], bound, "kg") scen.commit("Emission bound added") -def assert_function(scen: Scenario, year: Union[int, str]) -> None: +def assert_function(scen: Scenario, year: int | str) -> None: var_em = scen.var("EMISS", {"node": "node"}).set_index(["year"])["lvl"] bound_em = scen.par("bound_emission", {"type_year": year}).at[0, "value"] diff --git a/message_ix/tests/test_feature_duration_time.py b/message_ix/tests/test_feature_duration_time.py index 415fbba4c..3208f98bb 100644 --- a/message_ix/tests/test_feature_duration_time.py +++ b/message_ix/tests/test_feature_duration_time.py @@ -7,7 +7,6 @@ """ from itertools import product -from typing import Union from ixmp import Platform @@ -126,7 +125,7 @@ def model_generator( # "output" for h in times_out: out = com_dict[tec]["output"] - out_spec: list[Union[int, str]] = [ + out_spec: list[int | str] = [ yr, yr, "standard", diff --git a/message_ix/tests/test_feature_price_commodity.py b/message_ix/tests/test_feature_price_commodity.py index 8ee9c3ef2..89dc6f50b 100644 --- a/message_ix/tests/test_feature_price_commodity.py +++ b/message_ix/tests/test_feature_price_commodity.py @@ -1,12 +1,10 @@ -from typing import Optional, Union - import pytest from ixmp import Platform from message_ix import ModelError, Scenario -def model_setup(scen: Scenario, var_cost: Optional[int] = 1) -> None: +def model_setup(scen: Scenario, var_cost: int | None = 1) -> None: scen.add_set("node", "node") scen.add_set("commodity", "comm") scen.add_set("level", "level") @@ -16,8 +14,8 @@ def model_setup(scen: Scenario, var_cost: Optional[int] = 1) -> None: scen.add_set("technology", "tec") scen.add_set("mode", "mode") - tec_specs: list[Union[int, str]] = ["node", "tec", 2020, 2020, "mode"] - output_specs: list[Union[int, str]] = ["node", "comm", "level", "year", "year"] + tec_specs: list[int | str] = ["node", "tec", 2020, 2020, "mode"] + output_specs: list[int | str] = ["node", "comm", "level", "year", "year"] scen.add_par("output", tec_specs + output_specs, 1, "GWa") scen.add_par("var_cost", tec_specs + ["year"], var_cost, "USD/GWa") diff --git a/message_ix/tests/test_feature_price_emission.py b/message_ix/tests/test_feature_price_emission.py index cf3158498..4dc6ae5a4 100644 --- a/message_ix/tests/test_feature_price_emission.py +++ b/message_ix/tests/test_feature_price_emission.py @@ -1,5 +1,3 @@ -from typing import Union - import numpy.testing as npt import pytest from ixmp import Platform @@ -94,13 +92,13 @@ def add_two_tecs(scen: Scenario, years: list[int]) -> None: def add_many_tecs(scen: Scenario, years: list[int], n: int = 50) -> None: """add a range of dirty-to-clean technologies to the scenario""" - output_specs: list[Union[int, str]] = ["node", "comm", "level", "year", "year"] + output_specs: list[int | str] = ["node", "comm", "level", "year", "year"] for i in range(n + 1): t = "tec{}".format(i) scen.add_set("technology", t) for y in years: - tec_specs: list[Union[int, str]] = ["node", t, y, y, "mode"] + tec_specs: list[int | str] = ["node", t, y, y, "mode"] # variable costs grow quadratically over technologies # to get rid of the curse of linearity c = (10 * i / n) ** 2 * (1.045) ** (y - years[0]) diff --git a/message_ix/tests/test_feature_storage.py b/message_ix/tests/test_feature_storage.py index 86357d53a..4820940df 100644 --- a/message_ix/tests/test_feature_storage.py +++ b/message_ix/tests/test_feature_storage.py @@ -11,7 +11,6 @@ import logging from itertools import product -from typing import Union import pandas as pd import pandas.testing as pdt @@ -34,12 +33,12 @@ def model_setup(scen: Scenario, years: list[int]) -> None: scen.add_set("type_year", years) scen.add_set("technology", ["wind_ppl", "gas_ppl"]) scen.add_set("mode", "M1") - output_specs: list[Union[int, str]] = ["node", "electr", "level", "year", "year"] + output_specs: list[int | str] = ["node", "electr", "level", "year", "year"] # Two technologies, one cheaper than the other var_cost = {"wind_ppl": 0, "gas_ppl": 2} for year, (tec, cost) in product(years, var_cost.items()): scen.add_par("demand", ["node", "electr", "level", year, "year"], 1, "GWa") - tec_specs: list[Union[int, str]] = ["node", tec, year, year, "M1"] + tec_specs: list[int | str] = ["node", tec, year, year, "M1"] scen.add_par("output", tec_specs + output_specs, 1, "GWa") scen.add_par("var_cost", tec_specs + ["year"], cost, "USD/GWa") @@ -55,7 +54,7 @@ def add_seasonality(scen: Scenario, time_duration: dict[str, float]) -> None: # A function for modifying model parameters after adding sub-annual time steps def year_to_time( - scen: Scenario, parname: str, time_share: Union[dict[str, float], dict[str, int]] + scen: Scenario, parname: str, time_share: dict[str, float] | dict[str, int] ) -> None: old = scen.par(parname) scen.remove_par(parname, old) diff --git a/message_ix/tests/test_feature_vintage_and_active_years.py b/message_ix/tests/test_feature_vintage_and_active_years.py index 244a7d79c..f683b2244 100644 --- a/message_ix/tests/test_feature_vintage_and_active_years.py +++ b/message_ix/tests/test_feature_vintage_and_active_years.py @@ -1,6 +1,5 @@ from collections.abc import Sequence from functools import lru_cache -from typing import Optional import numpy as np import pandas as pd @@ -41,7 +40,7 @@ def _setup( mp: Platform, years: Sequence[int], firstmodelyear: int, - tl_years: Optional[filter] = None, + tl_years: filter | None = None, ) -> tuple[Scenario, pd.DataFrame]: """Common setup for test of :meth:`.vintage_and_active_years`. @@ -75,7 +74,7 @@ def _setup( def _q( - df: pd.DataFrame, query: str, append: Optional[pd.DataFrame] = None + df: pd.DataFrame, query: str, append: pd.DataFrame | None = None ) -> pd.DataFrame: """Shorthand to query the results of :func:`_generate_yv_ya`. diff --git a/message_ix/tests/test_macro.py b/message_ix/tests/test_macro.py index e3b198b02..4d297a1d4 100644 --- a/message_ix/tests/test_macro.py +++ b/message_ix/tests/test_macro.py @@ -1,6 +1,6 @@ from collections.abc import Generator from pathlib import Path -from typing import Any, Literal, Union +from typing import Any, Literal import numpy as np import numpy.testing as npt @@ -227,7 +227,7 @@ def test_calc( w_data_path: Path, key: str, test: Literal["allclose", "equal"], - expected: Union[list[float], list[int]], + expected: list[float] | list[int], ) -> None: """Test calculation of intermediate values on a solved Westeros scenario.""" c = prepare_computer(westeros_solved, data=w_data_path) diff --git a/message_ix/tests/test_tutorials.py b/message_ix/tests/test_tutorials.py index e1150a368..59bf39655 100644 --- a/message_ix/tests/test_tutorials.py +++ b/message_ix/tests/test_tutorials.py @@ -4,7 +4,7 @@ from collections.abc import Generator from pathlib import Path from shutil import copyfile -from typing import Any, Optional, Union +from typing import Any import ixmp.backend import numpy.testing as npt @@ -47,11 +47,11 @@ # NOTE Pytest.param returns ParamSet, which is private and hard to type hint def _t( - group: Union[str, None], + group: str | None, basename: str, *, - check: Optional[list[tuple]] = None, - marks: Optional[list[pytest.MarkDecorator]] = None, + check: list[tuple] | None = None, + marks: list[pytest.MarkDecorator] | None = None, ) -> tuple: """Shorthand for defining tutorial test cases. @@ -144,7 +144,7 @@ def nb_path( yield tutorial_path.joinpath(*request.param).with_suffix(".ipynb") -def default_args() -> dict[str, Union[int, str]]: +def default_args() -> dict[str, int | str]: """Default arguments for :func:`.run_notebook.""" if GHA: # Use a longer timeout diff --git a/message_ix/tests/tools/test_add_year.py b/message_ix/tests/tools/test_add_year.py index e35bc6251..907c676b2 100644 --- a/message_ix/tests/tools/test_add_year.py +++ b/message_ix/tests/tools/test_add_year.py @@ -1,5 +1,5 @@ from collections.abc import Callable, Generator -from typing import Any, Union +from typing import Any import pytest from click.testing import Result @@ -27,12 +27,12 @@ def base_scen_mp( scen.add_set("year", years) scen.add_set("technology", "tec") scen.add_set("mode", "mode") - output_specs: list[Union[int, str]] = ["node", "comm", "level", "year", "year"] + output_specs: list[int | str] = ["node", "comm", "level", "year", "year"] for yr, value in data.items(): scen.add_par("demand", ["node", "comm", "level", yr, "year"], 1, "GWa") scen.add_par("technical_lifetime", ["node", "tec", yr], 10, "y") - tec_specs: list[Union[int, str]] = ["node", "tec", yr, yr, "mode"] + tec_specs: list[int | str] = ["node", "tec", yr, yr, "mode"] scen.add_par("output", tec_specs + output_specs, 1, "-") scen.add_par("var_cost", tec_specs + ["year"], value, "USD/GWa") diff --git a/message_ix/tools/add_year/__init__.py b/message_ix/tools/add_year/__init__.py index e5ad7094b..92c50d7f9 100644 --- a/message_ix/tools/add_year/__init__.py +++ b/message_ix/tools/add_year/__init__.py @@ -12,7 +12,7 @@ # %% I) Importing required packages import logging -from typing import Literal, Optional, Union +from typing import Literal import numpy as np import pandas as pd @@ -24,12 +24,12 @@ # %% II) Utility functions for dataframe manupulation def intpol( - y1: Union[float, pd.Series, pd.DataFrame], - y2: Union[float, pd.Series, pd.DataFrame], + y1: float | pd.Series | pd.DataFrame, + y2: float | pd.Series | pd.DataFrame, x1: int, x2: int, x: int, -) -> Union[float, pd.Series, pd.DataFrame]: +) -> float | pd.Series | pd.DataFrame: """Interpolate between (*x1*, *y1*) and (*x2*, *y2*) at *x*. Parameters @@ -52,7 +52,7 @@ def slice_df( idx: list[str], level: str, locator: list, - value: Union[int, str, None], + value: int | str | None, ) -> pd.DataFrame: """Slice a MultiIndex DataFrame and set a value to a specific level. @@ -76,9 +76,7 @@ def slice_df( return df.set_index(idx) -def mask_df( - df: pd.DataFrame, index: tuple[Union[int, str], ...], count: int, value -) -> None: +def mask_df(df: pd.DataFrame, index: tuple[int | str, ...], count: int, value) -> None: """Create a mask for removing extra values from *df*.""" df.loc[ index, @@ -110,15 +108,15 @@ def add_year( sc_ref: Scenario, sc_new: Scenario, years_new: list[int], - firstyear_new: Optional[int] = None, - lastyear_new: Optional[int] = None, + firstyear_new: int | None = None, + lastyear_new: int | None = None, macro: bool = False, - baseyear_macro: Optional[int] = None, - parameter: Union[list[str], Literal["all"]] = "all", - region: Union[list[str], Literal["all"]] = "all", + baseyear_macro: int | None = None, + parameter: list[str] | Literal["all"] = "all", + region: list[str] | Literal["all"] = "all", rewrite: bool = True, unit_check: bool = True, - extrapol_neg: Optional[float] = None, + extrapol_neg: float | None = None, bound_extend: bool = True, ) -> None: """Add years to *sc_ref* to produce *sc_new*. @@ -292,9 +290,9 @@ def add_year_set( # noqa: C901 sc_ref: Scenario, sc_new: Scenario, years_new: list[int], - firstyear_new: Optional[int] = None, - lastyear_new: Optional[int] = None, - baseyear_macro: Optional[int] = None, + firstyear_new: int | None = None, + lastyear_new: int | None = None, + baseyear_macro: int | None = None, ) -> None: """Add new years to sets. @@ -340,7 +338,7 @@ def add_year_set( # noqa: C901 baseyear_macro ) - yr_pair: list[list[Union[int, str]]] = [] + yr_pair: list[list[int | str]] = [] for yr in years_new: yr_pair.append([yr, yr]) yr_pair.append(["cumulative", yr]) @@ -404,7 +402,7 @@ def add_year_par( extrapolate: bool = False, rewrite: bool = True, unit_check: bool = True, - extrapol_neg: Optional[float] = None, + extrapol_neg: float | None = None, bound_extend: bool = True, ) -> None: """Add new years to parameters. @@ -570,7 +568,7 @@ def interpolate_1d( # noqa: C901 year_col: str, value_col: str = "value", extrapolate: bool = False, - extrapol_neg: Optional[float] = None, + extrapol_neg: float | None = None, bound_extend: bool = True, ): """Interpolate data with one year dimension. @@ -729,8 +727,8 @@ def interpolate_2d( # noqa: C901 par_tec: pd.DataFrame, value_col: str = "value", extrapolate: bool = False, - extrapol_neg: Optional[float] = None, - year_diff: Optional[list[int]] = None, + extrapol_neg: float | None = None, + year_diff: list[int] | None = None, bound_extend: bool = True, ): """Interpolate parameters with two dimensions related year. diff --git a/message_ix/tools/lp_diag/cli.py b/message_ix/tools/lp_diag/cli.py index 2843d28d1..415b3329f 100644 --- a/message_ix/tools/lp_diag/cli.py +++ b/message_ix/tools/lp_diag/cli.py @@ -16,7 +16,6 @@ import sys # for redirecting stdout from datetime import datetime as dt from pathlib import Path -from typing import Optional import click @@ -52,7 +51,7 @@ help="Magnitude order of the upper tail (default: 5).", ) @click.option("--outp", "fn_outp", metavar="PATH", help="Path for file output.") -def main(w_dir: Path, prob_id: Path, fn_outp: Optional[Path], lo_tail, up_tail) -> None: +def main(w_dir: Path, prob_id: Path, fn_outp: Path | None, lo_tail, up_tail) -> None: """Diagnostics of basic properties of LP problems stored in the MPS format. \b diff --git a/message_ix/util/__init__.py b/message_ix/util/__init__.py index 06683aa5e..1261286d9 100644 --- a/message_ix/util/__init__.py +++ b/message_ix/util/__init__.py @@ -4,7 +4,7 @@ from collections import ChainMap, defaultdict from collections.abc import Mapping from pathlib import Path -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING import pandas as pd from ixmp.backend import ItemType @@ -23,7 +23,7 @@ def copy_model( set_default: bool = False, quiet: bool = False, *, - source_dir: Optional[Path] = None, + source_dir: Path | None = None, ) -> None: """Copy the MESSAGE GAMS files to a new `path`. diff --git a/message_ix/util/gams_io.py b/message_ix/util/gams_io.py index fb08d089e..d98ade055 100644 --- a/message_ix/util/gams_io.py +++ b/message_ix/util/gams_io.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Literal, Optional, Union, cast +from typing import TYPE_CHECKING, Literal, cast import pandas as pd @@ -33,7 +33,7 @@ # } # This must be convertible to pd.DataFrame -MESSAGE_IX_VERSION: dict[str, Union[list[float], list[int], list[str]]] = { +MESSAGE_IX_VERSION: dict[str, list[float] | list[int] | list[str]] = { "version_part": ["major", "minor"], "value": [2, 0], } @@ -115,9 +115,7 @@ def _compose_resource_grade_map( return resource_grade_map -def _handle_renames( - df: pd.DataFrame, renames: Optional[dict[str, str]] -) -> pd.DataFrame: +def _handle_renames(df: pd.DataFrame, renames: dict[str, str] | None) -> pd.DataFrame: """Rename columns from `df` according to `renames`, if present.""" return df.rename(columns=renames) if renames else df @@ -125,8 +123,8 @@ def _handle_renames( def _handle_empty_parameter( scenario: "Scenario", source_name: str, - columns: Optional[list[str]], - renames: Optional[dict[str, str]] = None, + columns: list[str] | None, + renames: dict[str, str] | None = None, ) -> pd.DataFrame: """Create an empty pd.DataFrame with correct column names.""" _columns = columns or scenario.idx_names(name=source_name) @@ -136,9 +134,9 @@ def _handle_empty_parameter( def _compose_records( scenario: "Scenario", - sources: dict[str, Optional[list[str]]], - filters: Optional[HelperFilterInfo], - renames: Optional[dict[str, str]], + sources: dict[str, list[str] | None], + filters: HelperFilterInfo | None, + renames: dict[str, str] | None, ) -> pd.DataFrame: """Compose the records for an auxiliary IndexSet/Table. @@ -149,9 +147,9 @@ def _compose_records( sources: dict A mapping to specify the sources. Keys are parameter names, values are optional columns to limit the data to. - filters: Optional[HelperFilterInfo] + filters: HelperFilterInfo Optionally, specify a filter that limits a single column to specific values. - renames: Optional[dict[str, str]] + renames: dict[str, str] Optionally, rename columns of the records to align them properly. """ # Create a list to collect records @@ -195,7 +193,7 @@ def _compose_records( def _compose_map_tec_time( - scenario: "Scenario", sources: dict[str, Optional[list[str]]] + scenario: "Scenario", sources: dict[str, list[str] | None] ) -> pd.DataFrame: """Compose the records for an auxiliary IndexSet/Table. @@ -235,8 +233,8 @@ def _compose_map_tec_time( def _compose_map_resource( scenario: "Scenario", - sources: dict[str, Optional[list[str]]], - filters: Optional[HelperFilterInfo], + sources: dict[str, list[str] | None], + filters: HelperFilterInfo | None, resource_grade_map: dict[tuple[str, str], list[str]], ) -> pd.DataFrame: """Compose the records for an auxiliary IndexSet/Table. @@ -248,7 +246,7 @@ def _compose_map_resource( sources: dict A mapping to specify the sources. Keys are parameter names, values are optional columns to limit the data to. - filters: Optional[HelperFilterInfo] + filters: HelperFilterInfo Optionally, specify a filter that limits a single column to specific values. resource_grade_map: dict[list[str], list[str]] An auxiliary mapping from (`node`, `commodity`) to list of `grade` to construct diff --git a/message_ix/util/scenario_data.py b/message_ix/util/scenario_data.py index 9b4196143..906e4eeb7 100644 --- a/message_ix/util/scenario_data.py +++ b/message_ix/util/scenario_data.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from typing import Literal, Optional, Union +from typing import Literal # NOTE on transcribing from ixmp_source: # The data in this file is based on @@ -64,7 +64,7 @@ class DefaultIndexSetData: class TableInfo: name: str indexsets: list[str] - column_names: Optional[list[str]] = None + column_names: list[str] | None = None # List of required table names and indexsets they are linked to, including column_names @@ -200,11 +200,11 @@ class ParameterInfo: name: str gams_name: str # could default to None and use name in that case, maybe? indexsets: list[str] - column_names: Optional[list[str]] = None + column_names: list[str] | None = None write_to_gdx: bool = True is_tec: bool = False is_tec_act: bool = False - section: Optional[ + section: ( Literal[ "energyforms:", "demand:", @@ -213,7 +213,8 @@ class ParameterInfo: "variables:", "relationsX:", ] - ] = None + | None + ) = None # NOTE changing '2LDB' to 'toLDB' toLDB: bool = False mode: bool = False @@ -1372,7 +1373,7 @@ class ParameterInfo: @dataclass class DefaultParameterData: name: str - data: dict[str, Union[list[int], list[str]]] # based on current usage + data: dict[str, list[int] | list[str]] # based on current usage DEFAULT_PARAMETER_DATA = [ @@ -1386,8 +1387,8 @@ class DefaultParameterData: class VariableInfo: name: str gams_name: str - indexsets: Optional[list[str]] = None - column_names: Optional[list[str]] = None + indexsets: list[str] | None = None + column_names: list[str] | None = None # List of required variable names and indexsets they are linked to, including @@ -1534,14 +1535,14 @@ class VariableInfo: class HelperFilterInfo: column_name: str target_name: Literal["resource", "renewables", "stocks"] - target: Optional[list[str]] = None + target: list[str] | None = None @dataclass class HelperIndexSetInfo: name: str - sources: dict[str, Optional[list[str]]] - filters: Optional[HelperFilterInfo] = None + sources: dict[str, list[str] | None] + filters: HelperFilterInfo | None = None HELPER_INDEXSETS = [ @@ -1568,7 +1569,7 @@ class HelperIndexSetInfo: @dataclass class HelperTableInfo(HelperIndexSetInfo): - renames: Optional[dict[str, str]] = None + renames: dict[str, str] | None = None # NOTE on transcribing from ixmp_source: ixmp.Element.getVector() only returns keys, diff --git a/message_ix/util/scenario_setup.py b/message_ix/util/scenario_setup.py index 86aaf78e8..87a513589 100644 --- a/message_ix/util/scenario_setup.py +++ b/message_ix/util/scenario_setup.py @@ -1,5 +1,5 @@ import logging -from typing import TYPE_CHECKING, Any, Literal, TypeVar, Union, cast +from typing import TYPE_CHECKING, Any, Literal, TypeVar, cast import pandas as pd from ixmp import Platform @@ -161,7 +161,7 @@ def ensure_required_indexsets_have_data(scenario: "Scenario") -> None: def _maybe_add_to_table( table: "Table", - data: Union[dict[str, Any], pd.DataFrame], + data: dict[str, Any] | pd.DataFrame, backend: "Backend", ) -> None: """Add (parts of) `data` to `table` if they are missing.""" @@ -268,7 +268,7 @@ def _find_all_descendants(parent: T) -> list[T]: def _maybe_add_single_item_to_indexset( - indexset: "IndexSet", data: Union[float, int, str], backend: "Backend" + indexset: "IndexSet", data: float | int | str, backend: "Backend" ) -> None: """Add `data` to `indexset` if it is missing.""" if data not in list(indexset.data): @@ -277,7 +277,7 @@ def _maybe_add_single_item_to_indexset( def _maybe_add_list_to_indexset( indexset: "IndexSet", - data: Union[list[float], list[int], list[str]], + data: list[float] | list[int] | list[str], backend: "Backend", ) -> None: """Add missing parts of `data` to `indexset`.""" @@ -290,7 +290,7 @@ def _maybe_add_list_to_indexset( def _maybe_add_to_indexset( indexset: "IndexSet", - data: Union[float, int, str, list[float], list[int], list[str]], + data: float | int | str | list[float] | list[int] | list[str], backend: "Backend", ) -> None: """Add (parts of) `data` to `indexset` if they are missing.""" @@ -307,7 +307,7 @@ def _maybe_add_to_indexset( # slower than necessary (though likely not by much). Is the maintenance effort worth it? def _maybe_add_to_parameter( parameter: "Parameter", - data: Union[dict[str, Any], pd.DataFrame], + data: dict[str, Any] | pd.DataFrame, backend: "Backend", ) -> None: """Add (parts of) `data` to `parameter` if they are missing.""" From 7a0951a2ad2067e7323b5c46ba47d426dfaabad9 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Fri, 24 Oct 2025 14:03:22 +0200 Subject: [PATCH 4/8] Mark tests that fail without ixmp4 py3.14 support - Add mark 'ixmp4_209' used/handled in iiasa/ixmp#602. --- message_ix/tests/model/message/test_cap_comm.py | 1 + message_ix/tests/model/test_message.py | 2 ++ message_ix/tests/report/test_operator.py | 4 ++++ message_ix/tests/test_core.py | 10 ++++++---- message_ix/tests/test_feature_addon.py | 3 +++ message_ix/tests/test_feature_bound_activity_shares.py | 2 ++ message_ix/tests/test_feature_bound_emission.py | 3 +++ message_ix/tests/test_feature_capacity_factor.py | 2 ++ message_ix/tests/test_feature_duration_time.py | 3 +++ message_ix/tests/test_feature_price_commodity.py | 2 ++ message_ix/tests/test_feature_price_emission.py | 2 ++ message_ix/tests/test_feature_storage.py | 2 ++ message_ix/tests/test_feature_temporal_level.py | 2 ++ .../tests/test_feature_vintage_and_active_years.py | 3 +++ message_ix/tests/test_macro.py | 2 ++ message_ix/tests/test_message.py | 4 ++++ message_ix/tests/test_message_macro.py | 1 + message_ix/tests/test_report.py | 2 ++ message_ix/tests/test_testing.py | 3 +++ message_ix/tests/tools/test_sankey.py | 2 ++ message_ix/tests/util/test_gams_io.py | 5 ++--- message_ix/tests/util/test_tutorial.py | 3 +++ pyproject.toml | 1 + 23 files changed, 57 insertions(+), 7 deletions(-) diff --git a/message_ix/tests/model/message/test_cap_comm.py b/message_ix/tests/model/message/test_cap_comm.py index eb43db1b9..a86813a06 100644 --- a/message_ix/tests/model/message/test_cap_comm.py +++ b/message_ix/tests/model/message/test_cap_comm.py @@ -11,6 +11,7 @@ if TYPE_CHECKING: from ixmp import Platform +pytestmark = pytest.mark.ixmp4_209 COMMON: dict[str, int | str] = dict( commodity="coal", diff --git a/message_ix/tests/model/test_message.py b/message_ix/tests/model/test_message.py index 8dc1ad5d7..9424959a4 100644 --- a/message_ix/tests/model/test_message.py +++ b/message_ix/tests/model/test_message.py @@ -13,6 +13,8 @@ if TYPE_CHECKING: from ixmp import Platform +pytestmark = pytest.mark.ixmp4_209 + #: Periods: one historical, two in the time horizon Y = [0, 10, 20] diff --git a/message_ix/tests/report/test_operator.py b/message_ix/tests/report/test_operator.py index cd754b65f..1c40550eb 100644 --- a/message_ix/tests/report/test_operator.py +++ b/message_ix/tests/report/test_operator.py @@ -5,6 +5,7 @@ import matplotlib import pandas as pd import pyam +import pytest from dask.core import literal try: @@ -25,6 +26,7 @@ # NOTE These tests likely don't need to be parametrized +@pytest.mark.ixmp4_209 def test_as_message_df(test_mp: "Platform") -> None: q = random_qty(dict(c=3, h=2, nl=5)) q.units = "kg" @@ -84,6 +86,7 @@ def test_as_message_df(test_mp: "Platform") -> None: assert q.size == len(s.par("demand")) +@pytest.mark.ixmp4_209 def test_as_pyam(message_test_mp: "Platform") -> None: scen = Scenario(message_test_mp, **SCENARIO["dantzig"]) if not scen.has_solution(): @@ -101,6 +104,7 @@ def test_as_pyam(message_test_mp: "Platform") -> None: assert isinstance(p, pyam.IamDataFrame) +@pytest.mark.ixmp4_209 def test_concat(dantzig_reporter: Reporter) -> None: """pyam.concat() correctly passes through to ixmp…concat().""" rep = dantzig_reporter diff --git a/message_ix/tests/test_core.py b/message_ix/tests/test_core.py index 04cd4c418..0d36c45d3 100644 --- a/message_ix/tests/test_core.py +++ b/message_ix/tests/test_core.py @@ -17,6 +17,8 @@ from message_ix import Scenario from message_ix.testing import GHA, SCENARIO, make_dantzig, make_westeros +pytestmark = pytest.mark.ixmp4_209 + @pytest.fixture def dantzig_message_scenario( @@ -393,7 +395,7 @@ def test_add_cat_unique(message_test_mp: ixmp.Platform) -> None: assert [1963] == scen2.cat("year", "firstmodelyear") -def test_years_active(test_mp: ixmp.Platform) -> None: +def test_years_active0(test_mp: ixmp.Platform) -> None: test_mp.add_unit("year") scen = Scenario(test_mp, **SCENARIO["dantzig"], version="new") scen.add_set("node", "foo") @@ -439,7 +441,7 @@ def test_years_active(test_mp: ixmp.Platform) -> None: npt.assert_array_equal(result, years[1:-1]) -def test_years_active_extend(message_test_mp: ixmp.Platform) -> None: +def test_years_active1(message_test_mp: ixmp.Platform) -> None: scen = Scenario(message_test_mp, **SCENARIO["dantzig multi-year"]) # Existing time horizon @@ -464,7 +466,7 @@ def test_years_active_extend(message_test_mp: ixmp.Platform) -> None: npt.assert_array_equal(result, years[1:-1]) -def test_years_active_extended2(test_mp: ixmp.Platform) -> None: +def test_years_active2(test_mp: ixmp.Platform) -> None: test_mp.add_unit("year") scen = Scenario(test_mp, **SCENARIO["dantzig"], version="new") scen.add_set("node", "foo") @@ -511,7 +513,7 @@ def test_years_active_extended2(test_mp: ixmp.Platform) -> None: npt.assert_array_equal(result, years[-2]) -def test_years_active_extend3(test_mp: ixmp.Platform) -> None: +def test_years_active3(test_mp: ixmp.Platform) -> None: test_mp.add_unit("year") scen = Scenario(test_mp, **SCENARIO["dantzig"], version="new") scen.add_set("node", "foo") diff --git a/message_ix/tests/test_feature_addon.py b/message_ix/tests/test_feature_addon.py index f7a7451a8..2448cda67 100644 --- a/message_ix/tests/test_feature_addon.py +++ b/message_ix/tests/test_feature_addon.py @@ -1,10 +1,13 @@ import numpy as np import pandas as pd +import pytest from ixmp import Platform from message_ix import Scenario from message_ix.testing import SCENARIO +pytestmark = pytest.mark.ixmp4_209 + # First model year of the Dantzig scenario _year = 1963 diff --git a/message_ix/tests/test_feature_bound_activity_shares.py b/message_ix/tests/test_feature_bound_activity_shares.py index 4574981d6..d81457ead 100644 --- a/message_ix/tests/test_feature_bound_activity_shares.py +++ b/message_ix/tests/test_feature_bound_activity_shares.py @@ -11,6 +11,8 @@ from message_ix import ModelError, Scenario, make_df from message_ix.testing import make_dantzig +pytestmark = pytest.mark.ixmp4_209 + #: First model year of the :func:`.make_dantzig` scenario. Y0 = 1963 diff --git a/message_ix/tests/test_feature_bound_emission.py b/message_ix/tests/test_feature_bound_emission.py index 0a1f6f488..28b4671b7 100644 --- a/message_ix/tests/test_feature_bound_emission.py +++ b/message_ix/tests/test_feature_bound_emission.py @@ -1,7 +1,10 @@ +import pytest from ixmp import Platform from message_ix import Scenario +pytestmark = pytest.mark.ixmp4_209 + def model_setup(scen: Scenario, years: list[int]) -> None: scen.add_set("node", "node") diff --git a/message_ix/tests/test_feature_capacity_factor.py b/message_ix/tests/test_feature_capacity_factor.py index 510003854..bad03a3bf 100644 --- a/message_ix/tests/test_feature_capacity_factor.py +++ b/message_ix/tests/test_feature_capacity_factor.py @@ -6,6 +6,8 @@ from message_ix import ModelError, Scenario from message_ix.testing import make_subannual +pytestmark = pytest.mark.ixmp4_209 + def check_solution(scen: Scenario) -> None: """Perform several assertions about the solution of `scen`.""" diff --git a/message_ix/tests/test_feature_duration_time.py b/message_ix/tests/test_feature_duration_time.py index 3208f98bb..bccbaf212 100644 --- a/message_ix/tests/test_feature_duration_time.py +++ b/message_ix/tests/test_feature_duration_time.py @@ -8,10 +8,13 @@ from itertools import product +import pytest from ixmp import Platform from message_ix import Scenario +pytestmark = pytest.mark.ixmp4_209 + # A function for generating a simple model with sub-annual time slices def model_generator( diff --git a/message_ix/tests/test_feature_price_commodity.py b/message_ix/tests/test_feature_price_commodity.py index 89dc6f50b..fbab9db58 100644 --- a/message_ix/tests/test_feature_price_commodity.py +++ b/message_ix/tests/test_feature_price_commodity.py @@ -3,6 +3,8 @@ from message_ix import ModelError, Scenario +pytestmark = pytest.mark.ixmp4_209 + def model_setup(scen: Scenario, var_cost: int | None = 1) -> None: scen.add_set("node", "node") diff --git a/message_ix/tests/test_feature_price_emission.py b/message_ix/tests/test_feature_price_emission.py index 4dc6ae5a4..4a90994ca 100644 --- a/message_ix/tests/test_feature_price_emission.py +++ b/message_ix/tests/test_feature_price_emission.py @@ -4,6 +4,8 @@ from message_ix import Scenario, make_df +pytestmark = pytest.mark.ixmp4_209 + MODEL = "test_emissions_price" solve_args = {"equ_list": ["EMISSION_EQUIVALENCE"]} diff --git a/message_ix/tests/test_feature_storage.py b/message_ix/tests/test_feature_storage.py index 4820940df..20e62eeb0 100644 --- a/message_ix/tests/test_feature_storage.py +++ b/message_ix/tests/test_feature_storage.py @@ -23,6 +23,8 @@ from message_ix.testing import make_dantzig from message_ix.util import expand_dims +pytestmark = pytest.mark.ixmp4_209 + # A function for generating a simple MESSAGEix model with two technologies def model_setup(scen: Scenario, years: list[int]) -> None: diff --git a/message_ix/tests/test_feature_temporal_level.py b/message_ix/tests/test_feature_temporal_level.py index c54512d67..66a24ef94 100644 --- a/message_ix/tests/test_feature_temporal_level.py +++ b/message_ix/tests/test_feature_temporal_level.py @@ -15,6 +15,8 @@ from message_ix import ModelError, Scenario from message_ix.testing import make_subannual +pytestmark = pytest.mark.ixmp4_209 + # Values for the com_dict argument to make_subannual COM_DICT = { "gas_ppl": {"input": "fuel", "output": "electr"}, diff --git a/message_ix/tests/test_feature_vintage_and_active_years.py b/message_ix/tests/test_feature_vintage_and_active_years.py index f683b2244..c19ef0617 100644 --- a/message_ix/tests/test_feature_vintage_and_active_years.py +++ b/message_ix/tests/test_feature_vintage_and_active_years.py @@ -15,6 +15,9 @@ # tested here (and not just in ixmp). +pytestmark = pytest.mark.ixmp4_209 + + @lru_cache() def _generate_yv_ya(periods: tuple[int, ...]) -> pd.DataFrame: """All meaningful combinations of (vintage year, active year) given `periods`.""" diff --git a/message_ix/tests/test_macro.py b/message_ix/tests/test_macro.py index 4d297a1d4..705c65051 100644 --- a/message_ix/tests/test_macro.py +++ b/message_ix/tests/test_macro.py @@ -14,6 +14,8 @@ from message_ix.report import Quantity from message_ix.testing import SCENARIO, make_westeros +pytestmark = pytest.mark.ixmp4_209 + # NOTE These tests maybe don't need to be parametrized # Do the following depend on otherwise untested Scenario functions? # Scenario.add_macro() diff --git a/message_ix/tests/test_message.py b/message_ix/tests/test_message.py index e3e65b310..5226aaeb8 100644 --- a/message_ix/tests/test_message.py +++ b/message_ix/tests/test_message.py @@ -3,6 +3,7 @@ from typing import TYPE_CHECKING import ixmp +import pytest from ixmp.backend.jdbc import JDBCBackend from message_ix.message import MESSAGE @@ -11,6 +12,9 @@ from ixmp import Platform +pytestmark = pytest.mark.ixmp4_209 + + class TestMESSAGE: """Tests of :class:`.MESSAGE`.""" diff --git a/message_ix/tests/test_message_macro.py b/message_ix/tests/test_message_macro.py index f606f8ea0..5aa207b7e 100644 --- a/message_ix/tests/test_message_macro.py +++ b/message_ix/tests/test_message_macro.py @@ -41,6 +41,7 @@ class _MM(MESSAGE_MACRO): ): _MM() + @pytest.mark.ixmp4_209 def test_initialize(self, test_mp: Platform) -> None: # MESSAGE_MACRO.initialize() runs Scenario( diff --git a/message_ix/tests/test_report.py b/message_ix/tests/test_report.py index bc62cd5c6..6b8e568f4 100644 --- a/message_ix/tests/test_report.py +++ b/message_ix/tests/test_report.py @@ -25,6 +25,8 @@ from message_ix.report import Reporter, configure from message_ix.testing import SCENARIO, make_dantzig, make_westeros +pytestmark = pytest.mark.ixmp4_209 + # NOTE These tests maybe don't need to be parametrized. # Does `Reporter.from_scenario()` depend on otherwise untested Scenario functions? diff --git a/message_ix/tests/test_testing.py b/message_ix/tests/test_testing.py index e9371a14d..cff6fa532 100644 --- a/message_ix/tests/test_testing.py +++ b/message_ix/tests/test_testing.py @@ -4,6 +4,8 @@ from message_ix import ModelError, Scenario from message_ix.testing import make_austria, make_dantzig, make_westeros +pytestmark = pytest.mark.ixmp4_209 + @pytest.mark.parametrize( "kwargs", @@ -24,6 +26,7 @@ def test_make_austria( assert isinstance(s, Scenario) +@pytest.mark.ixmp4_209 @pytest.mark.parametrize( "kwargs", ( diff --git a/message_ix/tests/tools/test_sankey.py b/message_ix/tests/tools/test_sankey.py index 84dd3e9c7..0e5baf17d 100644 --- a/message_ix/tests/tools/test_sankey.py +++ b/message_ix/tests/tools/test_sankey.py @@ -11,6 +11,8 @@ if TYPE_CHECKING: import pyam +pytestmark = pytest.mark.ixmp4_209 + # NOTE This test likely doesn't need to be parametrized diff --git a/message_ix/tests/util/test_gams_io.py b/message_ix/tests/util/test_gams_io.py index 545773837..3f4dd21a6 100644 --- a/message_ix/tests/util/test_gams_io.py +++ b/message_ix/tests/util/test_gams_io.py @@ -1,14 +1,11 @@ import pandas as pd import pytest from ixmp import Platform -from ixmp.testing import min_ixmp4_version from message_ix import Scenario from message_ix.testing import make_westeros from message_ix.util.scenario_data import HELPER_INDEXSETS, HELPER_TABLES -pytestmark = min_ixmp4_version - def test_store_message_version() -> None: from ixmp.util.ixmp4 import ContainerData @@ -23,6 +20,7 @@ def test_store_message_version() -> None: @pytest.mark.ixmp4 +@pytest.mark.ixmp4_209 def test_add_default_data_to_container_data_list(test_mp: Platform) -> None: from ixmp.util.ixmp4 import ContainerData @@ -66,6 +64,7 @@ def test_add_default_data_to_container_data_list(test_mp: Platform) -> None: @pytest.mark.ixmp4 +@pytest.mark.ixmp4_209 def test_add_auxiliary_items_to_container_data_list( test_mp: Platform, request: pytest.FixtureRequest ) -> None: diff --git a/message_ix/tests/util/test_tutorial.py b/message_ix/tests/util/test_tutorial.py index 608e50a04..c03dd7886 100644 --- a/message_ix/tests/util/test_tutorial.py +++ b/message_ix/tests/util/test_tutorial.py @@ -1,5 +1,6 @@ from typing import TYPE_CHECKING +import pytest from ixmp.report import Key from message_ix import Reporter, Scenario, make_df @@ -10,6 +11,8 @@ import pytest from ixmp import Platform +pytestmark = pytest.mark.ixmp4_209 + # NOTE This test likely doesn't need to be parametrized def test_prepare_plots(dantzig_reporter: Reporter) -> None: diff --git a/pyproject.toml b/pyproject.toml index 8ab1dbc73..d56e45b37 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -119,6 +119,7 @@ filterwarnings = [ ] markers = [ "ixmp4: tests exclusive to IXMP4Backend.", + "ixmp4_209: https://github.com/iiasa/ixmp4/pull/209", "jdbc: tests exclusive to JDBCBackend.", "nightly: Slow-running nightly tests of particular scenarios.", "rmessageix: test of the message_ix R interface.", From 0de557a3ba36af55bcc1a5ade67cf9c9cbecea3f Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Fri, 24 Oct 2025 14:15:07 +0200 Subject: [PATCH 5/8] Remove Python 3.9 compatibility code --- message_ix/tests/test_core.py | 4 +-- message_ix/tests/test_report.py | 41 ++++++++------------------ message_ix/tests/tools/test_migrate.py | 7 ----- message_ix/tools/migrate.py | 3 +- message_ix/tools/sankey.py | 6 +--- 5 files changed, 15 insertions(+), 46 deletions(-) diff --git a/message_ix/tests/test_core.py b/message_ix/tests/test_core.py index 0d36c45d3..1003c3967 100644 --- a/message_ix/tests/test_core.py +++ b/message_ix/tests/test_core.py @@ -1,4 +1,3 @@ -import sys from collections.abc import Generator from copy import deepcopy from pathlib import Path @@ -155,8 +154,7 @@ def test_backends_available() -> None: """Check that the expected set of backends are available within GHA workflows.""" from ixmp.backend import available - exp = {"ixmp4", "jdbc"} if sys.version_info >= (3, 10) else {"jdbc"} - assert exp <= set(available()) + assert {"ixmp4", "jdbc"} <= set(available()) def test_year_int(test_mp: ixmp.Platform, request: pytest.FixtureRequest) -> None: diff --git a/message_ix/tests/test_report.py b/message_ix/tests/test_report.py index 6b8e568f4..057711fc6 100644 --- a/message_ix/tests/test_report.py +++ b/message_ix/tests/test_report.py @@ -1,6 +1,5 @@ import logging import re -import sys from collections import defaultdict from collections.abc import Generator from functools import partial @@ -355,34 +354,18 @@ def add_tm(df, name="Activity"): assert not any(c in df2.columns for c in ["h", "m", "t"]) # Variable names were formatted by the callback - reg_var = ( - pd.DataFrame( - [ - ["seattle", "Activity|canning_plant|production"], - ["seattle", "Activity|transport_from_seattle|to_new-york"], - ["seattle", "Activity|transport_from_seattle|to_chicago"], - ["seattle", "Activity|transport_from_seattle|to_topeka"], - ["san-diego", "Activity|canning_plant|production"], - ["san-diego", "Activity|transport_from_san-diego|to_new-york"], - ["san-diego", "Activity|transport_from_san-diego|to_chicago"], - ["san-diego", "Activity|transport_from_san-diego|to_topeka"], - ], - columns=["region", "variable"], - ) - if sys.version_info >= (3, 10) - else pd.DataFrame( - [ - ["san-diego", "Activity|canning_plant|production"], - ["san-diego", "Activity|transport_from_san-diego|to_chicago"], - ["san-diego", "Activity|transport_from_san-diego|to_new-york"], - ["san-diego", "Activity|transport_from_san-diego|to_topeka"], - ["seattle", "Activity|canning_plant|production"], - ["seattle", "Activity|transport_from_seattle|to_chicago"], - ["seattle", "Activity|transport_from_seattle|to_new-york"], - ["seattle", "Activity|transport_from_seattle|to_topeka"], - ], - columns=["region", "variable"], - ) + reg_var = pd.DataFrame( + [ + ["seattle", "Activity|canning_plant|production"], + ["seattle", "Activity|transport_from_seattle|to_new-york"], + ["seattle", "Activity|transport_from_seattle|to_chicago"], + ["seattle", "Activity|transport_from_seattle|to_topeka"], + ["san-diego", "Activity|canning_plant|production"], + ["san-diego", "Activity|transport_from_san-diego|to_new-york"], + ["san-diego", "Activity|transport_from_san-diego|to_chicago"], + ["san-diego", "Activity|transport_from_san-diego|to_topeka"], + ], + columns=["region", "variable"], ) assert_frame_equal(df2[["region", "variable"]], reg_var) diff --git a/message_ix/tests/tools/test_migrate.py b/message_ix/tests/tools/test_migrate.py index bfc0fb514..843e94f0e 100644 --- a/message_ix/tests/tools/test_migrate.py +++ b/message_ix/tests/tools/test_migrate.py @@ -1,7 +1,5 @@ """Tests of :mod:`message_ix.tools.migrate`.""" -import sys - import pytest from ixmp import Platform @@ -10,11 +8,6 @@ @pytest.mark.jdbc -@pytest.mark.xfail( - condition=sys.version_info < (3, 10), - raises=ImportError, - reason="Uses itertools.pairwise, added in Python 3.10", -) def test_v311( caplog: pytest.LogCaptureFixture, request: pytest.FixtureRequest, test_mp: Platform ) -> None: diff --git a/message_ix/tools/migrate.py b/message_ix/tools/migrate.py index d7ba60d50..9209394d2 100644 --- a/message_ix/tools/migrate.py +++ b/message_ix/tools/migrate.py @@ -1,6 +1,7 @@ """Tools for migrating :class:`Scenario` data across versions of :mod:`message_ix`.""" import logging +from itertools import pairwise from typing import TYPE_CHECKING if TYPE_CHECKING: @@ -25,8 +26,6 @@ def initial_new_capacity_up_v311( *All* values in initial_new_capacity_up are rewritten in this way. """ - # TODO Move this to top level when dropping support for Python 3.9 - from itertools import pairwise if migration_applied(s, "initial_new_capacity_up_v311"): log.info( diff --git a/message_ix/tools/sankey.py b/message_ix/tools/sankey.py index f655e1688..12b00ce0c 100644 --- a/message_ix/tools/sankey.py +++ b/message_ix/tools/sankey.py @@ -1,11 +1,7 @@ import logging from typing import TYPE_CHECKING -try: - from pyam.str import get_variable_components -except ImportError: # Python < 3.10 → pyam-iamc < 3 - from pyam.utils import get_variable_components - +from pyam.str import get_variable_components if TYPE_CHECKING: import pyam From fda4eaf16ef67d54fd86e834c81656a36d620134 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Fri, 24 Oct 2025 14:47:06 +0200 Subject: [PATCH 6/8] Use match/case instead of nested elif in lp_diag --- message_ix/tools/lp_diag/__init__.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/message_ix/tools/lp_diag/__init__.py b/message_ix/tools/lp_diag/__init__.py index b57279ccd..a2a634cd5 100644 --- a/message_ix/tools/lp_diag/__init__.py +++ b/message_ix/tools/lp_diag/__init__.py @@ -647,19 +647,18 @@ def row_att( print(f"{sec_name} value {val} ignored for neutral row {row_name}.") return attr = self.seq_row.get(row_seq) # [row_name, lo_bnd, up_bnd, row_type] - if sec_name == "rhs": # process the RHS value - if row_type == "G": # update lo_bnd + match sec_name, row_type: + case "rhs", "G": # update lo_bnd attr[1] = val - elif row_type == "L": # update up_bnd + case "rhs", "L": # update up_bnd attr[2] = val - elif row_type == "E": # update both bounds + case "rhs", "E": # update both bounds attr[1] = attr[2] = val - else: # process the ranges value - if row_type == "G": # update up_bnd + case "ranges", "G": # update up_bnd attr[2] = attr[1] + abs(val) - elif row_type == "L": # update lo_bnd + case "ranges", "L": # update lo_bnd attr[1] = attr[2] - abs(val) - elif row_type == "E": # update both bounds + case "ranges", "E": # update both bounds if val > 0: attr[2] = attr[1] + val else: From df02efec4fdbf93d15b02f7fcf1f5fe987c5df1a Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 27 Oct 2025 09:16:11 +0100 Subject: [PATCH 7/8] Use "agg" matplotlib backend on GHA/Windows Avoid TCL errors possibly due to python/cpython#125235 and/or astral-sh/uv#7036. --- message_ix/testing/__init__.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/message_ix/testing/__init__.py b/message_ix/testing/__init__.py index 73a36399f..e7a1f7783 100644 --- a/message_ix/testing/__init__.py +++ b/message_ix/testing/__init__.py @@ -1,5 +1,6 @@ import io import os +import platform from collections.abc import Callable, Generator from itertools import product from pathlib import Path @@ -28,12 +29,21 @@ def pytest_configure(config: pytest.Config) -> None: """Force iam-units to use a distinct cache for each worker. - Work around for https://github.com/hgrecco/flexcache/issues/6 / - https://github.com/IAMconsortium/units/issues/54. + Work arounds for: + + 1. https://github.com/hgrecco/flexcache/issues/6 / + https://github.com/IAMconsortium/units/issues/54. + 2. https://github.com/python/cpython/issues/125235, + https://github.com/astral-sh/uv/issues/7036, or similar. """ name = f"iam-units-{os.environ.get('PYTEST_XDIST_WORKER', '')}".rstrip("-") os.environ["IAM_UNITS_CACHE"] = str(config.cache.mkdir(name)) + if GHA and platform.system() == "Windows": + import matplotlib + + matplotlib.use("agg") + def pytest_report_header(config: pytest.Config, start_path: Path) -> str: """Add the message_ix import path to the pytest report header.""" From 5bd58c3d32ed61740745841530c5543821bf6a30 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Fri, 24 Oct 2025 14:50:02 +0200 Subject: [PATCH 8/8] Add #985 to release notes --- RELEASE_NOTES.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst index e62255058..1a9710a36 100644 --- a/RELEASE_NOTES.rst +++ b/RELEASE_NOTES.rst @@ -22,6 +22,8 @@ Adjust any imports like the following: All changes ----------- +- :mod:`message_ix` is tested and compatible with `Python 3.14 `__ (:pull:`985`). +- Support for Python 3.9 is dropped (:pull:`985`), as it has reached end-of-life. - Add representation of commodity flows associated with construction and retirement of technology capacity (:pull:`451`). - New parameters @@ -48,6 +50,7 @@ All changes - Document the :ref:`minimum version of Java ` required for :class:`ixmp.JDBCBackend ` (:pull:`962`). - Improve type hinting (:pull:`963`). - Fix capitalization in auxiliary_settings.gms to enable GDX output file compression on MacOS and Linux. (:pull:`965`) + All changes -----------