From 44f0c883a47a056c4a899541c5273a740bd8243a Mon Sep 17 00:00:00 2001 From: Daniel Heinesen Date: Mon, 30 Sep 2024 14:47:30 +0200 Subject: [PATCH 1/7] First draft of new climatology config --- pyaerocom/climatology_config.py | 46 +++++++++++++++++++++++ pyaerocom/colocation/colocation_3d.py | 11 +++--- pyaerocom/colocation/colocation_setup.py | 20 ++++++++-- pyaerocom/colocation/colocation_utils.py | 42 +++++++++++++-------- pyaerocom/config.py | 2 +- pyaerocom/stationdata.py | 40 +++++++++++++++++--- tests/colocation/test_colocation_3d.py | 15 ++++++-- tests/colocation/test_colocation_utils.py | 39 +++++++++++++++---- tests/colocation/test_colocator.py | 6 ++- 9 files changed, 179 insertions(+), 42 deletions(-) create mode 100644 pyaerocom/climatology_config.py diff --git a/pyaerocom/climatology_config.py b/pyaerocom/climatology_config.py new file mode 100644 index 000000000..1ebecfefc --- /dev/null +++ b/pyaerocom/climatology_config.py @@ -0,0 +1,46 @@ +from pydantic import ( + BaseModel, + ConfigDict, + Field, + ValidationError, + field_validator, + model_validator, +) + +from pyaerocom import const + + +class ClimatologyConfig(BaseModel): + """ + Holds the configuration for the climatology + + Attributes + ------------- + start : int, optional + Start year of the climatology + stop : int, optional + Stop year of the climatology + resample_how : str, optional + How to resample the climatology. Must be mean or median. + freq : str, optional + Which frequency the climatology should have + mincount : dict, optional + Number of values should be present for the data to be used in the climatology. + Dict where freqs are the keys and the count is the values + + """ + + start: int = const.CLIM_START + stop: int = const.CLIM_STOP + + resample_how: str = const.CLIM_RESAMPLE_HOW + freq: str = const.CLIM_FREQ + mincount: dict = const.CLIM_MIN_COUNT + + @field_validator("resample_how") + @classmethod + def validate_resample_how(cls, v): + if v in ["mean", "median"]: + return v + + raise ValidationError diff --git a/pyaerocom/colocation/colocation_3d.py b/pyaerocom/colocation/colocation_3d.py index aa29e0336..0bb96a57b 100644 --- a/pyaerocom/colocation/colocation_3d.py +++ b/pyaerocom/colocation/colocation_3d.py @@ -14,6 +14,7 @@ from pyaerocom import __version__ as pya_ver from pyaerocom import const +from pyaerocom.climatology_config import ClimatologyConfig from pyaerocom._lowlevel_helpers import LayerLimits, RegridResDeg from pyaerocom.exceptions import ( DataUnitError, @@ -311,7 +312,7 @@ def colocate_vertical_profile_gridded( update_baseyear_gridded: int = None, min_num_obs: int | dict | None = None, colocate_time: bool = False, - use_climatology_ref: bool = False, + use_climatology_ref: dict = False, resample_how: str | dict = None, colocation_layer_limits: tuple[LayerLimits, ...] | None = None, profile_layer_limits: tuple[LayerLimits, ...] | None = None, @@ -412,10 +413,10 @@ def colocate_vertical_profile_gridded( data = data.resample_time(str(ts_type), min_num_obs=min_num_obs, how=resample_how) ts_type_data = ts_type - if use_climatology_ref: # pragma: no cover - col_freq = "monthly" - obs_start = const.CLIM_START - obs_stop = const.CLIM_STOP + if isinstance(use_climatology_ref, ClimatologyConfig): # pragma: no cover + col_freq = use_climatology_ref.freq + obs_start = use_climatology_ref.start + obs_stop = use_climatology_ref.stop else: col_freq = str(ts_type) obs_start = start diff --git a/pyaerocom/colocation/colocation_setup.py b/pyaerocom/colocation/colocation_setup.py index ed74838cc..d22bb1664 100644 --- a/pyaerocom/colocation/colocation_setup.py +++ b/pyaerocom/colocation/colocation_setup.py @@ -17,6 +17,7 @@ ) from pyaerocom import const +from pyaerocom.climatology_config import ClimatologyConfig from pyaerocom._lowlevel_helpers import LayerLimits, RegridResDeg from pyaerocom.config import ALL_REGION_NAME from pyaerocom.helpers import start_stop @@ -109,9 +110,9 @@ class ColocationSetup(BaseModel): obs_data_dir : str, optional location of obs data. If None, attempt to infer obs location based on obs ID. - obs_use_climatology : bool - BETA if True, pyaerocom default climatology is computed from observation - stations (so far only possible for unrgidded / gridded colocation). + obs_use_climatology : ClimatologyConfig | bool, optional + Configuration for climatology. If True is given, a default configuration is made. + With False, climatology is turned off obs_vert_type : str AeroCom vertical code encoded in the model filenames (only AeroCom 3 and later). Specifies which model file should be read in case there are @@ -379,7 +380,18 @@ def validate_basedirs(cls, v): obs_name: str | None = None obs_data_dir: Path | str | None = None - obs_use_climatology: bool = False + obs_use_climatology: ClimatologyConfig | bool = False + + @field_validator("obs_use_climatology") + @classmethod + def validate_obs_use_climatology(cls, v): + if isinstance(v, ClimatologyConfig): + return v + + if v == True: + return ClimatologyConfig() + + return v obs_cache_only: bool = False # only relevant if obs is ungridded obs_vert_type: str | None = None diff --git a/pyaerocom/colocation/colocation_utils.py b/pyaerocom/colocation/colocation_utils.py index 2780fdfc8..2d54437c8 100644 --- a/pyaerocom/colocation/colocation_utils.py +++ b/pyaerocom/colocation/colocation_utils.py @@ -12,6 +12,7 @@ from pyaerocom import __version__ as pya_ver from pyaerocom import const +from pyaerocom.climatology_config import ClimatologyConfig from pyaerocom._lowlevel_helpers import RegridResDeg from pyaerocom.exceptions import ( DataUnitError, @@ -427,8 +428,9 @@ def _colocate_site_data_helper( to aggregate from hourly to daily, rather than the mean. min_num_obs : int or dict, optional minimum number of observations for resampling of time - use_climatology_ref : bool - if True, climatological timeseries are used from observations + use_climatology_ref : ClimateConfig | bool, optional + If provided, the climatology will be calculated from the config + Raises ------ @@ -448,8 +450,18 @@ def _colocate_site_data_helper( var, ts_type=ts_type, how=resample_how, min_num_obs=min_num_obs, inplace=True )[var] - if use_climatology_ref: - obs_ts = stat_data_ref.calc_climatology(var_ref, min_num_obs=min_num_obs)[var_ref] + if isinstance(use_climatology_ref, ClimatologyConfig): + # clim_min_obs = use_climatology_ref.get("min_num_obs", None) + # if clim_min_obs is None: + # use_climatology_ref["min_num_obs"] = min_num_obs + obs_ts = stat_data_ref.calc_climatology( + var_ref, + start=use_climatology_ref.start, + stop=use_climatology_ref.stop, + clim_mincount=use_climatology_ref.mincount, + resample_how=use_climatology_ref.resample_how, + clim_freq=use_climatology_ref.freq, + )[var_ref] else: obs_ts = stat_data_ref.resample_time( var_ref, @@ -501,15 +513,15 @@ def _colocate_site_data_helper_timecol( to aggregate from hourly to daily, rather than the mean. min_num_obs : int or dict, optional minimum number of observations for resampling of time - use_climatology_ref : bool - if True, NotImplementedError is raised + use_climatology_ref: ClimateConfig | bool + if provided, NotImplementedError is raised Raises ------ TemporalResolutionError if model or obs sampling frequency is lower than desired output frequency NotImplementedError - if input arg `use_climatology_ref` is True. + if input arg `use_climatology_ref` is provided. Returns ------- @@ -517,10 +529,10 @@ def _colocate_site_data_helper_timecol( dataframe containing the colocated input data (column names are data and ref) """ - if use_climatology_ref: + if isinstance(use_climatology_ref, ClimatologyConfig): raise NotImplementedError( "Using observation climatology in colocation with option " - "colocate_time=True is not available yet ..." + "use_climatology_ref is not available yet ..." ) grid_tst = stat_data.get_var_ts_type(var) @@ -672,8 +684,8 @@ def colocate_gridded_ungridded( if True and if original time resolution of data is higher than desired time resolution (`ts_type`), then both datasets are colocated in time *before* resampling to lower resolution. - use_climatology_ref : bool - if True, climatological timeseries are used from observations + use_climatology_ref : ClimateConfig | bool, optional. + Configuration for calculating the climatology. If set to a bool, this will not be done resample_how : str or dict string specifying how data should be aggregated when resampling in time. Default is "mean". Can also be a nested dictionary, e.g. @@ -757,10 +769,10 @@ def colocate_gridded_ungridded( data = data.resample_time(str(ts_type), min_num_obs=min_num_obs, how=resample_how) ts_type_data = ts_type - if use_climatology_ref: - col_freq = "monthly" - obs_start = const.CLIM_START - obs_stop = const.CLIM_STOP + if isinstance(use_climatology_ref, ClimatologyConfig): # pragma: no cover + col_freq = use_climatology_ref.freq + obs_start = use_climatology_ref.start + obs_stop = use_climatology_ref.stop else: col_freq = str(ts_type) obs_start = start diff --git a/pyaerocom/config.py b/pyaerocom/config.py index 186d6d057..5516a7c30 100644 --- a/pyaerocom/config.py +++ b/pyaerocom/config.py @@ -143,7 +143,7 @@ class Config: CLIM_START = 2005 CLIM_STOP = 2015 - CLIM_FREQ = "daily" + CLIM_FREQ = "monthly" CLIM_RESAMPLE_HOW = "mean" # median, ... # as a function of climatological frequency CLIM_MIN_COUNT = dict( diff --git a/pyaerocom/stationdata.py b/pyaerocom/stationdata.py index 5dde3a083..c1249de01 100644 --- a/pyaerocom/stationdata.py +++ b/pyaerocom/stationdata.py @@ -8,7 +8,12 @@ import xarray as xr from pyaerocom import const -from pyaerocom._lowlevel_helpers import BrowseDict, dict_to_str, list_to_shortstr, merge_dicts +from pyaerocom._lowlevel_helpers import ( + BrowseDict, + dict_to_str, + list_to_shortstr, + merge_dicts, +) from pyaerocom.exceptions import ( CoordinateError, DataDimensionError, @@ -379,7 +384,11 @@ def get_station_coords(self, force_single_value=True): return output def get_meta( - self, force_single_value=True, quality_check=True, add_none_vals=False, add_meta_keys=None + self, + force_single_value=True, + quality_check=True, + add_none_vals=False, + add_meta_keys=None, ): """Return meta-data as dictionary @@ -732,10 +741,18 @@ def _merge_vardata_2d(self, other, var_name, resample_how=None, min_num_obs=None ts_type = self._check_ts_types_for_merge(other, var_name) s0 = self.resample_time( - var_name, ts_type=ts_type, how=resample_how, min_num_obs=min_num_obs, inplace=True + var_name, + ts_type=ts_type, + how=resample_how, + min_num_obs=min_num_obs, + inplace=True, )[var_name].dropna() s1 = other.resample_time( - var_name, ts_type=ts_type, how=resample_how, min_num_obs=min_num_obs, inplace=True + var_name, + ts_type=ts_type, + how=resample_how, + min_num_obs=min_num_obs, + inplace=True, )[var_name].dropna() info = other.var_info[var_name] @@ -1038,7 +1055,11 @@ def calc_climatology( clim_freq = "monthly" data = self.resample_time( - var_name, ts_type=clim_freq, how=resample_how, min_num_obs=min_num_obs, inplace=False + var_name, + ts_type=clim_freq, + how=resample_how, + min_num_obs=min_num_obs, + inplace=False, ) ts = data.to_timeseries(var_name) @@ -1049,9 +1070,16 @@ def calc_climatology( if clim_mincount is None: clim_mincount = const.CLIM_MIN_COUNT[clim_freq] + if isinstance(clim_mincount, dict): + clim_mincount = clim_mincount[clim_freq] clim = calc_climatology( - ts, start, stop, min_count=clim_mincount, set_year=set_year, resample_how=resample_how + ts, + start, + stop, + min_count=clim_mincount, + set_year=set_year, + resample_how=resample_how, ) new = StationData() diff --git a/tests/colocation/test_colocation_3d.py b/tests/colocation/test_colocation_3d.py index 7d09fd1ef..b6cb54900 100644 --- a/tests/colocation/test_colocation_3d.py +++ b/tests/colocation/test_colocation_3d.py @@ -23,13 +23,22 @@ @pytest.fixture def fake_model_data_with_altitude(): longitude = iris.coords.DimCoord( - np.linspace(-15, 25, 20), var_name="lon", standard_name="longitude", units="degrees" + np.linspace(-15, 25, 20), + var_name="lon", + standard_name="longitude", + units="degrees", ) latitude = iris.coords.DimCoord( - np.linspace(50, 55, 10), var_name="lat", standard_name="latitude", units="degrees" + np.linspace(50, 55, 10), + var_name="lat", + standard_name="latitude", + units="degrees", ) altitude = iris.coords.DimCoord( - np.linspace(0, 60000, 10000), var_name="alt", standard_name="altitude", units="meters" + np.linspace(0, 60000, 10000), + var_name="alt", + standard_name="altitude", + units="meters", ) time = iris.coords.DimCoord( np.arange(18892, 18892 + 7, 1), diff --git a/tests/colocation/test_colocation_utils.py b/tests/colocation/test_colocation_utils.py index d529e82f9..9b88418d7 100644 --- a/tests/colocation/test_colocation_utils.py +++ b/tests/colocation/test_colocation_utils.py @@ -5,6 +5,7 @@ from cf_units import Unit from pyaerocom import GriddedData, const, helpers +from pyaerocom.climatology_config import ClimatologyConfig from pyaerocom.colocation.colocated_data import ColocatedData from pyaerocom.colocation.colocation_utils import ( _colocate_site_data_helper, @@ -77,10 +78,30 @@ def test__regrid_gridded(data_tm5): @pytest.mark.parametrize( "stat_data,stat_data_ref,var,var_ref,ts_type,resample_how,min_num_obs, use_climatology_ref,num_valid", [ - (S4, S3, "concpm10", "concpm10", "monthly", "mean", {"monthly": {"daily": 25}}, False, 10), - (S3, S4, "concpm10", "concpm10", "monthly", "mean", {"monthly": {"daily": 25}}, False, 24), - (S1, S2, "concpm10", "concpm10", "monthly", "mean", 25, False, 12), - (S2, S1, "concpm10", "concpm10", "monthly", "mean", 25, False, 11), + ( + S4, + S3, + "concpm10", + "concpm10", + "monthly", + "mean", + {"monthly": {"daily": 25}}, + False, + 10, + ), + ( + S3, + S4, + "concpm10", + "concpm10", + "monthly", + "mean", + {"monthly": {"daily": 25}}, + False, + 24, + ), + (S1, S2, "concpm10", "concpm10", "monthly", "mean", 25, {}, 12), + (S2, S1, "concpm10", "concpm10", "monthly", "mean", 25, {}, 11), ], ) def test__colocate_site_data_helper_timecol( @@ -152,7 +173,8 @@ def test_colocate_gridded_ungridded_new_var(data_tm5, aeronetsunv3lev2_subset): ), ( dict( - filter_name=f"{ALL_REGION_NAME}-wMOUNTAINS", min_num_obs=const.OBS_MIN_NUM_RESAMPLE + filter_name=f"{ALL_REGION_NAME}-wMOUNTAINS", + min_num_obs=const.OBS_MIN_NUM_RESAMPLE, ), "monthly", (2, 12, 11), @@ -162,7 +184,7 @@ def test_colocate_gridded_ungridded_new_var(data_tm5, aeronetsunv3lev2_subset): ( dict( filter_name=f"{ALL_REGION_NAME}-noMOUNTAINS", - use_climatology_ref=True, + use_climatology_ref=ClimatologyConfig(), min_num_obs=const.OBS_MIN_NUM_RESAMPLE, ), "monthly", @@ -213,7 +235,10 @@ def test_colocate_gridded_ungridded_nonglobal(aeronetsunv3lev2_subset): for time in times: time_coord = iris.coords.DimCoord(time, units=time_unit, standard_name="time") cube = helpers.make_dummy_cube_latlon( - lat_res_deg=1, lon_res_deg=1, lat_range=[30.05, 81.95], lon_range=[-29.5, 89.95] + lat_res_deg=1, + lon_res_deg=1, + lat_range=[30.05, 81.95], + lon_range=[-29.5, 89.95], ) cube.add_aux_coord(time_coord) cubes.append(cube) diff --git a/tests/colocation/test_colocator.py b/tests/colocation/test_colocator.py index 27ba371ca..d41f3ec79 100644 --- a/tests/colocation/test_colocator.py +++ b/tests/colocation/test_colocator.py @@ -8,6 +8,7 @@ from pyaerocom.colocation.colocation_setup import ColocationSetup from pyaerocom.colocation.colocator import Colocator from pyaerocom.config import ALL_REGION_NAME +from pyaerocom.climatology_config import ClimatologyConfig from pyaerocom.exceptions import ColocationError, ColocationSetupError from pyaerocom.io.aux_read_cubes import add_cubes from pyaerocom.io.mscw_ctm.reader import ReadMscwCtm @@ -227,7 +228,10 @@ def test_Colocator_run_gridded_gridded(setup): 0.002, ), ( - dict(model_use_vars={"od550aer": "abs550aer"}, obs_use_climatology=True), + dict( + model_use_vars={"od550aer": "abs550aer"}, + obs_use_climatology=ClimatologyConfig(start=0, stop=9999), + ), "abs550aer", "od550aer", (2, 12, 16), From f903e94a4311ebae58ad1902fe3bf79af1da3986 Mon Sep 17 00:00:00 2001 From: Daniel Heinesen Date: Mon, 30 Sep 2024 14:53:01 +0200 Subject: [PATCH 2/7] Small changes to field val for climateconfig --- pyaerocom/colocation/colocation_setup.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pyaerocom/colocation/colocation_setup.py b/pyaerocom/colocation/colocation_setup.py index d22bb1664..8866487d8 100644 --- a/pyaerocom/colocation/colocation_setup.py +++ b/pyaerocom/colocation/colocation_setup.py @@ -388,10 +388,13 @@ def validate_obs_use_climatology(cls, v): if isinstance(v, ClimatologyConfig): return v - if v == True: - return ClimatologyConfig() + if isinstance(v, bool): + if v: + return ClimatologyConfig() + else: + return v - return v + raise ValidationError obs_cache_only: bool = False # only relevant if obs is ungridded obs_vert_type: str | None = None From ef5cc9fade39dbdcfd7366e129bae7d80b86bfc9 Mon Sep 17 00:00:00 2001 From: Daniel Heinesen Date: Mon, 30 Sep 2024 14:54:12 +0200 Subject: [PATCH 3/7] Removes unused deps --- pyaerocom/climatology_config.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/pyaerocom/climatology_config.py b/pyaerocom/climatology_config.py index 1ebecfefc..d8c6414c6 100644 --- a/pyaerocom/climatology_config.py +++ b/pyaerocom/climatology_config.py @@ -1,10 +1,7 @@ from pydantic import ( BaseModel, - ConfigDict, - Field, ValidationError, field_validator, - model_validator, ) from pyaerocom import const From 2b5c8008d0cc3281468ad0b53c904fa7f0a1a8bb Mon Sep 17 00:00:00 2001 From: Daniel Heinesen Date: Fri, 11 Oct 2024 06:50:49 +0200 Subject: [PATCH 4/7] Adds set_year --- pyaerocom/climatology_config.py | 29 +++++++++++++---------- pyaerocom/colocation/colocation_utils.py | 1 + pyaerocom/config.py | 2 +- tests/colocation/test_colocation_utils.py | 2 +- tests/colocation/test_colocator.py | 2 +- 5 files changed, 20 insertions(+), 16 deletions(-) diff --git a/pyaerocom/climatology_config.py b/pyaerocom/climatology_config.py index d8c6414c6..afa58cf30 100644 --- a/pyaerocom/climatology_config.py +++ b/pyaerocom/climatology_config.py @@ -1,8 +1,6 @@ -from pydantic import ( - BaseModel, - ValidationError, - field_validator, -) +from pydantic import BaseModel, ValidationError, field_validator + +from typing import Literal from pyaerocom import const @@ -30,14 +28,19 @@ class ClimatologyConfig(BaseModel): start: int = const.CLIM_START stop: int = const.CLIM_STOP - resample_how: str = const.CLIM_RESAMPLE_HOW - freq: str = const.CLIM_FREQ - mincount: dict = const.CLIM_MIN_COUNT + set_year: int | None = None - @field_validator("resample_how") + @field_validator("set_year") @classmethod - def validate_resample_how(cls, v): - if v in ["mean", "median"]: - return v + def validate_set_year(cls, v): + if v is None: + return int((cls.stop - cls.start) // 2 + cls.start) + 1 + + if v > cls.stop or v < cls.start: + raise ValidationError - raise ValidationError + return v + + resample_how: Literal["mean", "median"] = const.CLIM_RESAMPLE_HOW + freq: str = const.CLIM_FREQ + mincount: dict = const.CLIM_MIN_COUNT diff --git a/pyaerocom/colocation/colocation_utils.py b/pyaerocom/colocation/colocation_utils.py index 2d54437c8..493bffd53 100644 --- a/pyaerocom/colocation/colocation_utils.py +++ b/pyaerocom/colocation/colocation_utils.py @@ -461,6 +461,7 @@ def _colocate_site_data_helper( clim_mincount=use_climatology_ref.mincount, resample_how=use_climatology_ref.resample_how, clim_freq=use_climatology_ref.freq, + set_year=use_climatology_ref.set_year, )[var_ref] else: obs_ts = stat_data_ref.resample_time( diff --git a/pyaerocom/config.py b/pyaerocom/config.py index 5516a7c30..186d6d057 100644 --- a/pyaerocom/config.py +++ b/pyaerocom/config.py @@ -143,7 +143,7 @@ class Config: CLIM_START = 2005 CLIM_STOP = 2015 - CLIM_FREQ = "monthly" + CLIM_FREQ = "daily" CLIM_RESAMPLE_HOW = "mean" # median, ... # as a function of climatological frequency CLIM_MIN_COUNT = dict( diff --git a/tests/colocation/test_colocation_utils.py b/tests/colocation/test_colocation_utils.py index 9b88418d7..9400bbde1 100644 --- a/tests/colocation/test_colocation_utils.py +++ b/tests/colocation/test_colocation_utils.py @@ -184,7 +184,7 @@ def test_colocate_gridded_ungridded_new_var(data_tm5, aeronetsunv3lev2_subset): ( dict( filter_name=f"{ALL_REGION_NAME}-noMOUNTAINS", - use_climatology_ref=ClimatologyConfig(), + use_climatology_ref=ClimatologyConfig(freq="monthly"), min_num_obs=const.OBS_MIN_NUM_RESAMPLE, ), "monthly", diff --git a/tests/colocation/test_colocator.py b/tests/colocation/test_colocator.py index d41f3ec79..5000f0268 100644 --- a/tests/colocation/test_colocator.py +++ b/tests/colocation/test_colocator.py @@ -211,7 +211,7 @@ def test_Colocator_run_gridded_gridded(setup): dict( model_use_vars={"od550aer": "abs550aer"}, model_use_climatology=True, - obs_use_climatology=True, + obs_use_climatology=True, # ClimatologyConfig(start=0, stop=9999), ), "abs550aer", "od550aer", From 7cd2dc5423f501eefa5b07b1cf792eaeca50d024 Mon Sep 17 00:00:00 2001 From: Daniel Heinesen Date: Fri, 11 Oct 2024 07:11:29 +0200 Subject: [PATCH 5/7] WIP --- pyaerocom/colocation/colocation_utils.py | 4 +--- pyaerocom/config.py | 2 +- pyaerocom/stationdata.py | 5 ++++- tests/colocation/test_colocator.py | 4 ++-- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/pyaerocom/colocation/colocation_utils.py b/pyaerocom/colocation/colocation_utils.py index 493bffd53..4d44f0ec8 100644 --- a/pyaerocom/colocation/colocation_utils.py +++ b/pyaerocom/colocation/colocation_utils.py @@ -451,13 +451,11 @@ def _colocate_site_data_helper( )[var] if isinstance(use_climatology_ref, ClimatologyConfig): - # clim_min_obs = use_climatology_ref.get("min_num_obs", None) - # if clim_min_obs is None: - # use_climatology_ref["min_num_obs"] = min_num_obs obs_ts = stat_data_ref.calc_climatology( var_ref, start=use_climatology_ref.start, stop=use_climatology_ref.stop, + min_num_obs=min_num_obs, clim_mincount=use_climatology_ref.mincount, resample_how=use_climatology_ref.resample_how, clim_freq=use_climatology_ref.freq, diff --git a/pyaerocom/config.py b/pyaerocom/config.py index 186d6d057..5516a7c30 100644 --- a/pyaerocom/config.py +++ b/pyaerocom/config.py @@ -143,7 +143,7 @@ class Config: CLIM_START = 2005 CLIM_STOP = 2015 - CLIM_FREQ = "daily" + CLIM_FREQ = "monthly" CLIM_RESAMPLE_HOW = "mean" # median, ... # as a function of climatological frequency CLIM_MIN_COUNT = dict( diff --git a/pyaerocom/stationdata.py b/pyaerocom/stationdata.py index c1249de01..9e605b168 100644 --- a/pyaerocom/stationdata.py +++ b/pyaerocom/stationdata.py @@ -1048,7 +1048,10 @@ def calc_climatology( if ts_type < TsType( clim_freq ): # current resolution is lower than input climatological freq - supported = list(const.CLIM_MIN_COUNT) + if clim_mincount is None: + supported = list(const.CLIM_MIN_COUNT) + else: + supported = list(clim_mincount) if str(ts_type) in supported: clim_freq = str(ts_type) else: # use monthly diff --git a/tests/colocation/test_colocator.py b/tests/colocation/test_colocator.py index 5000f0268..ccf545ed7 100644 --- a/tests/colocation/test_colocator.py +++ b/tests/colocation/test_colocator.py @@ -211,7 +211,7 @@ def test_Colocator_run_gridded_gridded(setup): dict( model_use_vars={"od550aer": "abs550aer"}, model_use_climatology=True, - obs_use_climatology=True, # ClimatologyConfig(start=0, stop=9999), + obs_use_climatology=True, ), "abs550aer", "od550aer", @@ -230,7 +230,7 @@ def test_Colocator_run_gridded_gridded(setup): ( dict( model_use_vars={"od550aer": "abs550aer"}, - obs_use_climatology=ClimatologyConfig(start=0, stop=9999), + obs_use_climatology=True, ), "abs550aer", "od550aer", From d35a7d04776528f1bf6277d3b356da592e3d2130 Mon Sep 17 00:00:00 2001 From: Daniel Heinesen Date: Fri, 11 Oct 2024 07:39:40 +0200 Subject: [PATCH 6/7] Tests run now, but colocation_utils:772 seems like an old hack --- pyaerocom/colocation/colocation_utils.py | 2 +- pyaerocom/config.py | 2 +- tests/colocation/test_colocation_utils.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyaerocom/colocation/colocation_utils.py b/pyaerocom/colocation/colocation_utils.py index 4d44f0ec8..2f6eecd24 100644 --- a/pyaerocom/colocation/colocation_utils.py +++ b/pyaerocom/colocation/colocation_utils.py @@ -769,7 +769,7 @@ def colocate_gridded_ungridded( ts_type_data = ts_type if isinstance(use_climatology_ref, ClimatologyConfig): # pragma: no cover - col_freq = use_climatology_ref.freq + col_freq = "monthly" # use_climatology_ref.freq obs_start = use_climatology_ref.start obs_stop = use_climatology_ref.stop else: diff --git a/pyaerocom/config.py b/pyaerocom/config.py index 5516a7c30..186d6d057 100644 --- a/pyaerocom/config.py +++ b/pyaerocom/config.py @@ -143,7 +143,7 @@ class Config: CLIM_START = 2005 CLIM_STOP = 2015 - CLIM_FREQ = "monthly" + CLIM_FREQ = "daily" CLIM_RESAMPLE_HOW = "mean" # median, ... # as a function of climatological frequency CLIM_MIN_COUNT = dict( diff --git a/tests/colocation/test_colocation_utils.py b/tests/colocation/test_colocation_utils.py index 9400bbde1..9b88418d7 100644 --- a/tests/colocation/test_colocation_utils.py +++ b/tests/colocation/test_colocation_utils.py @@ -184,7 +184,7 @@ def test_colocate_gridded_ungridded_new_var(data_tm5, aeronetsunv3lev2_subset): ( dict( filter_name=f"{ALL_REGION_NAME}-noMOUNTAINS", - use_climatology_ref=ClimatologyConfig(freq="monthly"), + use_climatology_ref=ClimatologyConfig(), min_num_obs=const.OBS_MIN_NUM_RESAMPLE, ), "monthly", From bbc313d9a838a2f4a7660e3d903b4758aca5fc0b Mon Sep 17 00:00:00 2001 From: Daniel Heinesen Date: Fri, 11 Oct 2024 07:44:32 +0200 Subject: [PATCH 7/7] WIP --- tests/colocation/test_colocator.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/colocation/test_colocator.py b/tests/colocation/test_colocator.py index ccf545ed7..7cb55fc72 100644 --- a/tests/colocation/test_colocator.py +++ b/tests/colocation/test_colocator.py @@ -8,7 +8,6 @@ from pyaerocom.colocation.colocation_setup import ColocationSetup from pyaerocom.colocation.colocator import Colocator from pyaerocom.config import ALL_REGION_NAME -from pyaerocom.climatology_config import ClimatologyConfig from pyaerocom.exceptions import ColocationError, ColocationSetupError from pyaerocom.io.aux_read_cubes import add_cubes from pyaerocom.io.mscw_ctm.reader import ReadMscwCtm