Fix forecast length

dnora_destine/ice/ice_readers.py

@@ -14,4 +14,4 @@ class ECMWF(ProductReader):
         default_data_source=DataSource.REMOTE,
     )
 
-    file_structure = FileStructure(stride=24, hours_per_file=24)
+    file_structure = FileStructure(stride=24, hours_per_file=97)

dnora_destine/polytope_functions.py

@@ -9,27 +9,30 @@
 from dnora.wind import Wind
 from dnora.ice import Ice
 from dnora.type_manager.dnora_types import DnoraDataType
-
+from dnora.read.ds_read_functions import setup_temp_dir
 import os
 
 import glob
 
 
-# This is defined here because the dnora-version doesn't have the ability to keep old files
-def setup_temp_dir(
-    data_type: DnoraDataType, reader_name: str, clean_old_files: bool = True
-) -> str:
-    """Sets up a temporery directory for fimex files and cleans out possible old files"""
-    temp_folder = f"dnora_{data_type.name.lower()}_temp"
-    if not os.path.isdir(temp_folder):
-        os.mkdir(temp_folder)
-        print("Creating folder %s..." % temp_folder)
-    if clean_old_files:
-        msg.plain("Removing old files from temporary folder...")
-        for f in glob.glob(f"dnora_{data_type.name.lower()}_temp/{reader_name}*.*"):
-            os.remove(f)
+def get_destine_steps(start_time: str, end_time: str) -> tuple[str, list[int]]:
+    """DestinE data in daily runs, so first step (0) is 00:00
+
+    Function calculates which steps are needed to cover exactly start_time and end_time
+
+    returns the start date and list of steps
+    """
+    start_time = pd.to_datetime(start_time)
+    end_time = pd.to_datetime(end_time)
+
+    date_str = start_time.strftime("%Y%m%d")
+    start_of_day = pd.to_datetime(f"{date_str} 00:00:00")
 
-    return temp_folder
+    first_step = int(pd.to_timedelta(start_time - start_of_day).total_seconds() / 3600)
+    last_step = int(pd.to_timedelta(end_time - start_of_day).total_seconds() / 3600) + 1
+    all_steps = range(first_step, last_step)
+
+    return date_str, list(all_steps)
 
 
 def download_ecmwf_from_destine(start_time, end_time, lon, lat, folder: str) -> str:
@@ -46,6 +49,9 @@ def download_ecmwf_from_destine(start_time, end_time, lon, lat, folder: str) ->
         raise e
     c = Client(address="polytope.lumi.apps.dte.destination-earth.eu")
 
+    date_str, steps = get_destine_steps(start_time, end_time)
+    steps = "/".join([f"{h:.0f}" for h in steps])
+
     filename = f"{folder}/ECMWF_temp.grib"  # Switch to this in production. Then the files will be cleaned out
     # filename = f"{folder}/destine_temp.grib"
     request_winds = {
@@ -57,15 +63,14 @@ def download_ecmwf_from_destine(start_time, end_time, lon, lat, folder: str) ->
         "levtype": "sfc",
         "param": "165/166",
         "time": "00",
-        "step": "0/1/2/3/4/5/6/7/8/9/10/11/12/13/14/15/16/17/18/19/20/21/22/23",
+        "step": steps,  # "0/1/2/3/4/5/6/7/8/9/10/11/12/13/14/15/16/17/18/19/20/21/22/23",
         "area": [
             int(np.ceil(lat[1])),
             int(np.floor(lon[0])),
             int(np.floor(lat[0])),
             int(np.ceil(lon[1])),
         ],
     }
-    date_str = start_time.strftime("%Y%m%d")
     request_winds["date"] = date_str
 
     c.retrieve("destination-earth", request_winds, filename)
@@ -97,7 +102,10 @@ def ds_polytope_read(
     yi = np.arange(min(lats), max(lats), native_dlat)
     Xi, Yi = np.meshgrid(xi, yi)
 
-    Nt = len(ds.step)
+    try:
+        Nt = len(ds.step)
+    except TypeError:
+        Nt = 1
     u10i = np.zeros((Nt, len(yi), len(xi)))
     v10i = np.zeros((Nt, len(yi), len(xi)))
     # If this becomes slow, we need to think about 3D interpolation / resuing weights
@@ -109,7 +117,7 @@ def ds_polytope_read(
             list(zip(lons, lats)), v10[n, :], (Xi, Yi), method="nearest"
         )
 
-    data = Wind(lon=xi, lat=yi, time=ds.time + ds.step)
+    data = Wind(lon=xi, lat=yi, time=(ds.time + ds.step).values)
     data.set_u(u10i)
     data.set_v(v10i)
     lo, la = utils.grid.expand_area(
@@ -129,15 +137,13 @@ def download_ecmwf_wave_from_destine(
 
     if end_time is None:
         params = "140221"
-        steps = "0"
         end_time = start_time
     else:
         params = "140229/140230/140231"
-        # Because the reader is set up to do daily chunks, the start and end time will always be in the same day
-        days = [str(l) for l in range(start_time.hour, end_time.hour + 1)]
-        steps = "/".join(days)
-        # steps = "0/1/2/3/4/5/6/7/8/9/10/11/12/13/14/15/16/17/18/19/20/21/22/23"
-    end_time = pd.Timestamp(end_time)
+    #    end_time = pd.Timestamp(end_time)
+
+    date_str, steps = get_destine_steps(start_time, end_time)
+    steps = "/".join([f"{h:.0f}" for h in steps])
 
     try:
         from polytope.api import Client
@@ -161,9 +167,7 @@ def download_ecmwf_wave_from_destine(
         "time": "00",
         "step": steps,
     }
-    date_str = start_time.strftime("%Y%m%d")
     request_waves["date"] = date_str
-
     c.retrieve("destination-earth", request_waves, filename)
 
 
@@ -228,6 +232,8 @@ def download_ecmwf_ice_from_destine(start_time, end_time, lon, lat, folder: str)
 
     filename = f"{folder}/ECMWF_temp_ice.grib"  # Switch to this in production. Then the files will be cleaned out
     # filename = f"{folder}/destine_temp.grib"
+    date_str, steps = get_destine_steps(start_time, end_time)
+    steps = "/".join([f"{h:.0f}" for h in steps])
     request_ice = {
         "class": "d1",
         "expver": "0001",
@@ -237,15 +243,14 @@ def download_ecmwf_ice_from_destine(start_time, end_time, lon, lat, folder: str)
         "levtype": "sfc",
         "param": "31",
         "time": "00",
-        "step": "0/1/2/3/4/5/6/7/8/9/10/11/12/13/14/15/16/17/18/19/20/21/22/23",
+        "step": steps,
         "area": [
             int(np.ceil(lat[1])),
             int(np.floor(lon[0])),
             int(np.floor(lat[0])),
             int(np.ceil(lon[1])),
         ],
     }
-    date_str = start_time.strftime("%Y%m%d")
     request_ice["date"] = date_str
 
     c.retrieve("destination-earth", request_ice, filename)
@@ -269,15 +274,17 @@ def ds_ice_polytope_read(
     lons, lats, sic = (
         ds.siconc.longitude.values,
         ds.siconc.latitude.values,
-        ds.siconc.values,
+        np.atleast_2d(ds.siconc.values),
         # ds.sit.values,
     )
     native_dlon, native_dlat = 1 / 8, 1 / 30
     xi = np.arange(min(lons), max(lons), native_dlon)
     yi = np.arange(min(lats), max(lats), native_dlat)
     Xi, Yi = np.meshgrid(xi, yi)
-
-    Nt = len(ds.step)
+    try:
+        Nt = len(ds.step)
+    except TypeError:
+        Nt = 1
     sici = np.zeros((Nt, len(yi), len(xi)))
     # siti = np.zeros((Nt, len(yi), len(xi)))
     # If this becomes slow, we need to think about 3D interpolation / resuing weights
@@ -289,7 +296,7 @@ def ds_ice_polytope_read(
         #    list(zip(lons, lats)), sit[n, :], (Xi, Yi), method="nearest"
         # )
 
-    data = Ice(lon=xi, lat=yi, time=ds.time + ds.step)
+    data = Ice(lon=xi, lat=yi, time=(ds.time + ds.step).values)
     data.set_sic(sici)
     # data.set_sit(siti)
     lo, la = utils.grid.expand_area(

dnora_destine/spectra/spectral_readers.py

@@ -35,7 +35,7 @@ class ECMWF(SpectralProductReader):
 
     file_structure = FileStructure(
         stride=24,
-        hours_per_file=24,
+        hours_per_file=97,
     )
 
     def post_processing(self):

dnora_destine/wind/wind_readers.py

@@ -15,4 +15,4 @@ class ECMWF(ProductReader):
         default_data_source=DataSource.REMOTE,
     )
 
-    file_structure = FileStructure(stride=24, hours_per_file=24)
+    file_structure = FileStructure(stride=24, hours_per_file=97)

tests/test_utils/test_get_steps.py

@@ -0,0 +1,68 @@
+from dnora_destine.polytope_functions import get_destine_steps
+
+
+def test_trivial():
+    start_time = "2025-10-10 00:00"
+    end_time = "2025-10-10 00:00"
+    date_str, steps = get_destine_steps(start_time, end_time)
+    assert date_str == "20251010"
+    assert steps == [0]
+
+
+def test_one_hour():
+    start_time = "2025-10-10 00:00"
+    end_time = "2025-10-10 01:00"
+    date_str, steps = get_destine_steps(start_time, end_time)
+    assert date_str == "20251010"
+    assert steps == [0, 1]
+
+
+def test_24h():
+    start_time = "2025-10-10 00:00"
+    end_time = "2025-10-11 00:00"
+    date_str, steps = get_destine_steps(start_time, end_time)
+    assert date_str == "20251010"
+    assert steps == list(range(25))
+    assert steps == [
+        0,
+        1,
+        2,
+        3,
+        4,
+        5,
+        6,
+        7,
+        8,
+        9,
+        10,
+        11,
+        12,
+        13,
+        14,
+        15,
+        16,
+        17,
+        18,
+        19,
+        20,
+        21,
+        22,
+        23,
+        24,
+    ]
+
+
+def test_96h():
+    start_time = "2025-10-10 00:00"
+    end_time = "2025-10-14 00:00"
+    date_str, steps = get_destine_steps(start_time, end_time)
+    assert date_str == "20251010"
+    assert steps == list(range(97))
+
+
+def test_start_not_00():
+    start_time = "2025-10-10 03:00"
+    end_time = "2025-10-11 05:00"
+    date_str, steps = get_destine_steps(start_time, end_time)
+    assert date_str == "20251010"
+    assert steps == list(range(3, 30))