Added fixes and TODOs for EPIC module compatibility. Fixed issue on test_results in xml files

Author: doc78

pytest.ini

@@ -14,6 +14,4 @@ env =
     D:NUMEXPR_MAX_THREADS=12
 ;    D:PCRASTER_NR_WORKER_THREADS=4 gives errors with pcraster.multicore - need changes in many hydro modules
 ; run pytest tests/ -m "slow" before release !!!
-;addopts = -ra -x -l --cov=lisflood --cov-config=.coveragerc -m "not slow" -vv
-
-addopts = --no-cov -m "slow or not slow" 
+addopts = -ra -x -l --cov=lisflood --cov-config=.coveragerc -m "not slow" -vv

src/lisflood/Lisflood_dynamic.py

@@ -27,15 +27,11 @@
 
 from .global_modules.settings import CDFFlags, LisSettings, MaskInfo
 
-#import time  # CR: speed test
-
 class LisfloodModel_dyn(DynamicModel):
 
     # =========== DYNAMIC ====================================================
 
     def dynamic(self):
-        #t0tot=time.time()  # CR: speed test
-
         """ Dynamic part of LISFLOOD
             calls the dynamic part of the hydrological modules
         """
@@ -260,6 +256,4 @@ def splitlanduse(array1, array2=None, array3=None):
         self.stateVar_module.dynamic()
         if option['wateruse'] and option['indicator'] and self.monthend: 
             self.indicatorcalc_module.dynamic_setzero()
-        # setting monthly and yearly dindicator to zero at the end of the month (year)
-
-        #print(' - total time for this step: ', (time.time() - t0tot))  # CR: speed test
+        # setting monthly and yearly dindicator to zero at the end of the month (year)

src/lisflood/Lisflood_initial.py

@@ -365,16 +365,22 @@ def defsoil(self, name_1, name_2=None, name_3=None, coords=None):
             coords = self.coord_landuse
         data = self.allocateDataArray(coords)
         values_1 = readInputWithBackup(name_1)
-        if (list(coords.keys())[0]=="landuse") or (list(coords.keys())[0]=="runoff"):
-            data.values[0][:] = values_1
-            data.values[1][:] = readInputWithBackup(name_2, values_1)
-            data.values[2][:] = readInputWithBackup(name_3, values_1)
+        option = self.settings.options
+        if option.get('cropsEPIC'):
+            labels = list(coords.values())[0]
+            data.loc[labels[0],:] = values_1
+            data.loc[labels[1],:] = readInputWithBackup(name_2, values_1)
+            data.loc[labels[2],:] = readInputWithBackup(name_3, values_1)
         else:
-            raise Exception("Coords key not found!")
+            if (list(coords.keys())[0]=="landuse") or (list(coords.keys())[0]=="runoff"):
+                data.values[0][:] = values_1
+                data.values[1][:] = readInputWithBackup(name_2, values_1)
+                data.values[2][:] = readInputWithBackup(name_3, values_1)
+            else:
+                raise Exception("Coords key not found!")
         return data
 
     def deffraction(self, variable):
-         """Weighted sum over the soil fractions of each pixel"""
-         #ax_veg = variable.dims.index("vegetation")
-         ax_veg=0
-         return _vegSum(ax_veg, variable.values, self.SoilFraction.values)
+        """Weighted sum over the soil fractions of each pixel"""
+        ax_veg = variable.dims.index("vegetation")
+        return _vegSum(ax_veg, variable.values, self.SoilFraction.values)

src/lisflood/global_modules/add1.py

@@ -53,6 +53,10 @@ def __new__(cls, input_array):
     @property
     def values(self):
         return self
+    
+    @property
+    def dims(self):
+        return ['vegetation']
 
 
 def defsoil(name1, name2=None, name3=None):

src/lisflood/hydrological_modules/soil.py

@@ -267,7 +267,7 @@ def splitlanduse(array1, array2=None, array3=None):
 
         for veg, luse in self.var.VEGETATION_LANDUSE.items():
             iveg = self.var.vegetation.index(veg)
-            iluse = self.var.epic_settings.soil_uses.index(luse)
+            iluse = self.var.SOIL_USES.index(luse)
             ini_1a = np.where(ThetaInit1aValue[iveg] == -9999, self.var.WFC1a[iluse], ThetaInit1aValue[iveg] * self.var.SoilDepth1a[iluse])
             self.var.W1a[iveg] = np.where(self.var.PoreSpaceNotZero1a[iluse], ini_1a, 0)
             ini_1b = np.where(ThetaInit1bValue[iveg] == -9999, self.var.WFC1b[iluse], ThetaInit1bValue[iveg] * self.var.SoilDepth1b[iluse])
@@ -498,8 +498,9 @@ def dynamic_perpixel(self):
         for landuse, veg_list in self.var.LANDUSE_VEGETATION.items():
             iveg_list,iveg_list_pres,ilanduse = self.var.get_indexes_from_landuse_and_veg_list_GLOBAL(landuse, veg_list)
             self.var.Theta[iveg_list] = self.var.SoilFraction[iveg_list_pres] * tot_sm[iveg_list] / self.var.SoilDepthTotal[ilanduse]
-        soil_fract_sum = np.sum(self.var.SoilFraction,0)
-        self.var.ThetaAll = np.where(soil_fract_sum > 0, np.sum(self.var.Theta,0) / soil_fract_sum, 0)
+        ax_veg=self.var.SoilFraction.dims.index("vegetation")
+        soil_fract_sum = np.sum(self.var.SoilFraction,ax_veg)
+        self.var.ThetaAll = np.where(soil_fract_sum > 0, np.sum(self.var.Theta,ax_veg) / soil_fract_sum, 0)
 
         self.var.SeepTopToSubPixelA = self.var.deffraction(self.var.SeepTopToSubA)
         self.var.SeepTopToSubPixelB = self.var.deffraction(self.var.SeepTopToSubB)

src/lisflood/hydrological_modules/soilloop.py

@@ -464,7 +464,7 @@ def initial(self):
             self.is_paddy_irrig = np.zeros(len(self.var.vegetation), bool)
 
         self.index_landuse_all = np.array([self.var.SOIL_USES.index(self.var.VEGETATION_LANDUSE[v]) for v in self.var.vegetation]) #######################
-        self.index_landuse_prescr = np.array([self.var.SOIL_USES.index(self.var.VEGETATION_LANDUSE[v]) for v in self.var.prescribed_vegetation]) #######################
+        self.index_landuse_prescr = np.array([self.var.SOIL_USES.index(self.var.VEGETATION_LANDUSE[v]) for v in self.var.PRESCRIBED_VEGETATION]) #######################
         self.is_irrigated = np.array([self.var.VEGETATION_LANDUSE[v] == 'Irrigated' for v in self.var.vegetation]) #######################
         if option["cropsEPIC"]: #######################
             self.is_paddy_irrig = np.array([v in self.var.crop_module.paddy_crops for v in self.var.vegetation]) #######################

src/lisflood/hydrological_modules/surface_routing.py

@@ -122,10 +122,16 @@ def dynamic(self):
 
         self.var.SurfaceRunSoil = self.var.allocateDataArray([self.var.dim_landuse, self.var.dim_pixel])
         for landuse, veg_list in self.var.LANDUSE_VEGETATION.items():
+            # TODO: CR: to ensure compatibility with EPIC, get ax_veg for the sum operation from the actual dimesions on the variable
+            # self.var.SurfaceRunSoil.loc[landuse] = (self.var.SoilFraction.loc[veg_list] * \
+            #         np.maximum(self.var.AvailableWaterForInfiltration.loc[veg_list] - self.var.Infiltration.loc[veg_list], 0)).sum("vegetation")
             iveg_list,iveg_list_pres,ilanduse = self.var.get_indexes_from_landuse_and_veg_list_GLOBAL(landuse, veg_list)
             self.var.SurfaceRunSoil.values[ilanduse] = np.sum((self.var.SoilFraction.values[iveg_list_pres] * \
                     np.maximum(self.var.AvailableWaterForInfiltration.values[iveg_list] - self.var.Infiltration.values[iveg_list],0)),0)
 
+        # TODO: CR: to ensure compatibility with EPIC, get ax_veg for the sum operation from the actual dimesions on the variable 
+        # (probably implementing two different NumpyModified classes for vegetation and for landuse, checking also for any drop of performances)
+        # self.var.SurfaceRunoff = self.var.DirectRunoff + self.var.SurfaceRunSoil.sum("landuse").values
         self.var.SurfaceRunoff = self.var.DirectRunoff + np.sum(self.var.SurfaceRunSoil.values,0)
 
         # Surface runoff for this time step (mm)
@@ -142,6 +148,9 @@ def dynamic(self):
         # Routing of overland flow to channel using kinematic wave
         # Note that all 'new' water is added as side-flow
         SideflowDirect = self.var.DirectRunoff * self.var.MMtoM3 * self.var.InvPixelLength * self.var.InvDtSec
+        # TODO: CR: to ensure compatibility with EPIC, get ax_veg for the sum operation from the actual dimesions on the variable 
+        # SideflowOther = self.var.SurfaceRunSoil.loc[["Rainfed","Irrigated"]].sum("landuse").values * self.var.MMtoM3 * self.var.InvPixelLength * self.var.InvDtSec
+        # SideflowForest = self.var.SurfaceRunSoil.loc["Forest"].values * self.var.MMtoM3 * self.var.InvPixelLength * self.var.InvDtSec
         ilusevalues = []
         for luse in ["Rainfed","Irrigated"]:
             ilusevalues.append(self.var.epic_settings.soil_uses.index(luse))

src/lisflood/hydrological_modules/waterabstraction.py

@@ -349,9 +349,7 @@ def dynamic(self):
                 self.var.abstraction_required_irrigation_M3 = maskinfo.in_zero()  
             else: # irrigation demand = transpiration deficit multiplied by anti-salinity factor   
                 irr_pre = "Irrigated_prescribed" # It applies only to prescribed irrigation fraction (EPIC simulates specific irrigated crops, if any) 
-                landuse_irr_pre = self.var.VEGETATION_LANDUSE[irr_pre] # landuse corresponding to vegetation fraction
-                ivegIrrigatedPrescribed = self.var.vegetation.index(irr_pre)
-                iluseIrrigatedPrescribed = self.var.epic_settings.soil_uses.index(landuse_irr_pre)
+                ivegIrrigatedPrescribed, iluseIrrigatedPrescribed, _ = self.var.get_landuse_and_indexes_from_vegetation_GLOBAL(irr_pre)
 
                 self.var.Ta.values[ivegIrrigatedPrescribed] = np.maximum(np.minimum(self.var.RWS.values[ivegIrrigatedPrescribed] * self.var.potential_transpiration.values[ivegIrrigatedPrescribed],
                                                              self.var.W1.values[ivegIrrigatedPrescribed] - self.var.WWP1.values[iluseIrrigatedPrescribed]),  maskinfo.in_zero())                 

tests/data/LF_ETRS89_UseCase/settings/cold.xml

@@ -1112,7 +1112,7 @@
                 </comment>
             </textvar>
 
-            <textvar name="LeakageFraction" value="0.2">
+            <textvar name="LeakageFraction" value="$(PathRoot)/maps/ad_leakage">
                 <comment>
                     0.2
                     $(PathRoot)/maps/leakage.map
@@ -4407,7 +4407,7 @@ LFBINDING: MORE LOW-LEVEL CONTROL OVER MODEL IN- AND OUTPUT
                     </comment>
                 </textvar>
 
-                <textvar name="FractionGroundwaterUsed" value="$(PathMapsLanduse)/fracgwusedNew">
+                <textvar name="FractionGroundwaterUsed" value="$(PathMapsLanduse)/fracgwused">
                     <comment>
                         groundwater used fraction of a pixel (0-1)
                     </comment>

tests/data/LF_ETRS89_UseCase/settings/prerun.xml

@@ -411,13 +411,13 @@ The option "numCPUs_parallelNumba" may take the following values:
                 **************************************************************
             </comment>
 
-            <textvar name="PathOut" value="$(PathRoot)/out/">
+            <textvar name="PathOut" value="$(PathRoot)/out">
                 <comment>
                     Output path
                 </comment>
             </textvar>
 
-            <textvar name="PathInit" value="$(PathRoot)/out/">
+            <textvar name="PathInit" value="$(PathRoot)/out">
                 <comment>
                     Path of the initial value maps e.g. lzavin.map
                 </comment>
@@ -1071,7 +1071,7 @@ The option "numCPUs_parallelNumba" may take the following values:
                 </comment>
             </textvar>
 
-            <textvar name="LeakageFraction" value="0.2">
+            <textvar name="LeakageFraction" value="$(PathRoot)/maps/ad_leakage">
                 <comment>
                     Fraction of leakage of public water supply (0=no leakage, 1=100% leakage)
                 </comment>
@@ -4322,7 +4322,7 @@ LFBINDING: MORE LOW-LEVEL CONTROL OVER MODEL IN- AND OUTPUT
                     </comment>
                 </textvar>
 
-                <textvar name="FractionGroundwaterUsed" value="$(PathMapsLanduse)/fracgwusedNew">
+                <textvar name="FractionGroundwaterUsed" value="$(PathMapsLanduse)/fracgwused">
                     <comment>
                         groundwater used fraction of a pixel (0-1)
                     </comment>