Adding climate change in Nevergrad/PCSE

mypy.ini

@@ -3,7 +3,8 @@
 [mypy-scipy.*,requests,pandas,compiler_gym,compiler_gym.*,gym_anm,matplotlib.*,pytest,cma,bayes_opt.*,torchvision.models,torch.*,mpl_toolkits.*,fcmaes.*,tqdm,pillow,PIL,PIL.Image,sklearn.*,pyomo.*,pyproj,IOHexperimenter.*,tensorflow,koncept.models,cv2,imquality,imquality.brisque,lpips,mixsimulator.*,networkx.*,cdt.*,pymoo,pymoo.*,bayes_optim.*,olympus.*,pymoo,pymoo.*,pybullet,pybullet_envs,pybulletgym,pyvirtualdisplay,nlopt,aquacrop.*]
 ignore_missing_imports = True
 
-[mypy-nevergrad.functions.rl.agents,torchvision,torchvision.*,nevergrad.functions.games.*,nevergrad.functions.multiobjective.pyhv,nevergrad.optimization.test_doc,,pymoo,pymoo.*,pybullet,pybullet_envs,pybulletgym,pyvirtualdisplay,nlopt,pcse.*,yaml.*,aquacrop.*,nevergrad.functions.gym.tuple_gym_env,nevergrad.functions.gym.multigym]
+[mypy-nevergrad.functions.rl.*,torchvision,torchvision.*,nevergrad.functions.games.*,nevergrad.functions.multiobjective.pyhv,nevergrad.optimization.test_doc,pymoo,pymoo.*,pybullet,pybullet_envs,pybulletgym,pyvirtualdisplay,nlopt,pcse.*,yaml.*,aquacrop.*,nevergrad.functions.gym.multigym.nevergrad.functions.gym.tuple_gym_env,pymoo,pymoo.*,pybullet,pybullet_envs,pybulletgym,pyvirtualdisplay,pcse.*,yaml.*,aquacrop.*,nlopt]
+#[mypy-nevergrad.functions.rl.*,torchvision,torchvision.*,nevergrad.functions.games.*,nevergrad.functions.multiobjective.pyhv,nevergrad.optimization.test_doc,,nevergrad.functions.gym.tuple_gym_env,nevergrad.functions.gym.multigym]
 #[mypy-nevergrad.functions.rl.*,torchvision,torchvision.*,nevergrad.functions.games.*,nevergrad.functions.multiobjective.pyhv,nevergrad.optimization.test_doc,nevergrad.functions.gym.multigym,nevergrad.functions.gym.tuple_gym_env]
 #[mypy-nevergrad.functions.rl.*,torchvision,torchvision.*,nevergrad.functions.games.*,nevergrad.functions.multiobjective.pyhv,nevergrad.optimization.test_doc,nevergrad.functions.gym.multigym,nevergrad.functions.gym.tuple_gym_env,pcse.*]
 #[mypy-nevergrad.functions.rl.*,torchvision,torchvision.*,nevergrad.functions.games.*,nevergrad.functions.multiobjective.pyhv,nevergrad.optimization.test_doc,nevergrad.functions.gym.multigym,nevergrad.functions.gym.tuple_gym_env,pcse.*,yaml.*]

nevergrad/benchmark/experiments.py

@@ -1554,6 +1554,18 @@ def kenya_new_many_crop_and_variety_irrigation(seed: tp.Optional[int] = None) ->
     return irrigation(seed, kenya=True, variety_choice=True, multi_crop=True, year_min=2016, year_max=2021)
 
 
+@registry.register
+def kenya_future_many_crop_and_variety_irrigation(seed: tp.Optional[int] = None) -> tp.Iterator[Experiment]:
+    return irrigation(seed, kenya=True, variety_choice=True, multi_crop=True, year_min=2026, year_max=2031)
+
+
+@registry.register
+def kenya_farfuture_many_crop_and_variety_irrigation(
+    seed: tp.Optional[int] = None,
+) -> tp.Iterator[Experiment]:
+    return irrigation(seed, kenya=True, variety_choice=True, multi_crop=True, year_min=2036, year_max=2041)
+
+
 @registry.register
 def kenya_old_many_crop_and_variety_irrigation(seed: tp.Optional[int] = None) -> tp.Iterator[Experiment]:
     return irrigation(seed, kenya=True, variety_choice=True, multi_crop=True, year_min=1996, year_max=2001)
@@ -1591,7 +1603,7 @@ def crop_simulator(seed: tp.Optional[int] = None) -> tp.Iterator[Experiment]:
     Low dimensional problem, only 2 vars. This is optimization for model identification: we want
     to find parameters so that the simulation matches observations.
     """
-    funcs = [CropSimulator()]
+    funcs = [Pcse()]
     seedg = create_seed_generator(seed)
     optims = ["DE", "PSO", "CMA", "NGOpt"]
     for budget in [25, 50, 100, 200]:
@@ -1604,14 +1616,35 @@ def crop_simulator(seed: tp.Optional[int] = None) -> tp.Iterator[Experiment]:
                             yield xp
 
 
+# @registry.register
+# def irrigation(seed: tp.Optional[int] = None, benin: bool = False, variety_choice: bool = False) -> tp.Iterator[Experiment]:
+#    """Irrigation simulator. Maximize leaf area index,
+#    so that you get a lot of primary production.
+#    Sequential or 30 workers."""
+##    funcs = [Irrigation(i, benin=benin, variety_choice=variety_choice) for i in range(23)]
+#    seedg = create_seed_generator(seed)
+#    optims = get_optimizers("basics", seed=next(seedg))
+#    optims = ["DiagonalCMA", "CMA", "DE", "PSO", "TwoPointsDE", "DiscreteLenglerOnePlusOne"]
+#    optims += ["NGOptRW", "NGTuned"]
+#    for budget in [250]: #, 50, 100, 200]:
+#        for num_workers in [1]: #, 30, 60]:
+#            if num_workers < budget:
+#                for algo in optims:
+#                    for fu in funcs:
+#                        xp = Experiment(fu, algo, budget, num_workers=num_workers, seed=next(seedg))
+#                        skip_ci(reason="Too slow")
+#                        if not xp.is_incoherent:
+#                            yield xp
+
+
 @registry.register
 def pcse(seed: tp.Optional[int] = None) -> tp.Iterator[Experiment]:
     """Crop simulator.
 
     Low dimensional problem, only 2 vars. This is optimization for model identification: we want
     to find parameters so that the simulation matches observations.
     """
-    funcs = [CropSimulator()]
+    funcs = [Pcse()]
     seedg = create_seed_generator(seed)
     optims = get_optimizers("basics", seed=next(seedg))
     for budget in [25, 50, 100, 200]:

nevergrad/functions/irrigation/irrigation.py

@@ -10,6 +10,8 @@
 """
 
 
+import copy
+from datetime import date
 from pathlib import Path
 import urllib.request
 import numpy as np
@@ -260,13 +262,96 @@ def __init__(
                     latitude=self.location.latitude, longitude=self.location.longitude
                 )
                 WPD[self.address] = self.weatherdataprovider
+            self.wdp_extend(self.weatherdataprovider)
             self.set_data(symmetry, k, rice)
             v = [self.meta_total_yield(np.random.rand(self.this_dimension)) for _ in range(5)]
             if min(v) != max(v):
                 break
             self.variant_choice[symmetry] = k
         print(f"we work on {self.cropname} with variety {self.cropvariety} in {self.address}.")
 
+    def wdp_extend(self, wdp):
+
+        my_keys = copy.deepcopy(list(wdp.store.keys()))
+
+        def we_have(y, m, d):
+            try:
+                date(y, m, d)
+            except ValueError:
+                return False
+            return (date(y, m, d), 0) in my_keys
+
+        def get_val(y, m, d, slot):
+            my_vals = [
+                getattr(wdp.store[(date(y_, m, d), 0)], slot)
+                for y_ in range(y - 4, y + 5)
+                if we_have(y_, m, d)
+            ]
+            assert len(my_vals) >= 3, f"big issue with data {y} {m} {d}!"
+            return sum(my_vals) / len(my_vals)
+
+        def extrapolate(wdp, y, m, od):
+            d = od
+            if m == 2 and d == 29:
+                d = 28
+            if we_have(y, m, d):
+                return
+            first_y = None
+            last_y = None
+            container = None
+            for y_ in range(y - 40, y + 1):
+                if not we_have(y_, m, d):
+                    continue
+                if we_have(y_, m, d):
+                    last_y = y_
+                    if first_y is None:
+                        first_y = y_
+            container = copy.deepcopy(wdp.store[(date(last_y, m, d), 0)])
+            assert container is not None, "No data found even in 40 years !"
+            for slot in container.__slots__:
+                if slot not in [
+                    "IRRAD",
+                    "TMIN",
+                    "TMAX",
+                    "VAP",
+                    "RAIN",
+                    "E0",
+                    "ES0",
+                    "ET0",
+                    "WIND",
+                    "SNOWDEPTH",
+                    "TEMP",
+                    "TMINRA",
+                ]:
+                    continue
+                try:
+                    value = get_val(last_y, m, d, slot)
+                except AttributeError:
+                    continue
+                delta = (get_val(last_y, m, d, slot) - get_val(first_y, m, d, slot)) / (last_y - first_y)
+                value += delta * (y - last_y)
+                vmin, vmax = container.ranges[slot]
+                if value < vmin:
+                    value = vmin
+                if value > vmax:
+                    value = vmax
+                setattr(container, slot, value)
+            wdp.store[(date(y, m, od), 0)] = container
+
+        # Extension for the future
+        for y in range(2020, 2050):
+            for m in range(1, 13):
+                for d in range(1, 32):
+                    try:
+                        date(y, m, d)
+                    except ValueError:
+                        continue
+                    extrapolate(self.weatherdataprovider, y, m, d)
+                    ok = [k for k in my_keys if k[0].year == y and k[0].month == m and k[0].day == d]
+                    assert len(ok) < 2
+                    assert len(ok) >= (1 if we_have(y, m, d) else 0)
+
+
     def set_data(self, symmetry: int, k: int, rice: bool):
         crop_types = [crop for crop, variety in self.cropd.get_crops_varieties().items()]
         crop_types = [c for c in crop_types if "obacco" not in c]
@@ -324,7 +409,7 @@ def total_yield(self, x: np.ndarray, year: int = 2006):
         ]
         c = self.total_irrigation
         if len(x) == 10:
-            c = 0
+            c = 0.0
         a0 = c * x[4] / (x[4] + x[5] + x[6] + x[7])
         a1 = c * x[5] / (x[4] + x[5] + x[6] + x[7])
         a2 = c * x[6] / (x[4] + x[5] + x[6] + x[7])

nevergrad/functions/pcse/__init__.py

@@ -5,4 +5,4 @@
 
 # Based on https://github.com/ajwdewit/pcse_notebooks
 # MIT License.
-from .pcse import CropSimulator as CropSimulator
+from .pcse import Pcse as Pcse

nevergrad/functions/pcse/pcse.py

@@ -146,6 +146,7 @@ def __call__(self, par_values, grad=None):
         # print("Objective function value with default parameters (%s): %s" % (defaults, error))
         TDWI_range = [0.1, 0.6]
         SPAN_range = [30, 40]
+
         param = ng.p.Array(
             shape=(2,), lower=(TDWI_range[0], SPAN_range[0]), upper=(TDWI_range[1], SPAN_range[1])
         ).set_name("2hp")

nevergrad/functions/pcse/test_pcse.py

@@ -8,7 +8,7 @@
 
 
 def test_pcse() -> None:
-    func = pcse.CropSimulator()
+    func = pcse.Pcse()
     x = 0 * np.random.rand(func.dimension)
     value = func(x)
     value2 = func(x)

nevergrad/optimization/test_optimizerlib.py

@@ -737,9 +737,11 @@ def test_ngopt_selection(
                 # assert choice == "MetaTuneRecentering"
                 assert "MetaTuneRecentering" in optim_string
             if num_workers > 1:
-                # assert choice not in ["SQP", "Cobyla"]
                 assert "SQP" not in optim_string and "Cobyla" not in optim_string
-        assert opt._info()["sub-optim"] in optim_string
+        if "CMA" not in choice:
+            assert choice == opt._info()["sub-optim"]
+        else:
+            assert choice in opt._info()["sub-optim"]
 
 
 def test_bo_ordering() -> None:

requirements/bench.txt

@@ -44,4 +44,4 @@ box2d-py>=2.3.5
 glfw
 mujoco
 pcse>=5.5.0
-nlopt
+nlopt