Merge branch 'develop' into feature/suppress_xarray_cast_warnings

.github/scripts/make_release.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 """This script is part of the GitHub CI make-release pipeline
 
-It reads the version number from climada/_version.py and then uses the `gh` cli
+It reads the version number from climada*/_version.py and then uses the `gh` cli
 to create the new release.
 
 """

.github/scripts/prepare_release.py

@@ -39,8 +39,11 @@ def bump_version_number(version_number: str, level: str) -> str:
     major, minor, patch = version_number.split(".")
     if level == "major":
         major = str(int(major)+1)
+        minor = "0"
+        patch = "0"
     elif level == "minor":
         minor = str(int(minor)+1)
+        patch = "0"
     elif level == "patch":
         patch = str(int(patch)+1)
     else:
@@ -111,7 +114,7 @@ def update_changelog(nvn):
                             if "release date: " in line.lower():
                                 today = time.strftime("%Y-%m-%d")
                                 lines[i] = f"Release date: {today}"
-                    changelog.write("\n".join(lines).replace("\n\n", "\n"))
+                    changelog.write(re.sub("\n+$", "\n", "\n".join(lines)))
                     changelog.write("\n")
     return GitFile('CHANGELOG.md')
 

.github/scripts/setup_devbranch.py

@@ -104,8 +104,9 @@ def setup_devbranch():
     Just changes files, all `git` commands are in the setup_devbranch.sh file.
     """
     main_version = get_last_version().strip('v')
-
-    dev_version = f"{main_version}-dev"
+    semver = main_version.split(".")
+    semver[-1] = f"{int(semver[-1]) + 1}-dev"
+    dev_version = ".".join(semver)
 
     update_setup(dev_version)
     update_version(dev_version)

.github/workflows/ci.yml

@@ -0,0 +1,70 @@
+name: GitHub CI
+
+# Execute this for every push
+on: [push]
+
+# Use bash explicitly for being able to enter the conda environment
+defaults:
+  run:
+    shell: bash -l {0}
+
+jobs:
+  build-and-test:
+    name: Build Env, Install, Unit Tests
+    runs-on: ubuntu-latest
+    permissions:
+      # For publishing results
+      checks: write
+
+    # Run this test for different Python versions
+    strategy:
+      # Do not abort other tests if only a single one fails
+      fail-fast: false
+      matrix:
+        python-version: ["3.9", "3.10", "3.11"]
+
+    steps:
+      -
+        name: Checkout Repo
+        uses: actions/checkout@v3
+      -
+        # Store the current date to use it as cache key for the environment
+        name: Get current date
+        id: date
+        run: echo "date=$(date +%Y-%m-%d)" >> "${GITHUB_OUTPUT}"
+      -
+        name: Create Environment with Mamba
+        uses: mamba-org/setup-micromamba@v1
+        with:
+          environment-name: climada_env_${{ matrix.python-version }}
+          environment-file: requirements/env_climada.yml
+          create-args: >-
+            python=${{ matrix.python-version }}
+            make
+          init-shell: >-
+            bash
+          # Persist environment for branch, Python version, single day
+          cache-environment-key: env-${{ github.ref }}-${{ matrix.python-version }}-${{ steps.date.outputs.date }}
+      -
+        name: Install CLIMADA
+        run: |
+          python -m pip install ".[test]"
+      -
+        name: Run Unit Tests
+        run: |
+          make unit_test
+      -
+        name: Publish Test Results
+        uses: EnricoMi/publish-unit-test-result-action@v2
+        if: always()
+        with:
+          junit_files: tests_xml/tests.xml
+          check_name: "Unit Test Results Python ${{ matrix.python-version }}"
+          comment_mode: "off"
+      -
+        name: Upload Coverage Reports
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: coverage-report-unittests-py${{ matrix.python-version }}
+          path: coverage/

.github/workflows/make-release.yml

@@ -28,10 +28,10 @@ jobs:
         with:
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
       - name: Prepare new release
-        run: .github/scripts/prepare_release.py ${{ inputs.level }}
+        run: python .github/scripts/prepare_release.py ${{ inputs.level }}
         env:
           GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
       - name: Publish
-        run: .github/scripts/make_release.py
+        run: python .github/scripts/make_release.py
         env:
           GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}

.github/workflows/postrelease-setup-devbranch.yml

@@ -19,6 +19,6 @@ jobs:
         with:
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
       - name: Update files
-        run: .github/scripts/setup_devbranch.sh
+        run: bash .github/scripts/setup_devbranch.sh
         env:
           GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}

CHANGELOG.md

@@ -6,20 +6,99 @@ Release date: YYYY-MM-DD
 
 Code freeze date: YYYY-MM-DD
 
+### Description
+
+### Dependency Changes
+
+### Added
+
+### Changed
+
+### Fixed
+
+### Deprecated
+
+### Removed
+
+## 4.0.1
+
+Release date: 2023-09-27
+
+### Dependency Changes
+
+Added:
+
+- `matplotlib-base` None → >=3.8
+
+Changed:
+
+- `geopandas` >=0.13 → >=0.14
+- `pandas` >=1.5,<2.0 &rarr; >=2.1
+
+Removed:
+
+- `matplotlib` >=3.7
+
+### Changed
+
+- Rearranged file-system structure: `data` directory moved into `climada` package directory. [#781](https://github.com/CLIMADA-project/climada_python/pull/781)
+
+### Fixed
+
+- `climada.util.coordinates.get_country_code` bug, occurring with non-standard longitudinal coordinates around the anti-meridian. [#770](https://github.com/CLIMADA-project/climada_python/issues/770)
+
+## 4.0.0
+
+Release date: 2023-09-01
+
 ### Dependency Updates
 
 Added:
 
 - `pytest` [#726](https://github.com/CLIMADA-project/climada_python/pull/726)
 - `pytest-cov` [#726](https://github.com/CLIMADA-project/climada_python/pull/726)
 - `pytest-subtests` [#726](https://github.com/CLIMADA-project/climada_python/pull/726)
+- `unittest-xml-reporting`
 
 Changed:
 
+- `cartopy` >=0.20.0,<0.20.3 &rarr; >=0.21
+- `cfgrib` >=0.9.7,<0.9.10 &rarr; =0.9.9
+- `contextily` >=1.0 &rarr; >=1.3
+- `dask` >=2.25 &rarr; >=2023
+- `eccodes` [auto] &rarr; =2.27
+- `gdal` !=3.4.1 &rarr; >=3.6
+- `geopandas` >=0.8 &rarr; >=0.13
+- `h5py` >=2.10 &rarr; >=3.8
+- `haversine` >=2.3 &rarr; >=2.8
+- `matplotlib` >=3.2,< 3.6 &rarr; >=3.7
+- `netcdf4` >=1.5 &rarr; >=1.6
+- `numba` >=0.51,!=0.55.0 &rarr; >=0.57
+- `openpyxl` >=3.0 &rarr; >=3.1
+- `pandas-datareader` >=0.9 &rarr; >=0.10
+- `pathos` >=0.2 &rarr; >=0.3
+- `pint` >=0.15 &rarr; >=0.22
+- `proj` !=9.0.0 &rarr; >=9.1
+- `pycountry` >=20.7 &rarr; >=22.3
+- `pytables` >=3.6 &rarr; >=3.7
+- `rasterio` >=1.2.7,<1.3 &rarr; >=1.3
+- `requests` >=2.24 &rarr; >=2.31
+- `salib` >=1.3.0 &rarr; >=1.4
+- `scikit-learn` >=1.0 &rarr; >=1.2
+- `scipy` >=1.6 &rarr; >=1.10
+- `sparse` >=0.13 &rarr; >=0.14
+- `statsmodels` >=0.11 &rarr; >=0.14
+- `tabulate` >=0.8 &rarr; >=0.9
+- `tqdm` >=4.48 &rarr; >=4.65
+- `xarray` >=0.13 &rarr; >=2023.5
+- `xlrd` >=1.2 &rarr; >=2.0
+- `xlsxwriter` >=1.3 &rarr; >=3.1
+
 Removed:
 
 - `nbsphinx` [#712](https://github.com/CLIMADA-project/climada_python/pull/712)
 - `pandoc` [#712](https://github.com/CLIMADA-project/climada_python/pull/712)
+- `xmlrunner`
 
 ### Added
 
@@ -29,7 +108,10 @@ Removed:
 - 'Extra' requirements `doc`, `test`, and `dev` for Python package [#712](https://github.com/CLIMADA-project/climada_python/pull/712)
 - Added method `Exposures.centroids_total_value` to replace the functionality of `Exposures.affected_total_value`. This method is temporary and deprecated. [#702](https://github.com/CLIMADA-project/climada_python/pull/702)
 - New method `climada.util.api_client.Client.purge_cache`: utility function to remove outdated files from the local file system to free disk space.
-  ([#737](https://github.com/CLIMADA-project/climada_python/pull/737))
+([#737](https://github.com/CLIMADA-project/climada_python/pull/737))
+- New attribute `climada.hazard.Hazard.haz_type`: used for assigning impacts to hazards. In previous versions this information was stored in the now removed `climada.hazard.tag.Tag` class. [#736](https://github.com/CLIMADA-project/climada_python/pull/736)
+- New attribute `climada.entity.exposures.Exposures.description`: used for setting the default title in plots from plotting mathods `plot_hexbin` and `plot_scatter`. In previous versions this information was stored in the deprecated `climada.entity.tag.Tag` class. [#756](https://github.com/CLIMADA-project/climada_python/pull/756)
+- Added advanced examples in unsequa tutorial for coupled input variables and for handling efficiently the loading of multiple large files [#766](https://github.com/CLIMADA-project/climada_python/pull/766)
 
 ### Changed
 
@@ -50,30 +132,47 @@ Removed:
 - Use `pytest` for executing tests [#726](https://github.com/CLIMADA-project/climada_python/pull/726)
 - Users can opt-out of the climada specific logging definitions and freely configure logging to their will, by setting the config value `logging.managed` to `false`. [#724](https://github.com/CLIMADA-project/climada_python/pull/724)
 - Add option to read additional variables from IBTrACS when using `TCTracks.from_ibtracs_netcdf` [#728](https://github.com/CLIMADA-project/climada_python/pull/728)
-- The `haz_type` attribute has been moved from `climada.hazard.tag.Tag` to `climada.hazard.Hazard` itself. [#736](https://github.com/CLIMADA-project/climada_python/pull/736)
 - New file format for `TCTracks` I/O with better performance. This change is not backwards compatible: If you stored `TCTracks` objects with `TCTracks.write_hdf5`, reload the original data and store them again. [#735](https://github.com/CLIMADA-project/climada_python/pull/735)
 - Add option to load only a subset when reading TC tracks using `TCTracks.from_simulations_emanuel`. [#741](https://github.com/CLIMADA-project/climada_python/pull/741)
 - Set `save_mat` to `False` in the `unsequa` module [#746](https://github.com/CLIMADA-project/climada_python/pull/746)
 - `list_dataset_infos` from `climada.util.api_client.Client`: the `properties` argument, a `dict`, can now have `None` as values. Before, only strings and lists of strings were allowed. Setting a particular property to `None` triggers a search for datasets where this property is not assigned. [#752](https://github.com/CLIMADA-project/climada_python/pull/752)
 - Reduce memory requirements of `TropCyclone.from_tracks` [#749](https://github.com/CLIMADA-project/climada_python/pull/749)
 - Support for different wind speed and pressure units in `TCTracks` when running `TropCyclone.from_tracks` [#749](https://github.com/CLIMADA-project/climada_python/pull/749)
+- The title of plots created by the `Exposures` methods `plot_hexbin` and `plot_scatter` can be set as a method argument. [#756](https://github.com/CLIMADA-project/climada_python/pull/756)
+- Changed the parallel package from Pathos to Multiproess in the unsequa module [#763](https://github.com/CLIMADA-project/climada_python/pull/763)
+- Updated installation instructions to use conda for core and petals [#776](https://github.com/CLIMADA-project/climada_python/pull/776)
 
 ### Fixed
 
 - `util.lines_polys_handler` solve polygon disaggregation issue in metre-based projection [#666](https://github.com/CLIMADA-project/climada_python/pull/666)
 - Problem with `pyproj.CRS` as `Impact` attribute, [#706](https://github.com/CLIMADA-project/climada_python/issues/706). Now CRS is always stored as `str` in WKT format.
+- Correctly handle assertion errors in `Centroids.values_from_vector_files` and fix the associated test [#768](https://github.com/CLIMADA-project/climada_python/pull/768/)
+- Text in `Forecast` class plots can now be adjusted [#769](https://github.com/CLIMADA-project/climada_python/issues/769)
+- `Impact.impact_at_reg` now supports impact matrices where all entries are zero [#773](https://github.com/CLIMADA-project/climada_python/pull/773)
+- upgrade pathos 0.3.0 -> 0.3.1 issue [#761](https://github.com/CLIMADA-project/climada_python/issues/761) (for unsequa module [#763](https://github.com/CLIMADA-project/climada_python/pull/763))
+- Fix bugs with pandas 2.0 (iteritems -> items, append -> concat) (fix issue [#700](https://github.com/CLIMADA-project/climada_python/issues/700) for unsequa module) [#763](https://github.com/CLIMADA-project/climada_python/pull/763))
+- Remove matplotlib styles in unsequa module (fixes issue [#758](https://github.com/CLIMADA-project/climada_python/issues/758)) [#763](https://github.com/CLIMADA-project/climada_python/pull/763)
 
 ### Deprecated
 
 - `Centroids.from_geodataframe` and `Centroids.from_pix_bounds` [#721](https://github.com/CLIMADA-project/climada_python/pull/721)
 - `Impact.tot_value`: Use `Exposures.affected_total_value` to compute the total value affected by a hazard intensity above a custom threshold [#702](https://github.com/CLIMADA-project/climada_python/pull/702)
-- `climada.hazard.tag.Tag` and `climada.entity.tag.Tag`. [#736](https://github.com/CLIMADA-project/climada_python/pull/736). They were unified into `climada.util.tag.Tag`. Note: the latter is to be deprecated and removed in a future version as well.
+- `climada.entity.tag.Tag`. [#779](https://github.com/CLIMADA-project/climada_python/pull/779). The class is not used anymore but had to be kept for reading Exposures HDF5 files that were created with previous versions of CLIMADA.
 
 ### Removed
 
 - `Centroids.set_raster_from_pix_bounds` [#721](https://github.com/CLIMADA-project/climada_python/pull/721)
 - `requirements/env_developer.yml` environment specs. Use 'extra' requirements when installing the Python package instead [#712](https://github.com/CLIMADA-project/climada_python/pull/712)
-- `Impact.tag` attribute. This change is not backwards-compatible with respect to the files written and read by the `Impact` class [#743](https://github.com/CLIMADA-project/climada_python/pull/743)
+- The `climada.entitity.tag.Tag` class, together with `Impact.tag`, `Exposures.tag`, `ImpactFuncSet.tag`, `MeasuresSet.tag`, `Hazard.tag` attributes.
+This may break backwards-compatibility with respect to the files written and read by the `Impact` class.
+[#736](https://github.com/CLIMADA-project/climada_python/pull/736),
+[#743](https://github.com/CLIMADA-project/climada_python/pull/743),
+[#753](https://github.com/CLIMADA-project/climada_python/pull/753),
+[#754](https://github.com/CLIMADA-project/climada_python/pull/754),
+[#756](https://github.com/CLIMADA-project/climada_python/pull/756),
+[#767](https://github.com/CLIMADA-project/climada_python/pull/767),
+[#779](https://github.com/CLIMADA-project/climada_python/pull/779)
+- `impact.tot_value` attribute removed from unsequa module [#763](https://github.com/CLIMADA-project/climada_python/pull/763)
 
 ## v3.3.2
 
@@ -101,10 +200,6 @@ Patch-relaese with altered base config file so that the basic installation test
 
 Release date: 2023-02-17
 
-Code freeze date: 2023-02-05
-
-### Description
-
 ### Dependency Changes
 
 new:
@@ -194,4 +289,3 @@ updated:
 - `climada.enginge.impact.Impact.calc()` and `climada.enginge.impact.Impact.calc_impact_yearset()`
 [#436](https://github.com/CLIMADA-project/climada_python/pull/436).
 
-### Removed

Makefile

@@ -29,11 +29,11 @@ install_test : ## Test installation was successful
 
 .PHONY : data_test
 data_test : ## Test data APIs
-	python test_data_api.py
+	python script/jenkins/test_data_api.py
 
 .PHONY : notebook_test
 notebook_test : ## Test notebooks in doc/tutorial
-	python test_notebooks.py
+	python script/jenkins/test_notebooks.py report
 
 .PHONY : integ_test
 integ_test : ## Integration tests execution with xml reports