Bump Python / Numpy supported versions (#166)

benbovy · web-flow · commit 4a392cac9192 · 2024-10-10T11:11:05.000+02:00
* python-wheels: bump action versions * bump supported Python versions Follow https://scientific-python.org/specs/spec-0000/ * bump python version in conda environments * support numpy 2.0 Already supported by recents versions of pybind11 and xtensor-python, but we need to build against numpy 2.x in order to make fastscapelib-python work with numpy 1.xx and 2.x. * support numpy 1.24 at minimum Follow https://scientific-python.org/specs/spec-0000/ * bump xtensor(-python) downloaded versions * make numba dependency optional Allow building python 3.13 wheels without support for numba flow kernels for now. * fix import error + fix typing issue * fix doc builds (API) * doc: fix doxygen warnings * misc fixes * ci: add numpy 1.xx compat test * try fix doc build error No such error when building locally?! * fix pip install numpy version * try tracking doc build error on RTD * try fix RTD - add numba to doc environment - revert api hidden (it works locally so it should work on RTD) - doc envrionment: bump python version to 3.11 (maybe will solve the annotation warning similar to sphinx-doc/sphinx#11460 ?)
diff --git a/.github/workflows/python-wheels.yml b/.github/workflows/python-wheels.yml
@@ -19,7 +19,7 @@ jobs:
         run: pipx run build --sdist
 
       - name: Upload SDist
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           path: dist/*.tar.gz
 
@@ -35,7 +35,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.16
+        uses: pypa/cibuildwheel@v2.21
         env:
           # skip PyPy and muslinux (for now)
           CIBW_SKIP: "pp* *musllinux*"
@@ -60,7 +60,7 @@ jobs:
     if: github.event_name == 'release' && github.event.action == 'published'
     steps:
       - name: Get dist files
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: artifact
           path: dist
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -64,7 +64,7 @@ jobs:
       fail-fast: false
       matrix:
         os: ["ubuntu-20.04", "ubuntu-latest", "macos-latest", "windows-latest"]
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.10", "3.11", "3.12"]
     steps:
       - name: Checkout repo
         uses: actions/checkout@v3
@@ -85,3 +85,24 @@ jobs:
       - name: Run tests
         run: |
           pytest python/fastscapelib/tests -vv --color=yes
+
+  test_npy:
+    name: Test Python (Numpy 1.xx compat)
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v3
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Build and install Fastscapelib Python (Numpy 2.x)
+        run: |
+          python -m pip install . --config-settings=cmake.define.FS_DOWNLOAD_XTENSOR_PYTHON=ON
+
+      - name: Run tests (Numpy 1.xx)
+        run: |
+          python -m pip install numpy==1.26.4 pytest
+          pytest python/fastscapelib/tests -vv --color=yes
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -39,14 +39,14 @@ if(SKBUILD AND FS_DOWNLOAD_XTENSOR_PYTHON)
 
     FetchContent_Declare(xtensor
         GIT_REPOSITORY https://github.com/xtensor-stack/xtensor
-        GIT_TAG 0.24.6
+        GIT_TAG 0.25.0
         GIT_SHALLOW TRUE)
     set(CPP17 ON CACHE BOOL "Enable C++17 for xtensor" FORCE)
     FetchContent_MakeAvailable(xtensor)
 
     FetchContent_Declare(xtensor-python
         GIT_REPOSITORY https://github.com/xtensor-stack/xtensor-python
-        GIT_TAG 0.26.1
+        GIT_TAG 0.27.0
         GIT_SHALLOW TRUE)
     FetchContent_MakeAvailable(xtensor-python)
 
diff --git a/doc/environment.yml b/doc/environment.yml
@@ -2,13 +2,14 @@ name: fastscapelib-docs
 channels:
   - conda-forge
 dependencies:
-  - python=3.9
+  - python=3.11
   - breathe
   - cmake
   - xtensor-python
   - pybind11
   - scikit-build-core
   - numpy
+  - numba
   - matplotlib-base
   - pygalmesh
   - sphinx
diff --git a/doc/source/api_python/hidden.rst b/doc/source/api_python/hidden.rst
@@ -7,5 +7,5 @@
 
    MSTMethod
    MSTRouteMethod
-   flow.numba_flow_kernel.NumbaFlowKernel
-   flow.numba_flow_kernel.NumbaFlowKernelData
+   flow.numba.flow_kernel.NumbaFlowKernel
+   flow.numba.flow_kernel.NumbaFlowKernelData
diff --git a/doc/source/install.md b/doc/source/install.md
@@ -130,7 +130,7 @@ options.
 
 ## Install the Python Library
 
-Fastscapelib's Python package requires Python (3.9+) and [numpy].
+Fastscapelib's Python package requires Python (3.10+) and [numpy].
 
 (install-python-binary)=
 
diff --git a/environment-python-dev.yml b/environment-python-dev.yml
@@ -2,7 +2,7 @@ name: fastscapelib-python-dev
 channels:
   - conda-forge
 dependencies:
-  - python>=3.8
+  - python>=3.10
   - numpy
   - numba
   - xtensor-python
diff --git a/include/fastscapelib/flow/flow_graph.hpp b/include/fastscapelib/flow/flow_graph.hpp
@@ -251,7 +251,7 @@ namespace fastscapelib
          * @tparam FK The flow kernel type.
          * @tparam FKD The flow kernel data type.
          * @param kernel The flow kernel object to apply along the graph.
-         * @param kernel_data The object holding or referencing input and output data
+         * @param data The object holding or referencing input and output data
          * used by the flow kernel.
          *
          */
diff --git a/include/fastscapelib/utils/thread_pool.hpp b/include/fastscapelib/utils/thread_pool.hpp
@@ -57,6 +57,7 @@ namespace fastscapelib
              * @param first_index_ The first index in the range.
              * @param index_after_last_ The index after the last index in the range.
              * @param num_blocks_ The desired number of blocks to divide the range into.
+             * @param min_size_ The minimum size of each blocks (ignored if zero, default).
              */
             blocks(const T& first_index_,
                    const T& index_after_last_,
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,10 +1,11 @@
 [build-system]
 requires = [
     "scikit-build-core",
-    "pybind11",
-    # TODO: replace by "numpy>=1.25.0,<2" when dropping py3.8 support.
-    # https://github.com/scipy/oldest-supported-numpy/issues/76
-    "oldest-supported-numpy",
+    # pybind11 2.12 added support for numpy 2.0
+    # pybind11 doesn't require numpy at build time, but xtensor-python does!
+    # packages built with numpy 2.x are compatible with numpy 1.xx
+    "pybind11>=2.12,<3",
+    "numpy>=2.0,<3",
 ]
 build-backend = "scikit_build_core.build"
 
@@ -21,14 +22,17 @@ authors = [
 maintainers = [
     {name = "Fastscapelib contributors"},
 ]
-requires-python = ">=3.8"
-dependencies = ["numpy", "numba"]
+requires-python = ">=3.10"
+dependencies = [
+    "numpy>=1.24",
+]
 
 [project.urls]
 Home = "https://fastscapelib.readthedocs.io"
 Repository = "https://github.com/fastscape-lem/fastscapelib"
 
 [project.optional-dependencies]
+numba = ["numba"]
 test = ["pytest>=6.0"]
 
 [tool.scikit-build]
@@ -40,6 +44,12 @@ test-extras = "test"
 test-command = "pytest {project}/python/fastscapelib/tests"
 build-verbosity = 1
 
+[[tool.cibuildwheel.overrides]]
+# TODO: remove when numba supports python 3.13
+select = "cp3{10,11,12}-*"
+inherit.test-requires = "append"
+test-requires = ["numba"]
+
 [tool.cibuildwheel.macos]
 environment = {SKBUILD_CMAKE_ARGS='-DFS_DOWNLOAD_XTENSOR_PYTHON=ON', MACOSX_DEPLOYMENT_TARGET=10.13}
 
diff --git a/python/fastscapelib/eroders/flow_kernel_eroder.py b/python/fastscapelib/eroders/flow_kernel_eroder.py
@@ -1,14 +1,19 @@
-import inspect
+from __future__ import annotations
 
-import numba as nb
+import inspect
+from typing import TYPE_CHECKING, Any
 
 from fastscapelib.flow import FlowGraph, FlowGraphTraversalDir
-from fastscapelib.flow.numba_flow_kernel import (
-    NumbaFlowKernel,
-    NumbaFlowKernelData,
-    NumbaJittedClass,
-    create_flow_kernel,
-)
+from fastscapelib.flow.numba import create_flow_kernel
+
+if TYPE_CHECKING:
+    import numba as nb
+
+    from fastscapelib.flow.numba.flow_kernel import (
+        NumbaFlowKernel,
+        NumbaFlowKernelData,
+        NumbaJittedClass,
+    )
 
 
 class FlowKernelEroderMeta(type):
@@ -46,7 +51,7 @@ def __new__(cls, name, bases, dct):
 
 
 class FlowKernelEroder(metaclass=FlowKernelEroderMeta):
-    spec: dict[str, nb.types.Type]
+    spec: dict[str, nb.types.Type | tuple[nb.types.Type, Any]]
     outputs: list[str]
     apply_dir: FlowGraphTraversalDir
     _flow_graph: FlowGraph
diff --git a/python/fastscapelib/flow/__init__.py b/python/fastscapelib/flow/__init__.py
@@ -15,7 +15,7 @@
     _FlowKernelData,
 )
 
-from fastscapelib.flow.numba_flow_kernel import create_flow_kernel
+from fastscapelib.flow.numba import create_flow_kernel
 
 __all__ = [
     "FlowDirection",
diff --git a/python/fastscapelib/flow/__init__.pyi b/python/fastscapelib/flow/__init__.pyi
@@ -3,7 +3,7 @@ from typing import Any, ClassVar, List, Union, overload
 import numpy as np
 import numpy.typing as npt
 
-from fastscapelib.flow.numba_flow_kernel import NumbaFlowKernel, NumbaFlowKernelData
+from fastscapelib.flow.numba import NumbaFlowKernel, NumbaFlowKernelData
 from fastscapelib.grid import ProfileGrid, RasterGrid, TriMesh
 
 Grid = Union[ProfileGrid, RasterGrid, TriMesh]
diff --git a/python/fastscapelib/flow/numba/__init__.py b/python/fastscapelib/flow/numba/__init__.py
@@ -0,0 +1,86 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, Callable, Iterable
+
+if TYPE_CHECKING:
+    import numba as nb
+
+    from fastscapelib.flow import FlowGraph, FlowGraphTraversalDir
+    from fastscapelib.flow.numba.flow_kernel import (
+        NumbaFlowKernel,
+        NumbaFlowKernelData,
+        NumbaJittedClass,
+    )
+
+
+def create_flow_kernel(
+    flow_graph: FlowGraph,
+    kernel_func: Callable[[NumbaJittedClass], int],
+    spec: dict[str, nb.types.Type | tuple[nb.types.Type, Any]],
+    apply_dir: FlowGraphTraversalDir,
+    outputs: Iterable[str] = (),
+    max_receivers: int = -1,
+    n_threads: int = 1,
+    print_generated_code: bool = False,
+    print_stats: bool = False,
+) -> tuple[NumbaFlowKernel, NumbaFlowKernelData]:
+    """Creates a numba flow kernel.
+
+    Parameters
+    ----------
+    flow_graph : :py:class:`~fastscapelib.FlowGraph`
+        A flow graph object.
+    kernel_func : callable
+        A Python function to apply to each node of the graph. It must take one
+        input argument (numba jit-compiled class) that holds or references input
+        and output kernel data of one graph node. It should return an integer.
+    spec : dict
+        Dictionary where keys are kernel input and output variable names and
+        values are either variable types (i.e. numba scalar or array types) or
+        variable (type, value) tuples (only for scalar variables).
+    apply_dir : :py:class:`~fastscapelib.FlowGraphTraversalDir.`
+        The direction and order in which the flow kernel will be applied along
+        the graph.
+    outputs : iterable
+        The names of the kernel output variables. All names given here must be
+        also present in ``spec``.
+    max_receivers : int
+        Maximum number of flow receiver nodes per graph node. Setting this number
+        to 1 may speed up the application of the kernel function along the graph.
+        Setting this number to -1 (default) will use the maximum number defined
+        from the flow graph object and is generally safer.
+    n_threads : int
+        Number of threads to use for applying the kernel function in parallel
+        along the flow graph (default: 1, the kernel will be applied sequentially).
+        A value > 1 may be useful for kernel functions that are computationally
+        expensive. For trivial kernel functions it is generally more efficient to
+        apply the kernel sequentially (i.e., ``n_threads = 1``).
+    print_generated_code : bool
+        If True, prints the code used to create the numba jit-compiled classes for
+        managing kernel data (default: False). Useful for debugging.
+    print_stats : bool
+        If True, prints a small report on kernel creation performance
+        (default: False).
+
+    Returns
+    -------
+    flow_kernel : :py:class:`~fastscapelib.flow.numba.flow_kernel.NumbaFlowKernel`
+        An object used to apply the flow kernel.
+    flow_kernel_data : :py:class:`~fastscapelib.flow.numba.flow_kernel.NumbaFlowKernelData`
+        An object used to manage flow kernel data.
+
+    """
+    from fastscapelib.flow.numba.flow_kernel import NumbaFlowKernelFactory
+
+    factory = NumbaFlowKernelFactory(
+        flow_graph,
+        kernel_func,
+        spec,
+        apply_dir,
+        outputs=outputs,
+        max_receivers=max_receivers,
+        n_threads=n_threads,
+        print_generated_code=print_generated_code,
+        print_stats=print_stats,
+    )
+    return factory.kernel, factory.data
diff --git a/python/fastscapelib/flow/numba/flow_kernel.py b/python/fastscapelib/flow/numba/flow_kernel.py
diff --git a/python/fastscapelib/tests/test_flow_kernel.py b/python/fastscapelib/tests/test_flow_kernel.py
diff --git a/python/src/flow_graph.cpp b/python/src/flow_graph.cpp

Original file line number	Diff line number	Diff line change
`@@ -251,7 +251,7 @@ namespace fastscapelib`
`251`	`251`	`* @tparam FK The flow kernel type.`
`252`	`252`	`* @tparam FKD The flow kernel data type.`
`253`	`253`	`* @param kernel The flow kernel object to apply along the graph.`
`254`		`- * @param kernel_data The object holding or referencing input and output data`
	`254`	`+ * @param data The object holding or referencing input and output data`
`255`	`255`	`* used by the flow kernel.`
`256`	`256`	`*`
`257`	`257`	`*/`