diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml new file mode 100644 index 0000000000..bcd30af93c --- /dev/null +++ b/.github/workflows/benchmarks.yml @@ -0,0 +1,78 @@ +# This workflow was adapted from xarray. See licenses/XARRAY_LICENSE for license details. +name: Benchmark + +on: + pull_request: + types: [opened, reopened, synchronize, labeled] + workflow_dispatch: + +env: + PR_HEAD_LABEL: ${{ github.event.pull_request.head.label }} + +jobs: + benchmark: + if: ${{ contains( github.event.pull_request.labels.*.name, 'run-benchmark') && github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' }} + name: Linux + runs-on: ubuntu-24.04 + env: + ASV_DIR: "./asv_bench" + CONDA_ENV_FILE: environment.yml + + steps: + # We need the full repo to avoid this issue + # https://github.com/actions/checkout/issues/23 + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up conda environment + uses: mamba-org/setup-micromamba@v2 + with: + micromamba-version: "1.5.10-0" + environment-file: ${{env.CONDA_ENV_FILE}} + environment-name: parcels-dev + cache-environment: true + cache-environment-key: "${{runner.os}}-${{runner.arch}}-py${{env.PYTHON_VERSION}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}-benchmark" + # add "build" because of https://github.com/airspeed-velocity/asv/issues/1385 + create-args: >- + python-build + asv + mamba<=1.5.10 + + - name: Run benchmarks + shell: bash -l {0} + id: benchmark + env: + OPENBLAS_NUM_THREADS: 1 + MKL_NUM_THREADS: 1 + OMP_NUM_THREADS: 1 + ASV_FACTOR: 1.5 + ASV_SKIP_SLOW: 1 + run: | + set -x + # ID this runner + asv machine --yes + echo "Baseline: ${{ github.event.pull_request.base.sha }} (${{ github.event.pull_request.base.label }})" + echo "Contender: ${GITHUB_SHA} ($PR_HEAD_LABEL)" + # Run benchmarks for current commit against base + ASV_OPTIONS="--split --factor $ASV_FACTOR" + asv continuous $ASV_OPTIONS ${{ github.event.pull_request.base.sha }} ${GITHUB_SHA} \ + | sed "/Traceback \|failed$\|PERFORMANCE DECREASED/ s/^/::error::/" \ + | tee benchmarks.log + # # Report and export results for subsequent steps + # if grep "Traceback \|failed\|PERFORMANCE DECREASED" benchmarks.log > /dev/null ; then + # exit 1 + # fi + working-directory: ${{ env.ASV_DIR }} + + - name: Add instructions to artifact + if: always() + run: | + cp benchmarks/README_CI.md benchmarks.log .asv/results/ + working-directory: ${{ env.ASV_DIR }} + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: asv-benchmark-results-${{ runner.os }} + path: ${{ env.ASV_DIR }}/.asv/results diff --git a/.gitignore b/.gitignore index c34fabf59c..26f8a5a2e5 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,10 @@ lib/ bin/ parcels_examples +# asv environments +asv_bench/.asv +asv_bench/pkgs + *.so *.log *.nc diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 96f33d1339..5b1cf0d621 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,6 +9,7 @@ repos: - id: check-json types: [text] files: \.(json|ipynb)$ + exclude: ^asv_bench/asv\.conf\.json$ - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.7.2 hooks: diff --git a/asv_bench/asv.conf.json b/asv_bench/asv.conf.json new file mode 100644 index 0000000000..c4f09d4bb4 --- /dev/null +++ b/asv_bench/asv.conf.json @@ -0,0 +1,206 @@ +{ + // The version of the config file format. Do not change, unless + // you know what you are doing. + "version": 1, + + // The name of the project being benchmarked + "project": "parcels", + + // The project's homepage + "project_url": "http://docs.oceanparcels.org/", + + // The URL or local path of the source code repository for the + // project being benchmarked + "repo": "https://github.com/oceanparcels/parcels.git", + + // The Python project's subdirectory in your repo. If missing or + // the empty string, the project is assumed to be located at the root + // of the repository. + // "repo_subdir": "", + + // Customizable commands for building, installing, and + // uninstalling the project. See asv.conf.json documentation. + // + // "install_command": ["in-dir={env_dir} python -mpip install {wheel_file}"], + // "uninstall_command": ["return-code=any python -mpip uninstall -y {project}"], + // fix for bad builds + // https://github.com/airspeed-velocity/asv/issues/1389#issuecomment-2076131185 + "build_command": [ + "python -m build", + "python -mpip wheel --no-deps --no-build-isolation --no-index -w {build_cache_dir} {build_dir}" + ], + // List of branches to benchmark. If not provided, defaults to "master" + // (for git) or "default" (for mercurial). + "branches": ["main"], // for git + // "branches": ["default"], // for mercurial + + // The DVCS being used. If not set, it will be automatically + // determined from "repo" by looking at the protocol in the URL + // (if remote), or by looking for special directories, such as + // ".git" (if local). + "dvcs": "git", + + // The tool to use to create environments. May be "conda", + // "virtualenv" or other value depending on the plugins in use. + // If missing or the empty string, the tool will be automatically + // determined by looking for tools on the PATH environment + // variable. + "environment_type": "mamba", + "conda_channels": ["conda-forge"], + + // timeout in seconds for installing any dependencies in environment + // defaults to 10 min + "install_timeout": 600, + + // the base URL to show a commit for the project. + "show_commit_url": "https://github.com/oceanparcels/parcels/commit/", + + // The Pythons you'd like to test against. If not provided, defaults + // to the current version of Python used to run `asv`. + "pythons": ["3.11"], + + // The list of conda channel names to be searched for benchmark + // dependency packages in the specified order + // "conda_channels": ["conda-forge", "defaults"], + + // A conda environment file that is used for environment creation. + // "conda_environment_file": "environment.yml", + + // The matrix of dependencies to test. Each key of the "req" + // requirements dictionary is the name of a package (in PyPI) and + // the values are version numbers. An empty list or empty string + // indicates to just test against the default (latest) + // version. null indicates that the package is to not be + // installed. If the package to be tested is only available from + // PyPi, and the 'environment_type' is conda, then you can preface + // the package name by 'pip+', and the package will be installed + // via pip (with all the conda available packages installed first, + // followed by the pip installed packages). + // + // The ``@env`` and ``@env_nobuild`` keys contain the matrix of + // environment variables to pass to build and benchmark commands. + // An environment will be created for every combination of the + // cartesian product of the "@env" variables in this matrix. + // Variables in "@env_nobuild" will be passed to every environment + // during the benchmark phase, but will not trigger creation of + // new environments. A value of ``null`` means that the variable + // will not be set for the current combination. + // + // "matrix": { + // "req": { + // "numpy": ["1.6", "1.7"], + // "six": ["", null], // test with and without six installed + // "pip+emcee": [""] // emcee is only available for install with pip. + // }, + // "env": {"ENV_VAR_1": ["val1", "val2"]}, + // "env_nobuild": {"ENV_VAR_2": ["val3", null]}, + // }, + + // using dependencies listed in parcels feedstock + "matrix": { + "setuptools_scm": [""], + "cftime": [""], + "cgen": [""], + "dask": [""], + "matplotlib-base": [""], + "netcdf4": [""], + "numpy": [""], + "platformdirs": [""], + "psutil": [""], + "pymbolic": [""], + "pytest": [""], + "scipy": [""], + // "trajan": [""], + "tqdm": [""], + "xarray": [""], + "zarr": [""] + }, + + // Combinations of libraries/python versions can be excluded/included + // from the set to test. Each entry is a dictionary containing additional + // key-value pairs to include/exclude. + // + // An exclude entry excludes entries where all values match. The + // values are regexps that should match the whole string. + // + // An include entry adds an environment. Only the packages listed + // are installed. The 'python' key is required. The exclude rules + // do not apply to includes. + // + // In addition to package names, the following keys are available: + // + // - python + // Python version, as in the *pythons* variable above. + // - environment_type + // Environment type, as above. + // - sys_platform + // Platform, as in sys.platform. Possible values for the common + // cases: 'linux2', 'win32', 'cygwin', 'darwin'. + // - req + // Required packages + // - env + // Environment variables + // - env_nobuild + // Non-build environment variables + // + // "exclude": [ + // {"python": "3.2", "sys_platform": "win32"}, // skip py3.2 on windows + // {"environment_type": "conda", "req": {"six": null}}, // don't run without six on conda + // {"env": {"ENV_VAR_1": "val2"}}, // skip val2 for ENV_VAR_1 + // ], + // + // "include": [ + // // additional env for python2.7 + // {"python": "2.7", "req": {"numpy": "1.8"}, "env_nobuild": {"FOO": "123"}}, + // // additional env if run on windows+conda + // {"platform": "win32", "environment_type": "conda", "python": "2.7", "req": {"libpython": ""}}, + // ], + + // The directory (relative to the current directory) that benchmarks are + // stored in. If not provided, defaults to "benchmarks" + // "benchmark_dir": "benchmarks", + + // The directory (relative to the current directory) to cache the Python + // environments in. If not provided, defaults to "env" + "env_dir": ".asv/env", + + // The directory (relative to the current directory) that raw benchmark + // results are stored in. If not provided, defaults to "results". + "results_dir": ".asv/results", + + // The directory (relative to the current directory) that the html tree + // should be written to. If not provided, defaults to "html". + "html_dir": ".asv/html" + + // The number of characters to retain in the commit hashes. + // "hash_length": 8, + + // `asv` will cache results of the recent builds in each + // environment, making them faster to install next time. This is + // the number of builds to keep, per environment. + // "build_cache_size": 2, + + // The commits after which the regression search in `asv publish` + // should start looking for regressions. Dictionary whose keys are + // regexps matching to benchmark names, and values corresponding to + // the commit (exclusive) after which to start looking for + // regressions. The default is to start from the first commit + // with results. If the commit is `null`, regression detection is + // skipped for the matching benchmark. + // + // "regressions_first_commits": { + // "some_benchmark": "352cdf", // Consider regressions only after this commit + // "another_benchmark": null, // Skip regression detection altogether + // }, + + // The thresholds for relative change in results, after which `asv + // publish` starts reporting regressions. Dictionary of the same + // form as in ``regressions_first_commits``, with values + // indicating the thresholds. If multiple entries match, the + // maximum is taken. If no entry matches, the default is 5%. + // + // "regressions_thresholds": { + // "some_benchmark": 0.01, // Threshold of 1% + // "another_benchmark": 0.5, // Threshold of 50% + // }, +} diff --git a/asv_bench/benchmarks/README_CI.md b/asv_bench/benchmarks/README_CI.md new file mode 100644 index 0000000000..9c35e8a93b --- /dev/null +++ b/asv_bench/benchmarks/README_CI.md @@ -0,0 +1,122 @@ +# Benchmark CI + + + + + +## How it works + +The `asv` suite can be run for any PR on GitHub Actions (check workflow `.github/workflows/benchmarks.yml`) by adding a `run-benchmark` label to said PR. This will trigger a job that will run the benchmarking suite for the current PR head (merged commit) against the PR base (usually `main`). + +We use `asv continuous` to run the job, which runs a relative performance measurement. This means that there's no state to be saved and that regressions are only caught in terms of performance ratio (absolute numbers are available but they are not useful since we do not use stable hardware over time). `asv continuous` will: + +- Compile `scikit-image` for _both_ commits. We use `ccache` to speed up the process, and `mamba` is used to create the build environments. +- Run the benchmark suite for both commits, _twice_ (since `processes=2` by default). +- Generate a report table with performance ratios: + - `ratio=1.0` -> performance didn't change. + - `ratio<1.0` -> PR made it slower. + - `ratio>1.0` -> PR made it faster. + +Due to the sensitivity of the test, we cannot guarantee that false positives are not produced. In practice, values between `(0.7, 1.5)` are to be considered part of the measurement noise. When in doubt, running the benchmark suite one more time will provide more information about the test being a false positive or not. + +## Running the benchmarks on GitHub Actions + +1. On a PR, add the label `run-benchmark`. +2. The CI job will be started. Checks will appear in the usual dashboard panel above the comment box. +3. If more commits are added, the label checks will be grouped with the last commit checks _before_ you added the label. +4. Alternatively, you can always go to the `Actions` tab in the repo and [filter for `workflow:Benchmark`](https://github.com/scikit-image/scikit-image/actions?query=workflow%3ABenchmark). Your username will be assigned to the `actor` field, so you can also filter the results with that if you need it. + +## The artifacts + +The CI job will also generate an artifact. This is the `.asv/results` directory compressed in a zip file. Its contents include: + +- `fv-xxxxx-xx/`. A directory for the machine that ran the suite. It contains three files: + - `.json`, `.json`: the benchmark results for each commit, with stats. + - `machine.json`: details about the hardware. +- `benchmarks.json`: metadata about the current benchmark suite. +- `benchmarks.log`: the CI logs for this run. +- This README. + +## Re-running the analysis + +Although the CI logs should be enough to get an idea of what happened (check the table at the end), one can use `asv` to run the analysis routines again. + +1. Uncompress the artifact contents in the repo, under `.asv/results`. This is, you should see `.asv/results/benchmarks.log`, not `.asv/results/something_else/benchmarks.log`. Write down the machine directory name for later. +2. Run `asv show` to see your available results. You will see something like this: + +``` +$> asv show + +Commits with results: + +Machine : Jaimes-MBP +Environment: conda-py3.9-cython-numpy1.20-scipy + + 00875e67 + +Machine : fv-az95-499 +Environment: conda-py3.7-cython-numpy1.17-pooch-scipy + + 8db28f02 + 3a305096 +``` + +3. We are interested in the commits for `fv-az95-499` (the CI machine for this run). We can compare them with `asv compare` and some extra options. `--sort ratio` will show largest ratios first, instead of alphabetical order. `--split` will produce three tables: improved, worsened, no changes. `--factor 1.5` tells `asv` to only complain if deviations are above a 1.5 ratio. `-m` is used to indicate the machine ID (use the one you wrote down in step 1). Finally, specify your commit hashes: baseline first, then contender! + +``` +$> asv compare --sort ratio --split --factor 1.5 -m fv-az95-499 8db28f02 3a305096 + +Benchmarks that have stayed the same: + + before after ratio + [8db28f02] [3a305096] + + n/a n/a n/a benchmark_restoration.RollingBall.time_rollingball_ndim + 1.23±0.04ms 1.37±0.1ms 1.12 benchmark_transform_warp.WarpSuite.time_to_float64(, 128, 3) + 5.07±0.1μs 5.59±0.4μs 1.10 benchmark_transform_warp.ResizeLocalMeanSuite.time_resize_local_mean(, (192, 192, 192), (192, 192, 192)) + 1.23±0.02ms 1.33±0.1ms 1.08 benchmark_transform_warp.WarpSuite.time_same_type(, 128, 3) + 9.45±0.2ms 10.1±0.5ms 1.07 benchmark_rank.Rank3DSuite.time_3d_filters('majority', (32, 32, 32)) + 23.0±0.9ms 24.6±1ms 1.07 benchmark_interpolation.InterpolationResize.time_resize((80, 80, 80), 0, 'symmetric', , True) + 38.7±1ms 41.1±1ms 1.06 benchmark_transform_warp.ResizeLocalMeanSuite.time_resize_local_mean(, (2048, 2048), (192, 192, 192)) + 4.97±0.2μs 5.24±0.2μs 1.05 benchmark_transform_warp.ResizeLocalMeanSuite.time_resize_local_mean(, (2048, 2048), (2048, 2048)) + 4.21±0.2ms 4.42±0.3ms 1.05 benchmark_rank.Rank3DSuite.time_3d_filters('gradient', (32, 32, 32)) + +... +``` + +If you want more details on a specific test, you can use `asv show`. Use `-b pattern` to filter which tests to show, and then specify a commit hash to inspect: + +``` +$> asv show -b time_to_float64 8db28f02 + +Commit: 8db28f02 + +benchmark_transform_warp.WarpSuite.time_to_float64 [fv-az95-499/conda-py3.7-cython-numpy1.17-pooch-scipy] + ok + =============== ============= ========== ============= ========== ============ ========== ============ ========== ============ + -- N / order + --------------- -------------------------------------------------------------------------------------------------------------- + dtype_in 128 / 0 128 / 1 128 / 3 1024 / 0 1024 / 1 1024 / 3 4096 / 0 4096 / 1 4096 / 3 + =============== ============= ========== ============= ========== ============ ========== ============ ========== ============ + numpy.uint8 2.56±0.09ms 523±30μs 1.28±0.05ms 130±3ms 28.7±2ms 81.9±3ms 2.42±0.01s 659±5ms 1.48±0.01s + numpy.uint16 2.48±0.03ms 530±10μs 1.28±0.02ms 130±1ms 30.4±0.7ms 81.1±2ms 2.44±0s 653±3ms 1.47±0.02s + numpy.float32 2.59±0.1ms 518±20μs 1.27±0.01ms 127±3ms 26.6±1ms 74.8±2ms 2.50±0.01s 546±10ms 1.33±0.02s + numpy.float64 2.48±0.04ms 513±50μs 1.23±0.04ms 134±3ms 30.7±2ms 85.4±2ms 2.55±0.01s 632±4ms 1.45±0.01s + =============== ============= ========== ============= ========== ============ ========== ============ ========== ============ + started: 2021-07-06 06:14:36, duration: 1.99m +``` + +## Other details + +### Skipping slow or demanding tests + +To minimize the time required to run the full suite, we trimmed the parameter matrix in some cases and, in others, directly skipped tests that ran for too long or require too much memory. Unlike `pytest`, `asv` does not have a notion of marks. However, you can `raise NotImplementedError` in the setup step to skip a test. In that vein, a new private function is defined at `benchmarks.__init__`: `_skip_slow`. This will check if the `ASV_SKIP_SLOW` environment variable has been defined. If set to `1`, it will raise `NotImplementedError` and skip the test. To implement this behavior in other tests, you can add the following attribute: + +```python +from . import _skip_slow # this function is defined in benchmarks.__init__ + +def time_something_slow(): + pass + +time_something.setup = _skip_slow +``` diff --git a/asv_bench/benchmarks/__init__.py b/asv_bench/benchmarks/__init__.py new file mode 100644 index 0000000000..e91191dd19 --- /dev/null +++ b/asv_bench/benchmarks/__init__.py @@ -0,0 +1,19 @@ +import os + + +def _skip_slow(): + """ + Use this function to skip slow or highly demanding tests. + + Use it as a `Class.setup` method or a `function.setup` attribute. + + Examples + -------- + >>> from . import _skip_slow + >>> def time_something_slow(): + ... pass + ... + >>> time_something.setup = _skip_slow + """ + if os.environ.get("ASV_SKIP_SLOW", "0") == "1": + raise NotImplementedError("Skipping this test...") diff --git a/asv_bench/benchmarks/benchmark_argofloat.py b/asv_bench/benchmarks/benchmark_argofloat.py new file mode 100644 index 0000000000..c795398ccc --- /dev/null +++ b/asv_bench/benchmarks/benchmark_argofloat.py @@ -0,0 +1,133 @@ +from datetime import timedelta + +import numpy as np +import xarray as xr + +from parcels import AdvectionRK4, FieldSet, JITParticle, ParticleSet, StatusCode, Variable + + +def ArgoVerticalMovement(particle, fieldset, time): + driftdepth = 1000 # maximum depth in m + maxdepth = 2000 # maximum depth in m + vertical_speed = 0.10 # sink and rise speed in m/s + cycletime = 10 * 86400 # total time of cycle in seconds + drifttime = 9 * 86400 # time of deep drift in seconds + + if particle.cycle_phase == 0: + # Phase 0: Sinking with vertical_speed until depth is driftdepth + particle_ddepth += vertical_speed * particle.dt + if particle.depth + particle_ddepth >= driftdepth: + particle_ddepth = driftdepth - particle.depth + particle.cycle_phase = 1 + + elif particle.cycle_phase == 1: + # Phase 1: Drifting at depth for drifttime seconds + particle.drift_age += particle.dt + if particle.drift_age >= drifttime: + particle.drift_age = 0 # reset drift_age for next cycle + particle.cycle_phase = 2 + + elif particle.cycle_phase == 2: + # Phase 2: Sinking further to maxdepth + particle_ddepth += vertical_speed * particle.dt + if particle.depth + particle_ddepth >= maxdepth: + particle_ddepth = maxdepth - particle.depth + particle.cycle_phase = 3 + + elif particle.cycle_phase == 3: + # Phase 3: Rising with vertical_speed until at surface + particle_ddepth -= vertical_speed * particle.dt + # particle.temp = fieldset.temp[time, particle.depth, particle.lat, particle.lon] # if fieldset has temperature + if particle.depth + particle_ddepth <= fieldset.mindepth: + particle_ddepth = fieldset.mindepth - particle.depth + # particle.temp = 0./0. # reset temperature to NaN at end of sampling cycle + particle.cycle_phase = 4 + + elif particle.cycle_phase == 4: + # Phase 4: Transmitting at surface until cycletime is reached + if particle.cycle_age > cycletime: + particle.cycle_phase = 0 + particle.cycle_age = 0 + + if particle.state == StatusCode.Evaluate: + particle.cycle_age += particle.dt # update cycle_age + + +class ArgoFloatJIT: + particle_type = JITParticle + + def setup(self): + self.runtime_days = 45 + time = np.datetime64("2025-01-01") + np.arange(self.runtime_days + 1) * np.timedelta64(1, "D") + lon = np.linspace(-180, 180, 120) + lat = np.linspace(-90, 90, 100) + + Lon, Lat = np.meshgrid(lon, lat) + + # Create large-scale gyre flow + U_gyre = np.cos(np.radians(Lat)) * np.sin(np.radians(Lon)) # Zonal flow + V_gyre = -np.sin(np.radians(Lat)) * np.cos(np.radians(Lon)) # Meridional flow + + f = 2 * 7.2921e-5 * np.sin(np.radians(Lat)) + + U_coriolis = U_gyre * (1 - 0.5 * np.abs(f)) + V_coriolis = V_gyre * (1 - 0.5 * np.abs(f)) + + noise_level = 0.1 # Adjust for more or less variability + U_noise = noise_level * np.random.randn(*U_coriolis.shape) + V_noise = noise_level * np.random.randn(*V_coriolis.shape) + + # Final realistic U and V velocity fields + U_final = U_coriolis + U_noise + V_final = V_coriolis + V_noise + + depth = np.linspace(0, 2000, 100) + + U_val = np.tile(U_final[None, None, :, :], (len(time), len(depth), 1, 1)) # Repeat for each time step + V_val = np.tile(V_final[None, None, :, :], (len(time), len(depth), 1, 1)) + + U = xr.DataArray( + U_val, + dims=["time", "depth", "lat", "lon"], + coords={"time": time, "depth": depth, "lat": lat, "lon": lon}, + name="U_velocity", + ) + + V = xr.DataArray( + V_val, + dims=["time", "depth", "lat", "lon"], + coords={"time": time, "depth": depth, "lat": lat, "lon": lon}, + name="V_velocity", + ) + + ds = xr.Dataset({"U": U, "V": V}) + + variables = { + "U": "U", + "V": "V", + } + dimensions = {"lat": "lat", "lon": "lon", "time": "time", "depth": "depth"} + fieldset = FieldSet.from_xarray_dataset(ds, variables, dimensions) + # uppermost layer in the hydrodynamic data + fieldset.mindepth = fieldset.U.depth[0] + # Define a new Particle type including extra Variables + + ArgoParticle = self.particle_type.add_variables( + [ + Variable("cycle_phase", dtype=np.int32, initial=0.0), + Variable("cycle_age", dtype=np.float32, initial=0.0), + Variable("drift_age", dtype=np.float32, initial=0.0), + ] + ) + + self.pset = ParticleSet(fieldset=fieldset, pclass=ArgoParticle, lon=[32], lat=[-31], depth=[0]) + + def time_run_many_timesteps(self): + self.pset.execute( + [ArgoVerticalMovement, AdvectionRK4], runtime=timedelta(days=self.runtime_days), dt=timedelta(seconds=30) + ) + + +# How do we derive benchmarks ? +# class ArgoFloatScipy(ArgoFloatJIT): +# particle_type = ScipyParticle diff --git a/asv_bench/benchmarks/benchmarks_integration.py b/asv_bench/benchmarks/benchmarks_integration.py new file mode 100644 index 0000000000..27c9540b7e --- /dev/null +++ b/asv_bench/benchmarks/benchmarks_integration.py @@ -0,0 +1,69 @@ +from datetime import timedelta + +import numpy as np + +from parcels import AdvectionRK4, FieldSet, ParticleSet, ScipyParticle + + +def time_advection2d(): + """Flat 2D zonal flow that increases linearly with depth from 0 m/s to 1 m/s. + + Time-taking variant. + """ + xdim = ydim = zdim = 2 + npart = 11 + + dimensions = { + "lon": np.linspace(0.0, 1e4, xdim, dtype=np.float32), + "lat": np.linspace(0.0, 1e4, ydim, dtype=np.float32), + "depth": np.linspace(0.0, 1.0, zdim, dtype=np.float32), + } + data = { + "U": np.ones((xdim, ydim, zdim), dtype=np.float32), + "V": np.zeros((xdim, ydim, zdim), dtype=np.float32), + } + data["U"][:, :, 0] = 0.0 + fieldset = FieldSet.from_data(data, dimensions, mesh="flat", transpose=True) + + pset = ParticleSet( + fieldset, + pclass=ScipyParticle, + lon=np.zeros(npart), + lat=np.zeros(npart) + 1e2, + depth=np.linspace(0, 1, npart), + ) + + pset.execute(AdvectionRK4, runtime=timedelta(hours=2), dt=timedelta(seconds=30)) + assert np.allclose(pset.depth * pset.time, pset.lon, atol=1.0e-1) + + +def peakmem_advection2d(): + """Flat 2D zonal flow that increases linearly with depth from 0 m/s to 1 m/s. + + Peak-Mem-taking variant. + """ + xdim = ydim = zdim = 2 + npart = 11 + + dimensions = { + "lon": np.linspace(0.0, 1e4, xdim, dtype=np.float32), + "lat": np.linspace(0.0, 1e4, ydim, dtype=np.float32), + "depth": np.linspace(0.0, 1.0, zdim, dtype=np.float32), + } + data = { + "U": np.ones((xdim, ydim, zdim), dtype=np.float32), + "V": np.zeros((xdim, ydim, zdim), dtype=np.float32), + } + data["U"][:, :, 0] = 0.0 + fieldset = FieldSet.from_data(data, dimensions, mesh="flat", transpose=True) + + pset = ParticleSet( + fieldset, + pclass=ScipyParticle, + lon=np.zeros(npart), + lat=np.zeros(npart) + 1e2, + depth=np.linspace(0, 1, npart), + ) + + pset.execute(AdvectionRK4, runtime=timedelta(hours=2), dt=timedelta(seconds=30)) + assert np.allclose(pset.depth * pset.time, pset.lon, atol=1.0e-1) diff --git a/asv_bench/benchmarks/benchmarks_nemo_curvilinear.py b/asv_bench/benchmarks/benchmarks_nemo_curvilinear.py new file mode 100644 index 0000000000..70836030fc --- /dev/null +++ b/asv_bench/benchmarks/benchmarks_nemo_curvilinear.py @@ -0,0 +1,53 @@ +from datetime import timedelta + +import numpy as np + +import parcels + +# ptype = {"scipy": parcels.ScipyParticle, "jit": parcels.JITParticle} +# advection = {"RK4": parcels.AdvectionRK4, "AA": parcels.AdvectionAnalytical} +# path_nemo = "~/Documents/PhD/projects/2025-02_parcels_benchmarking/NemoCurvilinear_data" +path_nemo = parcels.download_example_dataset("NemoCurvilinear_data") + + +class NemoCurvilinearJIT: + particle_type = parcels.JITParticle + + def setup(self): + filenames = { + "U": { + "lon": f"{path_nemo}/mesh_mask.nc4", + "lat": f"{path_nemo}/mesh_mask.nc4", + "data": f"{path_nemo}/U_purely_zonal-ORCA025_grid_U.nc4", + }, + "V": { + "lon": f"{path_nemo}/mesh_mask.nc4", + "lat": f"{path_nemo}/mesh_mask.nc4", + "data": f"{path_nemo}/V_purely_zonal-ORCA025_grid_V.nc4", + }, + } + variables = {"U": "U", "V": "V"} + + dimensions = {"lon": "glamf", "lat": "gphif", "time": "time_counter"} + + fieldset = parcels.FieldSet.from_nemo(filenames, variables, dimensions, allow_time_extrapolation=True) + + # Start 20 particles on a meridional line at 180W + npart = 20 + lonp = -180 * np.ones(npart) + latp = [i for i in np.linspace(-70, 85, npart)] + + self.pset = parcels.ParticleSet.from_list(fieldset, self.particle_type, lon=lonp, lat=latp) + # pfile = parcels.ParticleFile("nemo_particles", pset, outputdt=timedelta(days=1)) + + def time_run_experiment(self): + self.pset.execute( + parcels.AdvectionRK4, + runtime=timedelta(days=30), + dt=timedelta(hours=6), + # output_file=pfile, + ) + + +class NemoCurvilinearScipy(NemoCurvilinearJIT): + particle_type = parcels.ScipyParticle diff --git a/asv_bench/benchmarks/benchmarks_particle_execution.py b/asv_bench/benchmarks/benchmarks_particle_execution.py new file mode 100644 index 0000000000..47a7b47846 --- /dev/null +++ b/asv_bench/benchmarks/benchmarks_particle_execution.py @@ -0,0 +1,76 @@ +from datetime import timedelta + +import numpy as np + +from parcels import AdvectionRK4, FieldSet, JITParticle, ParticleSet, ScipyParticle + + +class ParticleExecutionJIT: + timeout = 240 + + def setup(self): + xdim = ydim = zdim = 2 + npart = 1_000 + + dimensions = { + "lon": np.linspace(0.0, 1e4, xdim, dtype=np.float32), + "lat": np.linspace(0.0, 1e4, ydim, dtype=np.float32), + "depth": np.linspace(0.0, 1.0, zdim, dtype=np.float32), + } + data = { + "U": np.ones((xdim, ydim, zdim), dtype=np.float32), + "V": np.zeros((xdim, ydim, zdim), dtype=np.float32), + } + data["U"][:, :, 0] = 0.0 + fieldset = FieldSet.from_data(data, dimensions, mesh="flat", transpose=True) + + self.pset = ParticleSet( + fieldset, + pclass=JITParticle, + lon=np.zeros(npart), + lat=np.zeros(npart) + 1e2, + depth=np.linspace(0, 1, npart), + ) + # trigger compilation + self.pset.execute(AdvectionRK4, runtime=0, dt=timedelta(seconds=5)) + + def time_run_single_timestep(self): + self.pset.execute(AdvectionRK4, runtime=timedelta(seconds=1 * 5), dt=timedelta(seconds=5)) + + def time_run_many_timesteps(self): + self.pset.execute(AdvectionRK4, runtime=timedelta(seconds=100 * 5), dt=timedelta(seconds=5)) + + +class ParticleExecutionScipy: + timeout = 240 + + def setup(self): + xdim = ydim = zdim = 2 + npart = 1_000 + + dimensions = { + "lon": np.linspace(0.0, 1e4, xdim, dtype=np.float32), + "lat": np.linspace(0.0, 1e4, ydim, dtype=np.float32), + "depth": np.linspace(0.0, 1.0, zdim, dtype=np.float32), + } + data = { + "U": np.ones((xdim, ydim, zdim), dtype=np.float32), + "V": np.zeros((xdim, ydim, zdim), dtype=np.float32), + } + data["U"][:, :, 0] = 0.0 + fieldset = FieldSet.from_data(data, dimensions, mesh="flat", transpose=True) + + self.pset = ParticleSet( + fieldset, + pclass=ScipyParticle, + lon=np.zeros(npart), + lat=np.zeros(npart) + 1e2, + depth=np.linspace(0, 1, npart), + ) + self.pset.execute(AdvectionRK4, runtime=timedelta(seconds=1 * 5), dt=timedelta(seconds=5)) + + def time_run_single_timestep(self): + self.pset.execute(AdvectionRK4, runtime=timedelta(seconds=1 * 5), dt=timedelta(seconds=5)) + + def time_run_many_timesteps(self): + self.pset.execute(AdvectionRK4, runtime=timedelta(seconds=100 * 5), dt=timedelta(seconds=5)) diff --git a/docs/community/benchmarking.md b/docs/community/benchmarking.md new file mode 100644 index 0000000000..32b074c3f4 --- /dev/null +++ b/docs/community/benchmarking.md @@ -0,0 +1,22 @@ +# Benchmarking + +Parcels comes with an [asv](https://asv.readthedocs.io/en/latest/) benchmarking suite to monitor the performance of the project over it's lifespan. + +The benchmarking is run in CI using GitHub Actions (similar to other projects like xarray, scikit-image, and pandas), using a ratio to determine performance regressions instead of raw outputs. More on the reliability of benchmarking in CI can be seen at [this blog post](https://labs.quansight.org/blog/2021/08/github-actions-benchmarks). Due to the reliance of CI for benchmarking, the benchmarks are small such that they test the core functionality of Parcels. Large scale simulation benchmarking is an avenue for future development. + +## Setup + +The asv benchmarks require these dependencies: + +`conda install -c conda-forge asv>0.6 libmambapy<2 conda-build` + +Progress on compatibility of asv with libmambapy 2 is documented at [this issue](https://github.com/airspeed-velocity/asv/issues/1438). + +## Running the benchmarks + +The benchmarks are located in the `asv_bench` folder can be run locally using the following commands: + +```bash +cd asv_bench +asv run +``` diff --git a/docs/community/contributing.rst b/docs/community/contributing.rst index f8728d1df6..c61da113ce 100644 --- a/docs/community/contributing.rst +++ b/docs/community/contributing.rst @@ -74,7 +74,21 @@ From there: - create a git branch, implement, commit, and push your changes - `create a pull request `_ (PR) into ``master`` of the original repo making sure to link to the issue that you are working on. Not yet finished with your feature but still want feedback on how you're going? Then mark it as "draft" and ``@ping`` a maintainer. See our `maintainer notes `_ to see our PR review workflow. -If you made changes to the documentation, and want to render a local version, you can run the command ``sphinx-autobuild --ignore "*.zip" docs docs/_build`` to create a server to automatically rebuild the documentation when you make changes. +Here is a short overview of a few different commands that we use during development: + +.. code-block:: bash + + # Run unit-tests + pytest + + # Run typechecking + mypy + + # Build the documentation. This launches a server, automatically rebuilt when changes are made + sphinx-autobuild --ignore "*.zip" docs docs/_build + + +To run performance benchmarks, see the `the benchmarking page `_. Code guidelines ~~~~~~~~~~~~~~~ diff --git a/docs/community/index.rst b/docs/community/index.rst index 3121d6cab9..0ccdd3bb12 100644 --- a/docs/community/index.rst +++ b/docs/community/index.rst @@ -18,3 +18,4 @@ See the sections in the primary sidebar and below to explore. :maxdepth: 1 maintainer + benchmarking