From e9a91bb45c6d4d74a18a5fe5f2d989c1c5e47999 Mon Sep 17 00:00:00 2001 From: Alexander Clausen Date: Wed, 13 Mar 2024 19:37:04 +0100 Subject: [PATCH 1/5] Add CI configuration - Support Python 3.12 - Drop support for old Python (which is already being dropped by LiberTEM) - Collect coverage during testing --- .github/dependabot.yml | 9 +++++++++ .github/workflows/ci.yml | 40 ++++++++++++++++++++++++++++++++++++++++ pyproject.toml | 7 +++---- 3 files changed, 52 insertions(+), 4 deletions(-) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/ci.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..2d67f9a --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,9 @@ +# Please see the documentation for all configuration options: +# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" # Location of package manifests + schedule: + interval: "weekly" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..544c0b2 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,40 @@ +name: CI + +on: + push: + branches: + - master + - 'v[0-9]*' + tags: + - 'v[0-9]*' + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + tests: + strategy: + matrix: + python_version: ["3.9", "3.10", "3.11", "3.12"] + os: [ubuntu-latest, windows-latest, macos-14] + exclude: + - os: macos-14 + python_version: "3.9" + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: Choose Python version ${{ matrix.python_version }} + uses: actions/setup-python@v5 + with: + python-version: '${{ matrix.python_version }}' + - name: install the package + run: python3 -m pip install .[test] + - name: run pytest + run: pytest --cov=microscope_calibration --cov-report=xml --cov-report=term tests/ + - name: submit code coverage + uses: codecov/codecov-action@v4 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + diff --git a/pyproject.toml b/pyproject.toml index acd87a5..f49a31a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "microscope-calibration" description = "Tools to calibrate a microscope" license = {file = "LICENSE"} keywords = ["microscopy", "LiberTEM", "TEM", "STEM"] -requires-python = ">=3.7" +requires-python = ">=3.9" dynamic = ["version", "readme"] dependencies = [ "numpy", @@ -20,11 +20,10 @@ dependencies = [ ] classifiers = [ "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Development Status :: 3 - Alpha", ] authors = [ @@ -35,7 +34,7 @@ authors = [ repository = "https://github.com/LiberTEM/Microscope-Calibration" [project.optional-dependencies] -test = ["pytest"] +test = ["pytest", "pytest-cov"] [tool.setuptools.dynamic] version = {attr = "microscope_calibration.__version__"} From 39c0a1349e01c1c8e51087764156b6d3ad7cdda6 Mon Sep 17 00:00:00 2001 From: Alexander Clausen Date: Wed, 13 Mar 2024 19:39:06 +0100 Subject: [PATCH 2/5] Fix some tests - `minimize` -> `optimize`; move `method` parameter into `minimizer_kwargs` - The `callback` now gets the current loss as fourth argument --- tests/test_overfocus.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/test_overfocus.py b/tests/test_overfocus.py index 0d11a5b..73e8ca7 100644 --- a/tests/test_overfocus.py +++ b/tests/test_overfocus.py @@ -9,7 +9,7 @@ from microscope_calibration.common.stem_overfocus import ( OverfocusParams, make_model, get_translation_matrix ) -from microscope_calibration.util.optimize import make_overfocus_loss_function, minimize +from microscope_calibration.util.optimize import make_overfocus_loss_function, optimize from microscope_calibration.udf.stem_overfocus import OverfocusUDF from libertem.api import Context @@ -441,14 +441,14 @@ def test_optimize(): dataset=ds, overfocus_udf=ref_udf, ) - res = minimize(loss=loss) + res = optimize(loss=loss) res_params = make_new_params(res.x) assert_allclose(res_params['scan_rotation'], params['scan_rotation'], atol=0.1) assert_allclose(res_params['overfocus'], params['overfocus'], rtol=0.1) valdict = {'val': False} - def callback(args, new_params, udf_results): + def callback(args, new_params, udf_results, current_loss): if valdict['val']: pass else: @@ -467,7 +467,10 @@ def callback(args, new_params, udf_results): extra_udfs=(OverfocusUDF(params), ), plots=(), ) - res = minimize(loss=loss, method='SLSQP', bounds=[(-10, 10), (-10, 10)]) + res = optimize( + loss=loss, minimizer_kwargs={'method': 'SLSQP'}, + bounds=[(-10, 10), (-10, 10)], + ) res_params = make_new_params(res.x) assert_allclose(res_params['scan_rotation'], params['scan_rotation'], atol=0.1) assert_allclose(res_params['overfocus'], params['overfocus'], rtol=0.1) From 6c6beb8c01440d21f1cdcc7de6d3f6f880a5c16d Mon Sep 17 00:00:00 2001 From: Alexander Clausen Date: Wed, 13 Mar 2024 19:41:48 +0100 Subject: [PATCH 3/5] Add pre-commit config and run it on all files --- .github/workflows/ci.yml | 1 - .pre-commit-config.yaml | 19 +++++++++++++++++++ packaging/.gitignore | 1 - packaging/build.sh | 2 +- packaging/run.sh | 1 - packaging/update-requirements.sh | 1 - src/microscope_calibration/util/optimize.py | 3 ++- .../util/stem_overfocus_sim.py | 4 ++-- 8 files changed, 24 insertions(+), 8 deletions(-) create mode 100644 .pre-commit-config.yaml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 544c0b2..db4cb3c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,4 +37,3 @@ jobs: uses: codecov/codecov-action@v4 env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..7d55d17 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,19 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-added-large-files +- repo: https://github.com/pycqa/flake8 + rev: 7.0.0 + hooks: + - id: flake8 +- repo: https://github.com/asottile/pyupgrade + rev: v3.15.1 + hooks: + - id: pyupgrade + args: ["--py39-plus"] diff --git a/packaging/.gitignore b/packaging/.gitignore index 92fa03a..1ca47ef 100644 --- a/packaging/.gitignore +++ b/packaging/.gitignore @@ -1,2 +1 @@ Microscope-Calibration - diff --git a/packaging/build.sh b/packaging/build.sh index d2f6b1e..0da7e85 100755 --- a/packaging/build.sh +++ b/packaging/build.sh @@ -6,5 +6,5 @@ mkdir packaging/Microscope-Calibration; \ git archive --format=tar HEAD | tar xvf - -C packaging/Microscope-Calibration \ ) -apptainer build --force overfocus.sif overfocus.def +apptainer build --force overfocus.sif overfocus.def apptainer overlay create --size 1024 overfocus.sif diff --git a/packaging/run.sh b/packaging/run.sh index 26c52a5..f5ffa45 100755 --- a/packaging/run.sh +++ b/packaging/run.sh @@ -1,3 +1,2 @@ #!/bin/sh apptainer run --writable overfocus.sif - diff --git a/packaging/update-requirements.sh b/packaging/update-requirements.sh index 895e303..16360d0 100755 --- a/packaging/update-requirements.sh +++ b/packaging/update-requirements.sh @@ -1,3 +1,2 @@ #!/bin/sh uv pip compile --python-version=3.11 requirements.in ../pyproject.toml > requirements.txt - diff --git a/src/microscope_calibration/util/optimize.py b/src/microscope_calibration/util/optimize.py index 8923770..fbd0044 100644 --- a/src/microscope_calibration/util/optimize.py +++ b/src/microscope_calibration/util/optimize.py @@ -1,7 +1,8 @@ import numpy as np from scipy. optimize import shgo from skimage.measure import blur_effect -from typing import TYPE_CHECKING, Iterable, Callable, Optional +from typing import TYPE_CHECKING, Callable, Optional +from collections.abc import Iterable from microscope_calibration.common.stem_overfocus import OverfocusParams diff --git a/src/microscope_calibration/util/stem_overfocus_sim.py b/src/microscope_calibration/util/stem_overfocus_sim.py index 84248a3..a2bcbff 100644 --- a/src/microscope_calibration/util/stem_overfocus_sim.py +++ b/src/microscope_calibration/util/stem_overfocus_sim.py @@ -21,7 +21,7 @@ def smiley(size): y, x = np.ogrid[-size//2:size//2, -size//2:size//2] outline = (((y*1.2)**2 + x**2) > (110/256*size)**2) & \ - ((((y*1.2)**2 + x**2) < (120/256*size)**2)) + (((y*1.2)**2 + x**2) < (120/256*size)**2) obj[outline] = 0.0 left_eye = ((y + 40/256*size)**2 + (x + 40/256*size)**2) < (20/256*size)**2 @@ -35,7 +35,7 @@ def smiley(size): obj[nose] = (0.05j * x + 0.05j * y)[nose] mouth = (((y*1)**2 + x**2) > (50/256*size)**2) & \ - ((((y*1)**2 + x**2) < (70/256*size)**2)) & \ + (((y*1)**2 + x**2) < (70/256*size)**2) & \ (y > 20/256*size) obj[mouth] = 0 From 05c690178894b21db36a42a81bf915949045c3d4 Mon Sep 17 00:00:00 2001 From: Alexander Clausen Date: Wed, 13 Mar 2024 19:56:03 +0100 Subject: [PATCH 4/5] Add additional `numba_coverage` job --- .github/workflows/ci.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index db4cb3c..1ed6308 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,3 +37,20 @@ jobs: uses: codecov/codecov-action@v4 env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + + numba_coverage: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: 'Choose Python version 3.12' + uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: install the package + run: python3 -m pip install .[test] + - name: run pytest + run: NUMBA_DISABLE_JIT=1 pytest --cov=microscope_calibration --cov-report=xml --cov-report=term tests/ + - name: submit code coverage + uses: codecov/codecov-action@v4 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} From ba7d4ef7edeb97c0b065843f437fe9dc97186ee6 Mon Sep 17 00:00:00 2001 From: Dieter Weber Date: Mon, 18 Mar 2024 10:09:56 +0100 Subject: [PATCH 5/5] Temporary patch for https://github.com/TemGym/TemGym/issues/13 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f49a31a..0b1850d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ dependencies = [ "numpy", "libertem", "typing-extensions", - "temgymbasic@git+https://github.com/TemGym/TemGym", + "temgymbasic@git+https://github.com/TemGym/TemGym@03062b098a50ea72f0ae9a2438b861d7a62691ee", "numba", "scikit-image", "scipy",