Publish wheels for Python 3.13, fix CI

.circleci/config.yml

@@ -20,7 +20,7 @@ jobs:
             TWINE_USERNAME: __token__
           command: |
             # install deps
-            python3 -m pip install --user cibuildwheel==2.16.5 twine
+            python3 -m pip install --user cibuildwheel==2.23.0 twine
             # build wheels
             python3 -m cibuildwheel --output-dir wheelhouse
             # upload wheels to PyPI

.github/workflows/build.yml

@@ -23,44 +23,48 @@ jobs:
       matrix:
         os: [ubuntu-20.04, windows-2019, macos-13]
         cibw-build-prefix: ['*']
+        cibw-build-kind: [all]  # for artifact name only
         include:
           - os: ubuntu-20.04
             cibw-build-prefix: cp*
+            cibw-build-kind: cp
           - os: ubuntu-20.04
             cibw-build-prefix: pp*
+            cibw-build-kind: pp
         exclude:
           - os: ubuntu-20.04
             cibw-build-prefix: '*'
+            cibw-build-kind: all
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Restore mtimes of files in repo
         uses: chetan/[email protected]
       - name: Cargo home cache
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         env:
           cache-name: cargo-home
         with:
           path: ~/cargo-home-dirs
-          key: ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.cibw-build-prefix }}-v10.0
+          key: ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.cibw-build-prefix }}-v11.0
           restore-keys: |
-            ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.cibw-build-prefix }}-v10.
+            ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.cibw-build-prefix }}-v11.
       - name: Rust build cache
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         env:
           cache-name: rust-build
         with:
           path: ~/cargo-target-dirs
-          key: ${{ env.cache-name }}-${{ runner.os }}-v10.0-${{ matrix.cibw-build-prefix }}-${{ hashFiles('**/Cargo.toml') }}
+          key: ${{ env.cache-name }}-${{ runner.os }}-v11.0-${{ matrix.cibw-build-prefix }}-${{ hashFiles('**/Cargo.toml') }}
           restore-keys: |
-            ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.cibw-build-prefix }}-v10.
+            ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.cibw-build-prefix }}-v11.
       - name: Save env vars for cibuildwheel
         run: >
           echo "MAIN_DIR=$PWD" >> "$GITHUB_ENV";
           echo "OUTER_HOME=$HOME" >> "$GITHUB_ENV";
           cat "$GITHUB_ENV"
         shell: bash
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.16.5
+        uses: pypa/cibuildwheel@v2.23.0
         env:
           CIBW_BUILD: >
             ${{ matrix.cibw-build-prefix }}
@@ -92,6 +96,7 @@ jobs:
             CARGO_HOME_UNIX="$HOME/.cargo"
             RUSTUP_HOME_UNIX="$HOME/.rustup"
             CARGO_TARGET_DIR_UNIX="$HOME/target"
+            MACOSX_DEPLOYMENT_TARGET="10.12"
           CIBW_ENVIRONMENT_WINDOWS: >
             HOST_HOME_DIR="$HOME"
             HOST_PROJ_DIR="$GITHUB_WORKSPACE"
@@ -123,33 +128,36 @@ jobs:
             {delocate_archs} -w {dest_dir} {wheel};
             bash $PROJ_DIR/ci-utils/after-build.sh
           CIBW_BUILD_VERBOSITY: 1
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
+          name: artifact-${{ runner.os }}-${{ matrix.cibw-build-kind }}
           path: ./wheelhouse/*.whl
 
   build_sdist:
     name: Build source distribution
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Build sdist
         run: pipx run build --sdist
       - name: Install tool to post-process sdist to make extension optional
         run: pip install 'postprocess-sdist-make-rust-ext-opt>=0.2,<0.3'
       - name: Post-process sdist to make extension optional
         run: postprocess-sdist-make-rust-ext-opt -fO dist dist/*.tar.gz
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
+          name: artifact-source
           path: dist/*.tar.gz
 
   upload_pypi:
     needs: [build_wheels, build_sdist]
     runs-on: ubuntu-latest
     if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
     steps:
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
-          name: artifact
+          pattern: artifact-*
+          merge-multiple: true
           path: dist
       - uses: pypa/[email protected]
         with:

.github/workflows/test.yml

@@ -32,13 +32,13 @@ jobs:
             python-version: "pypy3.9"
             path: ~/.cache/pip
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
       - name: Restore Rust/Cargo cache
-        uses: actions/cache/restore@v3
+        uses: actions/cache/restore@v4
         with:
           path: |
             ~/.cargo/bin/
@@ -48,7 +48,7 @@ jobs:
             target/
           key: ${{ runner.os }}-rust-v4-${{ hashFiles('**/Cargo.lock') }}
       - name: Restore Python/pip cache
-        uses: actions/cache/restore@v3
+        uses: actions/cache/restore@v4
         with:
           path: ${{ matrix.path }}
           key: ${{ runner.os }}-python-v3-${{ hashFiles('**/requirements.txt') }}
@@ -68,7 +68,7 @@ jobs:
         run: |
           pytest
       - name: Save Rust/Cargo cache
-        uses: actions/cache/save@v3
+        uses: actions/cache/save@v4
         if: always()
         with:
           path: |
@@ -79,7 +79,7 @@ jobs:
             target/
           key: ${{ runner.os }}-rust-v4-${{ hashFiles('**/Cargo.lock') }}
       - name: Save Python/pip cache
-        uses: actions/cache/save@v3
+        uses: actions/cache/save@v4
         if: always()
         with:
           path: ${{ matrix.path }}

Cargo.toml

@@ -11,7 +11,7 @@ crate-type = ["cdylib"]
 [dependencies]
 num-bigint = ">=0.4.3,<0.5"
 owned_chars = ">=0.3.2,<0.4"
-pyo3 = { version = ">=0.16.5,<0.17", features = ["num-bigint"] }
+pyo3 = { version = ">=0.18,<0.19", features = ["num-bigint"] }
 thiserror = ">=1.0.37,<2"
 utf8-chars = ">=2.0.2,<3"
 compact_str = ">=0.7.1,<0.8"
@@ -24,7 +24,7 @@ utf8-width = ">=0.1.6,<0.2"
 rstest = ">=0.18.1,<0.19"
 
 [build-dependencies]
-pyo3-build-config = { version = "= 0.16.6", features = ["resolve-config"] }
+pyo3-build-config = { version = "= 0.18.3", features = ["resolve-config"] }
 
 # workaround for linkage errors when running cargo test:
 # https://pyo3.rs/v0.18.1/faq#i-cant-run-cargo-test-or-i-cant-build-in-a-cargo-workspace-im-having-linker-issues-like-symbol-not-found-or-undefined-reference-to-_pyexc_systemerror

ci-utils/before-build-linux.sh

@@ -1,5 +1,11 @@
 #!/bin/bash
 
+# install libatomic (required for newer rustup versions to work on older
+# CentOS-based manylinux images - may have to be removed again once
+# cibuildwheel updates the default manylinux image used from 2014 to sth.
+# newer)
+yum install -y libatomic
+
 # copy file mtimes from host
 shopt -s dotglob
 cp -r -a --attributes-only "$HOST_PROJ_DIR"/* "$PROJ_DIR"/

src/lib.rs

@@ -157,7 +157,7 @@ enum Token {
 #[pymethods]
 impl RustTokenizer {
     #[new]
-    #[args("*", buffering = -1, correct_cursor = "true")]
+    #[pyo3(signature = (stream, *, buffering = -1, correct_cursor = true))]
     fn new(stream: PyObject, buffering: i64, correct_cursor: bool) -> PyResult<Self> {
         let buffering_mode = if buffering < 0 {
             BufferingMode::DontCare