diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7b31e10..6c66f0f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -62,7 +62,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -v -e .[test] + pip install -v -e .[test,stubtest] shell: bash - name: Run Cargo tests run: | @@ -72,6 +72,23 @@ jobs: - name: Run pytest tests run: | pytest + - name: Run Mypy stubtest + # XXX doesn't work on Pypy due to some symtable issues and on Windows + # with Python 3.8 due to pathlib issues... + if: > + !startsWith(matrix.python-version, 'pypy') + && !( + startsWith(matrix.python-version, '3.8') + && startsWith(matrix.os, 'windows-') + ) + run: > + stubtest + ${{ + !startsWith(matrix.python-version, '3.8') + && '--ignore-disjoint-bases' + || '' + }} + --allowlist stubtest-allowlist json_stream_rs_tokenizer - name: Save Rust/Cargo cache uses: actions/cache/save@v4 if: always() diff --git a/json_stream_rs_tokenizer/__init__.py b/json_stream_rs_tokenizer/__init__.py index 9ad8d6e..31140d0 100644 --- a/json_stream_rs_tokenizer/__init__.py +++ b/json_stream_rs_tokenizer/__init__.py @@ -24,14 +24,14 @@ class TokenType: # included only for backwards-compatibility - to the outside world, bigint # is now always supported via fallback to conversion in Python - def supports_bigint(): + def supports_bigint() -> bool: return True if _supports_bigint(): RustTokenizer = _RustTokenizer else: - class RustTokenizer: + class RustTokenizer: # type: ignore[no-redef] """ Rust tokenizer (fallback wrapper for integer conversion) """ @@ -111,7 +111,7 @@ def load(fp, persistent=False): """ Run json-stream's `load` but using the Rust tokenizer. """ - import json_stream + import json_stream # type: ignore[import-untyped] return json_stream.load( fp, persistent, tokenizer=rust_tokenizer_or_raise() @@ -122,6 +122,6 @@ def visit(fp, visitor): """ Run json-stream's `visit` but using the Rust tokenizer. """ - import json_stream + import json_stream # type: ignore[import-untyped] return json_stream.visit(fp, visitor, tokenizer=rust_tokenizer_or_raise()) diff --git a/json_stream_rs_tokenizer/benchmark/__main__.py b/json_stream_rs_tokenizer/benchmark/__main__.py index 3be98aa..7d6f1b1 100644 --- a/json_stream_rs_tokenizer/benchmark/__main__.py +++ b/json_stream_rs_tokenizer/benchmark/__main__.py @@ -1,9 +1,9 @@ try: from .cli import main -except ImportError as e: +except ImportError as _e: raise ImportError( "benchmark dependencies not installed, please consult the README" - ) from e + ) from _e if __name__ == "__main__": exit(main()) diff --git a/json_stream_rs_tokenizer/benchmark/app.py b/json_stream_rs_tokenizer/benchmark/app.py index 19d9a21..8f2b6ee 100644 --- a/json_stream_rs_tokenizer/benchmark/app.py +++ b/json_stream_rs_tokenizer/benchmark/app.py @@ -4,10 +4,11 @@ from pathlib import Path from tempfile import TemporaryDirectory -import json_stream as js -from contexttimer import Timer -from json_stream.tokenizer import tokenize as pure_python_tokenizer -from json_stream_to_standard_types import to_standard_types +import json_stream as js # type: ignore[import-untyped] +from contexttimer import Timer # type: ignore[import-untyped] +from json_stream.tokenizer import tokenize as pure_python_tokenizer # type: ignore[import-untyped] +from json_stream_to_standard_types import to_standard_types # type: ignore[import-untyped] + from tqdm import tqdm import json_stream_rs_tokenizer as jsrs diff --git a/json_stream_rs_tokenizer/benchmark/cli.py b/json_stream_rs_tokenizer/benchmark/cli.py index ca0b0db..a16a947 100644 --- a/json_stream_rs_tokenizer/benchmark/cli.py +++ b/json_stream_rs_tokenizer/benchmark/cli.py @@ -1,7 +1,7 @@ from sys import stderr import typer -from si_prefix import si_parse +from si_prefix import si_parse # type: ignore[import-untyped] from . import app diff --git a/json_stream_rs_tokenizer/json_stream_rs_tokenizer.pyi b/json_stream_rs_tokenizer/json_stream_rs_tokenizer.pyi new file mode 100644 index 0000000..f8aeb8d --- /dev/null +++ b/json_stream_rs_tokenizer/json_stream_rs_tokenizer.pyi @@ -0,0 +1,26 @@ +""" +Manually written type hints stub file until PyO3 supports stub generation. + +See https://pyo3.rs/v0.27.1/python-typing-hints.html +""" +from typing import Any, IO, final + +@final +class RustTokenizer: + # TODO: buffering default is actually -1 but Mypy insists on it being + # ellipsis... + def __new__( + cls, stream: IO[Any], *, buffering: int = ..., correct_cursor: bool = True + ) -> RustTokenizer: ... + + def park_cursor(self) -> None: ... + + @property + def remainder(self) -> str | bytes: ... + +def supports_bigint() -> bool: ... + +__all__ = [ + "RustTokenizer", + "supports_bigint", +] diff --git a/json_stream_rs_tokenizer/py.typed b/json_stream_rs_tokenizer/py.typed new file mode 100644 index 0000000..b648ac9 --- /dev/null +++ b/json_stream_rs_tokenizer/py.typed @@ -0,0 +1 @@ +partial diff --git a/setup.py b/setup.py index ccd8fbd..3b6bd51 100644 --- a/setup.py +++ b/setup.py @@ -47,6 +47,10 @@ "json-stream-rs-tokenizer[benchmark]", "json-stream==2.3.2", ], + "stubtest": [ + "mypy>=1,<2", + "types-tqdm", # not specifying version b/c it should match tqdm + ], }, classifiers=[ "Programming Language :: Rust", diff --git a/stubtest-allowlist b/stubtest-allowlist new file mode 100644 index 0000000..04d2331 --- /dev/null +++ b/stubtest-allowlist @@ -0,0 +1 @@ +json_stream_rs_tokenizer.RustTokenizer-redefinition