Add type hints (#145)

* Add type hints

Author: smheidrich

.github/workflows/test.yml

@@ -62,7 +62,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -v -e .[test]
+          pip install -v -e .[test,stubtest]
         shell: bash
       - name: Run Cargo tests
         run: |
@@ -72,6 +72,23 @@ jobs:
       - name: Run pytest tests
         run: |
           pytest
+      - name: Run Mypy stubtest
+        # XXX doesn't work on Pypy due to some symtable issues and on Windows
+        #     with Python 3.8 due to pathlib issues...
+        if: >
+          !startsWith(matrix.python-version, 'pypy')
+          && !(
+            startsWith(matrix.python-version, '3.8')
+            && startsWith(matrix.os, 'windows-')
+          )
+        run: >
+          stubtest
+          ${{
+            !startsWith(matrix.python-version, '3.8')
+              && '--ignore-disjoint-bases'
+              || ''
+          }}
+          --allowlist stubtest-allowlist json_stream_rs_tokenizer
       - name: Save Rust/Cargo cache
         uses: actions/cache/save@v4
         if: always()

json_stream_rs_tokenizer/__init__.py

@@ -24,14 +24,14 @@ class TokenType:
 
     # included only for backwards-compatibility - to the outside world, bigint
     # is now always supported via fallback to conversion in Python
-    def supports_bigint():
+    def supports_bigint() -> bool:
         return True
 
     if _supports_bigint():
         RustTokenizer = _RustTokenizer
     else:
 
-        class RustTokenizer:
+        class RustTokenizer:  # type: ignore[no-redef]
             """
             Rust tokenizer (fallback wrapper for integer conversion)
             """
@@ -111,7 +111,7 @@ def load(fp, persistent=False):
     """
     Run json-stream's `load` but using the Rust tokenizer.
     """
-    import json_stream
+    import json_stream  # type: ignore[import-untyped]
 
     return json_stream.load(
         fp, persistent, tokenizer=rust_tokenizer_or_raise()
@@ -122,6 +122,6 @@ def visit(fp, visitor):
     """
     Run json-stream's `visit` but using the Rust tokenizer.
     """
-    import json_stream
+    import json_stream  # type: ignore[import-untyped]
 
     return json_stream.visit(fp, visitor, tokenizer=rust_tokenizer_or_raise())

json_stream_rs_tokenizer/benchmark/__main__.py

@@ -1,9 +1,9 @@
 try:
     from .cli import main
-except ImportError as e:
+except ImportError as _e:
     raise ImportError(
         "benchmark dependencies not installed, please consult the README"
-    ) from e
+    ) from _e
 
 if __name__ == "__main__":
     exit(main())

json_stream_rs_tokenizer/benchmark/app.py

@@ -4,10 +4,11 @@
 from pathlib import Path
 from tempfile import TemporaryDirectory
 
-import json_stream as js
-from contexttimer import Timer
-from json_stream.tokenizer import tokenize as pure_python_tokenizer
-from json_stream_to_standard_types import to_standard_types
+import json_stream as js  # type: ignore[import-untyped]
+from contexttimer import Timer  # type: ignore[import-untyped]
+from json_stream.tokenizer import tokenize as pure_python_tokenizer  # type: ignore[import-untyped]
+from json_stream_to_standard_types import to_standard_types  # type: ignore[import-untyped]
+
 from tqdm import tqdm
 
 import json_stream_rs_tokenizer as jsrs

json_stream_rs_tokenizer/benchmark/cli.py

@@ -1,7 +1,7 @@
 from sys import stderr
 
 import typer
-from si_prefix import si_parse
+from si_prefix import si_parse  # type: ignore[import-untyped]
 
 from . import app
 

json_stream_rs_tokenizer/json_stream_rs_tokenizer.pyi

@@ -0,0 +1,26 @@
+"""
+Manually written type hints stub file until PyO3 supports stub generation.
+
+See https://pyo3.rs/v0.27.1/python-typing-hints.html
+"""
+from typing import Any, IO, final
+
+@final
+class RustTokenizer:
+  # TODO: buffering default is actually -1 but Mypy insists on it being
+  #       ellipsis...
+  def __new__(
+    cls, stream: IO[Any], *, buffering: int = ..., correct_cursor: bool = True
+  ) -> RustTokenizer: ...
+
+  def park_cursor(self) -> None: ...
+
+  @property
+  def remainder(self) -> str | bytes: ...
+
+def supports_bigint() -> bool: ...
+
+__all__ = [
+  "RustTokenizer",
+  "supports_bigint",
+]

json_stream_rs_tokenizer/py.typed

@@ -0,0 +1 @@
+partial

setup.py

@@ -47,6 +47,10 @@
             "json-stream-rs-tokenizer[benchmark]",
             "json-stream==2.3.2",
         ],
+        "stubtest": [
+            "mypy>=1,<2",
+            "types-tqdm",  # not specifying version b/c it should match tqdm
+        ],
     },
     classifiers=[
         "Programming Language :: Rust",

stubtest-allowlist

@@ -0,0 +1 @@
+json_stream_rs_tokenizer.RustTokenizer-redefinition