Implement string streaming via sep. class

Allows json-stream's JsonStringReader unit tests to work for us, too.

Author: smheidrich

json_stream_rs_tokenizer/__init__.py

@@ -5,6 +5,7 @@
     "ExtensionException",
     "ExtensionUnavailable",
     "RequestedFeatureUnavailable",
+    "JsonStringReader",
 ]
 
 
@@ -20,6 +21,7 @@ class TokenType:
     from .json_stream_rs_tokenizer import (
         RustTokenizer as _RustTokenizer,
         supports_bigint as _supports_bigint,
+        JsonStringReader,
     )
 
     # included only for backwards-compatibility - to the outside world, bigint
@@ -90,7 +92,7 @@ def rust_tokenizer_or_raise(requires_bigint=True, **kwargs):
         ExtensionUnavailable: If the Rust extension is not available.
         RequestedFeatureUnavailable: If a requested feature is not available.
     """
-    supported_kwargs = {"buffering"}
+    supported_kwargs = {"buffering", "strings_as_files"}
     unsupported = kwargs.keys() - supported_kwargs
     if unsupported:
         raise RequestedFeatureUnavailable(

src/int.rs

@@ -18,12 +18,14 @@ pub enum ParseIntError {
 use num_bigint::BigInt;
 
 #[cfg(not(any(Py_LIMITED_API, PyPy)))]
+#[derive(Clone)]
 pub enum AppropriateInt {
     Normal(i64),
     Big(BigInt),
 }
 
 #[cfg(all(any(Py_LIMITED_API, PyPy)))]
+#[derive(Clone)]
 pub enum AppropriateInt {
     Normal(i64),
     Big(String), // to be converted into int on the Python side