From 3864e35b3df713caac763dc5b203120f4e86fefb Mon Sep 17 00:00:00 2001 From: smheidrich Date: Sun, 26 Oct 2025 14:26:38 +0100 Subject: [PATCH 01/10] Update PyO3 to 0.19 --- Cargo.lock | 24 ++++++++++++------------ Cargo.toml | 4 ++-- src/lib.rs | 1 - src/suitable_stream.rs | 4 ++-- 4 files changed, 16 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 192ab54..6154c1b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -243,9 +243,9 @@ checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" [[package]] name = "memoffset" -version = "0.8.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" dependencies = [ "autocfg", ] @@ -339,9 +339,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.18.3" +version = "0.19.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3b1ac5b3731ba34fdaa9785f8d74d17448cd18f30cf19e0c7e7b1fdb5272109" +checksum = "e681a6cfdc4adcc93b4d3cf993749a4552018ee0a9b65fc0ccfad74352c72a38" dependencies = [ "cfg-if", "indoc", @@ -357,9 +357,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.18.3" +version = "0.19.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cb946f5ac61bb61a5014924910d936ebd2b23b705f7a4a3c40b05c720b079a3" +checksum = "076c73d0bc438f7a4ef6fdd0c3bb4732149136abd952b110ac93e4edb13a6ba5" dependencies = [ "once_cell", "target-lexicon", @@ -367,9 +367,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.18.3" +version = "0.19.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd4d7c5337821916ea2a1d21d1092e8443cf34879e53a0ac653fbb98f44ff65c" +checksum = "e53cee42e77ebe256066ba8aa77eff722b3bb91f3419177cf4cd0f304d3284d9" dependencies = [ "libc", "pyo3-build-config", @@ -377,9 +377,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.18.3" +version = "0.19.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d39c55dab3fc5a4b25bbd1ac10a2da452c4aca13bb450f22818a002e29648d" +checksum = "dfeb4c99597e136528c6dd7d5e3de5434d1ceaf487436a3f03b2d56b6fc9efd1" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -389,9 +389,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.18.3" +version = "0.19.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97daff08a4c48320587b5224cc98d609e3c27b6d437315bd40b605c98eeb5918" +checksum = "947dc12175c254889edc0c02e399476c2f652b4b9ebd123aa655c224de259536" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 51a5389..304e1c4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ crate-type = ["cdylib"] [dependencies] num-bigint = ">=0.4.3,<0.5" owned_chars = ">=0.3.2,<0.4" -pyo3 = { version = ">=0.18,<0.19", features = ["num-bigint"] } +pyo3 = { version = ">=0.19,<0.20", features = ["num-bigint"] } thiserror = ">=1.0.37,<2" utf8-chars = ">=2.0.2,<3" compact_str = ">=0.7.1,<0.8" @@ -24,7 +24,7 @@ utf8-width = ">=0.1.6,<0.2" rstest = ">=0.18.1,<0.19" [build-dependencies] -pyo3-build-config = { version = "= 0.18.3", features = ["resolve-config"] } +pyo3-build-config = { version = ">=0.19,<0.20", features = ["resolve-config"] } # workaround for linkage errors when running cargo test: # https://pyo3.rs/v0.18.1/faq#i-cant-run-cargo-test-or-i-cant-build-in-a-cargo-workspace-im-having-linker-issues-like-symbol-not-found-or-undefined-reference-to-_pyexc_systemerror diff --git a/src/lib.rs b/src/lib.rs index 2e373c0..8f8e92a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -96,7 +96,6 @@ enum State { /// unrelated to the actual tokenization progress. For seekable streams, the /// improvement shouldn't be noticable. #[pyclass] -#[pyo3(text_signature = "(stream, *, buffering=-1, correct_cursor=True)")] struct RustTokenizer { stream: Box, completed: bool, diff --git a/src/suitable_stream.rs b/src/suitable_stream.rs index 6d86e82..4e06c24 100644 --- a/src/suitable_stream.rs +++ b/src/suitable_stream.rs @@ -31,9 +31,9 @@ enum ReadReturnType { fn determine_read_return_type(stream: &PyObject) -> PyResult { Python::with_gil(|py| -> PyResult { let read_result = stream.as_ref(py).call_method1("read", (0,))?; - Ok(if read_result.is_instance_of::()? { + Ok(if read_result.is_instance_of::() { ReadReturnType::String - } else if read_result.is_instance_of::()? { + } else if read_result.is_instance_of::() { ReadReturnType::Bytes } else { ReadReturnType::Other(format!("{}", read_result.get_type())) From 54b10e3e8ed8ae33a4bf546e1b1692278bb45bb5 Mon Sep 17 00:00:00 2001 From: smheidrich Date: Sun, 26 Oct 2025 14:30:47 +0100 Subject: [PATCH 02/10] Update PyO3 to 0.20 --- Cargo.lock | 50 ++++++++++++++++++++++++++++++++++---------------- Cargo.toml | 4 ++-- 2 files changed, 36 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6154c1b..e326e48 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -175,11 +175,20 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + [[package]] name = "indoc" -version = "1.0.9" +version = "2.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa799dd5ed20a7e349f3b4639aa80d74549c81716d9ec4f994c9b5815598306" +checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706" +dependencies = [ + "rustversion", +] [[package]] name = "io-extras" @@ -328,6 +337,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "portable-atomic" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" + [[package]] name = "proc-macro2" version = "1.0.103" @@ -339,9 +354,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.19.2" +version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e681a6cfdc4adcc93b4d3cf993749a4552018ee0a9b65fc0ccfad74352c72a38" +checksum = "53bdbb96d49157e65d45cc287af5f32ffadd5f4761438b527b055fb0d4bb8233" dependencies = [ "cfg-if", "indoc", @@ -349,6 +364,7 @@ dependencies = [ "memoffset", "num-bigint", "parking_lot", + "portable-atomic", "pyo3-build-config", "pyo3-ffi", "pyo3-macros", @@ -357,9 +373,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.19.2" +version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "076c73d0bc438f7a4ef6fdd0c3bb4732149136abd952b110ac93e4edb13a6ba5" +checksum = "deaa5745de3f5231ce10517a1f5dd97d53e5a2fd77aa6b5842292085831d48d7" dependencies = [ "once_cell", "target-lexicon", @@ -367,9 +383,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.19.2" +version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e53cee42e77ebe256066ba8aa77eff722b3bb91f3419177cf4cd0f304d3284d9" +checksum = "62b42531d03e08d4ef1f6e85a2ed422eb678b8cd62b762e53891c05faf0d4afa" dependencies = [ "libc", "pyo3-build-config", @@ -377,25 +393,27 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.19.2" +version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfeb4c99597e136528c6dd7d5e3de5434d1ceaf487436a3f03b2d56b6fc9efd1" +checksum = "7305c720fa01b8055ec95e484a6eca7a83c841267f0dd5280f0c8b8551d2c158" dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 1.0.109", + "syn 2.0.108", ] [[package]] name = "pyo3-macros-backend" -version = "0.19.2" +version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "947dc12175c254889edc0c02e399476c2f652b4b9ebd123aa655c224de259536" +checksum = "7c7e9b68bb9c3149c5b0cade5d07f953d6d125eb4337723c4ccdb665f1f96185" dependencies = [ + "heck", "proc-macro2", + "pyo3-build-config", "quote", - "syn 1.0.109", + "syn 2.0.108", ] [[package]] @@ -587,9 +605,9 @@ checksum = "462eeb75aeb73aea900253ce739c8e18a67423fadf006037cd3ff27e82748a06" [[package]] name = "unindent" -version = "0.1.11" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1766d682d402817b5ac4490b3c3002d91dfa0d22812f341609f97b08757359c" +checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" [[package]] name = "utf8-chars" diff --git a/Cargo.toml b/Cargo.toml index 304e1c4..439dd2e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ crate-type = ["cdylib"] [dependencies] num-bigint = ">=0.4.3,<0.5" owned_chars = ">=0.3.2,<0.4" -pyo3 = { version = ">=0.19,<0.20", features = ["num-bigint"] } +pyo3 = { version = ">=0.20,<0.21", features = ["num-bigint"] } thiserror = ">=1.0.37,<2" utf8-chars = ">=2.0.2,<3" compact_str = ">=0.7.1,<0.8" @@ -24,7 +24,7 @@ utf8-width = ">=0.1.6,<0.2" rstest = ">=0.18.1,<0.19" [build-dependencies] -pyo3-build-config = { version = ">=0.19,<0.20", features = ["resolve-config"] } +pyo3-build-config = { version = ">=0.20,<0.21", features = ["resolve-config"] } # workaround for linkage errors when running cargo test: # https://pyo3.rs/v0.18.1/faq#i-cant-run-cargo-test-or-i-cant-build-in-a-cargo-workspace-im-having-linker-issues-like-symbol-not-found-or-undefined-reference-to-_pyexc_systemerror From d41c0949b6d7b3e2691e07e3fbcceac8ee3c5147 Mon Sep 17 00:00:00 2001 From: smheidrich Date: Sun, 26 Oct 2025 15:00:05 +0100 Subject: [PATCH 03/10] Update PyO3 to 0.21 --- Cargo.lock | 20 +++++----- Cargo.toml | 4 +- src/lib.rs | 52 ++++++++++++++++++------- src/py_bytes_stream.rs | 35 ++++++++--------- src/py_err.rs | 9 ++--- src/py_text_stream.rs | 39 ++++++++----------- src/remainder.rs | 2 +- src/suitable_stream.rs | 13 ++++--- src/suitable_unbuffered_bytes_stream.rs | 7 ++-- 9 files changed, 99 insertions(+), 82 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e326e48..03e3ea4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -354,9 +354,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.20.3" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53bdbb96d49157e65d45cc287af5f32ffadd5f4761438b527b055fb0d4bb8233" +checksum = "a5e00b96a521718e08e03b1a622f01c8a8deb50719335de3f60b3b3950f069d8" dependencies = [ "cfg-if", "indoc", @@ -373,9 +373,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.20.3" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "deaa5745de3f5231ce10517a1f5dd97d53e5a2fd77aa6b5842292085831d48d7" +checksum = "7883df5835fafdad87c0d888b266c8ec0f4c9ca48a5bed6bbb592e8dedee1b50" dependencies = [ "once_cell", "target-lexicon", @@ -383,9 +383,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.20.3" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b42531d03e08d4ef1f6e85a2ed422eb678b8cd62b762e53891c05faf0d4afa" +checksum = "01be5843dc60b916ab4dad1dca6d20b9b4e6ddc8e15f50c47fe6d85f1fb97403" dependencies = [ "libc", "pyo3-build-config", @@ -393,9 +393,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.20.3" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7305c720fa01b8055ec95e484a6eca7a83c841267f0dd5280f0c8b8551d2c158" +checksum = "77b34069fc0682e11b31dbd10321cbf94808394c56fd996796ce45217dfac53c" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -405,9 +405,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.20.3" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c7e9b68bb9c3149c5b0cade5d07f953d6d125eb4337723c4ccdb665f1f96185" +checksum = "08260721f32db5e1a5beae69a55553f56b99bd0e1c3e6e0a5e8851a9d0f5a85c" dependencies = [ "heck", "proc-macro2", diff --git a/Cargo.toml b/Cargo.toml index 439dd2e..6bee6f1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ crate-type = ["cdylib"] [dependencies] num-bigint = ">=0.4.3,<0.5" owned_chars = ">=0.3.2,<0.4" -pyo3 = { version = ">=0.20,<0.21", features = ["num-bigint"] } +pyo3 = { version = ">=0.21,<0.22", features = ["num-bigint"] } thiserror = ">=1.0.37,<2" utf8-chars = ">=2.0.2,<3" compact_str = ">=0.7.1,<0.8" @@ -24,7 +24,7 @@ utf8-width = ">=0.1.6,<0.2" rstest = ">=0.18.1,<0.19" [build-dependencies] -pyo3-build-config = { version = ">=0.20,<0.21", features = ["resolve-config"] } +pyo3-build-config = { version = ">=0.21,<0.22", features = ["resolve-config"] } # workaround for linkage errors when running cargo test: # https://pyo3.rs/v0.18.1/faq#i-cant-run-cargo-test-or-i-cant-build-in-a-cargo-workspace-im-having-linker-issues-like-symbol-not-found-or-undefined-reference-to-_pyexc_systemerror diff --git a/src/lib.rs b/src/lib.rs index 8f8e92a..e10142b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -246,9 +246,7 @@ impl RustTokenizer { // final token Ok(Some(now_token)) } - None => { - Ok(None) - } + None => Ok(None), } } else { Ok(None) @@ -400,7 +398,10 @@ impl RustTokenizer { ))); } Err(ParseIntError::TooLargeOrSmall) => { - return Err(ParsingError::Limitation("Incapable of parsing integer due to platform constraint".to_string())); + return Err(ParsingError::Limitation( + "Incapable of parsing integer due to platform constraint" + .to_string(), + )); } } slf.advance = false; @@ -489,7 +490,9 @@ impl RustTokenizer { slf.advance = false; } _ => { - return Err(ParsingError::InvalidJson("A number must include only digits".to_string())); + return Err(ParsingError::InvalidJson( + "A number must include only digits".to_string(), + )); } }, State::FloatingPoint0 => match c { @@ -498,7 +501,10 @@ impl RustTokenizer { add_char = true; } _ => { - return Err(ParsingError::InvalidJson("A number with a decimal point must be followed by a fractional part".to_string())); + return Err(ParsingError::InvalidJson( + "A number with a decimal point must be followed by a fractional part" + .to_string(), + )); } }, State::False1 => match c { @@ -682,7 +688,9 @@ impl RustTokenizer { slf.unicode_buffer.push(c); } Eof => { - return Err(ParsingError::InvalidJson("Unterminated unicode literal at end of file".to_string())); + return Err(ParsingError::InvalidJson( + "Unterminated unicode literal at end of file".to_string(), + )); } } if slf.unicode_buffer.len() == 4 { @@ -716,10 +724,14 @@ impl RustTokenizer { slf.next_state = State::UnicodeSurrogateStringEscape; } Char(_) => { - return Err(ParsingError::InvalidJson("Unpaired UTF-16 surrogate".to_string())); + return Err(ParsingError::InvalidJson( + "Unpaired UTF-16 surrogate".to_string(), + )); } Eof => { - return Err(ParsingError::InvalidJson("Unpaired UTF-16 surrogate at end of file".to_string())); + return Err(ParsingError::InvalidJson( + "Unpaired UTF-16 surrogate at end of file".to_string(), + )); } }, State::UnicodeSurrogateStringEscape => match c { @@ -728,10 +740,14 @@ impl RustTokenizer { slf.next_state = State::UnicodeSurrogate; } Char(_) => { - return Err(ParsingError::InvalidJson("Unpaired UTF-16 surrogate".to_string())); + return Err(ParsingError::InvalidJson( + "Unpaired UTF-16 surrogate".to_string(), + )); } Eof => { - return Err(ParsingError::InvalidJson("Unpaired UTF-16 surrogate at end of file".to_string())); + return Err(ParsingError::InvalidJson( + "Unpaired UTF-16 surrogate at end of file".to_string(), + )); } }, State::UnicodeSurrogate => { @@ -740,7 +756,9 @@ impl RustTokenizer { slf.unicode_buffer.push(c); } Eof => { - return Err(ParsingError::InvalidJson("Unterminated unicode literal at end of file".to_string())); + return Err(ParsingError::InvalidJson( + "Unterminated unicode literal at end of file".to_string(), + )); } } if slf.unicode_buffer.len() == 4 { @@ -751,10 +769,14 @@ impl RustTokenizer { ))); }; if !is_surrogate(charcode) { - return Err(ParsingError::InvalidJson("Second half of UTF-16 surrogate pair is not a surrogate!".to_string())); + return Err(ParsingError::InvalidJson( + "Second half of UTF-16 surrogate pair is not a surrogate!".to_string(), + )); } let Some(prev_charcode) = slf.prev_charcode else { - return Err(ParsingError::InvalidJson("This should never happen, please report it as a bug...".to_string())); + return Err(ParsingError::InvalidJson( + "This should never happen, please report it as a bug...".to_string(), + )); }; c = Char(decode_surrogate_pair(prev_charcode, charcode).map_err(|_| { ParsingError::InvalidJson(format!( @@ -789,7 +811,7 @@ fn supports_bigint() -> PyResult { } #[pymodule] -fn json_stream_rs_tokenizer(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +fn json_stream_rs_tokenizer(_py: Python<'_>, m: &Bound) -> PyResult<()> { m.add_class::()?; m.add_wrapped(wrap_pyfunction!(supports_bigint))?; diff --git a/src/py_bytes_stream.rs b/src/py_bytes_stream.rs index abbf0ec..ece3dd1 100644 --- a/src/py_bytes_stream.rs +++ b/src/py_bytes_stream.rs @@ -1,5 +1,6 @@ use crate::py_common::PySeekWhence; use crate::py_err::TracebackDisplay; +use pyo3::types::PyAnyMethods; use pyo3::{PyObject, PyResult, Python}; use std::io; use std::io::{Read, Seek, SeekFrom}; @@ -23,19 +24,17 @@ impl Read for PyBytesStream { fn read(&mut self, buf: &mut [u8]) -> io::Result { let vec = Python::with_gil(|py| -> PyResult> { self.inner - .as_ref(py) + .bind(py) .call_method1("read", (buf.len(),))? .extract::>() }) .map_err(|e| { - io::Error::other( - format!( - "Error reading up to {} bytes from Python bytes stream: {}\n{}", - buf.len(), - e, - e.traceback_display(), - ), - ) + io::Error::other(format!( + "Error reading up to {} bytes from Python bytes stream: {}\n{}", + buf.len(), + e, + e.traceback_display(), + )) })?; buf[..vec.len()].clone_from_slice(&vec); Ok(vec.len()) @@ -51,20 +50,18 @@ impl Seek for PyBytesStream { }; Python::with_gil(|py| -> PyResult { self.inner - .as_ref(py) + .bind(py) .call_method1("seek", (offset, whence))? .extract::() }) .map_err(|e| { - io::Error::other( - format!( - "Error seeking to offset {} (from {:?}) in Python bytes stream: {}\n{}", - offset, - whence, - e, - e.traceback_display(), - ), - ) + io::Error::other(format!( + "Error seeking to offset {} (from {:?}) in Python bytes stream: {}\n{}", + offset, + whence, + e, + e.traceback_display(), + )) }) } } diff --git a/src/py_err.rs b/src/py_err.rs index d826f73..ba415f6 100644 --- a/src/py_err.rs +++ b/src/py_err.rs @@ -1,4 +1,4 @@ -use pyo3::{prelude::PyErr, types::PyTraceback, Python}; +use pyo3::{prelude::PyErr, types::PyTracebackMethods, Python}; /// Python error utilities. use std::fmt::{Display, Error, Formatter}; @@ -28,10 +28,9 @@ impl<'a> Display for PyErrTracebackDisplayer<'a> { fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), Error> { String::fmt( &Python::with_gil(|py| { - self.py_err.traceback(py).map_or( - Ok("(no traceback available)".to_string()), - PyTraceback::format, - ) + self.py_err + .traceback_bound(py) + .map_or(Ok("(no traceback available)".to_string()), |x| x.format()) }) .unwrap_or("(error getting traceback)".to_string()), f, diff --git a/src/py_text_stream.rs b/src/py_text_stream.rs index 07d9a57..8e13966 100644 --- a/src/py_text_stream.rs +++ b/src/py_text_stream.rs @@ -2,6 +2,7 @@ use crate::opaque_seek::{OpaqueSeek, OpaqueSeekFrom}; use crate::py_common::PySeekWhence; use crate::py_err::TracebackDisplay; use crate::read_string::ReadString; +use pyo3::types::PyAnyMethods; use pyo3::{IntoPy, PyObject, PyResult, Python}; use std::io; @@ -34,19 +35,17 @@ impl ReadString for PyTextStream { fn read_string(&mut self, size: usize) -> io::Result { Python::with_gil(|py| -> PyResult { self.inner - .as_ref(py) + .bind(py) .call_method1("read", (size,))? .extract::() }) .map_err(|e| { - io::Error::other( - format!( - "Error reading up to {} bytes from Python text stream: {}\n{}", - size, - e, - e.traceback_display(), - ), - ) + io::Error::other(format!( + "Error reading up to {} bytes from Python text stream: {}\n{}", + size, + e, + e.traceback_display(), + )) }) } } @@ -58,26 +57,22 @@ impl OpaqueSeek for PyTextStream { Python::with_gil(|py| { let (offset, whence) = match pos { OpaqueSeekFrom::Start(x) => (x, PySeekWhence::Set), - OpaqueSeekFrom::Current => { - (PyOpaqueSeekPos(0_u8.into_py(py)), PySeekWhence::Cur) - } + OpaqueSeekFrom::Current => (PyOpaqueSeekPos(0_u8.into_py(py)), PySeekWhence::Cur), OpaqueSeekFrom::End => (PyOpaqueSeekPos(0_u8.into_py(py)), PySeekWhence::End), }; match self .inner - .as_ref(py) + .bind(py) .call_method1("seek", (offset.clone(), whence)) { Ok(x) => Ok(PyOpaqueSeekPos(x.into_py(py))), - Err(e) => Err(io::Error::other( - format!( - "Error seeking to offset {:?} (from {:?}) in Python text stream: {}\n{}", - offset, - whence, - e, - e.traceback_display(), - ), - )), + Err(e) => Err(io::Error::other(format!( + "Error seeking to offset {:?} (from {:?}) in Python text stream: {}\n{}", + offset, + whence, + e, + e.traceback_display(), + ))), } }) } diff --git a/src/remainder.rs b/src/remainder.rs index d1a7696..314cd68 100644 --- a/src/remainder.rs +++ b/src/remainder.rs @@ -22,7 +22,7 @@ impl IntoPy for StreamData { fn into_py(self, py: Python<'_>) -> PyObject { match self { StreamData::Text(s) => s.into_py(py), - StreamData::Bytes(b) => PyBytes::new(py, b.as_slice()).into_py(py), + StreamData::Bytes(b) => PyBytes::new_bound(py, b.as_slice()).into_py(py), } } } diff --git a/src/suitable_stream.rs b/src/suitable_stream.rs index 4e06c24..c49b204 100644 --- a/src/suitable_stream.rs +++ b/src/suitable_stream.rs @@ -13,7 +13,7 @@ use crate::suitable_unbuffered_text_stream::SuitableUnbufferedTextStream; use crate::suitable_unseekable_buffered_bytes_stream::SuitableUnseekableBufferedBytesStream; use crate::suitable_unseekable_buffered_text_stream::SuitableUnseekableBufferedTextStream; use pyo3::exceptions::{PyTypeError, PyValueError}; -use pyo3::types::{PyBytes, PyString}; +use pyo3::types::{PyAnyMethods, PyBytes, PyString}; use pyo3::{PyObject, PyResult, Python}; const DEFAULT_BUFSIZE: usize = 8000; @@ -30,7 +30,7 @@ enum ReadReturnType { fn determine_read_return_type(stream: &PyObject) -> PyResult { Python::with_gil(|py| -> PyResult { - let read_result = stream.as_ref(py).call_method1("read", (0,))?; + let read_result = stream.bind(py).call_method1("read", (0,))?; Ok(if read_result.is_instance_of::() { ReadReturnType::String } else if read_result.is_instance_of::() { @@ -44,7 +44,7 @@ fn determine_read_return_type(stream: &PyObject) -> PyResult { fn is_seekable(stream: &PyObject) -> PyResult { Python::with_gil(|py| -> PyResult { stream - .as_ref(py) + .bind(py) .call_method1("seekable", ())? .extract::() }) @@ -85,8 +85,11 @@ fn decide_stream_settings( } else if seekable { StreamSettings::SeekableBuffered(bufsize) } else { - return Err(PyValueError::new_err("Incompatible stream requirements: correct_cursor and a buffer size > 1 \ - are only possible if the given stream is seekable, which this one is not".to_string())); + return Err(PyValueError::new_err( + "Incompatible stream requirements: correct_cursor and a buffer size > 1 \ + are only possible if the given stream is seekable, which this one is not" + .to_string(), + )); } } }) diff --git a/src/suitable_unbuffered_bytes_stream.rs b/src/suitable_unbuffered_bytes_stream.rs index fb8443a..a746b47 100644 --- a/src/suitable_unbuffered_bytes_stream.rs +++ b/src/suitable_unbuffered_bytes_stream.rs @@ -38,9 +38,10 @@ impl Utf8CharSource for SuitableUnbufferedBytesStream { // try to see if we're at the start of a unicode char: let n_bytes_in_char = get_width(buf[0]); if n_bytes_in_char == 0 { - return Err(io::Error::other( - format!("invalid UTF-8 start byte: {:x}", buf[0]), - )); + return Err(io::Error::other(format!( + "invalid UTF-8 start byte: {:x}", + buf[0] + ))); } // if we're inside a unicode char, we try and read its remaining bytes // (or until EOF, in which case from_utf8 below will return an error): From 484e79ff23246ca64047cb974d9767934ad09875 Mon Sep 17 00:00:00 2001 From: smheidrich Date: Sun, 26 Oct 2025 15:07:27 +0100 Subject: [PATCH 04/10] Update PyO3 to 0.22 --- Cargo.lock | 91 ++++++++---------------------------------------------- Cargo.toml | 6 ++-- 2 files changed, 17 insertions(+), 80 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 03e3ea4..0558676 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -23,12 +23,6 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" -[[package]] -name = "bitflags" -version = "2.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" - [[package]] name = "castaway" version = "0.2.4" @@ -177,9 +171,9 @@ checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" [[package]] name = "heck" -version = "0.4.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "indoc" @@ -235,15 +229,6 @@ version = "0.2.177" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" -[[package]] -name = "lock_api" -version = "0.4.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" -dependencies = [ - "scopeguard", -] - [[package]] name = "memchr" version = "2.7.6" @@ -302,29 +287,6 @@ dependencies = [ "delegate-attr", ] -[[package]] -name = "parking_lot" -version = "0.12.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" -dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "smallvec", - "windows-link", -] - [[package]] name = "pin-project-lite" version = "0.2.16" @@ -354,16 +316,16 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.21.2" +version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5e00b96a521718e08e03b1a622f01c8a8deb50719335de3f60b3b3950f069d8" +checksum = "f402062616ab18202ae8319da13fa4279883a2b8a9d9f83f20dbade813ce1884" dependencies = [ "cfg-if", "indoc", "libc", "memoffset", "num-bigint", - "parking_lot", + "once_cell", "portable-atomic", "pyo3-build-config", "pyo3-ffi", @@ -373,9 +335,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.21.2" +version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7883df5835fafdad87c0d888b266c8ec0f4c9ca48a5bed6bbb592e8dedee1b50" +checksum = "b14b5775b5ff446dd1056212d778012cbe8a0fbffd368029fd9e25b514479c38" dependencies = [ "once_cell", "target-lexicon", @@ -383,9 +345,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.21.2" +version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01be5843dc60b916ab4dad1dca6d20b9b4e6ddc8e15f50c47fe6d85f1fb97403" +checksum = "9ab5bcf04a2cdcbb50c7d6105de943f543f9ed92af55818fd17b660390fc8636" dependencies = [ "libc", "pyo3-build-config", @@ -393,9 +355,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.21.2" +version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77b34069fc0682e11b31dbd10321cbf94808394c56fd996796ce45217dfac53c" +checksum = "0fd24d897903a9e6d80b968368a34e1525aeb719d568dba8b3d4bfa5dc67d453" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -405,9 +367,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.21.2" +version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08260721f32db5e1a5beae69a55553f56b99bd0e1c3e6e0a5e8851a9d0f5a85c" +checksum = "36c011a03ba1e50152b4b394b479826cad97e7a21eb52df179cd91ac411cbfbe" dependencies = [ "heck", "proc-macro2", @@ -425,15 +387,6 @@ dependencies = [ "proc-macro2", ] -[[package]] -name = "redox_syscall" -version = "0.5.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" -dependencies = [ - "bitflags", -] - [[package]] name = "regex" version = "1.12.2" @@ -519,12 +472,6 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" -[[package]] -name = "scopeguard" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" - [[package]] name = "semver" version = "1.0.27" @@ -537,12 +484,6 @@ version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" -[[package]] -name = "smallvec" -version = "1.15.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" - [[package]] name = "static_assertions" version = "1.1.0" @@ -640,12 +581,6 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86bd8d4e895da8537e5315b8254664e6b769c4ff3db18321b297a1e7004392e3" -[[package]] -name = "windows-link" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" - [[package]] name = "windows-sys" version = "0.48.0" diff --git a/Cargo.toml b/Cargo.toml index 6bee6f1..dcdb7ab 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,9 @@ crate-type = ["cdylib"] [dependencies] num-bigint = ">=0.4.3,<0.5" owned_chars = ">=0.3.2,<0.4" -pyo3 = { version = ">=0.21,<0.22", features = ["num-bigint"] } +# TODO: Migrate off py-clone as recommended in +# https://pyo3.rs/v0.27.1/migration.html?highlight=bound#pyclone-is-now-gated-behind-the-py-clone-feature +pyo3 = { version = ">=0.22,<0.23", features = ["num-bigint", "py-clone"] } thiserror = ">=1.0.37,<2" utf8-chars = ">=2.0.2,<3" compact_str = ">=0.7.1,<0.8" @@ -24,7 +26,7 @@ utf8-width = ">=0.1.6,<0.2" rstest = ">=0.18.1,<0.19" [build-dependencies] -pyo3-build-config = { version = ">=0.21,<0.22", features = ["resolve-config"] } +pyo3-build-config = { version = ">=0.22,<0.23", features = ["resolve-config"] } # workaround for linkage errors when running cargo test: # https://pyo3.rs/v0.18.1/faq#i-cant-run-cargo-test-or-i-cant-build-in-a-cargo-workspace-im-having-linker-issues-like-symbol-not-found-or-undefined-reference-to-_pyexc_systemerror From 05a9df43d1250a1b45e97c8be552bc05bc71bb50 Mon Sep 17 00:00:00 2001 From: smheidrich Date: Tue, 28 Oct 2025 22:10:53 +0100 Subject: [PATCH 05/10] Update PyO3 to 0.23 --- Cargo.lock | 27 +++++++++++++-------- Cargo.toml | 5 ++-- src/int.rs | 20 +++++++++------ src/lib.rs | 55 +++++++++++++++++++++++++++++++++++------- src/py_common.rs | 14 +++++++---- src/py_err.rs | 2 +- src/py_text_stream.rs | 39 ++++++++++++++++++++---------- src/remainder.rs | 21 ++++++++++------ src/suitable_stream.rs | 2 +- 9 files changed, 129 insertions(+), 56 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0558676..c29a6ee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -217,6 +217,7 @@ dependencies = [ "pyo3-build-config", "rstest", "thiserror", + "unwrap-infallible", "utf8-chars", "utf8-io", "utf8-read", @@ -316,9 +317,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.22.6" +version = "0.23.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f402062616ab18202ae8319da13fa4279883a2b8a9d9f83f20dbade813ce1884" +checksum = "7778bffd85cf38175ac1f545509665d0b9b92a198ca7941f131f85f7a4f9a872" dependencies = [ "cfg-if", "indoc", @@ -335,9 +336,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.22.6" +version = "0.23.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b14b5775b5ff446dd1056212d778012cbe8a0fbffd368029fd9e25b514479c38" +checksum = "94f6cbe86ef3bf18998d9df6e0f3fc1050a8c5efa409bf712e661a4366e010fb" dependencies = [ "once_cell", "target-lexicon", @@ -345,9 +346,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.22.6" +version = "0.23.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ab5bcf04a2cdcbb50c7d6105de943f543f9ed92af55818fd17b660390fc8636" +checksum = "e9f1b4c431c0bb1c8fb0a338709859eed0d030ff6daa34368d3b152a63dfdd8d" dependencies = [ "libc", "pyo3-build-config", @@ -355,9 +356,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.22.6" +version = "0.23.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fd24d897903a9e6d80b968368a34e1525aeb719d568dba8b3d4bfa5dc67d453" +checksum = "fbc2201328f63c4710f68abdf653c89d8dbc2858b88c5d88b0ff38a75288a9da" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -367,9 +368,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.22.6" +version = "0.23.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36c011a03ba1e50152b4b394b479826cad97e7a21eb52df179cd91ac411cbfbe" +checksum = "fca6726ad0f3da9c9de093d6f116a93c1a38e417ed73bf138472cf4064f72028" dependencies = [ "heck", "proc-macro2", @@ -550,6 +551,12 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" +[[package]] +name = "unwrap-infallible" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "151ac09978d3c2862c4e39b557f4eceee2cc72150bc4cb4f16abf061b6e381fb" + [[package]] name = "utf8-chars" version = "2.1.0" diff --git a/Cargo.toml b/Cargo.toml index dcdb7ab..aeac51d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ num-bigint = ">=0.4.3,<0.5" owned_chars = ">=0.3.2,<0.4" # TODO: Migrate off py-clone as recommended in # https://pyo3.rs/v0.27.1/migration.html?highlight=bound#pyclone-is-now-gated-behind-the-py-clone-feature -pyo3 = { version = ">=0.22,<0.23", features = ["num-bigint", "py-clone"] } +pyo3 = { version = ">=0.23,<0.24", features = ["num-bigint", "py-clone"] } thiserror = ">=1.0.37,<2" utf8-chars = ">=2.0.2,<3" compact_str = ">=0.7.1,<0.8" @@ -21,12 +21,13 @@ utf8-io = ">=0.16.0,<0.17" #utf8-read = ">=0.4.0,<0.5" utf8-read = { git = "https://github.com/smheidrich/utf8-read-rs.git", branch = "configurable-chunk-size" } utf8-width = ">=0.1.6,<0.2" +unwrap-infallible = "0.1.5" [dev-dependencies] rstest = ">=0.18.1,<0.19" [build-dependencies] -pyo3-build-config = { version = ">=0.22,<0.23", features = ["resolve-config"] } +pyo3-build-config = { version = ">=0.23,<0.24", features = ["resolve-config"] } # workaround for linkage errors when running cargo test: # https://pyo3.rs/v0.18.1/faq#i-cant-run-cargo-test-or-i-cant-build-in-a-cargo-workspace-im-having-linker-issues-like-symbol-not-found-or-undefined-reference-to-_pyexc_systemerror diff --git a/src/int.rs b/src/int.rs index b0d2889..47d7e78 100644 --- a/src/int.rs +++ b/src/int.rs @@ -23,7 +23,7 @@ pub enum AppropriateInt { Big(BigInt), } -#[cfg(all(any(Py_LIMITED_API, PyPy)))] +#[cfg(any(Py_LIMITED_API, PyPy))] pub enum AppropriateInt { Normal(i64), Big(String), // to be converted into int on the Python side @@ -50,16 +50,22 @@ impl FromStr for AppropriateInt { } } -impl IntoPy for AppropriateInt { - fn into_py(self, py: Python<'_>) -> PyObject { - match self { - AppropriateInt::Normal(num) => num.into_py(py), - AppropriateInt::Big(num) => num.into_py(py), - } +impl<'py> IntoPyObject<'py> for AppropriateInt { + type Target = PyAny; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + Ok(match self { + AppropriateInt::Normal(num) => num.into_pyobject(py)?.into_any(), + AppropriateInt::Big(num) => num.into_pyobject(py)?.into_any(), + }) } } pub fn supports_bigint() -> bool { + // TODO: I think both of these *do* support BigInt in recent PyO3 versions => test & lift + // restriction #[cfg(any(Py_LIMITED_API, PyPy))] { return false; diff --git a/src/lib.rs b/src/lib.rs index e10142b..44a2f90 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,10 +10,12 @@ use crate::suitable_stream::{make_suitable_stream, SuitableStream}; use compact_str::CompactString; use pyo3::exceptions::{PyIOError, PyValueError}; use pyo3::prelude::*; +use pyo3::types::PyInt; use std::borrow::BorrowMut; use std::num::ParseFloatError; use std::str::FromStr; use thiserror::Error; +use unwrap_infallible::UnwrapInfallible; mod int; mod opaque_seek; @@ -97,7 +99,7 @@ enum State { /// improvement shouldn't be noticable. #[pyclass] struct RustTokenizer { - stream: Box, + stream: Box, completed: bool, advance: bool, token: String, @@ -116,9 +118,13 @@ fn is_delimiter(c: CharOrEof) -> bool { } } -impl IntoPy for TokenType { - fn into_py(self, py: Python<'_>) -> PyObject { - (self as u32).into_py(py) +impl<'py> IntoPyObject<'py> for TokenType { + type Target = PyInt; + type Output = Bound<'py, Self::Target>; + type Error = std::convert::Infallible; + + fn into_pyobject(self, py: Python<'py>) -> Result { + (self as u32).into_pyobject(py) } } @@ -128,6 +134,8 @@ pub enum ParsingError { InvalidJson(String), #[error("Error due to limitation: {0}")] Limitation(String), + #[error("Python error")] + PythonError(PyErr), #[error("Unknown error")] Unknown, } @@ -144,6 +152,12 @@ impl From for ParsingError { } } +impl From for ParsingError { + fn from(e: PyErr) -> ParsingError { + ParsingError::PythonError(e) + } +} + enum Token { Operator(String), String_(String), @@ -296,11 +310,34 @@ impl RustTokenizer { c: CharOrEof, ) -> Result)>, ParsingError> { match RustTokenizer::process_char(slf.borrow_mut(), c) { - Ok(Some(Token::Operator(s))) => Ok(Some((TokenType::Operator, Some(s.into_py(py))))), - Ok(Some(Token::String_(s))) => Ok(Some((TokenType::String_, Some(s.into_py(py))))), - Ok(Some(Token::Integer(n))) => Ok(Some((TokenType::Number, Some(n.into_py(py))))), - Ok(Some(Token::Float(f))) => Ok(Some((TokenType::Number, Some(f.into_py(py))))), - Ok(Some(Token::Boolean(b))) => Ok(Some((TokenType::Boolean, Some(b.into_py(py))))), + Ok(Some(Token::Operator(s))) => Ok(Some(( + TokenType::Operator, + Some(s.into_pyobject(py).unwrap_infallible().unbind().into_any()), + ))), + Ok(Some(Token::String_(s))) => Ok(Some(( + TokenType::String_, + Some(s.into_pyobject(py).unwrap_infallible().unbind().into_any()), + ))), + Ok(Some(Token::Integer(n))) => Ok(Some(( + TokenType::Number, + Some(n.into_pyobject(py)?.unbind().into_any()), + ))), + Ok(Some(Token::Float(f))) => Ok(Some(( + TokenType::Number, + Some(f.into_pyobject(py).unwrap_infallible().unbind().into_any()), + ))), + // TODO: Why do we need to_owned() for the bool but not the others? May be fixed in + // more recent PyO3 versions? + Ok(Some(Token::Boolean(b))) => Ok(Some(( + TokenType::Boolean, + Some( + b.into_pyobject(py) + .unwrap_infallible() + .to_owned() + .unbind() + .into_any(), + ), + ))), Ok(Some(Token::Null)) => Ok(Some((TokenType::Null, None))), Ok(None) => Ok(None), Err(e) => Err(e), diff --git a/src/py_common.rs b/src/py_common.rs index e8bced7..8af320f 100644 --- a/src/py_common.rs +++ b/src/py_common.rs @@ -1,5 +1,5 @@ -use pyo3::conversion::IntoPy; -use pyo3::{PyObject, Python}; +use pyo3::types::PyInt; +use pyo3::{Bound, IntoPyObject, Python}; #[derive(Debug, Clone, Copy)] pub enum PySeekWhence { @@ -8,8 +8,12 @@ pub enum PySeekWhence { End = 2, } -impl IntoPy for PySeekWhence { - fn into_py(self, py: Python<'_>) -> PyObject { - (self as u64).into_py(py) +impl<'py> IntoPyObject<'py> for PySeekWhence { + type Target = PyInt; + type Output = Bound<'py, Self::Target>; + type Error = std::convert::Infallible; + + fn into_pyobject(self, py: Python<'py>) -> Result { + (self as u32).into_pyobject(py) } } diff --git a/src/py_err.rs b/src/py_err.rs index ba415f6..9b18d77 100644 --- a/src/py_err.rs +++ b/src/py_err.rs @@ -29,7 +29,7 @@ impl<'a> Display for PyErrTracebackDisplayer<'a> { String::fmt( &Python::with_gil(|py| { self.py_err - .traceback_bound(py) + .traceback(py) .map_or(Ok("(no traceback available)".to_string()), |x| x.format()) }) .unwrap_or("(error getting traceback)".to_string()), diff --git a/src/py_text_stream.rs b/src/py_text_stream.rs index 8e13966..1654c2a 100644 --- a/src/py_text_stream.rs +++ b/src/py_text_stream.rs @@ -2,9 +2,10 @@ use crate::opaque_seek::{OpaqueSeek, OpaqueSeekFrom}; use crate::py_common::PySeekWhence; use crate::py_err::TracebackDisplay; use crate::read_string::ReadString; -use pyo3::types::PyAnyMethods; -use pyo3::{IntoPy, PyObject, PyResult, Python}; +use pyo3::types::{PyAny, PyAnyMethods}; +use pyo3::{IntoPyObject, Py, PyObject, PyResult, Python}; use std::io; +use unwrap_infallible::UnwrapInfallible; /// Python file-like object (= stream) that outputs text. pub struct PyTextStream { @@ -18,14 +19,8 @@ impl PyTextStream { } /// It is an error to do arithmetic on this number. -#[derive(Clone, Debug)] -pub struct PyOpaqueSeekPos(pub PyObject); - -impl IntoPy for PyOpaqueSeekPos { - fn into_py(self, _py: Python<'_>) -> PyObject { - self.0 - } -} +#[derive(Clone, Debug, IntoPyObject)] +pub struct PyOpaqueSeekPos(pub Py); impl ReadString for PyTextStream { // TODO Find out if there is a way to transfer this string in a zero-copy way from Py to Rs. @@ -57,15 +52,33 @@ impl OpaqueSeek for PyTextStream { Python::with_gil(|py| { let (offset, whence) = match pos { OpaqueSeekFrom::Start(x) => (x, PySeekWhence::Set), - OpaqueSeekFrom::Current => (PyOpaqueSeekPos(0_u8.into_py(py)), PySeekWhence::Cur), - OpaqueSeekFrom::End => (PyOpaqueSeekPos(0_u8.into_py(py)), PySeekWhence::End), + OpaqueSeekFrom::Current => ( + PyOpaqueSeekPos( + 0_u8.into_pyobject(py) + .unwrap_infallible() + .unbind() + .into_any(), + ), + PySeekWhence::Cur, + ), + OpaqueSeekFrom::End => ( + PyOpaqueSeekPos( + 0_u8.into_pyobject(py) + .unwrap_infallible() + .unbind() + .into_any(), + ), + PySeekWhence::End, + ), }; match self .inner .bind(py) .call_method1("seek", (offset.clone(), whence)) { - Ok(x) => Ok(PyOpaqueSeekPos(x.into_py(py))), + Ok(x) => Ok(PyOpaqueSeekPos( + x.into_pyobject(py).unwrap_infallible().unbind(), + )), Err(e) => Err(io::Error::other(format!( "Error seeking to offset {:?} (from {:?}) in Python text stream: {}\n{}", offset, diff --git a/src/remainder.rs b/src/remainder.rs index 314cd68..51d0403 100644 --- a/src/remainder.rs +++ b/src/remainder.rs @@ -1,5 +1,6 @@ -use pyo3::types::PyBytes; -use pyo3::{IntoPy, PyObject, Python}; +use pyo3::types::PyAny; +use pyo3::{Bound, IntoPyObject, PyErr, Python}; +use unwrap_infallible::UnwrapInfallible; pub enum StreamData { Text(String), @@ -18,11 +19,15 @@ pub trait Remainder { fn remainder(&self) -> StreamData; } -impl IntoPy for StreamData { - fn into_py(self, py: Python<'_>) -> PyObject { - match self { - StreamData::Text(s) => s.into_py(py), - StreamData::Bytes(b) => PyBytes::new_bound(py, b.as_slice()).into_py(py), - } +impl<'py> IntoPyObject<'py> for StreamData { + type Target = PyAny; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + Ok(match self { + StreamData::Text(s) => s.into_pyobject(py).unwrap_infallible().into_any(), + StreamData::Bytes(b) => b.as_slice().into_pyobject(py)?, + }) } } diff --git a/src/suitable_stream.rs b/src/suitable_stream.rs index c49b204..27480ec 100644 --- a/src/suitable_stream.rs +++ b/src/suitable_stream.rs @@ -99,7 +99,7 @@ pub fn make_suitable_stream( stream: PyObject, buffering: BufferingMode, correct_cursor: bool, -) -> PyResult> { +) -> PyResult> { let read_return_type: ReadReturnType = determine_read_return_type(&stream)?; let seekable: bool = is_seekable(&stream)?; let stream_settings = decide_stream_settings(correct_cursor, buffering, seekable)?; From 70d6324b05bfe09492c6f92fcbdc1f0b62b8f859 Mon Sep 17 00:00:00 2001 From: smheidrich Date: Tue, 28 Oct 2025 22:16:33 +0100 Subject: [PATCH 06/10] Update PyO3 to 0.24 --- Cargo.lock | 28 ++++++++++++++-------------- Cargo.toml | 6 ++++-- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c29a6ee..062be05 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 4 [[package]] name = "aho-corasick" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" dependencies = [ "memchr", ] @@ -317,9 +317,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.23.5" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7778bffd85cf38175ac1f545509665d0b9b92a198ca7941f131f85f7a4f9a872" +checksum = "e5203598f366b11a02b13aa20cab591229ff0a89fd121a308a5df751d5fc9219" dependencies = [ "cfg-if", "indoc", @@ -336,9 +336,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.23.5" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94f6cbe86ef3bf18998d9df6e0f3fc1050a8c5efa409bf712e661a4366e010fb" +checksum = "99636d423fa2ca130fa5acde3059308006d46f98caac629418e53f7ebb1e9999" dependencies = [ "once_cell", "target-lexicon", @@ -346,9 +346,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.23.5" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9f1b4c431c0bb1c8fb0a338709859eed0d030ff6daa34368d3b152a63dfdd8d" +checksum = "78f9cf92ba9c409279bc3305b5409d90db2d2c22392d443a87df3a1adad59e33" dependencies = [ "libc", "pyo3-build-config", @@ -356,9 +356,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.23.5" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbc2201328f63c4710f68abdf653c89d8dbc2858b88c5d88b0ff38a75288a9da" +checksum = "0b999cb1a6ce21f9a6b147dcf1be9ffedf02e0043aec74dc390f3007047cecd9" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -368,9 +368,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.23.5" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fca6726ad0f3da9c9de093d6f116a93c1a38e417ed73bf138472cf4064f72028" +checksum = "822ece1c7e1012745607d5cf0bcb2874769f0f7cb34c4cde03b9358eb9ef911a" dependencies = [ "heck", "proc-macro2", @@ -515,9 +515,9 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.12.16" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" +checksum = "df7f62577c25e07834649fc3b39fafdc597c0a3527dc1c60129201ccfcbaa50c" [[package]] name = "thiserror" diff --git a/Cargo.toml b/Cargo.toml index aeac51d..12a76d8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ num-bigint = ">=0.4.3,<0.5" owned_chars = ">=0.3.2,<0.4" # TODO: Migrate off py-clone as recommended in # https://pyo3.rs/v0.27.1/migration.html?highlight=bound#pyclone-is-now-gated-behind-the-py-clone-feature -pyo3 = { version = ">=0.23,<0.24", features = ["num-bigint", "py-clone"] } +pyo3 = { version = ">=0.24,<0.25", features = ["num-bigint", "py-clone"] } thiserror = ">=1.0.37,<2" utf8-chars = ">=2.0.2,<3" compact_str = ">=0.7.1,<0.8" @@ -21,13 +21,15 @@ utf8-io = ">=0.16.0,<0.17" #utf8-read = ">=0.4.0,<0.5" utf8-read = { git = "https://github.com/smheidrich/utf8-read-rs.git", branch = "configurable-chunk-size" } utf8-width = ">=0.1.6,<0.2" +# TODO: Rust nightly has this same feature; get rid of this dependency once it +# becomes stable unwrap-infallible = "0.1.5" [dev-dependencies] rstest = ">=0.18.1,<0.19" [build-dependencies] -pyo3-build-config = { version = ">=0.23,<0.24", features = ["resolve-config"] } +pyo3-build-config = { version = ">=0.24,<0.25", features = ["resolve-config"] } # workaround for linkage errors when running cargo test: # https://pyo3.rs/v0.18.1/faq#i-cant-run-cargo-test-or-i-cant-build-in-a-cargo-workspace-im-having-linker-issues-like-symbol-not-found-or-undefined-reference-to-_pyexc_systemerror From 43aa1a6a07db52ca63a846fc8c1d51efa6829dda Mon Sep 17 00:00:00 2001 From: smheidrich Date: Tue, 28 Oct 2025 22:20:13 +0100 Subject: [PATCH 07/10] Update PyO3 to 0.25 --- Cargo.lock | 21 ++++++++++----------- Cargo.toml | 4 ++-- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 062be05..dd701be 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -317,11 +317,10 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.24.2" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5203598f366b11a02b13aa20cab591229ff0a89fd121a308a5df751d5fc9219" +checksum = "8970a78afe0628a3e3430376fc5fd76b6b45c4d43360ffd6cdd40bdde72b682a" dependencies = [ - "cfg-if", "indoc", "libc", "memoffset", @@ -336,9 +335,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.24.2" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99636d423fa2ca130fa5acde3059308006d46f98caac629418e53f7ebb1e9999" +checksum = "458eb0c55e7ece017adeba38f2248ff3ac615e53660d7c71a238d7d2a01c7598" dependencies = [ "once_cell", "target-lexicon", @@ -346,9 +345,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.24.2" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78f9cf92ba9c409279bc3305b5409d90db2d2c22392d443a87df3a1adad59e33" +checksum = "7114fe5457c61b276ab77c5055f206295b812608083644a5c5b2640c3102565c" dependencies = [ "libc", "pyo3-build-config", @@ -356,9 +355,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.24.2" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b999cb1a6ce21f9a6b147dcf1be9ffedf02e0043aec74dc390f3007047cecd9" +checksum = "a8725c0a622b374d6cb051d11a0983786448f7785336139c3c94f5aa6bef7e50" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -368,9 +367,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.24.2" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "822ece1c7e1012745607d5cf0bcb2874769f0f7cb34c4cde03b9358eb9ef911a" +checksum = "4109984c22491085343c05b0dbc54ddc405c3cf7b4374fc533f5c3313a572ccc" dependencies = [ "heck", "proc-macro2", diff --git a/Cargo.toml b/Cargo.toml index 12a76d8..d6c7805 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ num-bigint = ">=0.4.3,<0.5" owned_chars = ">=0.3.2,<0.4" # TODO: Migrate off py-clone as recommended in # https://pyo3.rs/v0.27.1/migration.html?highlight=bound#pyclone-is-now-gated-behind-the-py-clone-feature -pyo3 = { version = ">=0.24,<0.25", features = ["num-bigint", "py-clone"] } +pyo3 = { version = ">=0.25,<0.26", features = ["num-bigint", "py-clone"] } thiserror = ">=1.0.37,<2" utf8-chars = ">=2.0.2,<3" compact_str = ">=0.7.1,<0.8" @@ -29,7 +29,7 @@ unwrap-infallible = "0.1.5" rstest = ">=0.18.1,<0.19" [build-dependencies] -pyo3-build-config = { version = ">=0.24,<0.25", features = ["resolve-config"] } +pyo3-build-config = { version = ">=0.25,<0.26", features = ["resolve-config"] } # workaround for linkage errors when running cargo test: # https://pyo3.rs/v0.18.1/faq#i-cant-run-cargo-test-or-i-cant-build-in-a-cargo-workspace-im-having-linker-issues-like-symbol-not-found-or-undefined-reference-to-_pyexc_systemerror From 40c1c6167fed85cff525392d6ccae4314546fa32 Mon Sep 17 00:00:00 2001 From: smheidrich Date: Tue, 28 Oct 2025 22:41:14 +0100 Subject: [PATCH 08/10] Update PyO3 to 0.26 --- Cargo.lock | 21 ++++++++++----------- Cargo.toml | 4 ++-- src/lib.rs | 6 +++--- src/py_bytes_stream.rs | 12 ++++++------ src/py_err.rs | 2 +- src/py_text_stream.rs | 10 +++++----- src/suitable_stream.rs | 14 +++++++------- 7 files changed, 34 insertions(+), 35 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dd701be..dd7668f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -317,9 +317,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.25.1" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8970a78afe0628a3e3430376fc5fd76b6b45c4d43360ffd6cdd40bdde72b682a" +checksum = "7ba0117f4212101ee6544044dae45abe1083d30ce7b29c4b5cbdfa2354e07383" dependencies = [ "indoc", "libc", @@ -335,19 +335,18 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.25.1" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "458eb0c55e7ece017adeba38f2248ff3ac615e53660d7c71a238d7d2a01c7598" +checksum = "4fc6ddaf24947d12a9aa31ac65431fb1b851b8f4365426e182901eabfb87df5f" dependencies = [ - "once_cell", "target-lexicon", ] [[package]] name = "pyo3-ffi" -version = "0.25.1" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7114fe5457c61b276ab77c5055f206295b812608083644a5c5b2640c3102565c" +checksum = "025474d3928738efb38ac36d4744a74a400c901c7596199e20e45d98eb194105" dependencies = [ "libc", "pyo3-build-config", @@ -355,9 +354,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.25.1" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8725c0a622b374d6cb051d11a0983786448f7785336139c3c94f5aa6bef7e50" +checksum = "2e64eb489f22fe1c95911b77c44cc41e7c19f3082fc81cce90f657cdc42ffded" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -367,9 +366,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.25.1" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4109984c22491085343c05b0dbc54ddc405c3cf7b4374fc533f5c3313a572ccc" +checksum = "100246c0ecf400b475341b8455a9213344569af29a3c841d29270e53102e0fcf" dependencies = [ "heck", "proc-macro2", diff --git a/Cargo.toml b/Cargo.toml index d6c7805..9c235d9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ num-bigint = ">=0.4.3,<0.5" owned_chars = ">=0.3.2,<0.4" # TODO: Migrate off py-clone as recommended in # https://pyo3.rs/v0.27.1/migration.html?highlight=bound#pyclone-is-now-gated-behind-the-py-clone-feature -pyo3 = { version = ">=0.25,<0.26", features = ["num-bigint", "py-clone"] } +pyo3 = { version = ">=0.26,<0.27", features = ["num-bigint", "py-clone"] } thiserror = ">=1.0.37,<2" utf8-chars = ">=2.0.2,<3" compact_str = ">=0.7.1,<0.8" @@ -29,7 +29,7 @@ unwrap-infallible = "0.1.5" rstest = ">=0.18.1,<0.19" [build-dependencies] -pyo3-build-config = { version = ">=0.25,<0.26", features = ["resolve-config"] } +pyo3-build-config = { version = ">=0.26,<0.27", features = ["resolve-config"] } # workaround for linkage errors when running cargo test: # https://pyo3.rs/v0.18.1/faq#i-cant-run-cargo-test-or-i-cant-build-in-a-cargo-workspace-im-having-linker-issues-like-symbol-not-found-or-undefined-reference-to-_pyexc_systemerror diff --git a/src/lib.rs b/src/lib.rs index 44a2f90..c7acaf5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -171,7 +171,7 @@ enum Token { impl RustTokenizer { #[new] #[pyo3(signature = (stream, *, buffering = -1, correct_cursor = true))] - fn new(stream: PyObject, buffering: i64, correct_cursor: bool) -> PyResult { + fn new(stream: Py, buffering: i64, correct_cursor: bool) -> PyResult { let buffering_mode = if buffering < 0 { BufferingMode::DontCare } else if buffering == 0 || buffering == 1 { @@ -199,7 +199,7 @@ impl RustTokenizer { fn __next__( mut slf: PyRefMut<'_, Self>, py: Python<'_>, - ) -> PyResult)>> { + ) -> PyResult>)>> { let mut now_token; loop { if slf.advance { @@ -308,7 +308,7 @@ impl RustTokenizer { slf: &mut Self, py: Python<'_>, c: CharOrEof, - ) -> Result)>, ParsingError> { + ) -> Result>)>, ParsingError> { match RustTokenizer::process_char(slf.borrow_mut(), c) { Ok(Some(Token::Operator(s))) => Ok(Some(( TokenType::Operator, diff --git a/src/py_bytes_stream.rs b/src/py_bytes_stream.rs index ece3dd1..77da7e0 100644 --- a/src/py_bytes_stream.rs +++ b/src/py_bytes_stream.rs @@ -1,17 +1,17 @@ use crate::py_common::PySeekWhence; use crate::py_err::TracebackDisplay; -use pyo3::types::PyAnyMethods; -use pyo3::{PyObject, PyResult, Python}; +use pyo3::types::{PyAny, PyAnyMethods}; +use pyo3::{Py, PyResult, Python}; use std::io; use std::io::{Read, Seek, SeekFrom}; /// Python file-like object (= stream) that outputs bytes. pub struct PyBytesStream { - inner: PyObject, + inner: Py, } impl PyBytesStream { - pub fn new(inner: PyObject) -> Self { + pub fn new(inner: Py) -> Self { PyBytesStream { inner } } } @@ -22,7 +22,7 @@ impl Read for PyBytesStream { // again in Python (so the lifetime can be entirely in our hands), which we can't because there // is no way to annotate such facts in Python. fn read(&mut self, buf: &mut [u8]) -> io::Result { - let vec = Python::with_gil(|py| -> PyResult> { + let vec = Python::attach(|py| -> PyResult> { self.inner .bind(py) .call_method1("read", (buf.len(),))? @@ -48,7 +48,7 @@ impl Seek for PyBytesStream { SeekFrom::Current(x) => (x, PySeekWhence::Cur), SeekFrom::End(x) => (x, PySeekWhence::End), }; - Python::with_gil(|py| -> PyResult { + Python::attach(|py| -> PyResult { self.inner .bind(py) .call_method1("seek", (offset, whence))? diff --git a/src/py_err.rs b/src/py_err.rs index 9b18d77..bdd4ac0 100644 --- a/src/py_err.rs +++ b/src/py_err.rs @@ -27,7 +27,7 @@ pub struct PyErrTracebackDisplayer<'a> { impl<'a> Display for PyErrTracebackDisplayer<'a> { fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), Error> { String::fmt( - &Python::with_gil(|py| { + &Python::attach(|py| { self.py_err .traceback(py) .map_or(Ok("(no traceback available)".to_string()), |x| x.format()) diff --git a/src/py_text_stream.rs b/src/py_text_stream.rs index 1654c2a..ffd639c 100644 --- a/src/py_text_stream.rs +++ b/src/py_text_stream.rs @@ -3,17 +3,17 @@ use crate::py_common::PySeekWhence; use crate::py_err::TracebackDisplay; use crate::read_string::ReadString; use pyo3::types::{PyAny, PyAnyMethods}; -use pyo3::{IntoPyObject, Py, PyObject, PyResult, Python}; +use pyo3::{IntoPyObject, Py, PyResult, Python}; use std::io; use unwrap_infallible::UnwrapInfallible; /// Python file-like object (= stream) that outputs text. pub struct PyTextStream { - inner: PyObject, + inner: Py, } impl PyTextStream { - pub fn new(inner: PyObject) -> Self { + pub fn new(inner: Py) -> Self { PyTextStream { inner } } } @@ -28,7 +28,7 @@ impl ReadString for PyTextStream { // again in Python (so the lifetime can be entirely in our hands), which we can't because there // is no way to annotate such facts in Python. fn read_string(&mut self, size: usize) -> io::Result { - Python::with_gil(|py| -> PyResult { + Python::attach(|py| -> PyResult { self.inner .bind(py) .call_method1("read", (size,))? @@ -49,7 +49,7 @@ impl OpaqueSeek for PyTextStream { type OpaqueSeekPos = PyOpaqueSeekPos; fn seek(&mut self, pos: OpaqueSeekFrom) -> io::Result { - Python::with_gil(|py| { + Python::attach(|py| { let (offset, whence) = match pos { OpaqueSeekFrom::Start(x) => (x, PySeekWhence::Set), OpaqueSeekFrom::Current => ( diff --git a/src/suitable_stream.rs b/src/suitable_stream.rs index 27480ec..d73f2a6 100644 --- a/src/suitable_stream.rs +++ b/src/suitable_stream.rs @@ -13,8 +13,8 @@ use crate::suitable_unbuffered_text_stream::SuitableUnbufferedTextStream; use crate::suitable_unseekable_buffered_bytes_stream::SuitableUnseekableBufferedBytesStream; use crate::suitable_unseekable_buffered_text_stream::SuitableUnseekableBufferedTextStream; use pyo3::exceptions::{PyTypeError, PyValueError}; -use pyo3::types::{PyAnyMethods, PyBytes, PyString}; -use pyo3::{PyObject, PyResult, Python}; +use pyo3::types::{PyAny, PyAnyMethods, PyBytes, PyString}; +use pyo3::{Py, PyResult, Python}; const DEFAULT_BUFSIZE: usize = 8000; @@ -28,8 +28,8 @@ enum ReadReturnType { Other(String), } -fn determine_read_return_type(stream: &PyObject) -> PyResult { - Python::with_gil(|py| -> PyResult { +fn determine_read_return_type(stream: &Py) -> PyResult { + Python::attach(|py| -> PyResult { let read_result = stream.bind(py).call_method1("read", (0,))?; Ok(if read_result.is_instance_of::() { ReadReturnType::String @@ -41,8 +41,8 @@ fn determine_read_return_type(stream: &PyObject) -> PyResult { }) } -fn is_seekable(stream: &PyObject) -> PyResult { - Python::with_gil(|py| -> PyResult { +fn is_seekable(stream: &Py) -> PyResult { + Python::attach(|py| -> PyResult { stream .bind(py) .call_method1("seekable", ())? @@ -96,7 +96,7 @@ fn decide_stream_settings( } pub fn make_suitable_stream( - stream: PyObject, + stream: Py, buffering: BufferingMode, correct_cursor: bool, ) -> PyResult> { From 0072cf68332b3aa650543bb6b021f423adab54f8 Mon Sep 17 00:00:00 2001 From: smheidrich Date: Tue, 28 Oct 2025 22:45:22 +0100 Subject: [PATCH 09/10] Update PyO3 to 0.27 --- Cargo.lock | 21 +++++++++++---------- Cargo.toml | 4 ++-- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dd7668f..3372faf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -317,14 +317,15 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.26.0" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ba0117f4212101ee6544044dae45abe1083d30ce7b29c4b5cbdfa2354e07383" +checksum = "37a6df7eab65fc7bee654a421404947e10a0f7085b6951bf2ea395f4659fb0cf" dependencies = [ "indoc", "libc", "memoffset", "num-bigint", + "num-traits", "once_cell", "portable-atomic", "pyo3-build-config", @@ -335,18 +336,18 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.26.0" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fc6ddaf24947d12a9aa31ac65431fb1b851b8f4365426e182901eabfb87df5f" +checksum = "f77d387774f6f6eec64a004eac0ed525aab7fa1966d94b42f743797b3e395afb" dependencies = [ "target-lexicon", ] [[package]] name = "pyo3-ffi" -version = "0.26.0" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "025474d3928738efb38ac36d4744a74a400c901c7596199e20e45d98eb194105" +checksum = "2dd13844a4242793e02df3e2ec093f540d948299a6a77ea9ce7afd8623f542be" dependencies = [ "libc", "pyo3-build-config", @@ -354,9 +355,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.26.0" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e64eb489f22fe1c95911b77c44cc41e7c19f3082fc81cce90f657cdc42ffded" +checksum = "eaf8f9f1108270b90d3676b8679586385430e5c0bb78bb5f043f95499c821a71" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -366,9 +367,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.26.0" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "100246c0ecf400b475341b8455a9213344569af29a3c841d29270e53102e0fcf" +checksum = "70a3b2274450ba5288bc9b8c1b69ff569d1d61189d4bff38f8d22e03d17f932b" dependencies = [ "heck", "proc-macro2", diff --git a/Cargo.toml b/Cargo.toml index 9c235d9..a11dd05 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ num-bigint = ">=0.4.3,<0.5" owned_chars = ">=0.3.2,<0.4" # TODO: Migrate off py-clone as recommended in # https://pyo3.rs/v0.27.1/migration.html?highlight=bound#pyclone-is-now-gated-behind-the-py-clone-feature -pyo3 = { version = ">=0.26,<0.27", features = ["num-bigint", "py-clone"] } +pyo3 = { version = ">=0.27,<0.28", features = ["num-bigint", "py-clone"] } thiserror = ">=1.0.37,<2" utf8-chars = ">=2.0.2,<3" compact_str = ">=0.7.1,<0.8" @@ -29,7 +29,7 @@ unwrap-infallible = "0.1.5" rstest = ">=0.18.1,<0.19" [build-dependencies] -pyo3-build-config = { version = ">=0.26,<0.27", features = ["resolve-config"] } +pyo3-build-config = { version = ">=0.27,<0.28", features = ["resolve-config"] } # workaround for linkage errors when running cargo test: # https://pyo3.rs/v0.18.1/faq#i-cant-run-cargo-test-or-i-cant-build-in-a-cargo-workspace-im-having-linker-issues-like-symbol-not-found-or-undefined-reference-to-_pyexc_systemerror From 0ec6721e6b7c0606858c332f887869fa67a6e729 Mon Sep 17 00:00:00 2001 From: smheidrich Date: Tue, 28 Oct 2025 22:52:29 +0100 Subject: [PATCH 10/10] Drop PyPy < 3.11 (PyO3 only supports 3.11+) --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d67d263..d7c343f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -31,10 +31,10 @@ jobs: path: ~/Library/Caches/pip # more combinations: - os: ubuntu-22.04 - python-version: "pypy3.9" + python-version: "pypy3.11" path: ~/.cache/pip - os: ubuntu-22.04-arm - python-version: "pypy3.9" + python-version: "pypy3.11" path: ~/.cache/pip steps: - uses: actions/checkout@v4