From c9e62002ec87c5fe56c3f77791ba8473f1fdcd0e Mon Sep 17 00:00:00 2001 From: Hanif Ariffin Date: Tue, 11 Feb 2025 16:09:38 +0800 Subject: [PATCH 001/295] Fixed the implementation of some math functions to match CPython closer. (#5510) Signed-off-by: Hanif Ariffin --- Lib/test/test_random.py | 2 -- stdlib/src/math.rs | 27 ++++++++++++++++++++++----- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_random.py b/Lib/test/test_random.py index 70bfbf09b5..5bd92f32c2 100644 --- a/Lib/test/test_random.py +++ b/Lib/test/test_random.py @@ -1012,8 +1012,6 @@ def gamma(z, sqrt2pi=(2.0*pi)**0.5): ]) class TestDistributions(unittest.TestCase): - # TODO: RUSTPYTHON ValueError: math domain error - @unittest.expectedFailure def test_zeroinputs(self): # Verify that distributions can handle a series of zero inputs' g = random.Random() diff --git a/stdlib/src/math.rs b/stdlib/src/math.rs index c1abc1a6f2..30b088f714 100644 --- a/stdlib/src/math.rs +++ b/stdlib/src/math.rs @@ -132,6 +132,9 @@ mod math { #[pyfunction] fn log(x: PyObjectRef, base: OptionalArg, vm: &VirtualMachine) -> PyResult { let base = base.map(|b| *b).unwrap_or(std::f64::consts::E); + if base.is_sign_negative() { + return Err(vm.new_value_error("math domain error".to_owned())); + } log2(x, vm).map(|logx| logx / base.log2()) } @@ -192,16 +195,18 @@ mod math { let x = *x; let y = *y; - if x < 0.0 && x.is_finite() && y.fract() != 0.0 && y.is_finite() { - return Err(vm.new_value_error("math domain error".to_owned())); - } - - if x == 0.0 && y < 0.0 && y != f64::NEG_INFINITY { + if x < 0.0 && x.is_finite() && y.fract() != 0.0 && y.is_finite() + || x == 0.0 && y < 0.0 && y != f64::NEG_INFINITY + { return Err(vm.new_value_error("math domain error".to_owned())); } let value = x.powf(y); + if x.is_finite() && y.is_finite() && value.is_infinite() { + return Err(vm.new_overflow_error("math range error".to_string())); + } + Ok(value) } @@ -212,6 +217,9 @@ mod math { return Ok(value); } if value.is_sign_negative() { + if value.is_zero() { + return Ok(-0.0f64); + } return Err(vm.new_value_error("math domain error".to_owned())); } Ok(value.sqrt()) @@ -260,6 +268,9 @@ mod math { #[pyfunction] fn cos(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult { + if x.is_infinite() { + return Err(vm.new_value_error("math domain error".to_owned())); + } call_math_func!(cos, x, vm) } @@ -394,11 +405,17 @@ mod math { #[pyfunction] fn sin(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult { + if x.is_infinite() { + return Err(vm.new_value_error("math domain error".to_owned())); + } call_math_func!(sin, x, vm) } #[pyfunction] fn tan(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult { + if x.is_infinite() { + return Err(vm.new_value_error("math domain error".to_owned())); + } call_math_func!(tan, x, vm) } From 2d5e4d89b071f0538564aaeb286640ba01a2e8a8 Mon Sep 17 00:00:00 2001 From: Noa Date: Tue, 11 Feb 2025 13:03:11 -0600 Subject: [PATCH 002/295] Update openssl to fix possible vulnerability --- Cargo.lock | 8 ++++---- stdlib/src/ssl.rs | 6 +++++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 70395fb1d5..4285718fa7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1396,9 +1396,9 @@ checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9" [[package]] name = "openssl" -version = "0.10.66" +version = "0.10.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9529f4786b70a3e8c61e11179af17ab6188ad8d0ded78c5529441ed39d4bd9c1" +checksum = "61cfb4e166a8bb8c9b55c500bc2308550148ece889be90f609377e58140f42c6" dependencies = [ "bitflags 2.6.0", "cfg-if", @@ -1437,9 +1437,9 @@ dependencies = [ [[package]] name = "openssl-sys" -version = "0.9.103" +version = "0.9.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f9e8deee91df40a943c71b917e5874b951d32a802526c85721ce3b776c929d6" +checksum = "8b22d5b84be05a8d6947c7cb71f7c849aa0f112acd4bf51c2a7c1c988ac0a9dc" dependencies = [ "cc", "libc", diff --git a/stdlib/src/ssl.rs b/stdlib/src/ssl.rs index 03222ff4f5..3622b74d1a 100644 --- a/stdlib/src/ssl.rs +++ b/stdlib/src/ssl.rs @@ -612,7 +612,11 @@ mod _ssl { Ok(pbuf.to_vec()) })?; ctx.set_alpn_select_callback(move |_, client| { - ssl::select_next_proto(&server, client).ok_or(ssl::AlpnError::NOACK) + let proto = + ssl::select_next_proto(&server, client).ok_or(ssl::AlpnError::NOACK)?; + let pos = memchr::memmem::find(client, proto) + .expect("selected alpn proto should be present in client protos"); + Ok(&client[pos..proto.len()]) }); Ok(()) } From 6e35e20e49d82ebcf0500d8b18d5dbcd0a9fd92a Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Tue, 11 Feb 2025 22:13:41 -0800 Subject: [PATCH 003/295] dependency bump Signed-off-by: Ashwin Naren --- Cargo.lock | 226 +++++++++++++++++++++++---------------- Cargo.toml | 48 ++++----- compiler/core/Cargo.toml | 2 +- derive-impl/Cargo.toml | 6 +- stdlib/Cargo.toml | 16 +-- 5 files changed, 170 insertions(+), 128 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4285718fa7..8f4c765e02 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,12 +2,6 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "Inflector" -version = "0.11.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" - [[package]] name = "adler2" version = "2.0.0" @@ -249,9 +243,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrono" -version = "0.4.38" +version = "0.4.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825" dependencies = [ "android-tzdata", "iana-time-zone", @@ -272,7 +266,7 @@ dependencies = [ "bitflags 1.3.2", "strsim", "textwrap 0.11.0", - "unicode-width", + "unicode-width 0.1.14", "vec_map", ] @@ -514,9 +508,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.20" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crunchy" @@ -572,7 +566,7 @@ checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.98", "unicode-xid", ] @@ -783,12 +777,12 @@ dependencies = [ [[package]] name = "gethostname" -version = "0.2.3" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1ebd34e35c46e00bb73e81363248d627782724609fe1b6396f553f68fe3862e" +checksum = "4fd4b8790c0792e3b11895efdf5f289ebe8b59107a6624f1cce68f24ff8c7035" dependencies = [ - "libc", - "winapi", + "rustix", + "windows-targets 0.52.6", ] [[package]] @@ -797,7 +791,7 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5" dependencies = [ - "unicode-width", + "unicode-width 0.1.14", ] [[package]] @@ -939,14 +933,14 @@ dependencies = [ [[package]] name = "is-macro" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2069faacbe981460232f880d26bf3c7634e322d49053aa48c27e3ae642f728f1" +checksum = "1d57a3e447e24c22647738e4607f1df1e0ec6f72e16182c4cd199f647cdfb0e4" dependencies = [ - "Inflector", + "heck", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.98", ] [[package]] @@ -967,6 +961,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.11" @@ -1007,6 +1010,12 @@ version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "507460a910eb7b32ee961886ff48539633b788a36b65692b95f225b844c82553" +[[package]] +name = "lambert_w" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b0c98033daa8d13aa2171722fd201bae337924189091f7988bdaff5301fa8c9" + [[package]] name = "lazy_static" version = "0.2.11" @@ -1051,9 +1060,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.159" +version = "0.2.169" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5" +checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" [[package]] name = "libffi" @@ -1136,9 +1145,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.22" +version = "0.4.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f" [[package]] name = "lz4_flex" @@ -1256,9 +1265,9 @@ checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "memmap2" -version = "0.5.10" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83faa42c0a078c393f6b29d5db232d8be22776a891f8f56e5284faee4a20b327" +checksum = "fd3f7eed9d3848f8b98834af67102b720745c4ec028fcd0aa0239277e7de374f" dependencies = [ "libc", ] @@ -1379,14 +1388,14 @@ checksum = "af1844ef2428cc3e1cb900be36181049ef3d3193c63e43026cfe202983b27a56" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.98", ] [[package]] name = "once_cell" -version = "1.19.0" +version = "1.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +checksum = "945462a4b81e43c4e3ba96bd7b49d834c6f61198356aa858733bc4acf3cbe62e" [[package]] name = "oorandom" @@ -1417,7 +1426,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.98", ] [[package]] @@ -1456,9 +1465,9 @@ checksum = "978aa494585d3ca4ad74929863093e87cac9790d81fe7aba2b3dc2890643a0fc" [[package]] name = "page_size" -version = "0.4.2" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eebde548fbbf1ea81a99b128872779c437752fb99f217c45245e1a61dcd9edcd" +checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" dependencies = [ "libc", "winapi", @@ -1573,7 +1582,7 @@ checksum = "52a40bc70c2c58040d2d8b167ba9a5ff59fc9dab7ad44771cfde3dcfde7a09c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.98", ] [[package]] @@ -1593,18 +1602,21 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.86" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99" dependencies = [ "unicode-ident", ] [[package]] name = "puruspe" -version = "0.2.5" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3804877ffeba468c806c2ad9057bbbae92e4b2c410c2f108baaa0042f241fa4c" +checksum = "5666f1226a41ebb7191b702947674f4814c9919da58a9f91a60090412664bbce" +dependencies = [ + "lambert_w", +] [[package]] name = "pyo3" @@ -1653,7 +1665,7 @@ dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.77", + "syn 2.0.98", ] [[package]] @@ -1666,14 +1678,14 @@ dependencies = [ "proc-macro2", "pyo3-build-config", "quote", - "syn 2.0.77", + "syn 2.0.98", ] [[package]] name = "quote" -version = "1.0.37" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" dependencies = [ "proc-macro2", ] @@ -1767,7 +1779,7 @@ checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" dependencies = [ "getrandom", "libredox", - "thiserror", + "thiserror 1.0.64", ] [[package]] @@ -1841,7 +1853,7 @@ dependencies = [ "pmutil", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.98", ] [[package]] @@ -1914,7 +1926,7 @@ dependencies = [ "bitflags 2.6.0", "indexmap", "insta", - "itertools 0.11.0", + "itertools 0.12.1", "log", "num-complex", "num-traits", @@ -1931,7 +1943,7 @@ dependencies = [ "ascii", "bitflags 2.6.0", "cfg-if", - "itertools 0.11.0", + "itertools 0.12.1", "libc", "lock_api", "malachite-base", @@ -1964,7 +1976,7 @@ name = "rustpython-compiler-core" version = "0.4.0" dependencies = [ "bitflags 2.6.0", - "itertools 0.11.0", + "itertools 0.12.1", "lz4_flex", "malachite-bigint", "num-complex", @@ -1985,7 +1997,7 @@ dependencies = [ name = "rustpython-derive-impl" version = "0.4.0" dependencies = [ - "itertools 0.11.0", + "itertools 0.12.1", "maplit", "once_cell", "proc-macro2", @@ -1995,7 +2007,7 @@ dependencies = [ "rustpython-parser-core", "syn 1.0.109", "syn-ext", - "textwrap 0.15.2", + "textwrap 0.16.1", ] [[package]] @@ -2030,7 +2042,7 @@ dependencies = [ "num-traits", "rustpython-compiler-core", "rustpython-derive", - "thiserror", + "thiserror 2.0.11", ] [[package]] @@ -2127,7 +2139,7 @@ dependencies = [ "gethostname", "hex", "indexmap", - "itertools 0.11.0", + "itertools 0.12.1", "junction", "libc", "libsqlite3-sys", @@ -2202,7 +2214,7 @@ dependencies = [ "hex", "indexmap", "is-macro", - "itertools 0.11.0", + "itertools 0.12.1", "junction", "libc", "log", @@ -2241,7 +2253,7 @@ dependencies = [ "static_assertions", "strum", "strum_macros", - "thiserror", + "thiserror 2.0.11", "thread_local", "timsort", "uname", @@ -2284,9 +2296,9 @@ checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" [[package]] name = "rustyline" -version = "14.0.0" +version = "15.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7803e8936da37efd9b6d4478277f4b2b9bb5cdb37a113e8d63222e58da647e63" +checksum = "2ee1e066dc922e513bda599c6ccb5f3bb2b0ea5870a579448f2622993f0a9a2f" dependencies = [ "bitflags 2.6.0", "cfg-if", @@ -2296,12 +2308,12 @@ dependencies = [ "libc", "log", "memchr", - "nix 0.28.0", + "nix 0.29.0", "radix_trie", "unicode-segmentation", - "unicode-width", + "unicode-width 0.2.0", "utf8parse", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -2321,9 +2333,9 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.24" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9aaafd5a2b6e3d657ff009d82fbd630b6bd54dd4eb06f21693925cdf80f9b8b" +checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" dependencies = [ "windows-sys 0.59.0", ] @@ -2342,9 +2354,9 @@ checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" [[package]] name = "serde" -version = "1.0.210" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" +checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" dependencies = [ "serde_derive", ] @@ -2373,13 +2385,13 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.210" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" +checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.98", ] [[package]] @@ -2458,9 +2470,9 @@ checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" [[package]] name = "socket2" -version = "0.5.7" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" +checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8" dependencies = [ "libc", "windows-sys 0.52.0", @@ -2480,21 +2492,21 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" [[package]] name = "strum" -version = "0.26.3" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +checksum = "ce1475c515a4f03a8a7129bb5228b81a781a86cb0b3fbbc19e1c556d491a401f" [[package]] name = "strum_macros" -version = "0.26.4" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +checksum = "9688894b43459159c82bfa5a5fa0435c19cbe3c9b427fa1dd7b1ce0c279b18a7" dependencies = [ "heck", "proc-macro2", "quote", "rustversion", - "syn 2.0.77", + "syn 2.0.98", ] [[package]] @@ -2516,9 +2528,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.77" +version = "2.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed" +checksum = "36147f1a48ae0ec2b5b3bc5b537d267457555a10dc06f3dbc8cb11ba3006d3b1" dependencies = [ "proc-macro2", "quote", @@ -2585,14 +2597,14 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" dependencies = [ - "unicode-width", + "unicode-width 0.1.14", ] [[package]] name = "textwrap" -version = "0.15.2" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7b3e525a49ec206798b40326a44121291b530c963cfb01018f63e135bac543d" +checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9" [[package]] name = "thiserror" @@ -2600,7 +2612,16 @@ version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84" dependencies = [ - "thiserror-impl", + "thiserror-impl 1.0.64", +] + +[[package]] +name = "thiserror" +version = "2.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d452f284b73e6d76dd36758a0c8684b1d5be31f92b89d07fd5822175732206fc" +dependencies = [ + "thiserror-impl 2.0.11", ] [[package]] @@ -2611,7 +2632,18 @@ checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.98", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.98", ] [[package]] @@ -2855,6 +2887,12 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" +[[package]] +name = "unicode-width" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" + [[package]] name = "unicode-xid" version = "0.2.6" @@ -2948,26 +2986,27 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a474f6281d1d70c17ae7aa6a613c87fce69a127e2624002df63dcb39d6cf6396" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" dependencies = [ "cfg-if", "once_cell", + "rustversion", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f89bb38646b4f81674e8f5c3fb81b562be1fd936d84320f3264486418519c79" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" dependencies = [ "bumpalo", "log", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.98", "wasm-bindgen-shared", ] @@ -2985,9 +3024,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cc6181fd9a7492eef6fef1f33961e3695e4579b9872a6f7c83aee556666d4fe" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2995,22 +3034,25 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.98", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] [[package]] name = "web-sys" @@ -3321,5 +3363,5 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.98", ] diff --git a/Cargo.toml b/Cargo.toml index f1391d000e..e838110e89 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -139,50 +139,50 @@ rustpython-format = { git = "https://github.com/RustPython/Parser.git", version # rustpython-format = { path = "../RustPython-parser/format" } ahash = "0.8.11" -ascii = "1.0" -bitflags = "2.4.1" +ascii = "1.1" +bitflags = "2.4.2" bstr = "1" cfg-if = "1.0" -chrono = "0.4.37" -crossbeam-utils = "0.8.19" +chrono = "0.4.39" +crossbeam-utils = "0.8.21" flame = "0.2.2" -getrandom = "0.2.12" +getrandom = "0.2.15" glob = "0.3" hex = "0.4.3" indexmap = { version = "2.2.6", features = ["std"] } insta = "1.38.0" -itertools = "0.11.0" -is-macro = "0.3.0" -junction = "1.0.0" -libc = "0.2.153" -log = "0.4.16" +itertools = "0.12.1" +is-macro = "0.3.7" +junction = "1.2.0" +libc = "0.2.169" +log = "0.4.25" nix = { version = "0.29", features = ["fs", "user", "process", "term", "time", "signal", "ioctl", "socket", "sched", "zerocopy", "dir", "hostname", "net", "poll"] } malachite-bigint = "0.2.3" malachite-q = "0.4.22" malachite-base = "0.4.22" -memchr = "2.7.2" -num-complex = "0.4.0" -num-integer = "0.1.44" +memchr = "2.7.4" +num-complex = "0.4.6" +num-integer = "0.1.46" num-traits = "0.2" num_enum = { version = "0.7", default-features = false } -once_cell = "1.19.0" -parking_lot = "0.12.1" -paste = "1.0.7" +once_cell = "1.20.3" +parking_lot = "0.12.3" +paste = "1.0.15" rand = "0.8.5" rustix = { version = "0.38", features = ["event"] } -rustyline = "14.0.0" +rustyline = "15.0.0" serde = { version = "1.0.133", default-features = false } -schannel = "0.1.22" +schannel = "0.1.27" static_assertions = "1.1" -strum = "0.26" -strum_macros = "0.26" +strum = "0.27" +strum_macros = "0.27" syn = "1.0.109" -thiserror = "1.0" -thread_local = "1.1.4" -unicode_names2 = "1.2.0" +thiserror = "2.0" +thread_local = "1.1.8" +unicode_names2 = "1.3.0" widestring = "1.1.0" windows-sys = "0.52.0" -wasm-bindgen = "0.2.92" +wasm-bindgen = "0.2.100" # Lints diff --git a/compiler/core/Cargo.toml b/compiler/core/Cargo.toml index 3d05fc2734..619ffcf61e 100644 --- a/compiler/core/Cargo.toml +++ b/compiler/core/Cargo.toml @@ -15,7 +15,7 @@ bitflags = { workspace = true } itertools = { workspace = true } malachite-bigint = { workspace = true } num-complex = { workspace = true } -serde = { version = "1.0.133", optional = true, default-features = false, features = ["derive"] } +serde = { version = "1.0.217", optional = true, default-features = false, features = ["derive"] } lz4_flex = "0.11" diff --git a/derive-impl/Cargo.toml b/derive-impl/Cargo.toml index a843f5d3c5..83a4bc4214 100644 --- a/derive-impl/Cargo.toml +++ b/derive-impl/Cargo.toml @@ -18,10 +18,10 @@ once_cell = { workspace = true } syn = { workspace = true, features = ["full", "extra-traits"] } maplit = "1.0.2" -proc-macro2 = "1.0.79" -quote = "1.0.18" +proc-macro2 = "1.0.93" +quote = "1.0.38" syn-ext = { version = "0.4.0", features = ["full"] } -textwrap = { version = "0.15.0", default-features = false } +textwrap = { version = "0.16.1", default-features = false } [lints] workspace = true \ No newline at end of file diff --git a/stdlib/Cargo.toml b/stdlib/Cargo.toml index 934c9b5cf1..3d91b6fab9 100644 --- a/stdlib/Cargo.toml +++ b/stdlib/Cargo.toml @@ -46,15 +46,15 @@ thread_local = { workspace = true } memchr = { workspace = true } base64 = "0.13.0" -csv-core = "0.1.10" +csv-core = "0.1.11" dyn-clone = "1.0.10" libz-sys = { version = "1.1", default-features = false, optional = true } -puruspe = "0.2.4" +puruspe = "0.3.0" xml-rs = "0.8.14" # random rand = { workspace = true } -rand_core = "0.6.3" +rand_core = "0.6.4" mt19937 = "2.0.1" # Crypto: @@ -92,8 +92,8 @@ uuid = { version = "1.1.2", features = ["v1", "fast-rng"] } # mmap [target.'cfg(all(unix, not(target_arch = "wasm32")))'.dependencies] -memmap2 = "0.5.4" -page_size = "0.4" +memmap2 = "0.5.10" +page_size = "0.6" [target.'cfg(all(unix, not(target_os = "redox"), not(target_os = "ios")))'.dependencies] termios = "0.3.3" @@ -102,8 +102,8 @@ termios = "0.3.3" rustix = { workspace = true } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] -gethostname = "0.2.3" -socket2 = { version = "0.5.6", features = ["all"] } +gethostname = "1.0.0" +socket2 = { version = "0.5.8", features = ["all"] } dns-lookup = "2" openssl = { version = "0.10.66", optional = true } openssl-sys = { version = "0.9.80", optional = true } @@ -136,7 +136,7 @@ features = [ ] [target.'cfg(target_os = "macos")'.dependencies] -system-configuration = "0.5.0" +system-configuration = "0.5.1" [lints] workspace = true From a46ce8ec3a9ab954d38f861ea43d0844b9976ddb Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Wed, 12 Feb 2025 21:11:01 -0800 Subject: [PATCH 004/295] Mark version 3.13.0 (#5495) * bump to 3.13.1 * fix some tests * strip left whitespace from doc * remove specific difflib test that was causing issues * fix test_enum Signed-off-by: Ashwin Naren --- .github/workflows/ci.yaml | 2 +- .github/workflows/cron-ci.yaml | 2 +- Cargo.lock | 472 +++++++++++++--------- DEVELOPMENT.md | 2 +- Lib/difflib.py | 19 - Lib/test/test_enum.py | 30 +- README.md | 2 +- common/src/str.rs | 31 ++ compiler/codegen/Cargo.toml | 1 + compiler/codegen/src/compile.rs | 42 +- extra_tests/snippets/builtin_object.py | 2 - extra_tests/snippets/stdlib_imghdr.py | 27 -- extra_tests/snippets/stdlib_subprocess.py | 12 +- extra_tests/snippets/stdlib_xdrlib.py | 12 - extra_tests/snippets/syntax_class.py | 4 +- extra_tests/snippets/syntax_doc.py | 15 + extra_tests/snippets/syntax_function2.py | 2 +- vm/src/builtins/str.rs | 28 +- vm/src/version.rs | 4 +- whats_left.py | 4 +- 20 files changed, 406 insertions(+), 307 deletions(-) delete mode 100644 extra_tests/snippets/stdlib_imghdr.py delete mode 100644 extra_tests/snippets/stdlib_xdrlib.py create mode 100644 extra_tests/snippets/syntax_doc.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ee8d274c18..7bbed2c3c2 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -105,7 +105,7 @@ env: test_weakref test_yield_from # Python version targeted by the CI. - PYTHON_VERSION: "3.12.3" + PYTHON_VERSION: "3.13.1" jobs: rust_tests: diff --git a/.github/workflows/cron-ci.yaml b/.github/workflows/cron-ci.yaml index 0880e2d249..60a06d80c7 100644 --- a/.github/workflows/cron-ci.yaml +++ b/.github/workflows/cron-ci.yaml @@ -7,7 +7,7 @@ name: Periodic checks/tasks env: CARGO_ARGS: --no-default-features --features stdlib,zlib,importlib,encodings,ssl,jit - PYTHON_VERSION: "3.12.0" + PYTHON_VERSION: "3.13.1" jobs: # codecov collects code coverage data from the rust tests, python snippets and python test suite. diff --git a/Cargo.lock b/Cargo.lock index 8f4c765e02..c238cf994f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -21,10 +21,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ "cfg-if", - "getrandom", + "getrandom 0.2.15", "once_cell", "version_check", - "zerocopy", + "zerocopy 0.7.35", ] [[package]] @@ -62,9 +62,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.89" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6" +checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" [[package]] name = "approx" @@ -109,9 +109,9 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "base64" @@ -127,9 +127,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.6.0" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36" [[package]] name = "blake2" @@ -151,9 +151,9 @@ dependencies = [ [[package]] name = "bstr" -version = "1.10.0" +version = "1.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40723b8fb387abc38f4f4a37c09073622e41dd12327033091ef8950659e6dc0c" +checksum = "531a9155a481e2ee699d4f98f43c0ca4ff8ee1bfd55c31e9e98fb29d2b176fe0" dependencies = [ "memchr", "regex-automata", @@ -162,15 +162,15 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.16.0" +version = "3.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" +checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" [[package]] name = "bytemuck" -version = "1.18.0" +version = "1.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94bbb0ad554ad961ddc5da507a12a29b14e4ae5bda06b19f575a3e6079d2e2ae" +checksum = "ef657dfab802224e671f5818e9a4935f9b1957ed18e58292690cc39e7a4092a3" [[package]] name = "byteorder" @@ -190,9 +190,9 @@ dependencies = [ [[package]] name = "bzip2-sys" -version = "0.1.11+1.0.8" +version = "0.1.12+1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +checksum = "72ebc2f1a417f01e1da30ef264ee86ae31d2dcd2d603ea283d3c244a883ca2a9" dependencies = [ "cc", "libc", @@ -216,9 +216,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.1.21" +version = "1.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07b1695e2c7e8fc85310cde85aeaab7e3097f593c91d209d3f9df76c928100f0" +checksum = "c7777341816418c02e033934a09f20dc0ccaf65a5201ef8a450ae0105a573fda" dependencies = [ "shlex", ] @@ -229,12 +229,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" -[[package]] -name = "cfg_aliases" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" - [[package]] name = "cfg_aliases" version = "0.2.1" @@ -281,14 +275,14 @@ dependencies = [ [[package]] name = "console" -version = "0.15.8" +version = "0.15.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb" +checksum = "ea3c6ecd8059b57859df5c69830340ed3c41d30e3da0c1cbed90a96ac853041b" dependencies = [ "encode_unicode", - "lazy_static 1.5.0", "libc", - "windows-sys 0.52.0", + "once_cell", + "windows-sys 0.59.0", ] [[package]] @@ -319,9 +313,9 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.14" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" dependencies = [ "libc", ] @@ -489,9 +483,9 @@ dependencies = [ [[package]] name = "crossbeam-deque" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" dependencies = [ "crossbeam-epoch", "crossbeam-utils", @@ -514,9 +508,9 @@ checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crunchy" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" +checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929" [[package]] name = "crypto-common" @@ -530,9 +524,9 @@ dependencies = [ [[package]] name = "csv" -version = "1.3.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe" +checksum = "acdc4883a9c96732e4733212c01447ebd805833b7275a73ca3ee080fd77afdaf" dependencies = [ "csv-core", "itoa", @@ -542,9 +536,9 @@ dependencies = [ [[package]] name = "csv-core" -version = "0.1.11" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" +checksum = "7d02f3b0da4c6504f86e9cd789d8dbafab48c2321be74e9987593de5a894d93d" dependencies = [ "memchr", ] @@ -616,9 +610,9 @@ dependencies = [ [[package]] name = "dyn-clone" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d6ef0072f8a535281e4876be788938b528e9a1d43900b82c2569af7da799125" +checksum = "feeef44e73baff3a26d371801df019877a9866a8c493d315ab00177843314f35" [[package]] name = "either" @@ -628,9 +622,9 @@ checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" [[package]] name = "encode_unicode" -version = "0.3.6" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" [[package]] name = "endian-type" @@ -657,12 +651,12 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.9" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -726,9 +720,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.33" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "324a1be68054ef05ad64b861cc9eaf1d623d2d8cb25b4bf2cb9cdd902b4bf253" +checksum = "c936bfdafb507ebbf50b8074c54fa31c5be9a1e7e5f467dd659697041407d07c" dependencies = [ "crc32fast", "libz-sys", @@ -803,15 +797,27 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "wasm-bindgen", ] [[package]] -name = "glob" +name = "getrandom" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.13.3+wasi-0.2.2", + "windows-targets 0.52.6", +] + +[[package]] +name = "glob" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" [[package]] name = "half" @@ -838,6 +844,12 @@ dependencies = [ "ahash", ] +[[package]] +name = "hashbrown" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" + [[package]] name = "heck" version = "0.5.0" @@ -873,11 +885,11 @@ checksum = "dfa686283ad6dd069f105e5ab091b04c62850d3e4cf5d67debad1933f55023df" [[package]] name = "home" -version = "0.5.9" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -905,12 +917,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.5.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5" +checksum = "8c9c992b02b5b4c94ea26e32fe5bccb7aa7d9f390ab5c1221ff895bc7ea8b652" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.15.2", ] [[package]] @@ -921,13 +933,14 @@ checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" [[package]] name = "insta" -version = "1.40.0" +version = "1.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6593a41c7a73841868772495db7dc1e8ecab43bb5c0b6da2059246c4b506ab60" +checksum = "71c1b125e30d93896b365e156c33dadfffab45ee8400afcbba4752f59de08a86" dependencies = [ "console", - "lazy_static 1.5.0", "linked-hash-map", + "once_cell", + "pin-project", "similar", ] @@ -972,16 +985,17 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.11" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" [[package]] name = "js-sys" -version = "0.3.70" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" dependencies = [ + "once_cell", "wasm-bindgen", ] @@ -1095,7 +1109,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "libc", ] @@ -1112,9 +1126,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.20" +version = "1.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2d16453e800a8cf6dd2fc3eb4bc99b786a9b90c663b8559a5b1a041bf89e472" +checksum = "df9b68e50e6e0b26f672573834882eb57759f6db9b3be2ea3c35c91188bb4eaa" dependencies = [ "cc", "pkg-config", @@ -1129,9 +1143,9 @@ checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" [[package]] name = "linux-raw-sys" -version = "0.4.14" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" [[package]] name = "lock_api" @@ -1160,11 +1174,11 @@ dependencies = [ [[package]] name = "mac_address" -version = "1.1.7" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8836fae9d0d4be2c8b4efcdd79e828a2faa058a90d005abf42f91cac5493a08e" +checksum = "c0aeb26bf5e836cc1c341c8106051b573f1766dfa05aa87f0b98be5e51b02303" dependencies = [ - "nix 0.28.0", + "nix", "winapi", ] @@ -1194,7 +1208,7 @@ version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ea0ed76adf7defc1a92240b5c36d5368cfe9251640dcce5bd2d0b7c1fd87aeb" dependencies = [ - "hashbrown", + "hashbrown 0.14.5", "itertools 0.11.0", "libm", "ryu", @@ -1265,9 +1279,9 @@ checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "memmap2" -version = "0.9.5" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd3f7eed9d3848f8b98834af67102b720745c4ec028fcd0aa0239277e7de374f" +checksum = "83faa42c0a078c393f6b29d5db232d8be22776a891f8f56e5284faee4a20b327" dependencies = [ "libc", ] @@ -1283,9 +1297,9 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.8.0" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +checksum = "b3b1c9bd4fe1f0f8b387f6eb9eb3b4a1aa26185e5750efb9140301703f62cd1b" dependencies = [ "adler2", ] @@ -1296,7 +1310,7 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "12ca7f22ed370d5991a9caec16a83187e865bc8a532f889670337d5a5689e3a1" dependencies = [ - "rand_core", + "rand_core 0.6.4", ] [[package]] @@ -1308,28 +1322,15 @@ dependencies = [ "smallvec", ] -[[package]] -name = "nix" -version = "0.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4" -dependencies = [ - "bitflags 2.6.0", - "cfg-if", - "cfg_aliases 0.1.1", - "libc", - "memoffset", -] - [[package]] name = "nix" version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "cfg-if", - "cfg_aliases 0.2.1", + "cfg_aliases", "libc", "memoffset", ] @@ -1409,7 +1410,7 @@ version = "0.10.70" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61cfb4e166a8bb8c9b55c500bc2308550148ece889be90f609377e58140f42c6" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "cfg-if", "foreign-types", "libc", @@ -1431,15 +1432,15 @@ dependencies = [ [[package]] name = "openssl-probe" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" [[package]] name = "openssl-src" -version = "300.3.2+3.3.2" +version = "300.4.2+3.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a211a18d945ef7e648cc6e0058f4c548ee46aab922ea203e0d30e966ea23647b" +checksum = "168ce4e058f975fe43e89d9ccf78ca668601887ae736090aacc23ae353c298e2" dependencies = [ "cc", ] @@ -1491,7 +1492,7 @@ checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.5.5", + "redox_syscall 0.5.8", "smallvec", "windows-targets 0.52.6", ] @@ -1504,18 +1505,18 @@ checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" [[package]] name = "phf" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" dependencies = [ "phf_shared", ] [[package]] name = "phf_codegen" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8d39688d359e6b34654d328e262234662d16cc0f60ec8dcbe5e718709342a5a" +checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" dependencies = [ "phf_generator", "phf_shared", @@ -1523,21 +1524,41 @@ dependencies = [ [[package]] name = "phf_generator" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" dependencies = [ "phf_shared", - "rand", + "rand 0.8.5", ] [[package]] name = "phf_shared" -version = "0.11.2" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher 1.0.1", +] + +[[package]] +name = "pin-project" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" +checksum = "dfe2e71e1471fe07709406bf725f710b02927c9c54b2b5b2ec0e8087d97c327d" dependencies = [ - "siphasher", + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6e859e6e5bd50440ab63c47e3ebabc90f26251f7c73c3d3e837b74a1cc3fa67" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.98", ] [[package]] @@ -1587,9 +1608,9 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.8.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d30538d42559de6b034bc76fd6dd4c38961b1ee5c6c56e3808c50128fdbc22ce" +checksum = "280dc24453071f1b63954171985a0b0d30058d287960968b9b2aca264c8d4ee6" [[package]] name = "ppv-lite86" @@ -1597,7 +1618,7 @@ version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" dependencies = [ - "zerocopy", + "zerocopy 0.7.35", ] [[package]] @@ -1620,9 +1641,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.22.4" +version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00e89ce2565d6044ca31a3eb79a334c3a79a841120a98f64eea9f579564cb691" +checksum = "f402062616ab18202ae8319da13fa4279883a2b8a9d9f83f20dbade813ce1884" dependencies = [ "cfg-if", "indoc", @@ -1638,9 +1659,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.22.4" +version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8afbaf3abd7325e08f35ffb8deb5892046fcb2608b703db6a583a5ba4cea01e" +checksum = "b14b5775b5ff446dd1056212d778012cbe8a0fbffd368029fd9e25b514479c38" dependencies = [ "once_cell", "target-lexicon", @@ -1648,9 +1669,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.22.4" +version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec15a5ba277339d04763f4c23d85987a5b08cbb494860be141e6a10a8eb88022" +checksum = "9ab5bcf04a2cdcbb50c7d6105de943f543f9ed92af55818fd17b660390fc8636" dependencies = [ "libc", "pyo3-build-config", @@ -1658,9 +1679,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.22.4" +version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15e0f01b5364bcfbb686a52fc4181d412b708a68ed20c330db9fc8d2c2bf5a43" +checksum = "0fd24d897903a9e6d80b968368a34e1525aeb719d568dba8b3d4bfa5dc67d453" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -1670,9 +1691,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.22.4" +version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a09b550200e1e5ed9176976d0060cbc2ea82dc8515da07885e7b8153a85caacb" +checksum = "36c011a03ba1e50152b4b394b479826cad97e7a21eb52df179cd91ac411cbfbe" dependencies = [ "heck", "proc-macro2", @@ -1713,8 +1734,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha", - "rand_core", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.0", + "zerocopy 0.8.17", ] [[package]] @@ -1724,7 +1756,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.0", ] [[package]] @@ -1733,7 +1775,17 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom", + "getrandom 0.2.15", +] + +[[package]] +name = "rand_core" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b08f3c9802962f7e1b25113931d94f43ed9725bebc59db9d0c3e9a23b67e15ff" +dependencies = [ + "getrandom 0.3.1", + "zerocopy 0.8.17", ] [[package]] @@ -1764,11 +1816,11 @@ checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" [[package]] name = "redox_syscall" -version = "0.5.5" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62871f2d65009c0256aed1b9cfeeb8ac272833c404e13d53d400cd0dad7a2ac0" +checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", ] [[package]] @@ -1777,9 +1829,9 @@ version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" dependencies = [ - "getrandom", + "getrandom 0.2.15", "libredox", - "thiserror 1.0.64", + "thiserror 1.0.69", ] [[package]] @@ -1796,9 +1848,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.6" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", @@ -1808,9 +1860,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.7" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" dependencies = [ "aho-corasick", "memchr", @@ -1819,9 +1871,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "region" @@ -1873,15 +1925,15 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.37" +version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -1923,7 +1975,7 @@ name = "rustpython-codegen" version = "0.4.0" dependencies = [ "ahash", - "bitflags 2.6.0", + "bitflags 2.8.0", "indexmap", "insta", "itertools 0.12.1", @@ -1931,6 +1983,7 @@ dependencies = [ "num-complex", "num-traits", "rustpython-ast", + "rustpython-common", "rustpython-compiler-core", "rustpython-parser", "rustpython-parser-core", @@ -1941,7 +1994,7 @@ name = "rustpython-common" version = "0.4.0" dependencies = [ "ascii", - "bitflags 2.6.0", + "bitflags 2.8.0", "cfg-if", "itertools 0.12.1", "libc", @@ -1954,9 +2007,9 @@ dependencies = [ "once_cell", "parking_lot", "radium", - "rand", + "rand 0.8.5", "rustpython-format", - "siphasher", + "siphasher 0.3.11", "volatile", "widestring", "windows-sys 0.52.0", @@ -1975,7 +2028,7 @@ dependencies = [ name = "rustpython-compiler-core" version = "0.4.0" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "itertools 0.12.1", "lz4_flex", "malachite-bigint", @@ -2023,7 +2076,7 @@ name = "rustpython-format" version = "0.4.0" source = "git+https://github.com/RustPython/Parser.git?rev=d2f137b372ec08ce4a243564a80f8f9153c45a23#d2f137b372ec08ce4a243564a80f8f9153c45a23" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "itertools 0.11.0", "malachite-bigint", "num-traits", @@ -2112,7 +2165,7 @@ dependencies = [ name = "rustpython-sre_engine" version = "0.4.0" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "num_enum", "optional", ] @@ -2150,7 +2203,7 @@ dependencies = [ "memchr", "memmap2", "mt19937", - "nix 0.29.0", + "nix", "num-complex", "num-integer", "num-traits", @@ -2163,8 +2216,8 @@ dependencies = [ "parking_lot", "paste", "puruspe", - "rand", - "rand_core", + "rand 0.8.5", + "rand_core 0.6.4", "rustix", "rustpython-common", "rustpython-derive", @@ -2199,7 +2252,7 @@ version = "0.4.0" dependencies = [ "ahash", "ascii", - "bitflags 2.6.0", + "bitflags 2.8.0", "bstr", "caseless", "cfg-if", @@ -2208,7 +2261,7 @@ dependencies = [ "exitcode", "flame", "flamer", - "getrandom", + "getrandom 0.2.15", "glob", "half 2.4.1", "hex", @@ -2221,7 +2274,7 @@ dependencies = [ "malachite-bigint", "memchr", "memoffset", - "nix 0.29.0", + "nix", "num-complex", "num-integer", "num-traits", @@ -2231,7 +2284,7 @@ dependencies = [ "optional", "parking_lot", "paste", - "rand", + "rand 0.8.5", "result-like", "rustc_version", "rustix", @@ -2290,9 +2343,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.17" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" +checksum = "f7c45b9784283f1b2e7fb61b42047c2fd678ef0960d4f6f1eba131594cc369d4" [[package]] name = "rustyline" @@ -2300,7 +2353,7 @@ version = "15.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2ee1e066dc922e513bda599c6ccb5f3bb2b0ea5870a579448f2622993f0a9a2f" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "cfg-if", "clipboard-win", "fd-lock", @@ -2308,7 +2361,7 @@ dependencies = [ "libc", "log", "memchr", - "nix 0.29.0", + "nix", "radix_trie", "unicode-segmentation", "unicode-width 0.2.0", @@ -2318,9 +2371,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.18" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" +checksum = "6ea1a2d0a644769cc99faa24c3ad26b379b786fe7c36fd3c546254801650e6dd" [[package]] name = "same-file" @@ -2348,9 +2401,9 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "semver" -version = "1.0.23" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" +checksum = "f79dfe2d285b0488816f30e700a7438c5a73d816b5b7d3ac72fbc48b0d185e03" [[package]] name = "serde" @@ -2396,9 +2449,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.128" +version = "1.0.138" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" +checksum = "d434192e7da787e94a6ea7e9670b26a036d0ca41e0b7efb2676dd32bae872949" dependencies = [ "itoa", "memchr", @@ -2446,9 +2499,9 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "similar" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1de1d4f81173b03af4c0cbed3c898f6bff5b870e4a7f5d6f4057d62a7a4b686e" +checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" [[package]] name = "siphasher" @@ -2456,6 +2509,12 @@ version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" +[[package]] +name = "siphasher" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" + [[package]] name = "slice-group-by" version = "0.3.1" @@ -2608,11 +2667,11 @@ checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9" [[package]] name = "thiserror" -version = "1.0.64" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ - "thiserror-impl 1.0.64", + "thiserror-impl 1.0.69", ] [[package]] @@ -2626,9 +2685,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "1.0.64" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", @@ -2694,9 +2753,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" +checksum = "022db8904dfa342efe721985167e9fcd16c29b226db4397ed752a761cfce81e8" dependencies = [ "tinyvec_macros", ] @@ -2862,9 +2921,9 @@ checksum = "623f59e6af2a98bdafeb93fa277ac8e1e40440973001ca15cf4ae1541cd16d56" [[package]] name = "unicode-ident" -version = "1.0.13" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" +checksum = "a210d160f08b701c8721ba1c726c11662f877ea6b7094007e1ca9a1041945034" [[package]] name = "unicode-normalization" @@ -2918,7 +2977,7 @@ dependencies = [ "getopts", "log", "phf_codegen", - "rand", + "rand 0.8.5", ] [[package]] @@ -2935,13 +2994,13 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.10.0" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" +checksum = "ced87ca4be083373936a67f8de945faa23b6b42384bd5b64434850802c6dccd0" dependencies = [ "atomic", - "getrandom", - "rand", + "getrandom 0.3.1", + "rand 0.9.0", ] [[package]] @@ -2984,6 +3043,15 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasi" +version = "0.13.3+wasi-0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26816d2e1a4a36a2940b96c5296ce403917633dff8f3440e9b236ed6f6bacad2" +dependencies = [ + "wit-bindgen-rt", +] + [[package]] name = "wasm-bindgen" version = "0.2.100" @@ -3012,12 +3080,13 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.43" +version = "0.4.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61e9300f63a621e96ed275155c108eb6f843b6a26d053f122ab69724559dc8ed" +checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" dependencies = [ "cfg-if", "js-sys", + "once_cell", "wasm-bindgen", "web-sys", ] @@ -3056,9 +3125,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.70" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26fdeaafd9bd129f65e7c031593c24d62186301e0c72c8978fa1678be7d532c0" +checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" dependencies = [ "js-sys", "wasm-bindgen", @@ -3339,11 +3408,20 @@ version = "0.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d135d17ab770252ad95e9a872d365cf3090e3be864a34ab46f48555993efc904" +[[package]] +name = "wit-bindgen-rt" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c" +dependencies = [ + "bitflags 2.8.0", +] + [[package]] name = "xml-rs" -version = "0.8.22" +version = "0.8.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af4e2e2f7cba5a093896c1e150fbfe177d1883e7448200efb81d40b9d339ef26" +checksum = "c5b940ebc25896e71dd073bad2dbaa2abfe97b0a391415e22ad1326d9c54e3c4" [[package]] name = "zerocopy" @@ -3352,7 +3430,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ "byteorder", - "zerocopy-derive", + "zerocopy-derive 0.7.35", +] + +[[package]] +name = "zerocopy" +version = "0.8.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa91407dacce3a68c56de03abe2760159582b846c6a4acd2f456618087f12713" +dependencies = [ + "zerocopy-derive 0.8.17", ] [[package]] @@ -3365,3 +3452,14 @@ dependencies = [ "quote", "syn 2.0.98", ] + +[[package]] +name = "zerocopy-derive" +version = "0.8.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06718a168365cad3d5ff0bb133aad346959a2074bd4a85c121255a11304a8626" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.98", +] diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 7c79a011ba..aa7d99eef3 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -25,7 +25,7 @@ RustPython requires the following: stable version: `rustup update stable` - If you do not have Rust installed, use [rustup](https://rustup.rs/) to do so. -- CPython version 3.12 or higher +- CPython version 3.13 or higher - CPython can be installed by your operating system's package manager, from the [Python website](https://www.python.org/downloads/), or using a third-party distribution, such as diff --git a/Lib/difflib.py b/Lib/difflib.py index ba0b256969..3425e438c9 100644 --- a/Lib/difflib.py +++ b/Lib/difflib.py @@ -1200,25 +1200,6 @@ def context_diff(a, b, fromfile='', tofile='', strings for 'fromfile', 'tofile', 'fromfiledate', and 'tofiledate'. The modification times are normally expressed in the ISO 8601 format. If not specified, the strings default to blanks. - - Example: - - >>> print(''.join(context_diff('one\ntwo\nthree\nfour\n'.splitlines(True), - ... 'zero\none\ntree\nfour\n'.splitlines(True), 'Original', 'Current')), - ... end="") - *** Original - --- Current - *************** - *** 1,4 **** - one - ! two - ! three - four - --- 1,4 ---- - + zero - one - ! tree - four """ _check_types(a, b, fromfile, tofile, fromfiledate, tofiledate, lineterm) diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index 3989b7d674..bff85a7ec2 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -1369,10 +1369,12 @@ class Inner(Enum): [Outer.a, Outer.b, Outer.Inner], ) + # TODO: RUSTPYTHON + @unittest.expectedFailure @unittest.skipIf( - python_version < (3, 13), - 'inner classes are still members', - ) + python_version < (3, 13), + 'inner classes are still members', + ) def test_nested_classes_in_enum_are_not_members(self): """Support locally-defined nested classes.""" class Outer(Enum): @@ -4555,20 +4557,24 @@ class Color(Enum): self.assertEqual(Color.green.value, 3) self.assertEqual(Color.yellow.value, 4) + # TODO: RUSTPYTHON + @unittest.expectedFailure @unittest.skipIf( - python_version < (3, 13), - 'mixed types with auto() will raise in 3.13', - ) + python_version < (3, 13), + 'inner classes are still members', + ) def test_auto_garbage_fail(self): with self.assertRaisesRegex(TypeError, 'will require all values to be sortable'): class Color(Enum): red = 'red' blue = auto() + # TODO: RUSTPYTHON + @unittest.expectedFailure @unittest.skipIf( - python_version < (3, 13), - 'mixed types with auto() will raise in 3.13', - ) + python_version < (3, 13), + 'inner classes are still members', + ) def test_auto_garbage_corrected_fail(self): with self.assertRaisesRegex(TypeError, 'will require all values to be sortable'): class Color(Enum): @@ -4598,9 +4604,9 @@ def _generate_next_value_(name, start, count, last): self.assertEqual(Color.blue.value, 'blue') @unittest.skipIf( - python_version < (3, 13), - 'auto() will return highest value + 1 in 3.13', - ) + python_version < (3, 13), + 'inner classes are still members', + ) def test_auto_with_aliases(self): class Color(Enum): red = auto() diff --git a/README.md b/README.md index 5b5b16d566..38e4d8fa8c 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ # [RustPython](https://rustpython.github.io/) -A Python-3 (CPython >= 3.12.0) Interpreter written in Rust :snake: :scream: +A Python-3 (CPython >= 3.13.0) Interpreter written in Rust :snake: :scream: :metal:. [![Build Status](https://github.com/RustPython/RustPython/workflows/CI/badge.svg)](https://github.com/RustPython/RustPython/actions?query=workflow%3ACI) diff --git a/common/src/str.rs b/common/src/str.rs index 3f9bf583b8..65b1e376cb 100644 --- a/common/src/str.rs +++ b/common/src/str.rs @@ -322,6 +322,37 @@ pub mod levenshtein { } } +/// Replace all tabs in a string with spaces, using the given tab size. +pub fn expandtabs(input: &str, tab_size: usize) -> String { + let tab_stop = tab_size; + let mut expanded_str = String::with_capacity(input.len()); + let mut tab_size = tab_stop; + let mut col_count = 0usize; + for ch in input.chars() { + match ch { + '\t' => { + let num_spaces = tab_size - col_count; + col_count += num_spaces; + let expand = " ".repeat(num_spaces); + expanded_str.push_str(&expand); + } + '\r' | '\n' => { + expanded_str.push(ch); + col_count = 0; + tab_size = 0; + } + _ => { + expanded_str.push(ch); + col_count += 1; + } + } + if col_count >= tab_size { + tab_size += tab_stop; + } + } + expanded_str +} + /// Creates an [`AsciiStr`][ascii::AsciiStr] from a string literal, throwing a compile error if the /// literal isn't actually ascii. /// diff --git a/compiler/codegen/Cargo.toml b/compiler/codegen/Cargo.toml index 0fe950be71..0817a95894 100644 --- a/compiler/codegen/Cargo.toml +++ b/compiler/codegen/Cargo.toml @@ -11,6 +11,7 @@ license.workspace = true [dependencies] rustpython-ast = { workspace = true, features=["unparse", "constant-optimization"] } +rustpython-common = { workspace = true } rustpython-parser-core = { workspace = true } rustpython-compiler-core = { workspace = true } diff --git a/compiler/codegen/src/compile.rs b/compiler/codegen/src/compile.rs index bcb67ca0f5..bd58a31be3 100644 --- a/compiler/codegen/src/compile.rs +++ b/compiler/codegen/src/compile.rs @@ -3368,17 +3368,51 @@ impl EmitArg for ir::BlockIdx { } } +/// Strips leading whitespace from a docstring. +/// +/// The code has been ported from `_PyCompile_CleanDoc` in cpython. +/// `inspect.cleandoc` is also a good reference, but has a few incompatibilities. +fn clean_doc(doc: &str) -> String { + let doc = rustpython_common::str::expandtabs(doc, 8); + // First pass: find minimum indentation of any non-blank lines + // after first line. + let margin = doc + .lines() + // Find the non-blank lines + .filter(|line| !line.trim().is_empty()) + // get the one with the least indentation + .map(|line| line.chars().take_while(|c| c == &' ').count()) + .min(); + if let Some(margin) = margin { + let mut cleaned = String::with_capacity(doc.len()); + // copy first line without leading whitespace + if let Some(first_line) = doc.lines().next() { + cleaned.push_str(first_line.trim_start()); + } + // copy subsequent lines without margin. + for line in doc.split('\n').skip(1) { + cleaned.push('\n'); + let cleaned_line = line.chars().skip(margin).collect::(); + cleaned.push_str(&cleaned_line); + } + + cleaned + } else { + doc.to_owned() + } +} + fn split_doc<'a>( body: &'a [located_ast::Stmt], opts: &CompileOpts, ) -> (Option, &'a [located_ast::Stmt]) { if let Some((located_ast::Stmt::Expr(expr), body_rest)) = body.split_first() { if let Some(doc) = try_get_constant_string(std::slice::from_ref(&expr.value)) { - if opts.optimize < 2 { - return (Some(doc), body_rest); + return if opts.optimize < 2 { + (Some(clean_doc(&doc)), body_rest) } else { - return (None, body_rest); - } + (None, body_rest) + }; } } (None, body) diff --git a/extra_tests/snippets/builtin_object.py b/extra_tests/snippets/builtin_object.py index ef83da83e2..5a12afbf45 100644 --- a/extra_tests/snippets/builtin_object.py +++ b/extra_tests/snippets/builtin_object.py @@ -7,9 +7,7 @@ class MyObject: assert myobj == myobj assert not myobj != myobj -object.__subclasshook__() == NotImplemented object.__subclasshook__(1) == NotImplemented -object.__subclasshook__(1, 2) == NotImplemented assert MyObject().__eq__(MyObject()) == NotImplemented assert MyObject().__ne__(MyObject()) == NotImplemented diff --git a/extra_tests/snippets/stdlib_imghdr.py b/extra_tests/snippets/stdlib_imghdr.py deleted file mode 100644 index 5ca524e269..0000000000 --- a/extra_tests/snippets/stdlib_imghdr.py +++ /dev/null @@ -1,27 +0,0 @@ -# unittest for modified imghdr.py -# Should be replace it into https://github.com/python/cpython/blob/main/Lib/test/test_imghdr.py -import os -import imghdr - - -TEST_FILES = ( - #('python.png', 'png'), - ('python.gif', 'gif'), - ('python.bmp', 'bmp'), - ('python.ppm', 'ppm'), - ('python.pgm', 'pgm'), - ('python.pbm', 'pbm'), - ('python.jpg', 'jpeg'), - ('python.ras', 'rast'), - #('python.sgi', 'rgb'), - ('python.tiff', 'tiff'), - ('python.xbm', 'xbm'), - ('python.webp', 'webp'), - ('python.exr', 'exr'), -) - -resource_dir = os.path.join(os.path.dirname(__file__), 'imghdrdata') - -for fname, expected in TEST_FILES: - res = imghdr.what(os.path.join(resource_dir, fname)) - assert res == expected \ No newline at end of file diff --git a/extra_tests/snippets/stdlib_subprocess.py b/extra_tests/snippets/stdlib_subprocess.py index b0323c8840..2e3aa7b2c1 100644 --- a/extra_tests/snippets/stdlib_subprocess.py +++ b/extra_tests/snippets/stdlib_subprocess.py @@ -32,7 +32,7 @@ def sleep(secs): assert p.poll() is None with assert_raises(subprocess.TimeoutExpired): - assert p.wait(1) + assert p.wait(1) p.wait() @@ -48,17 +48,17 @@ def sleep(secs): p.terminate() p.wait() if is_unix: - assert p.returncode == -signal.SIGTERM + assert p.returncode == -signal.SIGTERM else: - assert p.returncode == 1 + assert p.returncode == 1 p = subprocess.Popen(sleep(2)) p.kill() p.wait() if is_unix: - assert p.returncode == -signal.SIGKILL + assert p.returncode == -signal.SIGKILL else: - assert p.returncode == 1 + assert p.returncode == 1 p = subprocess.Popen(echo("test"), stdout=subprocess.PIPE) (stdout, stderr) = p.communicate() @@ -66,4 +66,4 @@ def sleep(secs): p = subprocess.Popen(sleep(5), stdout=subprocess.PIPE) with assert_raises(subprocess.TimeoutExpired): - p.communicate(timeout=1) + p.communicate(timeout=1) diff --git a/extra_tests/snippets/stdlib_xdrlib.py b/extra_tests/snippets/stdlib_xdrlib.py deleted file mode 100644 index 681cd77467..0000000000 --- a/extra_tests/snippets/stdlib_xdrlib.py +++ /dev/null @@ -1,12 +0,0 @@ -# This probably will be superceeded by the python unittests when that works. - -import xdrlib - -p = xdrlib.Packer() -p.pack_int(1337) - -d = p.get_buffer() - -print(d) - -# assert d == b'\x00\x00\x059' diff --git a/extra_tests/snippets/syntax_class.py b/extra_tests/snippets/syntax_class.py index 28d066a9e9..fb702f3304 100644 --- a/extra_tests/snippets/syntax_class.py +++ b/extra_tests/snippets/syntax_class.py @@ -50,7 +50,7 @@ def kungfu(x): assert x == 3 -assert Bar.__doc__ == " W00t " +assert Bar.__doc__ == "W00t " bar = Bar(42) assert bar.get_x.__doc__ == None @@ -147,7 +147,7 @@ class T3: test3 """ -assert T3.__doc__ == "\n test3\n " +assert T3.__doc__ == "\ntest3\n" class T4: diff --git a/extra_tests/snippets/syntax_doc.py b/extra_tests/snippets/syntax_doc.py new file mode 100644 index 0000000000..bdfc1fe778 --- /dev/null +++ b/extra_tests/snippets/syntax_doc.py @@ -0,0 +1,15 @@ + +def f1(): + """ + x + \ty + """ +assert f1.__doc__ == '\nx\ny\n' + +def f2(): + """ +\t x +\t\ty + """ + +assert f2.__doc__ == '\nx\n y\n' diff --git a/extra_tests/snippets/syntax_function2.py b/extra_tests/snippets/syntax_function2.py index ebea34fe58..dce4cb54eb 100644 --- a/extra_tests/snippets/syntax_function2.py +++ b/extra_tests/snippets/syntax_function2.py @@ -44,7 +44,7 @@ def f3(): """ pass -assert f3.__doc__ == "\n test3\n " +assert f3.__doc__ == "\ntest3\n" def f4(): "test4" diff --git a/vm/src/builtins/str.rs b/vm/src/builtins/str.rs index c0a096b211..52112f371d 100644 --- a/vm/src/builtins/str.rs +++ b/vm/src/builtins/str.rs @@ -1125,33 +1125,7 @@ impl PyStr { #[pymethod] fn expandtabs(&self, args: anystr::ExpandTabsArgs) -> String { - let tab_stop = args.tabsize(); - let mut expanded_str = String::with_capacity(self.byte_len()); - let mut tab_size = tab_stop; - let mut col_count = 0usize; - for ch in self.as_str().chars() { - match ch { - '\t' => { - let num_spaces = tab_size - col_count; - col_count += num_spaces; - let expand = " ".repeat(num_spaces); - expanded_str.push_str(&expand); - } - '\r' | '\n' => { - expanded_str.push(ch); - col_count = 0; - tab_size = 0; - } - _ => { - expanded_str.push(ch); - col_count += 1; - } - } - if col_count >= tab_size { - tab_size += tab_stop; - } - } - expanded_str + rustpython_common::str::expandtabs(self.as_str(), args.tabsize()) } #[pymethod] diff --git a/vm/src/version.rs b/vm/src/version.rs index 9a75f71142..3b6d9aa0ea 100644 --- a/vm/src/version.rs +++ b/vm/src/version.rs @@ -4,9 +4,9 @@ use chrono::{prelude::DateTime, Local}; use std::time::{Duration, UNIX_EPOCH}; -// = 3.12.0alpha +// = 3.13.0alpha pub const MAJOR: usize = 3; -pub const MINOR: usize = 12; +pub const MINOR: usize = 13; pub const MICRO: usize = 0; pub const RELEASELEVEL: &str = "alpha"; pub const RELEASELEVEL_N: usize = 0xA; diff --git a/whats_left.py b/whats_left.py index 4f087f89af..30b1de088e 100755 --- a/whats_left.py +++ b/whats_left.py @@ -35,8 +35,8 @@ implementation = platform.python_implementation() if implementation != "CPython": sys.exit(f"whats_left.py must be run under CPython, got {implementation} instead") -if sys.version_info[:2] < (3, 12): - sys.exit(f"whats_left.py must be run under CPython 3.12 or newer, got {implementation} {sys.version} instead") +if sys.version_info[:2] < (3, 13): + sys.exit(f"whats_left.py must be run under CPython 3.13 or newer, got {implementation} {sys.version} instead") def parse_args(): parser = argparse.ArgumentParser(description="Process some integers.") From a5364973d9e2a1ab3f5bd1882b5f506a7c80c2e8 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Thu, 13 Feb 2025 20:11:05 -0800 Subject: [PATCH 005/295] implement nt._supports_virtual_terminal Signed-off-by: Ashwin Naren --- vm/src/stdlib/nt.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/vm/src/stdlib/nt.rs b/vm/src/stdlib/nt.rs index 803cade630..d2785f42dd 100644 --- a/vm/src/stdlib/nt.rs +++ b/vm/src/stdlib/nt.rs @@ -43,6 +43,12 @@ pub(crate) mod module { || attr & FileSystem::FILE_ATTRIBUTE_DIRECTORY != 0)) } + #[pyfunction] + pub(super) fn _supports_virtual_terminal() -> PyResult { + // TODO: implement this + Ok(true) + } + #[derive(FromArgs)] pub(super) struct SymlinkArgs { src: OsPath, From e96557b3e12e878fa31b2ad1a098ba757b13fb28 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Thu, 13 Feb 2025 20:11:36 -0800 Subject: [PATCH 006/295] add _colorize.py at 3.13.2 Signed-off-by: Ashwin Naren --- Lib/_colorize.py | 67 ++++++++++++++++++ Lib/test/test__colorize.py | 135 +++++++++++++++++++++++++++++++++++++ 2 files changed, 202 insertions(+) create mode 100644 Lib/_colorize.py create mode 100644 Lib/test/test__colorize.py diff --git a/Lib/_colorize.py b/Lib/_colorize.py new file mode 100644 index 0000000000..70acfd4ad0 --- /dev/null +++ b/Lib/_colorize.py @@ -0,0 +1,67 @@ +import io +import os +import sys + +COLORIZE = True + + +class ANSIColors: + BOLD_GREEN = "\x1b[1;32m" + BOLD_MAGENTA = "\x1b[1;35m" + BOLD_RED = "\x1b[1;31m" + GREEN = "\x1b[32m" + GREY = "\x1b[90m" + MAGENTA = "\x1b[35m" + RED = "\x1b[31m" + RESET = "\x1b[0m" + YELLOW = "\x1b[33m" + + +NoColors = ANSIColors() + +for attr in dir(NoColors): + if not attr.startswith("__"): + setattr(NoColors, attr, "") + + +def get_colors(colorize: bool = False, *, file=None) -> ANSIColors: + if colorize or can_colorize(file=file): + return ANSIColors() + else: + return NoColors + + +def can_colorize(*, file=None) -> bool: + if file is None: + file = sys.stdout + + if not sys.flags.ignore_environment: + if os.environ.get("PYTHON_COLORS") == "0": + return False + if os.environ.get("PYTHON_COLORS") == "1": + return True + if os.environ.get("NO_COLOR"): + return False + if not COLORIZE: + return False + if os.environ.get("FORCE_COLOR"): + return True + if os.environ.get("TERM") == "dumb": + return False + + if not hasattr(file, "fileno"): + return False + + if sys.platform == "win32": + try: + import nt + + if not nt._supports_virtual_terminal(): + return False + except (ImportError, AttributeError): + return False + + try: + return os.isatty(file.fileno()) + except io.UnsupportedOperation: + return file.isatty() diff --git a/Lib/test/test__colorize.py b/Lib/test/test__colorize.py new file mode 100644 index 0000000000..056a5306ce --- /dev/null +++ b/Lib/test/test__colorize.py @@ -0,0 +1,135 @@ +import contextlib +import io +import sys +import unittest +import unittest.mock +import _colorize +from test.support.os_helper import EnvironmentVarGuard + + +@contextlib.contextmanager +def clear_env(): + with EnvironmentVarGuard() as mock_env: + for var in "FORCE_COLOR", "NO_COLOR", "PYTHON_COLORS": + mock_env.unset(var) + yield mock_env + + +def supports_virtual_terminal(): + if sys.platform == "win32": + return unittest.mock.patch("nt._supports_virtual_terminal", return_value=True) + else: + return contextlib.nullcontext() + + +class TestColorizeFunction(unittest.TestCase): + def test_colorized_detection_checks_for_environment_variables(self): + def check(env, fallback, expected): + with (self.subTest(env=env, fallback=fallback), + clear_env() as mock_env): + mock_env.update(env) + isatty_mock.return_value = fallback + stdout_mock.isatty.return_value = fallback + self.assertEqual(_colorize.can_colorize(), expected) + + with (unittest.mock.patch("os.isatty") as isatty_mock, + unittest.mock.patch("sys.stdout") as stdout_mock, + supports_virtual_terminal()): + stdout_mock.fileno.return_value = 1 + + for fallback in False, True: + check({}, fallback, fallback) + check({'TERM': 'dumb'}, fallback, False) + check({'TERM': 'xterm'}, fallback, fallback) + check({'TERM': ''}, fallback, fallback) + check({'FORCE_COLOR': '1'}, fallback, True) + check({'FORCE_COLOR': '0'}, fallback, True) + check({'FORCE_COLOR': ''}, fallback, fallback) + check({'NO_COLOR': '1'}, fallback, False) + check({'NO_COLOR': '0'}, fallback, False) + check({'NO_COLOR': ''}, fallback, fallback) + + check({'TERM': 'dumb', 'FORCE_COLOR': '1'}, False, True) + check({'FORCE_COLOR': '1', 'NO_COLOR': '1'}, True, False) + + for ignore_environment in False, True: + # Simulate running with or without `-E`. + flags = unittest.mock.MagicMock(ignore_environment=ignore_environment) + with unittest.mock.patch("sys.flags", flags): + check({'PYTHON_COLORS': '1'}, True, True) + check({'PYTHON_COLORS': '1'}, False, not ignore_environment) + check({'PYTHON_COLORS': '0'}, True, ignore_environment) + check({'PYTHON_COLORS': '0'}, False, False) + for fallback in False, True: + check({'PYTHON_COLORS': 'x'}, fallback, fallback) + check({'PYTHON_COLORS': ''}, fallback, fallback) + + check({'TERM': 'dumb', 'PYTHON_COLORS': '1'}, False, not ignore_environment) + check({'NO_COLOR': '1', 'PYTHON_COLORS': '1'}, False, not ignore_environment) + check({'FORCE_COLOR': '1', 'PYTHON_COLORS': '0'}, True, ignore_environment) + + @unittest.skipUnless(sys.platform == "win32", "requires Windows") + def test_colorized_detection_checks_on_windows(self): + with (clear_env(), + unittest.mock.patch("os.isatty") as isatty_mock, + unittest.mock.patch("sys.stdout") as stdout_mock, + supports_virtual_terminal() as vt_mock): + stdout_mock.fileno.return_value = 1 + isatty_mock.return_value = True + stdout_mock.isatty.return_value = True + + vt_mock.return_value = True + self.assertEqual(_colorize.can_colorize(), True) + vt_mock.return_value = False + self.assertEqual(_colorize.can_colorize(), False) + import nt + del nt._supports_virtual_terminal + self.assertEqual(_colorize.can_colorize(), False) + + def test_colorized_detection_checks_for_std_streams(self): + with (clear_env(), + unittest.mock.patch("os.isatty") as isatty_mock, + unittest.mock.patch("sys.stdout") as stdout_mock, + unittest.mock.patch("sys.stderr") as stderr_mock, + supports_virtual_terminal()): + stdout_mock.fileno.return_value = 1 + stderr_mock.fileno.side_effect = ZeroDivisionError + stderr_mock.isatty.side_effect = ZeroDivisionError + + isatty_mock.return_value = True + stdout_mock.isatty.return_value = True + self.assertEqual(_colorize.can_colorize(), True) + + isatty_mock.return_value = False + stdout_mock.isatty.return_value = False + self.assertEqual(_colorize.can_colorize(), False) + + def test_colorized_detection_checks_for_file(self): + with clear_env(), supports_virtual_terminal(): + + with unittest.mock.patch("os.isatty") as isatty_mock: + file = unittest.mock.MagicMock() + file.fileno.return_value = 1 + isatty_mock.return_value = True + self.assertEqual(_colorize.can_colorize(file=file), True) + isatty_mock.return_value = False + self.assertEqual(_colorize.can_colorize(file=file), False) + + # No file.fileno. + with unittest.mock.patch("os.isatty", side_effect=ZeroDivisionError): + file = unittest.mock.MagicMock(spec=['isatty']) + file.isatty.return_value = True + self.assertEqual(_colorize.can_colorize(file=file), False) + + # file.fileno() raises io.UnsupportedOperation. + with unittest.mock.patch("os.isatty", side_effect=ZeroDivisionError): + file = unittest.mock.MagicMock() + file.fileno.side_effect = io.UnsupportedOperation + file.isatty.return_value = True + self.assertEqual(_colorize.can_colorize(file=file), True) + file.isatty.return_value = False + self.assertEqual(_colorize.can_colorize(file=file), False) + + +if __name__ == "__main__": + unittest.main() From fde87a340caff2a7cea3d6805ec38cc0f0a49495 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Fri, 14 Feb 2025 23:03:58 -0800 Subject: [PATCH 007/295] Initial _ctypes implementation (#5519) * initial _ctypes implementation with _CData, get_errno, and set_errno Signed-off-by: Ashwin Naren --- Cargo.lock | 1 + extra_tests/snippets/builtins_ctypes.py | 105 ++++++++++++++++ vm/Cargo.toml | 3 +- vm/src/stdlib/ctypes.rs | 107 ++++++++++++++++ vm/src/stdlib/ctypes/base.rs | 161 ++++++++++++++++++++++++ vm/src/stdlib/mod.rs | 6 + 6 files changed, 382 insertions(+), 1 deletion(-) create mode 100644 extra_tests/snippets/builtins_ctypes.py create mode 100644 vm/src/stdlib/ctypes.rs create mode 100644 vm/src/stdlib/ctypes/base.rs diff --git a/Cargo.lock b/Cargo.lock index c238cf994f..2e3b216e73 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2258,6 +2258,7 @@ dependencies = [ "cfg-if", "chrono", "crossbeam-utils", + "errno", "exitcode", "flame", "flamer", diff --git a/extra_tests/snippets/builtins_ctypes.py b/extra_tests/snippets/builtins_ctypes.py new file mode 100644 index 0000000000..5df259cf82 --- /dev/null +++ b/extra_tests/snippets/builtins_ctypes.py @@ -0,0 +1,105 @@ +from _ctypes import sizeof +from _ctypes import _SimpleCData +from struct import calcsize as _calcsize + +def _check_size(typ, typecode=None): + # Check if sizeof(ctypes_type) against struct.calcsize. This + # should protect somewhat against a misconfigured libffi. + from struct import calcsize + if typecode is None: + # Most _type_ codes are the same as used in struct + typecode = typ._type_ + actual, required = sizeof(typ), calcsize(typecode) + if actual != required: + raise SystemError("sizeof(%s) wrong: %d instead of %d" % \ + (typ, actual, required)) + +class c_short(_SimpleCData): + _type_ = "h" +_check_size(c_short) + +class c_ushort(_SimpleCData): + _type_ = "H" +_check_size(c_ushort) + +class c_long(_SimpleCData): + _type_ = "l" +_check_size(c_long) + +class c_ulong(_SimpleCData): + _type_ = "L" +_check_size(c_ulong) + +if _calcsize("i") == _calcsize("l"): + # if int and long have the same size, make c_int an alias for c_long + c_int = c_long + c_uint = c_ulong +else: + class c_int(_SimpleCData): + _type_ = "i" + _check_size(c_int) + + class c_uint(_SimpleCData): + _type_ = "I" + _check_size(c_uint) + +class c_float(_SimpleCData): + _type_ = "f" +_check_size(c_float) + +class c_double(_SimpleCData): + _type_ = "d" +_check_size(c_double) + +class c_longdouble(_SimpleCData): + _type_ = "g" +if sizeof(c_longdouble) == sizeof(c_double): + c_longdouble = c_double + +if _calcsize("l") == _calcsize("q"): + # if long and long long have the same size, make c_longlong an alias for c_long + c_longlong = c_long + c_ulonglong = c_ulong +else: + class c_longlong(_SimpleCData): + _type_ = "q" + _check_size(c_longlong) + + class c_ulonglong(_SimpleCData): + _type_ = "Q" + ## def from_param(cls, val): + ## return ('d', float(val), val) + ## from_param = classmethod(from_param) + _check_size(c_ulonglong) + +class c_ubyte(_SimpleCData): + _type_ = "B" +c_ubyte.__ctype_le__ = c_ubyte.__ctype_be__ = c_ubyte +# backward compatibility: +##c_uchar = c_ubyte +_check_size(c_ubyte) + +class c_byte(_SimpleCData): + _type_ = "b" +c_byte.__ctype_le__ = c_byte.__ctype_be__ = c_byte +_check_size(c_byte) + +class c_char(_SimpleCData): + _type_ = "c" +c_char.__ctype_le__ = c_char.__ctype_be__ = c_char +_check_size(c_char) + +class c_char_p(_SimpleCData): + _type_ = "z" + def __repr__(self): + return "%s(%s)" % (self.__class__.__name__, c_void_p.from_buffer(self).value) +_check_size(c_char_p, "P") + +class c_void_p(_SimpleCData): + _type_ = "P" +c_voidp = c_void_p # backwards compatibility (to a bug) +_check_size(c_void_p) + +class c_bool(_SimpleCData): + _type_ = "?" +_check_size(c_bool) diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 6eaca281f8..1468651aeb 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -99,6 +99,8 @@ uname = "0.1.1" [target.'cfg(not(target_arch = "wasm32"))'.dependencies] rustyline = { workspace = true } which = "6" +errno = "0.3" +widestring = { workspace = true } [target.'cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))'.dependencies] num_cpus = "1.13.1" @@ -106,7 +108,6 @@ num_cpus = "1.13.1" [target.'cfg(windows)'.dependencies] junction = { workspace = true } schannel = { workspace = true } -widestring = { workspace = true } winreg = "0.52" [target.'cfg(windows)'.dependencies.windows] diff --git a/vm/src/stdlib/ctypes.rs b/vm/src/stdlib/ctypes.rs new file mode 100644 index 0000000000..d1fc26869f --- /dev/null +++ b/vm/src/stdlib/ctypes.rs @@ -0,0 +1,107 @@ +pub(crate) mod base; + +use crate::builtins::PyModule; +use crate::{PyRef, VirtualMachine}; + +pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { + let module = _ctypes::make_module(vm); + base::extend_module_nodes(vm, &module); + module +} + +#[pymodule] +pub(crate) mod _ctypes { + use super::base::PyCSimple; + use crate::builtins::PyTypeRef; + use crate::class::StaticType; + use crate::function::Either; + use crate::{AsObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine}; + use crossbeam_utils::atomic::AtomicCell; + use std::ffi::{ + c_double, c_float, c_int, c_long, c_longlong, c_schar, c_short, c_uchar, c_uint, c_ulong, + c_ulonglong, + }; + use std::mem; + use widestring::WideChar; + + pub fn get_size(ty: &str) -> usize { + match ty { + "u" => mem::size_of::(), + "c" | "b" => mem::size_of::(), + "h" => mem::size_of::(), + "H" => mem::size_of::(), + "i" => mem::size_of::(), + "I" => mem::size_of::(), + "l" => mem::size_of::(), + "q" => mem::size_of::(), + "L" => mem::size_of::(), + "Q" => mem::size_of::(), + "f" => mem::size_of::(), + "d" | "g" => mem::size_of::(), + "?" | "B" => mem::size_of::(), + "P" | "z" | "Z" => mem::size_of::(), + _ => unreachable!(), + } + } + + const SIMPLE_TYPE_CHARS: &str = "cbBhHiIlLdfguzZPqQ?"; + + pub fn new_simple_type( + cls: Either<&PyObjectRef, &PyTypeRef>, + vm: &VirtualMachine, + ) -> PyResult { + let cls = match cls { + Either::A(obj) => obj, + Either::B(typ) => typ.as_object(), + }; + + if let Ok(_type_) = cls.get_attr("_type_", vm) { + if _type_.is_instance((&vm.ctx.types.str_type).as_ref(), vm)? { + let tp_str = _type_.str(vm)?.to_string(); + + if tp_str.len() != 1 { + Err(vm.new_value_error( + format!("class must define a '_type_' attribute which must be a string of length 1, str: {tp_str}"), + )) + } else if !SIMPLE_TYPE_CHARS.contains(tp_str.as_str()) { + Err(vm.new_attribute_error(format!("class must define a '_type_' attribute which must be\n a single character string containing one of {SIMPLE_TYPE_CHARS}, currently it is {tp_str}."))) + } else { + Ok(PyCSimple { + _type_: tp_str, + _value: AtomicCell::new(vm.ctx.none()), + }) + } + } else { + Err(vm.new_type_error("class must define a '_type_' string attribute".to_string())) + } + } else { + Err(vm.new_attribute_error("class must define a '_type_' attribute".to_string())) + } + } + + #[pyfunction(name = "sizeof")] + pub fn size_of(tp: Either, vm: &VirtualMachine) -> PyResult { + match tp { + Either::A(type_) if type_.fast_issubclass(PyCSimple::static_type()) => { + let zelf = new_simple_type(Either::B(&type_), vm)?; + Ok(get_size(zelf._type_.as_str())) + } + Either::B(obj) if obj.has_attr("size_of_instances", vm)? => { + let size_of_method = obj.get_attr("size_of_instances", vm)?; + let size_of_return = size_of_method.call(vec![], vm)?; + Ok(usize::try_from_object(vm, size_of_return)?) + } + _ => Err(vm.new_type_error("this type has no size".to_string())), + } + } + + #[pyfunction] + fn get_errno() -> i32 { + errno::errno().0 + } + + #[pyfunction] + fn set_errno(value: i32) { + errno::set_errno(errno::Errno(value)); + } +} diff --git a/vm/src/stdlib/ctypes/base.rs b/vm/src/stdlib/ctypes/base.rs new file mode 100644 index 0000000000..0142c45345 --- /dev/null +++ b/vm/src/stdlib/ctypes/base.rs @@ -0,0 +1,161 @@ +use crate::builtins::{PyBytes, PyFloat, PyInt, PyModule, PyNone, PyStr}; +use crate::class::PyClassImpl; +use crate::{Py, PyObjectRef, PyResult, TryFromObject, VirtualMachine}; +use crossbeam_utils::atomic::AtomicCell; +use num_traits::ToPrimitive; +use rustpython_common::lock::PyRwLock; +use std::fmt::Debug; + +#[allow(dead_code)] +fn set_primitive(_type_: &str, value: &PyObjectRef, vm: &VirtualMachine) -> PyResult { + match _type_ { + "c" => { + if value + .clone() + .downcast_exact::(vm) + .is_ok_and(|v| v.len() == 1) + || value + .clone() + .downcast_exact::(vm) + .is_ok_and(|v| v.len() == 1) + || value + .clone() + .downcast_exact::(vm) + .map_or(Ok(false), |v| { + let n = v.as_bigint().to_i64(); + if let Some(n) = n { + Ok((0..=255).contains(&n)) + } else { + Ok(false) + } + })? + { + Ok(value.clone()) + } else { + Err(vm.new_type_error( + "one character bytes, bytearray or integer expected".to_string(), + )) + } + } + "u" => { + if let Ok(b) = value.str(vm).map(|v| v.to_string().chars().count() == 1) { + if b { + Ok(value.clone()) + } else { + Err(vm.new_type_error("one character unicode string expected".to_string())) + } + } else { + Err(vm.new_type_error(format!( + "unicode string expected instead of {} instance", + value.class().name() + ))) + } + } + "b" | "h" | "H" | "i" | "I" | "l" | "q" | "L" | "Q" => { + if value.clone().downcast_exact::(vm).is_ok() { + Ok(value.clone()) + } else { + Err(vm.new_type_error(format!( + "an integer is required (got type {})", + value.class().name() + ))) + } + } + "f" | "d" | "g" => { + if value.clone().downcast_exact::(vm).is_ok() { + Ok(value.clone()) + } else { + Err(vm.new_type_error(format!("must be real number, not {}", value.class().name()))) + } + } + "?" => Ok(PyObjectRef::from( + vm.ctx.new_bool(value.clone().try_to_bool(vm)?), + )), + "B" => { + if value.clone().downcast_exact::(vm).is_ok() { + Ok(vm.new_pyobj(u8::try_from_object(vm, value.clone())?)) + } else { + Err(vm.new_type_error(format!("int expected instead of {}", value.class().name()))) + } + } + "z" => { + if value.clone().downcast_exact::(vm).is_ok() + || value.clone().downcast_exact::(vm).is_ok() + { + Ok(value.clone()) + } else { + Err(vm.new_type_error(format!( + "bytes or integer address expected instead of {} instance", + value.class().name() + ))) + } + } + "Z" => { + if value.clone().downcast_exact::(vm).is_ok() { + Ok(value.clone()) + } else { + Err(vm.new_type_error(format!( + "unicode string or integer address expected instead of {} instance", + value.class().name() + ))) + } + } + _ => { + // "P" + if value.clone().downcast_exact::(vm).is_ok() + || value.clone().downcast_exact::(vm).is_ok() + { + Ok(value.clone()) + } else { + Err(vm.new_type_error("cannot be converted to pointer".to_string())) + } + } + } +} + +pub struct RawBuffer { + #[allow(dead_code)] + pub inner: Box<[u8]>, + #[allow(dead_code)] + pub size: usize, +} + +#[pyclass(name = "_CData", module = "_ctypes")] +pub struct PyCData { + _objects: AtomicCell>, + _buffer: PyRwLock, +} + +#[pyclass] +impl PyCData {} + +#[pyclass( + name = "_SimpleCData", + base = "PyCData", + module = "_ctypes" + // TODO: metaclass +)] +#[derive(PyPayload)] +pub struct PyCSimple { + pub _type_: String, + pub _value: AtomicCell, +} + +impl Debug for PyCSimple { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PyCSimple") + .field("_type_", &self._type_) + .finish() + } +} + +#[pyclass(flags(BASETYPE))] +impl PyCSimple {} + +pub fn extend_module_nodes(vm: &VirtualMachine, module: &Py) { + let ctx = &vm.ctx; + extend_module!(vm, module, { + "_CData" => PyCData::make_class(ctx), + "_SimpleCData" => PyCSimple::make_class(ctx), + }) +} diff --git a/vm/src/stdlib/mod.rs b/vm/src/stdlib/mod.rs index 12baee11f7..ca8f8f0bfd 100644 --- a/vm/src/stdlib/mod.rs +++ b/vm/src/stdlib/mod.rs @@ -37,6 +37,8 @@ pub mod posix; #[path = "posix_compat.rs"] pub mod posix; +#[cfg(any(target_family = "unix", target_family = "windows"))] +mod ctypes; #[cfg(windows)] pub(crate) mod msvcrt; #[cfg(all(unix, not(any(target_os = "android", target_os = "redox"))))] @@ -124,5 +126,9 @@ pub fn get_module_inits() -> StdlibMap { "_winapi" => winapi::make_module, "winreg" => winreg::make_module, } + #[cfg(any(target_family = "unix", target_family = "windows"))] + { + "_ctypes" => ctypes::make_module, + } } } From e8a3406624460abc25f4cc5ab6e80d52a822d04a Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sat, 15 Feb 2025 15:01:49 -0800 Subject: [PATCH 008/295] itertools upgrade --- Cargo.lock | 16 ++++++++-------- Cargo.toml | 2 +- compiler/codegen/src/compile.rs | 4 ++-- stdlib/src/math.rs | 2 +- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2e3b216e73..a62a5dfced 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -976,9 +976,9 @@ dependencies = [ [[package]] name = "itertools" -version = "0.12.1" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" dependencies = [ "either", ] @@ -1978,7 +1978,7 @@ dependencies = [ "bitflags 2.8.0", "indexmap", "insta", - "itertools 0.12.1", + "itertools 0.14.0", "log", "num-complex", "num-traits", @@ -1996,7 +1996,7 @@ dependencies = [ "ascii", "bitflags 2.8.0", "cfg-if", - "itertools 0.12.1", + "itertools 0.14.0", "libc", "lock_api", "malachite-base", @@ -2029,7 +2029,7 @@ name = "rustpython-compiler-core" version = "0.4.0" dependencies = [ "bitflags 2.8.0", - "itertools 0.12.1", + "itertools 0.14.0", "lz4_flex", "malachite-bigint", "num-complex", @@ -2050,7 +2050,7 @@ dependencies = [ name = "rustpython-derive-impl" version = "0.4.0" dependencies = [ - "itertools 0.12.1", + "itertools 0.14.0", "maplit", "once_cell", "proc-macro2", @@ -2192,7 +2192,7 @@ dependencies = [ "gethostname", "hex", "indexmap", - "itertools 0.12.1", + "itertools 0.14.0", "junction", "libc", "libsqlite3-sys", @@ -2268,7 +2268,7 @@ dependencies = [ "hex", "indexmap", "is-macro", - "itertools 0.12.1", + "itertools 0.14.0", "junction", "libc", "log", diff --git a/Cargo.toml b/Cargo.toml index e838110e89..958bc5eeff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -151,7 +151,7 @@ glob = "0.3" hex = "0.4.3" indexmap = { version = "2.2.6", features = ["std"] } insta = "1.38.0" -itertools = "0.12.1" +itertools = "0.14.0" is-macro = "0.3.7" junction = "1.2.0" libc = "0.2.169" diff --git a/compiler/codegen/src/compile.rs b/compiler/codegen/src/compile.rs index bd58a31be3..d650e998ea 100644 --- a/compiler/codegen/src/compile.rs +++ b/compiler/codegen/src/compile.rs @@ -2723,7 +2723,7 @@ impl Compiler { fn compile_keywords(&mut self, keywords: &[located_ast::Keyword]) -> CompileResult<()> { let mut size = 0; - let groupby = keywords.iter().group_by(|e| e.arg.is_none()); + let groupby = keywords.iter().chunk_by(|e| e.arg.is_none()); for (is_unpacking, sub_keywords) in &groupby { if is_unpacking { for keyword in sub_keywords { @@ -2886,7 +2886,7 @@ impl Compiler { (false, element) } }) - .group_by(|(starred, _)| *starred); + .chunk_by(|(starred, _)| *starred); for (starred, run) in &groups { let mut run_size = 0; diff --git a/stdlib/src/math.rs b/stdlib/src/math.rs index 30b088f714..21451203b5 100644 --- a/stdlib/src/math.rs +++ b/stdlib/src/math.rs @@ -356,7 +356,7 @@ mod math { .map(|x| (x / scale).powi(2)) .chain(std::iter::once(-norm * norm)) // Pairwise summation of floats gives less rounding error than a naive sum. - .tree_fold1(std::ops::Add::add) + .tree_reduce(std::ops::Add::add) .expect("expected at least 1 element"); norm = norm + correction / (2.0 * norm); } From 9e310934d3f1ad197d41815dc7f859bfb728b7d1 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Mon, 3 Feb 2025 16:20:47 -0800 Subject: [PATCH 009/295] fix panic Signed-off-by: Ashwin Naren --- vm/src/codecs.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/src/codecs.rs b/vm/src/codecs.rs index ff7bc48915..61be9f9176 100644 --- a/vm/src/codecs.rs +++ b/vm/src/codecs.rs @@ -623,7 +623,7 @@ fn surrogatepass_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<(PyOb // Try decoding a single surrogate character. If there are more, // let the codec call us again. let p = &s.as_bytes()[range.start..]; - if p.len() - range.start >= byte_length { + if p.len().saturating_sub(range.start) >= byte_length { match standard_encoding { StandardEncoding::Utf8 => { if (p[0] as u32 & 0xf0) == 0xe0 From 6788010f7d4c8897fa9b818f5ee118cd5c37627c Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Mon, 10 Feb 2025 14:29:52 -0800 Subject: [PATCH 010/295] windows-rs upgrade to 0.59 --- Cargo.lock | 191 ++++++++++++++++++++++++++++++++-------- Cargo.toml | 2 +- common/src/fileutils.rs | 14 +-- vm/Cargo.toml | 4 +- vm/src/stdlib/nt.rs | 5 +- vm/src/stdlib/winapi.rs | 4 + vm/src/stdlib/winreg.rs | 3 +- 7 files changed, 173 insertions(+), 50 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a62a5dfced..115451adfa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -216,9 +216,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.2.13" +version = "1.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7777341816418c02e033934a09f20dc0ccaf65a5201ef8a450ae0105a573fda" +checksum = "0c3d1b2e905a3a7b00a6141adb0e4c0bb941d11caf55349d863942a1cc44e3c9" dependencies = [ "shlex", ] @@ -645,9 +645,9 @@ dependencies = [ [[package]] name = "equivalent" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "errno" @@ -903,7 +903,7 @@ dependencies = [ "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "windows-core", + "windows-core 0.52.0", ] [[package]] @@ -1406,9 +1406,9 @@ checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9" [[package]] name = "openssl" -version = "0.10.70" +version = "0.10.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61cfb4e166a8bb8c9b55c500bc2308550148ece889be90f609377e58140f42c6" +checksum = "5e14130c6a98cd258fdcb0fb6d744152343ff729cbfcb28c656a9d12b999fbcd" dependencies = [ "bitflags 2.8.0", "cfg-if", @@ -1447,9 +1447,9 @@ dependencies = [ [[package]] name = "openssl-sys" -version = "0.9.105" +version = "0.9.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b22d5b84be05a8d6947c7cb71f7c849aa0f112acd4bf51c2a7c1c988ac0a9dc" +checksum = "8bb61ea9811cc39e3c2069f40b8b8e2e70d8569b361f879786cc7ed48b777cdd" dependencies = [ "cc", "libc", @@ -1745,8 +1745,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" dependencies = [ "rand_chacha 0.9.0", - "rand_core 0.9.0", - "zerocopy 0.8.17", + "rand_core 0.9.1", + "zerocopy 0.8.18", ] [[package]] @@ -1766,7 +1766,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core 0.9.0", + "rand_core 0.9.1", ] [[package]] @@ -1780,12 +1780,12 @@ dependencies = [ [[package]] name = "rand_core" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b08f3c9802962f7e1b25113931d94f43ed9725bebc59db9d0c3e9a23b67e15ff" +checksum = "a88e0da7a2c97baa202165137c158d0a2e824ac465d13d81046727b34cb247d3" dependencies = [ "getrandom 0.3.1", - "zerocopy 0.8.17", + "zerocopy 0.8.18", ] [[package]] @@ -2012,7 +2012,7 @@ dependencies = [ "siphasher 0.3.11", "volatile", "widestring", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -2242,7 +2242,7 @@ dependencies = [ "uuid", "widestring", "winapi", - "windows-sys 0.52.0", + "windows-sys 0.59.0", "xml-rs", ] @@ -2320,7 +2320,7 @@ dependencies = [ "which", "widestring", "windows", - "windows-sys 0.52.0", + "windows-sys 0.59.0", "winreg", ] @@ -2524,9 +2524,9 @@ checksum = "826167069c09b99d56f31e9ae5c99049e932a98c9dc2dac47645b08dbbf76ba7" [[package]] name = "smallvec" -version = "1.13.2" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" +checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd" [[package]] name = "socket2" @@ -2552,15 +2552,15 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" [[package]] name = "strum" -version = "0.27.0" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce1475c515a4f03a8a7129bb5228b81a781a86cb0b3fbbc19e1c556d491a401f" +checksum = "f64def088c51c9510a8579e3c5d67c65349dcf755e5479ad3d010aa6454e2c32" [[package]] name = "strum_macros" -version = "0.27.0" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9688894b43459159c82bfa5a5fa0435c19cbe3c9b427fa1dd7b1ce0c279b18a7" +checksum = "c77a8c5abcaf0f9ce05d62342b7d298c346515365c36b673df4ebe3ced01fde8" dependencies = [ "heck", "proc-macro2", @@ -3185,12 +3185,12 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows" -version = "0.52.0" +version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be" +checksum = "7f919aee0a93304be7f62e8e5027811bbba96bcb1de84d6618be56e43f8a32a1" dependencies = [ - "windows-core", - "windows-targets 0.52.6", + "windows-core 0.59.0", + "windows-targets 0.53.0", ] [[package]] @@ -3202,6 +3202,59 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-core" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "810ce18ed2112484b0d4e15d022e5f598113e220c53e373fb31e67e21670c1ce" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-result", + "windows-strings", + "windows-targets 0.53.0", +] + +[[package]] +name = "windows-implement" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83577b051e2f49a058c308f17f273b570a6a758386fc291b5f6a934dd84e48c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.98", +] + +[[package]] +name = "windows-interface" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb26fd936d991781ea39e87c3a27285081e3c0da5ca0fcbc02d368cc6f52ff01" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.98", +] + +[[package]] +name = "windows-result" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d08106ce80268c4067c0571ca55a9b4e9516518eaa1a1fe9b37ca403ae1d1a34" +dependencies = [ + "windows-targets 0.53.0", +] + +[[package]] +name = "windows-strings" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b888f919960b42ea4e11c2f408fadb55f78a9f236d5eef084103c8ce52893491" +dependencies = [ + "windows-targets 0.53.0", +] + [[package]] name = "windows-sys" version = "0.36.1" @@ -3266,13 +3319,29 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm", + "windows_i686_gnullvm 0.52.6", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] +[[package]] +name = "windows-targets" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e4c7e8ceaaf9cb7d7507c974735728ab453b67ef8f18febdd7c11fe59dca8b" +dependencies = [ + "windows_aarch64_gnullvm 0.53.0", + "windows_aarch64_msvc 0.53.0", + "windows_i686_gnu 0.53.0", + "windows_i686_gnullvm 0.53.0", + "windows_i686_msvc 0.53.0", + "windows_x86_64_gnu 0.53.0", + "windows_x86_64_gnullvm 0.53.0", + "windows_x86_64_msvc 0.53.0", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -3285,6 +3354,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" + [[package]] name = "windows_aarch64_msvc" version = "0.36.1" @@ -3303,6 +3378,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" + [[package]] name = "windows_i686_gnu" version = "0.36.1" @@ -3321,12 +3402,24 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" + [[package]] name = "windows_i686_msvc" version = "0.36.1" @@ -3345,6 +3438,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_i686_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" + [[package]] name = "windows_x86_64_gnu" version = "0.36.1" @@ -3363,6 +3462,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -3375,6 +3480,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" + [[package]] name = "windows_x86_64_msvc" version = "0.36.1" @@ -3393,14 +3504,20 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" + [[package]] name = "winreg" -version = "0.52.0" +version = "0.55.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5" +checksum = "cb5a765337c50e9ec252c2069be9bf91c7df47afb103b642ba3a53bf8101be97" dependencies = [ "cfg-if", - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] @@ -3436,11 +3553,11 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.17" +version = "0.8.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa91407dacce3a68c56de03abe2760159582b846c6a4acd2f456618087f12713" +checksum = "79386d31a42a4996e3336b0919ddb90f81112af416270cff95b5f5af22b839c2" dependencies = [ - "zerocopy-derive 0.8.17", + "zerocopy-derive 0.8.18", ] [[package]] @@ -3456,9 +3573,9 @@ dependencies = [ [[package]] name = "zerocopy-derive" -version = "0.8.17" +version = "0.8.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06718a168365cad3d5ff0bb133aad346959a2074bd4a85c121255a11304a8626" +checksum = "76331675d372f91bf8d17e13afbd5fe639200b73d01f0fc748bb059f9cca2db7" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 958bc5eeff..f7d7d896c5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -181,7 +181,7 @@ thiserror = "2.0" thread_local = "1.1.8" unicode_names2 = "1.3.0" widestring = "1.1.0" -windows-sys = "0.52.0" +windows-sys = "0.59.0" wasm-bindgen = "0.2.100" # Lints diff --git a/common/src/fileutils.rs b/common/src/fileutils.rs index dcb78675d8..296bb6be7c 100644 --- a/common/src/fileutils.rs +++ b/common/src/fileutils.rs @@ -25,7 +25,7 @@ pub mod windows { use crate::suppress_iph; use crate::windows::ToWideString; use libc::{S_IFCHR, S_IFDIR, S_IFMT}; - use std::ffi::{CString, OsStr, OsString}; + use std::ffi::{c_int, c_void, CString, OsStr, OsString}; use std::os::windows::ffi::OsStrExt; use std::sync::OnceLock; use windows_sys::core::PCWSTR; @@ -113,10 +113,10 @@ pub mod windows { if h.is_err() { unsafe { SetLastError(ERROR_INVALID_HANDLE) }; } - let h = h?; + let mut h = c_int::from(h? as i32); // reset stat? - let file_type = unsafe { GetFileType(h) }; + let file_type = unsafe { GetFileType(&mut h as *mut c_int as *mut c_void) }; if file_type == FILE_TYPE_UNKNOWN { return Err(std::io::Error::last_os_error()); } @@ -138,10 +138,10 @@ pub mod windows { let mut basic_info: FILE_BASIC_INFO = unsafe { std::mem::zeroed() }; let mut id_info: FILE_ID_INFO = unsafe { std::mem::zeroed() }; - if unsafe { GetFileInformationByHandle(h, &mut info) } == 0 + if unsafe { GetFileInformationByHandle(&mut h as *mut c_int as *mut c_void, &mut info) } == 0 || unsafe { GetFileInformationByHandleEx( - h, + &mut h as *mut c_int as *mut c_void, FileBasicInfo, &mut basic_info as *mut _ as *mut _, std::mem::size_of_val(&basic_info) as u32, @@ -153,7 +153,7 @@ pub mod windows { let p_id_info = if unsafe { GetFileInformationByHandleEx( - h, + &mut h as *mut c_int as *mut c_void, FileIdInfo, &mut id_info as *mut _ as *mut _, std::mem::size_of_val(&id_info) as u32, @@ -320,7 +320,7 @@ pub mod windows { .get_or_init(|| { let library_name = OsString::from("api-ms-win-core-file-l2-1-4").to_wide_with_nul(); let module = unsafe { LoadLibraryW(library_name.as_ptr()) }; - if module == 0 { + if module == c_int::from(0) as *mut c_int as *mut c_void { return None; } let name = CString::new("GetFileInformationByName").unwrap(); diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 1468651aeb..49eaabf96a 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -108,10 +108,10 @@ num_cpus = "1.13.1" [target.'cfg(windows)'.dependencies] junction = { workspace = true } schannel = { workspace = true } -winreg = "0.52" +winreg = "0.55" [target.'cfg(windows)'.dependencies.windows] -version = "0.52.0" +version = "0.59.0" features = [ "Win32_Foundation", "Win32_System_LibraryLoader", diff --git a/vm/src/stdlib/nt.rs b/vm/src/stdlib/nt.rs index d2785f42dd..64c04adbd6 100644 --- a/vm/src/stdlib/nt.rs +++ b/vm/src/stdlib/nt.rs @@ -25,6 +25,7 @@ pub(crate) mod module { mem::MaybeUninit, os::windows::ffi::{OsStrExt, OsStringExt}, }; + use std::ffi::{c_int, c_void}; use windows_sys::Win32::{ Foundation::{self, INVALID_HANDLE_VALUE}, Storage::FileSystem, @@ -150,7 +151,7 @@ pub(crate) mod module { } let h = unsafe { Threading::OpenProcess(Threading::PROCESS_ALL_ACCESS, 0, pid) }; - if h == 0 { + if h == c_int::from(0) as *mut c_int as *mut c_void { return Err(errno_err(vm)); } let ret = unsafe { Threading::TerminateProcess(h, sig) }; @@ -172,7 +173,7 @@ pub(crate) mod module { _ => return Err(vm.new_value_error("bad file descriptor".to_owned())), }; let h = unsafe { Console::GetStdHandle(stdhandle) }; - if h == 0 { + if h == c_int::from(0) as *mut c_int as *mut c_void { return Err(vm.new_os_error("handle cannot be retrieved".to_owned())); } if h == INVALID_HANDLE_VALUE { diff --git a/vm/src/stdlib/winapi.rs b/vm/src/stdlib/winapi.rs index ad6db12474..45ad6d12a7 100644 --- a/vm/src/stdlib/winapi.rs +++ b/vm/src/stdlib/winapi.rs @@ -3,6 +3,7 @@ pub(crate) use _winapi::make_module; #[pymodule] mod _winapi { + use std::ffi::{c_int, c_void}; use crate::{ builtins::PyStrRef, common::windows::ToWideString, @@ -116,6 +117,8 @@ mod _winapi { .to_pyresult(vm)?; target.assume_init() }; + let target = c_int::from(target as i32); + Ok(HANDLE(target)) } @@ -436,6 +439,7 @@ mod _winapi { #[pyfunction] fn GetModuleFileName(handle: isize, vm: &VirtualMachine) -> PyResult { let mut path: Vec = vec![0; MAX_PATH as usize]; + let handle = c_int::from(handle as i32) as *mut c_int as *mut c_void; let handle = HINSTANCE(handle); let length = diff --git a/vm/src/stdlib/winreg.rs b/vm/src/stdlib/winreg.rs index b368c43a3e..10a1ebc49e 100644 --- a/vm/src/stdlib/winreg.rs +++ b/vm/src/stdlib/winreg.rs @@ -35,6 +35,7 @@ mod winreg { use ::winreg::{enums::RegType, RegKey, RegValue}; use std::mem::ManuallyDrop; use std::{ffi::OsStr, io}; + use std::ffi::{c_int, c_void}; use windows_sys::Win32::Foundation; // access rights @@ -98,7 +99,7 @@ mod winreg { #[pymethod(magic)] fn bool(&self) -> bool { - self.key().raw_handle() != 0 + self.key().raw_handle() != c_int::from(0) as *mut c_int as *mut c_void } #[pymethod(magic)] fn enter(zelf: PyRef) -> PyRef { From 7e1568a1ffef0b44c47e98e465269e00693c207b Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sun, 16 Feb 2025 21:30:41 -0800 Subject: [PATCH 011/295] Revert "windows-rs upgrade to 0.59" This reverts commit 547530724e77a592734d8cd396115c4124d7a9f9. --- Cargo.lock | 191 ++++++++-------------------------------- Cargo.toml | 2 +- common/src/fileutils.rs | 14 +-- vm/Cargo.toml | 4 +- vm/src/stdlib/nt.rs | 5 +- vm/src/stdlib/winapi.rs | 4 - vm/src/stdlib/winreg.rs | 3 +- 7 files changed, 50 insertions(+), 173 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 115451adfa..a62a5dfced 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -216,9 +216,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.2.14" +version = "1.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c3d1b2e905a3a7b00a6141adb0e4c0bb941d11caf55349d863942a1cc44e3c9" +checksum = "c7777341816418c02e033934a09f20dc0ccaf65a5201ef8a450ae0105a573fda" dependencies = [ "shlex", ] @@ -645,9 +645,9 @@ dependencies = [ [[package]] name = "equivalent" -version = "1.0.2" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" @@ -903,7 +903,7 @@ dependencies = [ "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "windows-core 0.52.0", + "windows-core", ] [[package]] @@ -1406,9 +1406,9 @@ checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9" [[package]] name = "openssl" -version = "0.10.71" +version = "0.10.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e14130c6a98cd258fdcb0fb6d744152343ff729cbfcb28c656a9d12b999fbcd" +checksum = "61cfb4e166a8bb8c9b55c500bc2308550148ece889be90f609377e58140f42c6" dependencies = [ "bitflags 2.8.0", "cfg-if", @@ -1447,9 +1447,9 @@ dependencies = [ [[package]] name = "openssl-sys" -version = "0.9.106" +version = "0.9.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bb61ea9811cc39e3c2069f40b8b8e2e70d8569b361f879786cc7ed48b777cdd" +checksum = "8b22d5b84be05a8d6947c7cb71f7c849aa0f112acd4bf51c2a7c1c988ac0a9dc" dependencies = [ "cc", "libc", @@ -1745,8 +1745,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" dependencies = [ "rand_chacha 0.9.0", - "rand_core 0.9.1", - "zerocopy 0.8.18", + "rand_core 0.9.0", + "zerocopy 0.8.17", ] [[package]] @@ -1766,7 +1766,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core 0.9.1", + "rand_core 0.9.0", ] [[package]] @@ -1780,12 +1780,12 @@ dependencies = [ [[package]] name = "rand_core" -version = "0.9.1" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a88e0da7a2c97baa202165137c158d0a2e824ac465d13d81046727b34cb247d3" +checksum = "b08f3c9802962f7e1b25113931d94f43ed9725bebc59db9d0c3e9a23b67e15ff" dependencies = [ "getrandom 0.3.1", - "zerocopy 0.8.18", + "zerocopy 0.8.17", ] [[package]] @@ -2012,7 +2012,7 @@ dependencies = [ "siphasher 0.3.11", "volatile", "widestring", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -2242,7 +2242,7 @@ dependencies = [ "uuid", "widestring", "winapi", - "windows-sys 0.59.0", + "windows-sys 0.52.0", "xml-rs", ] @@ -2320,7 +2320,7 @@ dependencies = [ "which", "widestring", "windows", - "windows-sys 0.59.0", + "windows-sys 0.52.0", "winreg", ] @@ -2524,9 +2524,9 @@ checksum = "826167069c09b99d56f31e9ae5c99049e932a98c9dc2dac47645b08dbbf76ba7" [[package]] name = "smallvec" -version = "1.14.0" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" [[package]] name = "socket2" @@ -2552,15 +2552,15 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" [[package]] name = "strum" -version = "0.27.1" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f64def088c51c9510a8579e3c5d67c65349dcf755e5479ad3d010aa6454e2c32" +checksum = "ce1475c515a4f03a8a7129bb5228b81a781a86cb0b3fbbc19e1c556d491a401f" [[package]] name = "strum_macros" -version = "0.27.1" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c77a8c5abcaf0f9ce05d62342b7d298c346515365c36b673df4ebe3ced01fde8" +checksum = "9688894b43459159c82bfa5a5fa0435c19cbe3c9b427fa1dd7b1ce0c279b18a7" dependencies = [ "heck", "proc-macro2", @@ -3185,12 +3185,12 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows" -version = "0.59.0" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f919aee0a93304be7f62e8e5027811bbba96bcb1de84d6618be56e43f8a32a1" +checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be" dependencies = [ - "windows-core 0.59.0", - "windows-targets 0.53.0", + "windows-core", + "windows-targets 0.52.6", ] [[package]] @@ -3202,59 +3202,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "windows-core" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "810ce18ed2112484b0d4e15d022e5f598113e220c53e373fb31e67e21670c1ce" -dependencies = [ - "windows-implement", - "windows-interface", - "windows-result", - "windows-strings", - "windows-targets 0.53.0", -] - -[[package]] -name = "windows-implement" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83577b051e2f49a058c308f17f273b570a6a758386fc291b5f6a934dd84e48c1" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.98", -] - -[[package]] -name = "windows-interface" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb26fd936d991781ea39e87c3a27285081e3c0da5ca0fcbc02d368cc6f52ff01" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.98", -] - -[[package]] -name = "windows-result" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d08106ce80268c4067c0571ca55a9b4e9516518eaa1a1fe9b37ca403ae1d1a34" -dependencies = [ - "windows-targets 0.53.0", -] - -[[package]] -name = "windows-strings" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b888f919960b42ea4e11c2f408fadb55f78a9f236d5eef084103c8ce52893491" -dependencies = [ - "windows-targets 0.53.0", -] - [[package]] name = "windows-sys" version = "0.36.1" @@ -3319,29 +3266,13 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", + "windows_i686_gnullvm", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] -[[package]] -name = "windows-targets" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1e4c7e8ceaaf9cb7d7507c974735728ab453b67ef8f18febdd7c11fe59dca8b" -dependencies = [ - "windows_aarch64_gnullvm 0.53.0", - "windows_aarch64_msvc 0.53.0", - "windows_i686_gnu 0.53.0", - "windows_i686_gnullvm 0.53.0", - "windows_i686_msvc 0.53.0", - "windows_x86_64_gnu 0.53.0", - "windows_x86_64_gnullvm 0.53.0", - "windows_x86_64_msvc 0.53.0", -] - [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -3354,12 +3285,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" - [[package]] name = "windows_aarch64_msvc" version = "0.36.1" @@ -3378,12 +3303,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" - [[package]] name = "windows_i686_gnu" version = "0.36.1" @@ -3402,24 +3321,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" -[[package]] -name = "windows_i686_gnu" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" - [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" - [[package]] name = "windows_i686_msvc" version = "0.36.1" @@ -3438,12 +3345,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_i686_msvc" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" - [[package]] name = "windows_x86_64_gnu" version = "0.36.1" @@ -3462,12 +3363,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" - [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -3480,12 +3375,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" - [[package]] name = "windows_x86_64_msvc" version = "0.36.1" @@ -3504,20 +3393,14 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" - [[package]] name = "winreg" -version = "0.55.0" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb5a765337c50e9ec252c2069be9bf91c7df47afb103b642ba3a53bf8101be97" +checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5" dependencies = [ "cfg-if", - "windows-sys 0.59.0", + "windows-sys 0.48.0", ] [[package]] @@ -3553,11 +3436,11 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.18" +version = "0.8.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79386d31a42a4996e3336b0919ddb90f81112af416270cff95b5f5af22b839c2" +checksum = "aa91407dacce3a68c56de03abe2760159582b846c6a4acd2f456618087f12713" dependencies = [ - "zerocopy-derive 0.8.18", + "zerocopy-derive 0.8.17", ] [[package]] @@ -3573,9 +3456,9 @@ dependencies = [ [[package]] name = "zerocopy-derive" -version = "0.8.18" +version = "0.8.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76331675d372f91bf8d17e13afbd5fe639200b73d01f0fc748bb059f9cca2db7" +checksum = "06718a168365cad3d5ff0bb133aad346959a2074bd4a85c121255a11304a8626" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index f7d7d896c5..958bc5eeff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -181,7 +181,7 @@ thiserror = "2.0" thread_local = "1.1.8" unicode_names2 = "1.3.0" widestring = "1.1.0" -windows-sys = "0.59.0" +windows-sys = "0.52.0" wasm-bindgen = "0.2.100" # Lints diff --git a/common/src/fileutils.rs b/common/src/fileutils.rs index 296bb6be7c..dcb78675d8 100644 --- a/common/src/fileutils.rs +++ b/common/src/fileutils.rs @@ -25,7 +25,7 @@ pub mod windows { use crate::suppress_iph; use crate::windows::ToWideString; use libc::{S_IFCHR, S_IFDIR, S_IFMT}; - use std::ffi::{c_int, c_void, CString, OsStr, OsString}; + use std::ffi::{CString, OsStr, OsString}; use std::os::windows::ffi::OsStrExt; use std::sync::OnceLock; use windows_sys::core::PCWSTR; @@ -113,10 +113,10 @@ pub mod windows { if h.is_err() { unsafe { SetLastError(ERROR_INVALID_HANDLE) }; } - let mut h = c_int::from(h? as i32); + let h = h?; // reset stat? - let file_type = unsafe { GetFileType(&mut h as *mut c_int as *mut c_void) }; + let file_type = unsafe { GetFileType(h) }; if file_type == FILE_TYPE_UNKNOWN { return Err(std::io::Error::last_os_error()); } @@ -138,10 +138,10 @@ pub mod windows { let mut basic_info: FILE_BASIC_INFO = unsafe { std::mem::zeroed() }; let mut id_info: FILE_ID_INFO = unsafe { std::mem::zeroed() }; - if unsafe { GetFileInformationByHandle(&mut h as *mut c_int as *mut c_void, &mut info) } == 0 + if unsafe { GetFileInformationByHandle(h, &mut info) } == 0 || unsafe { GetFileInformationByHandleEx( - &mut h as *mut c_int as *mut c_void, + h, FileBasicInfo, &mut basic_info as *mut _ as *mut _, std::mem::size_of_val(&basic_info) as u32, @@ -153,7 +153,7 @@ pub mod windows { let p_id_info = if unsafe { GetFileInformationByHandleEx( - &mut h as *mut c_int as *mut c_void, + h, FileIdInfo, &mut id_info as *mut _ as *mut _, std::mem::size_of_val(&id_info) as u32, @@ -320,7 +320,7 @@ pub mod windows { .get_or_init(|| { let library_name = OsString::from("api-ms-win-core-file-l2-1-4").to_wide_with_nul(); let module = unsafe { LoadLibraryW(library_name.as_ptr()) }; - if module == c_int::from(0) as *mut c_int as *mut c_void { + if module == 0 { return None; } let name = CString::new("GetFileInformationByName").unwrap(); diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 49eaabf96a..1468651aeb 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -108,10 +108,10 @@ num_cpus = "1.13.1" [target.'cfg(windows)'.dependencies] junction = { workspace = true } schannel = { workspace = true } -winreg = "0.55" +winreg = "0.52" [target.'cfg(windows)'.dependencies.windows] -version = "0.59.0" +version = "0.52.0" features = [ "Win32_Foundation", "Win32_System_LibraryLoader", diff --git a/vm/src/stdlib/nt.rs b/vm/src/stdlib/nt.rs index 64c04adbd6..d2785f42dd 100644 --- a/vm/src/stdlib/nt.rs +++ b/vm/src/stdlib/nt.rs @@ -25,7 +25,6 @@ pub(crate) mod module { mem::MaybeUninit, os::windows::ffi::{OsStrExt, OsStringExt}, }; - use std::ffi::{c_int, c_void}; use windows_sys::Win32::{ Foundation::{self, INVALID_HANDLE_VALUE}, Storage::FileSystem, @@ -151,7 +150,7 @@ pub(crate) mod module { } let h = unsafe { Threading::OpenProcess(Threading::PROCESS_ALL_ACCESS, 0, pid) }; - if h == c_int::from(0) as *mut c_int as *mut c_void { + if h == 0 { return Err(errno_err(vm)); } let ret = unsafe { Threading::TerminateProcess(h, sig) }; @@ -173,7 +172,7 @@ pub(crate) mod module { _ => return Err(vm.new_value_error("bad file descriptor".to_owned())), }; let h = unsafe { Console::GetStdHandle(stdhandle) }; - if h == c_int::from(0) as *mut c_int as *mut c_void { + if h == 0 { return Err(vm.new_os_error("handle cannot be retrieved".to_owned())); } if h == INVALID_HANDLE_VALUE { diff --git a/vm/src/stdlib/winapi.rs b/vm/src/stdlib/winapi.rs index 45ad6d12a7..ad6db12474 100644 --- a/vm/src/stdlib/winapi.rs +++ b/vm/src/stdlib/winapi.rs @@ -3,7 +3,6 @@ pub(crate) use _winapi::make_module; #[pymodule] mod _winapi { - use std::ffi::{c_int, c_void}; use crate::{ builtins::PyStrRef, common::windows::ToWideString, @@ -117,8 +116,6 @@ mod _winapi { .to_pyresult(vm)?; target.assume_init() }; - let target = c_int::from(target as i32); - Ok(HANDLE(target)) } @@ -439,7 +436,6 @@ mod _winapi { #[pyfunction] fn GetModuleFileName(handle: isize, vm: &VirtualMachine) -> PyResult { let mut path: Vec = vec![0; MAX_PATH as usize]; - let handle = c_int::from(handle as i32) as *mut c_int as *mut c_void; let handle = HINSTANCE(handle); let length = diff --git a/vm/src/stdlib/winreg.rs b/vm/src/stdlib/winreg.rs index 10a1ebc49e..b368c43a3e 100644 --- a/vm/src/stdlib/winreg.rs +++ b/vm/src/stdlib/winreg.rs @@ -35,7 +35,6 @@ mod winreg { use ::winreg::{enums::RegType, RegKey, RegValue}; use std::mem::ManuallyDrop; use std::{ffi::OsStr, io}; - use std::ffi::{c_int, c_void}; use windows_sys::Win32::Foundation; // access rights @@ -99,7 +98,7 @@ mod winreg { #[pymethod(magic)] fn bool(&self) -> bool { - self.key().raw_handle() != c_int::from(0) as *mut c_int as *mut c_void + self.key().raw_handle() != 0 } #[pymethod(magic)] fn enter(zelf: PyRef) -> PyRef { From 630c1ff8d18fee8fe4e8ca1cbf4d25e0f30d2889 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sun, 16 Feb 2025 22:36:05 -0800 Subject: [PATCH 012/295] simple part of the bump Signed-off-by: Ashwin Naren --- Cargo.lock | 57 ++++++++++++++++++++++++++++++++++++----- vm/Cargo.toml | 4 +-- vm/src/stdlib/winreg.rs | 2 +- 3 files changed, 53 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a62a5dfced..0d9f468106 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -903,7 +903,7 @@ dependencies = [ "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "windows-core", + "windows-core 0.52.0", ] [[package]] @@ -3185,11 +3185,11 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows" -version = "0.52.0" +version = "0.57.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be" +checksum = "12342cb4d8e3b046f3d80effd474a7a02447231330ef77d71daa6fbc40681143" dependencies = [ - "windows-core", + "windows-core 0.57.0", "windows-targets 0.52.6", ] @@ -3202,6 +3202,49 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-core" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2ed2439a290666cd67ecce2b0ffaad89c2a56b976b736e6ece670297897832d" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-result", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-implement" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.98", +] + +[[package]] +name = "windows-interface" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.98", +] + +[[package]] +name = "windows-result" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-sys" version = "0.36.1" @@ -3395,12 +3438,12 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winreg" -version = "0.52.0" +version = "0.55.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5" +checksum = "cb5a765337c50e9ec252c2069be9bf91c7df47afb103b642ba3a53bf8101be97" dependencies = [ "cfg-if", - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 1468651aeb..72a874a821 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -108,10 +108,10 @@ num_cpus = "1.13.1" [target.'cfg(windows)'.dependencies] junction = { workspace = true } schannel = { workspace = true } -winreg = "0.52" +winreg = "0.55" [target.'cfg(windows)'.dependencies.windows] -version = "0.52.0" +version = "0.57.0" features = [ "Win32_Foundation", "Win32_System_LibraryLoader", diff --git a/vm/src/stdlib/winreg.rs b/vm/src/stdlib/winreg.rs index b368c43a3e..a67fd02b38 100644 --- a/vm/src/stdlib/winreg.rs +++ b/vm/src/stdlib/winreg.rs @@ -98,7 +98,7 @@ mod winreg { #[pymethod(magic)] fn bool(&self) -> bool { - self.key().raw_handle() != 0 + self.key().raw_handle() != core::ptr::null_mut() } #[pymethod(magic)] fn enter(zelf: PyRef) -> PyRef { From 38a6a8d98480e00c7182f9f2d0b4b3fcf6c26f95 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sun, 16 Feb 2025 22:49:25 -0800 Subject: [PATCH 013/295] duplicate windows-sys Signed-off-by: Ashwin Naren --- vm/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 72a874a821..f092e153a0 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -111,7 +111,7 @@ schannel = { workspace = true } winreg = "0.55" [target.'cfg(windows)'.dependencies.windows] -version = "0.57.0" +version = "0.52.0" features = [ "Win32_Foundation", "Win32_System_LibraryLoader", From 517ffed40167f02d1fffd2b49f02f2ce2191e8ba Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Mon, 17 Feb 2025 11:44:25 -0800 Subject: [PATCH 014/295] fix clippy lint Signed-off-by: Ashwin Naren --- vm/src/stdlib/winreg.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/src/stdlib/winreg.rs b/vm/src/stdlib/winreg.rs index a67fd02b38..747c58cfd8 100644 --- a/vm/src/stdlib/winreg.rs +++ b/vm/src/stdlib/winreg.rs @@ -98,7 +98,7 @@ mod winreg { #[pymethod(magic)] fn bool(&self) -> bool { - self.key().raw_handle() != core::ptr::null_mut() + !self.key().raw_handle().is_null() } #[pymethod(magic)] fn enter(zelf: PyRef) -> PyRef { From 9856d94f2d560f0d684d44a3e3357d70852bfc5b Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Mon, 17 Feb 2025 21:15:58 -0800 Subject: [PATCH 015/295] function to retrieve tz info on windows Signed-off-by: Ashwin Naren --- vm/Cargo.toml | 1 + vm/src/stdlib/time.rs | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/vm/Cargo.toml b/vm/Cargo.toml index f092e153a0..853338c372 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -116,6 +116,7 @@ features = [ "Win32_Foundation", "Win32_System_LibraryLoader", "Win32_System_Threading", + "Win32_System_Time", "Win32_UI_Shell", ] diff --git a/vm/src/stdlib/time.rs b/vm/src/stdlib/time.rs index 566650d0f2..e6581ce847 100644 --- a/vm/src/stdlib/time.rs +++ b/vm/src/stdlib/time.rs @@ -152,6 +152,17 @@ mod decl { Ok(get_perf_time(vm)?.as_nanos()) } + #[cfg(target_env = "msvc")] + #[cfg(not(target_arch = "wasm32"))] + fn get_tz_info() -> Time::TIME_ZONE_INFORMATION { + let mut info = Time::TIME_ZONE_INFORMATION::default(); + let info_ptr = &mut info as *mut Time::TIME_ZONE_INFORMATION; + let _ = unsafe { + Time::GetTimeZoneInformation(info_ptr) + }; + info + } + // #[pyfunction] // fn tzset() { // unsafe { super::_tzset() }; From 72338d578b4e604530a8dbff240208b0ad1de972 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Mon, 17 Feb 2025 21:16:17 -0800 Subject: [PATCH 016/295] tzname on windows Signed-off-by: Ashwin Naren --- vm/src/stdlib/time.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/vm/src/stdlib/time.rs b/vm/src/stdlib/time.rs index e6581ce847..d087cd3b1c 100644 --- a/vm/src/stdlib/time.rs +++ b/vm/src/stdlib/time.rs @@ -43,6 +43,9 @@ mod decl { DateTime, Datelike, Timelike, }; use std::time::Duration; + #[cfg(target_env = "msvc")] + #[cfg(not(target_arch = "wasm32"))] + use windows::Win32::System::Time; #[allow(dead_code)] pub(super) const SEC_TO_MS: i64 = 1000; @@ -195,6 +198,18 @@ mod decl { unsafe { (to_str(super::c_tzname[0]), to_str(super::c_tzname[1])) }.into_pytuple(vm) } + #[cfg(target_env = "msvc")] + #[cfg(not(target_arch = "wasm32"))] + #[pyattr] + fn tzname(vm: &VirtualMachine) -> crate::builtins::PyTupleRef { + use crate::builtins::tuple::IntoPyTuple; + let info = get_tz_info(); + let standard = widestring::decode_utf16_lossy(info.StandardName).filter(|&c| c != '\0').collect::(); + let daylight = widestring::decode_utf16_lossy(info.DaylightName).filter(|&c| c != '\0').collect::(); + let tz_name = (&*standard, &*daylight); + tz_name.into_pytuple(vm) + } + fn pyobj_to_date_time( value: Either, vm: &VirtualMachine, From 175afd97d8a5589d71bc7b7c0a8f683271425bc3 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Mon, 17 Feb 2025 21:19:36 -0800 Subject: [PATCH 017/295] time.timezone for windows Signed-off-by: Ashwin Naren --- vm/src/stdlib/time.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/vm/src/stdlib/time.rs b/vm/src/stdlib/time.rs index d087cd3b1c..e5398d498a 100644 --- a/vm/src/stdlib/time.rs +++ b/vm/src/stdlib/time.rs @@ -178,6 +178,16 @@ mod decl { unsafe { super::c_timezone } } + #[cfg(target_env = "msvc")] + #[cfg(not(target_arch = "wasm32"))] + #[pyattr] + fn timezone() -> i32 { + let info = get_tz_info(); + // https://users.rust-lang.org/t/accessing-tzname-and-similar-constants-in-windows/125771/3 + (info.Bias + info.StandardBias) * 60 + } + + #[cfg(not(target_os = "freebsd"))] #[cfg(not(target_env = "msvc"))] #[cfg(not(target_arch = "wasm32"))] From 4ed735b424d5d502472f5dad32fd4f1ee9bd60ad Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Mon, 17 Feb 2025 21:19:55 -0800 Subject: [PATCH 018/295] time.daylight for windows Signed-off-by: Ashwin Naren --- vm/src/stdlib/time.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/vm/src/stdlib/time.rs b/vm/src/stdlib/time.rs index e5398d498a..1277b6a7e1 100644 --- a/vm/src/stdlib/time.rs +++ b/vm/src/stdlib/time.rs @@ -196,6 +196,15 @@ mod decl { unsafe { super::c_daylight } } + #[cfg(target_env = "msvc")] + #[cfg(not(target_arch = "wasm32"))] + #[pyattr] + fn daylight() -> i32 { + let info = get_tz_info(); + // https://users.rust-lang.org/t/accessing-tzname-and-similar-constants-in-windows/125771/3 + (info.StandardBias != info.DaylightBias) as i32 + } + #[cfg(not(target_env = "msvc"))] #[cfg(not(target_arch = "wasm32"))] #[pyattr] From 69b1a9910f0d3adc56537ad67795d6a92ed81a95 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Mon, 17 Feb 2025 21:20:15 -0800 Subject: [PATCH 019/295] formatting Signed-off-by: Ashwin Naren --- vm/src/stdlib/time.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/vm/src/stdlib/time.rs b/vm/src/stdlib/time.rs index 1277b6a7e1..51411889a1 100644 --- a/vm/src/stdlib/time.rs +++ b/vm/src/stdlib/time.rs @@ -160,9 +160,7 @@ mod decl { fn get_tz_info() -> Time::TIME_ZONE_INFORMATION { let mut info = Time::TIME_ZONE_INFORMATION::default(); let info_ptr = &mut info as *mut Time::TIME_ZONE_INFORMATION; - let _ = unsafe { - Time::GetTimeZoneInformation(info_ptr) - }; + let _ = unsafe { Time::GetTimeZoneInformation(info_ptr) }; info } @@ -187,7 +185,6 @@ mod decl { (info.Bias + info.StandardBias) * 60 } - #[cfg(not(target_os = "freebsd"))] #[cfg(not(target_env = "msvc"))] #[cfg(not(target_arch = "wasm32"))] @@ -223,8 +220,12 @@ mod decl { fn tzname(vm: &VirtualMachine) -> crate::builtins::PyTupleRef { use crate::builtins::tuple::IntoPyTuple; let info = get_tz_info(); - let standard = widestring::decode_utf16_lossy(info.StandardName).filter(|&c| c != '\0').collect::(); - let daylight = widestring::decode_utf16_lossy(info.DaylightName).filter(|&c| c != '\0').collect::(); + let standard = widestring::decode_utf16_lossy(info.StandardName) + .filter(|&c| c != '\0') + .collect::(); + let daylight = widestring::decode_utf16_lossy(info.DaylightName) + .filter(|&c| c != '\0') + .collect::(); let tz_name = (&*standard, &*daylight); tz_name.into_pytuple(vm) } From f46697131258fcf3dd77a8a64e97e3e9f5ceb244 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Mon, 17 Feb 2025 21:22:10 -0800 Subject: [PATCH 020/295] clippy Signed-off-by: Ashwin Naren --- vm/src/stdlib/time.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vm/src/stdlib/time.rs b/vm/src/stdlib/time.rs index 51411889a1..5a493c3472 100644 --- a/vm/src/stdlib/time.rs +++ b/vm/src/stdlib/time.rs @@ -179,7 +179,7 @@ mod decl { #[cfg(target_env = "msvc")] #[cfg(not(target_arch = "wasm32"))] #[pyattr] - fn timezone() -> i32 { + fn timezone(_vm: &VirtualMachine) -> i32 { let info = get_tz_info(); // https://users.rust-lang.org/t/accessing-tzname-and-similar-constants-in-windows/125771/3 (info.Bias + info.StandardBias) * 60 @@ -196,7 +196,7 @@ mod decl { #[cfg(target_env = "msvc")] #[cfg(not(target_arch = "wasm32"))] #[pyattr] - fn daylight() -> i32 { + fn daylight(_vm: &VirtualMachine) -> i32 { let info = get_tz_info(); // https://users.rust-lang.org/t/accessing-tzname-and-similar-constants-in-windows/125771/3 (info.StandardBias != info.DaylightBias) as i32 From a71c16f8cb36c086bdc822955c50294945a17e39 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Mon, 17 Feb 2025 22:25:40 -0800 Subject: [PATCH 021/295] test colorize on ci Signed-off-by: Ashwin Naren --- .github/workflows/ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7bbed2c3c2..ed91125a0f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -34,6 +34,7 @@ env: # PLATFORM_INDEPENDENT_TESTS are tests that do not depend on the underlying OS. They are currently # only run on Linux to speed up the CI. PLATFORM_INDEPENDENT_TESTS: >- + test__colorize test_argparse test_array test_asyncgen From fa2acd7cde461b7c8aff0fe93365e4e4c809fc8b Mon Sep 17 00:00:00 2001 From: Noa Date: Mon, 17 Feb 2025 11:48:50 -0600 Subject: [PATCH 022/295] Update rand to 0.9 --- .github/workflows/ci.yaml | 3 ++- Cargo.lock | 20 ++++++++++---------- Cargo.toml | 4 ++-- common/src/hash.rs | 6 +++--- stdlib/Cargo.toml | 5 ++--- stdlib/src/random.rs | 8 +------- stdlib/src/uuid.rs | 20 +++----------------- vm/Cargo.toml | 4 ++-- vm/src/import.rs | 3 +-- vm/src/stdlib/os.rs | 2 +- wasm/lib/.cargo/config.toml | 5 +++++ wasm/lib/Cargo.toml | 5 ++++- wasm/wasm-unknown-test/.cargo/config.toml | 5 +++++ wasm/wasm-unknown-test/Cargo.toml | 1 + wasm/wasm-unknown-test/src/lib.rs | 8 ++++++++ 15 files changed, 50 insertions(+), 49 deletions(-) create mode 100644 wasm/lib/.cargo/config.toml create mode 100644 wasm/wasm-unknown-test/.cargo/config.toml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ed91125a0f..dc2dcc6c7e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -379,7 +379,8 @@ jobs: with: { wabt-version: "1.0.30" } - name: check wasm32-unknown without js run: | - cargo build --release --manifest-path wasm/wasm-unknown-test/Cargo.toml --target wasm32-unknown-unknown --verbose + cd wasm/wasm-unknown-test + cargo build --release --verbose if wasm-objdump -xj Import target/wasm32-unknown-unknown/release/wasm_unknown_test.wasm; then echo "ERROR: wasm32-unknown module expects imports from the host environment" >2 fi diff --git a/Cargo.lock b/Cargo.lock index 0d9f468106..87174aeac6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -808,8 +808,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi 0.13.3+wasi-0.2.2", + "wasm-bindgen", "windows-targets 0.52.6", ] @@ -1306,11 +1308,11 @@ dependencies = [ [[package]] name = "mt19937" -version = "2.0.1" +version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12ca7f22ed370d5991a9caec16a83187e865bc8a532f889670337d5a5689e3a1" +checksum = "df7151a832e54d2d6b2c827a20e5bcdd80359281cd2c354e725d4b82e7c471de" dependencies = [ - "rand_core 0.6.4", + "rand_core 0.9.0", ] [[package]] @@ -2007,7 +2009,7 @@ dependencies = [ "once_cell", "parking_lot", "radium", - "rand 0.8.5", + "rand 0.9.0", "rustpython-format", "siphasher 0.3.11", "volatile", @@ -2216,8 +2218,7 @@ dependencies = [ "parking_lot", "paste", "puruspe", - "rand 0.8.5", - "rand_core 0.6.4", + "rand 0.9.0", "rustix", "rustpython-common", "rustpython-derive", @@ -2262,7 +2263,7 @@ dependencies = [ "exitcode", "flame", "flamer", - "getrandom 0.2.15", + "getrandom 0.3.1", "glob", "half 2.4.1", "hex", @@ -2285,7 +2286,7 @@ dependencies = [ "optional", "parking_lot", "paste", - "rand 0.8.5", + "rand 0.9.0", "result-like", "rustc_version", "rustix", @@ -2329,6 +2330,7 @@ name = "rustpython_wasm" version = "0.4.0" dependencies = [ "console_error_panic_hook", + "getrandom 0.2.15", "js-sys", "rustpython-common", "rustpython-parser", @@ -3000,8 +3002,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced87ca4be083373936a67f8de945faa23b6b42384bd5b64434850802c6dccd0" dependencies = [ "atomic", - "getrandom 0.3.1", - "rand 0.9.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 958bc5eeff..bda7286afd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -146,7 +146,7 @@ cfg-if = "1.0" chrono = "0.4.39" crossbeam-utils = "0.8.21" flame = "0.2.2" -getrandom = "0.2.15" +getrandom = "0.3" glob = "0.3" hex = "0.4.3" indexmap = { version = "2.2.6", features = ["std"] } @@ -168,7 +168,7 @@ num_enum = { version = "0.7", default-features = false } once_cell = "1.20.3" parking_lot = "0.12.3" paste = "1.0.15" -rand = "0.8.5" +rand = "0.9" rustix = { version = "0.38", features = ["event"] } rustyline = "15.0.0" serde = { version = "1.0.133", default-features = false } diff --git a/common/src/hash.rs b/common/src/hash.rs index 4e87eff799..700d2ceef5 100644 --- a/common/src/hash.rs +++ b/common/src/hash.rs @@ -37,11 +37,11 @@ impl BuildHasher for HashSecret { } } -impl rand::distributions::Distribution for rand::distributions::Standard { +impl rand::distr::Distribution for rand::distr::StandardUniform { fn sample(&self, rng: &mut R) -> HashSecret { HashSecret { - k0: rng.gen(), - k1: rng.gen(), + k0: rng.random(), + k1: rng.random(), } } } diff --git a/stdlib/Cargo.toml b/stdlib/Cargo.toml index 3d91b6fab9..31f5bd18d4 100644 --- a/stdlib/Cargo.toml +++ b/stdlib/Cargo.toml @@ -54,8 +54,7 @@ xml-rs = "0.8.14" # random rand = { workspace = true } -rand_core = "0.6.4" -mt19937 = "2.0.1" +mt19937 = "3.1" # Crypto: digest = "0.10.3" @@ -88,7 +87,7 @@ bzip2 = { version = "0.4", optional = true } # uuid [target.'cfg(not(any(target_os = "ios", target_os = "android", target_os = "windows", target_arch = "wasm32", target_os = "redox")))'.dependencies] mac_address = "1.1.3" -uuid = { version = "1.1.2", features = ["v1", "fast-rng"] } +uuid = { version = "1.1.2", features = ["v1"] } # mmap [target.'cfg(all(unix, not(target_arch = "wasm32")))'.dependencies] diff --git a/stdlib/src/random.rs b/stdlib/src/random.rs index 1dfc4fcc30..35cbd94457 100644 --- a/stdlib/src/random.rs +++ b/stdlib/src/random.rs @@ -23,7 +23,7 @@ mod _random { impl Default for PyRng { fn default() -> Self { - PyRng::Std(Box::new(StdRng::from_entropy())) + PyRng::Std(Box::new(StdRng::from_os_rng())) } } @@ -46,12 +46,6 @@ mod _random { Self::MT(m) => m.fill_bytes(dest), } } - fn try_fill_bytes(&mut self, dest: &mut [u8]) -> Result<(), rand::Error> { - match self { - Self::Std(s) => s.try_fill_bytes(dest), - Self::MT(m) => m.try_fill_bytes(dest), - } - } } #[pyattr] diff --git a/stdlib/src/uuid.rs b/stdlib/src/uuid.rs index e5434da0e1..4cadbb787b 100644 --- a/stdlib/src/uuid.rs +++ b/stdlib/src/uuid.rs @@ -5,33 +5,19 @@ mod _uuid { use crate::{builtins::PyNone, vm::VirtualMachine}; use mac_address::get_mac_address; use once_cell::sync::OnceCell; - use rand::Rng; - use std::time::{Duration, SystemTime}; - use uuid::{ - v1::{Context, Timestamp}, - Uuid, - }; + use uuid::{timestamp::Timestamp, Context, Uuid}; fn get_node_id() -> [u8; 6] { match get_mac_address() { Ok(Some(_ma)) => get_mac_address().unwrap().unwrap().bytes(), - _ => rand::thread_rng().gen::<[u8; 6]>(), + _ => rand::random::<[u8; 6]>(), } } - pub fn now_unix_duration() -> Duration { - use std::time::UNIX_EPOCH; - - let now = SystemTime::now(); - now.duration_since(UNIX_EPOCH) - .expect("SystemTime before UNIX EPOCH!") - } - #[pyfunction] fn generate_time_safe() -> (Vec, PyNone) { static CONTEXT: Context = Context::new(0); - let now = now_unix_duration(); - let ts = Timestamp::from_unix(&CONTEXT, now.as_secs(), now.subsec_nanos()); + let ts = Timestamp::now(&CONTEXT); static NODE_ID: OnceCell<[u8; 6]> = OnceCell::new(); let unique_node_id = NODE_ID.get_or_init(get_node_id); diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 853338c372..5c10223bfc 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -23,7 +23,7 @@ ast = ["rustpython-ast"] codegen = ["rustpython-codegen", "ast"] parser = ["rustpython-parser", "ast"] serde = ["dep:serde"] -wasmbind = ["chrono/wasmbind", "getrandom/js", "wasm-bindgen"] +wasmbind = ["chrono/wasmbind", "getrandom/wasm_js", "wasm-bindgen"] [dependencies] rustpython-compiler = { workspace = true, optional = true } @@ -145,7 +145,7 @@ features = [ [target.'cfg(all(target_arch = "wasm32", target_os = "unknown"))'.dependencies] wasm-bindgen = { workspace = true, optional = true } -getrandom = { workspace = true, features = ["custom"] } +getrandom = { workspace = true } [build-dependencies] glob = { workspace = true } diff --git a/vm/src/import.rs b/vm/src/import.rs index f14ae31ef4..fbd563dabb 100644 --- a/vm/src/import.rs +++ b/vm/src/import.rs @@ -8,7 +8,6 @@ use crate::{ vm::{thread, VirtualMachine}, AsObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, }; -use rand::Rng; pub(crate) fn init_importlib_base(vm: &mut VirtualMachine) -> PyResult { flame_guard!("init importlib"); @@ -50,7 +49,7 @@ pub(crate) fn init_importlib_package(vm: &VirtualMachine, importlib: PyObjectRef let mut magic = get_git_revision().into_bytes(); magic.truncate(4); if magic.len() != 4 { - magic = rand::thread_rng().gen::<[u8; 4]>().to_vec(); + magic = rand::random::<[u8; 4]>().to_vec(); } let magic: PyObjectRef = vm.ctx.new_bytes(magic).into(); importlib_external.set_attr("MAGIC_NUMBER", magic, vm)?; diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 0b3f617552..6519af094d 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -978,7 +978,7 @@ pub(super) mod _os { return Err(vm.new_value_error("negative argument not allowed".to_owned())); } let mut buf = vec![0u8; size as usize]; - getrandom::getrandom(&mut buf).map_err(|e| match e.raw_os_error() { + getrandom::fill(&mut buf).map_err(|e| match e.raw_os_error() { Some(errno) => io::Error::from_raw_os_error(errno).into_pyexception(vm), None => vm.new_os_error("Getting random failed".to_owned()), })?; diff --git a/wasm/lib/.cargo/config.toml b/wasm/lib/.cargo/config.toml new file mode 100644 index 0000000000..ce1e7c694a --- /dev/null +++ b/wasm/lib/.cargo/config.toml @@ -0,0 +1,5 @@ +[build] +target = "wasm32-unknown-unknown" + +[target.wasm32-unknown-unknown] +rustflags = ["--cfg=getrandom_backend=\"wasm_js\""] diff --git a/wasm/lib/Cargo.toml b/wasm/lib/Cargo.toml index 1e5c37f4ef..4703cb9f4a 100644 --- a/wasm/lib/Cargo.toml +++ b/wasm/lib/Cargo.toml @@ -28,6 +28,9 @@ rustpython-parser = { workspace = true } serde = { workspace = true } wasm-bindgen = { workspace = true } +# remove once getrandom 0.2 is no longer otherwise in the dependency tree +getrandom = { version = "0.2", features = ["js"] } + console_error_panic_hook = "0.1" js-sys = "0.3" serde-wasm-bindgen = "0.3.1" @@ -47,4 +50,4 @@ web-sys = { version = "0.3", features = [ wasm-opt = false#["-O1"] [lints] -workspace = true \ No newline at end of file +workspace = true diff --git a/wasm/wasm-unknown-test/.cargo/config.toml b/wasm/wasm-unknown-test/.cargo/config.toml new file mode 100644 index 0000000000..f86ad96761 --- /dev/null +++ b/wasm/wasm-unknown-test/.cargo/config.toml @@ -0,0 +1,5 @@ +[build] +target = "wasm32-unknown-unknown" + +[target.wasm32-unknown-unknown] +rustflags = ["--cfg=getrandom_backend=\"custom\""] diff --git a/wasm/wasm-unknown-test/Cargo.toml b/wasm/wasm-unknown-test/Cargo.toml index f5e0b55786..ca8b15cfc5 100644 --- a/wasm/wasm-unknown-test/Cargo.toml +++ b/wasm/wasm-unknown-test/Cargo.toml @@ -8,6 +8,7 @@ crate-type = ["cdylib"] [dependencies] getrandom = { version = "0.2.12", features = ["custom"] } +getrandom_03 = { package = "getrandom", version = "0.3" } rustpython-vm = { path = "../../vm", default-features = false, features = ["compiler"] } [workspace] diff --git a/wasm/wasm-unknown-test/src/lib.rs b/wasm/wasm-unknown-test/src/lib.rs index fd043aea3a..cfdc445574 100644 --- a/wasm/wasm-unknown-test/src/lib.rs +++ b/wasm/wasm-unknown-test/src/lib.rs @@ -14,3 +14,11 @@ fn getrandom_always_fail(_buf: &mut [u8]) -> Result<(), getrandom::Error> { } getrandom::register_custom_getrandom!(getrandom_always_fail); + +#[unsafe(no_mangle)] +unsafe extern "Rust" fn __getrandom_v03_custom( + _dest: *mut u8, + _len: usize, +) -> Result<(), getrandom_03::Error> { + Err(getrandom_03::Error::UNSUPPORTED) +} From e2b0fe4266febafdc9ab34d558bfa8aef91b59cc Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Wed, 19 Feb 2025 17:50:10 -0800 Subject: [PATCH 023/295] _ctypes pt. 2 (#5524) * add __version__ * add more types/constants * shared library and ExternalLibs implementation * FreeLibrary for windows * fixed PyCSimple * LoadLibrary and FreeLibrary for non-windows * fault-tolerant float equality Signed-off-by: Ashwin Naren --- Cargo.lock | 124 +++++++++--------------- extra_tests/snippets/builtins_ctypes.py | 28 ++++++ vm/Cargo.toml | 1 + vm/src/stdlib/ctypes.rs | 116 +++++++++++++++++++++- vm/src/stdlib/ctypes/array.rs | 5 + vm/src/stdlib/ctypes/base.rs | 91 ++++++++++++++--- vm/src/stdlib/ctypes/function.rs | 24 +++++ vm/src/stdlib/ctypes/library.rs | 115 ++++++++++++++++++++++ vm/src/stdlib/ctypes/pointer.rs | 5 + vm/src/stdlib/ctypes/structure.rs | 5 + vm/src/stdlib/ctypes/union.rs | 5 + 11 files changed, 424 insertions(+), 95 deletions(-) create mode 100644 vm/src/stdlib/ctypes/array.rs create mode 100644 vm/src/stdlib/ctypes/function.rs create mode 100644 vm/src/stdlib/ctypes/library.rs create mode 100644 vm/src/stdlib/ctypes/pointer.rs create mode 100644 vm/src/stdlib/ctypes/structure.rs create mode 100644 vm/src/stdlib/ctypes/union.rs diff --git a/Cargo.lock b/Cargo.lock index 87174aeac6..11992d8d8c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -216,9 +216,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.2.13" +version = "1.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7777341816418c02e033934a09f20dc0ccaf65a5201ef8a450ae0105a573fda" +checksum = "0c3d1b2e905a3a7b00a6141adb0e4c0bb941d11caf55349d863942a1cc44e3c9" dependencies = [ "shlex", ] @@ -645,9 +645,9 @@ dependencies = [ [[package]] name = "equivalent" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "errno" @@ -905,7 +905,7 @@ dependencies = [ "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "windows-core 0.52.0", + "windows-core", ] [[package]] @@ -1028,9 +1028,9 @@ checksum = "507460a910eb7b32ee961886ff48539633b788a36b65692b95f225b844c82553" [[package]] name = "lambert_w" -version = "1.0.14" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b0c98033daa8d13aa2171722fd201bae337924189091f7988bdaff5301fa8c9" +checksum = "45bf98425154bfe790a47b72ac452914f6df9ebfb202bc59e089e29db00258cf" [[package]] name = "lazy_static" @@ -1099,6 +1099,16 @@ dependencies = [ "cc", ] +[[package]] +name = "libloading" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" +dependencies = [ + "cfg-if", + "windows-targets 0.52.6", +] + [[package]] name = "libm" version = "0.2.11" @@ -1408,9 +1418,9 @@ checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9" [[package]] name = "openssl" -version = "0.10.70" +version = "0.10.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61cfb4e166a8bb8c9b55c500bc2308550148ece889be90f609377e58140f42c6" +checksum = "5e14130c6a98cd258fdcb0fb6d744152343ff729cbfcb28c656a9d12b999fbcd" dependencies = [ "bitflags 2.8.0", "cfg-if", @@ -1449,9 +1459,9 @@ dependencies = [ [[package]] name = "openssl-sys" -version = "0.9.105" +version = "0.9.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b22d5b84be05a8d6947c7cb71f7c849aa0f112acd4bf51c2a7c1c988ac0a9dc" +checksum = "8bb61ea9811cc39e3c2069f40b8b8e2e70d8569b361f879786cc7ed48b777cdd" dependencies = [ "cc", "libc", @@ -1747,8 +1757,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" dependencies = [ "rand_chacha 0.9.0", - "rand_core 0.9.0", - "zerocopy 0.8.17", + "rand_core 0.9.1", + "zerocopy 0.8.18", ] [[package]] @@ -1768,7 +1778,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core 0.9.0", + "rand_core 0.9.1", ] [[package]] @@ -1782,12 +1792,12 @@ dependencies = [ [[package]] name = "rand_core" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b08f3c9802962f7e1b25113931d94f43ed9725bebc59db9d0c3e9a23b67e15ff" +checksum = "a88e0da7a2c97baa202165137c158d0a2e824ac465d13d81046727b34cb247d3" dependencies = [ "getrandom 0.3.1", - "zerocopy 0.8.17", + "zerocopy 0.8.18", ] [[package]] @@ -2272,6 +2282,7 @@ dependencies = [ "itertools 0.14.0", "junction", "libc", + "libloading", "log", "malachite-bigint", "memchr", @@ -2526,9 +2537,9 @@ checksum = "826167069c09b99d56f31e9ae5c99049e932a98c9dc2dac47645b08dbbf76ba7" [[package]] name = "smallvec" -version = "1.13.2" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" +checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd" [[package]] name = "socket2" @@ -2554,15 +2565,15 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" [[package]] name = "strum" -version = "0.27.0" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce1475c515a4f03a8a7129bb5228b81a781a86cb0b3fbbc19e1c556d491a401f" +checksum = "f64def088c51c9510a8579e3c5d67c65349dcf755e5479ad3d010aa6454e2c32" [[package]] name = "strum_macros" -version = "0.27.0" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9688894b43459159c82bfa5a5fa0435c19cbe3c9b427fa1dd7b1ce0c279b18a7" +checksum = "c77a8c5abcaf0f9ce05d62342b7d298c346515365c36b673df4ebe3ced01fde8" dependencies = [ "heck", "proc-macro2", @@ -2781,9 +2792,9 @@ dependencies = [ [[package]] name = "typenum" -version = "1.17.0" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" +checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" [[package]] name = "ucd" @@ -2997,9 +3008,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.13.1" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ced87ca4be083373936a67f8de945faa23b6b42384bd5b64434850802c6dccd0" +checksum = "8c1f41ffb7cf259f1ecc2876861a17e7142e63ead296f671f81f6ae85903e0d6" dependencies = [ "atomic", ] @@ -3185,11 +3196,11 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows" -version = "0.57.0" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12342cb4d8e3b046f3d80effd474a7a02447231330ef77d71daa6fbc40681143" +checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be" dependencies = [ - "windows-core 0.57.0", + "windows-core", "windows-targets 0.52.6", ] @@ -3202,49 +3213,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "windows-core" -version = "0.57.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2ed2439a290666cd67ecce2b0ffaad89c2a56b976b736e6ece670297897832d" -dependencies = [ - "windows-implement", - "windows-interface", - "windows-result", - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-implement" -version = "0.57.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.98", -] - -[[package]] -name = "windows-interface" -version = "0.57.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.98", -] - -[[package]] -name = "windows-result" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8" -dependencies = [ - "windows-targets 0.52.6", -] - [[package]] name = "windows-sys" version = "0.36.1" @@ -3479,11 +3447,11 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.17" +version = "0.8.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa91407dacce3a68c56de03abe2760159582b846c6a4acd2f456618087f12713" +checksum = "79386d31a42a4996e3336b0919ddb90f81112af416270cff95b5f5af22b839c2" dependencies = [ - "zerocopy-derive 0.8.17", + "zerocopy-derive 0.8.18", ] [[package]] @@ -3499,9 +3467,9 @@ dependencies = [ [[package]] name = "zerocopy-derive" -version = "0.8.17" +version = "0.8.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06718a168365cad3d5ff0bb133aad346959a2074bd4a85c121255a11304a8626" +checksum = "76331675d372f91bf8d17e13afbd5fe639200b73d01f0fc748bb059f9cca2db7" dependencies = [ "proc-macro2", "quote", diff --git a/extra_tests/snippets/builtins_ctypes.py b/extra_tests/snippets/builtins_ctypes.py index 5df259cf82..5bd6e5ef25 100644 --- a/extra_tests/snippets/builtins_ctypes.py +++ b/extra_tests/snippets/builtins_ctypes.py @@ -1,7 +1,29 @@ +import os as _os, sys as _sys + from _ctypes import sizeof from _ctypes import _SimpleCData from struct import calcsize as _calcsize +def create_string_buffer(init, size=None): + """create_string_buffer(aBytes) -> character array + create_string_buffer(anInteger) -> character array + create_string_buffer(aBytes, anInteger) -> character array + """ + if isinstance(init, bytes): + if size is None: + size = len(init)+1 + _sys.audit("ctypes.create_string_buffer", init, size) + buftype = c_char * size + buf = buftype() + buf.value = init + return buf + elif isinstance(init, int): + _sys.audit("ctypes.create_string_buffer", None, init) + buftype = c_char * init + buf = buftype() + return buf + raise TypeError(init) + def _check_size(typ, typecode=None): # Check if sizeof(ctypes_type) against struct.calcsize. This # should protect somewhat against a misconfigured libffi. @@ -103,3 +125,9 @@ class c_void_p(_SimpleCData): class c_bool(_SimpleCData): _type_ = "?" _check_size(c_bool) + +i = c_int(42) +f = c_float(3.14) +# s = create_string_buffer(b'\000' * 32) +assert i.value == 42 +assert abs(f.value - 3.14) < 1e-06 diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 5c10223bfc..895b29aa50 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -100,6 +100,7 @@ uname = "0.1.1" rustyline = { workspace = true } which = "6" errno = "0.3" +libloading = "0.8" widestring = { workspace = true } [target.'cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))'.dependencies] diff --git a/vm/src/stdlib/ctypes.rs b/vm/src/stdlib/ctypes.rs index d1fc26869f..027a680951 100644 --- a/vm/src/stdlib/ctypes.rs +++ b/vm/src/stdlib/ctypes.rs @@ -1,11 +1,34 @@ +pub(crate) mod array; pub(crate) mod base; +pub(crate) mod function; +pub(crate) mod library; +pub(crate) mod pointer; +pub(crate) mod structure; +pub(crate) mod union; use crate::builtins::PyModule; -use crate::{PyRef, VirtualMachine}; +use crate::class::PyClassImpl; +use crate::stdlib::ctypes::base::{PyCData, PyCSimple, PySimpleMeta}; +use crate::{Py, PyRef, VirtualMachine}; + +pub fn extend_module_nodes(vm: &VirtualMachine, module: &Py) { + let ctx = &vm.ctx; + PySimpleMeta::make_class(ctx); + extend_module!(vm, module, { + "_CData" => PyCData::make_class(ctx), + "_SimpleCData" => PyCSimple::make_class(ctx), + "Array" => array::PyCArray::make_class(ctx), + "CFuncPtr" => function::PyCFuncPtr::make_class(ctx), + "_Pointer" => pointer::PyCPointer::make_class(ctx), + "_pointer_type_cache" => ctx.new_dict(), + "Structure" => structure::PyCStructure::make_class(ctx), + "Union" => union::PyCUnion::make_class(ctx), + }) +} pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { let module = _ctypes::make_module(vm); - base::extend_module_nodes(vm, &module); + extend_module_nodes(vm, &module); module } @@ -15,6 +38,7 @@ pub(crate) mod _ctypes { use crate::builtins::PyTypeRef; use crate::class::StaticType; use crate::function::Either; + use crate::stdlib::ctypes::library; use crate::{AsObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine}; use crossbeam_utils::atomic::AtomicCell; use std::ffi::{ @@ -24,6 +48,57 @@ pub(crate) mod _ctypes { use std::mem; use widestring::WideChar; + #[pyattr(name = "__version__")] + const __VERSION__: &str = "1.1.0"; + + // TODO: get properly + #[pyattr(name = "RTLD_LOCAL")] + const RTLD_LOCAL: i32 = 0; + + // TODO: get properly + #[pyattr(name = "RTLD_GLOBAL")] + const RTLD_GLOBAL: i32 = 0; + + #[cfg(target_os = "windows")] + #[pyattr(name = "SIZEOF_TIME_T")] + pub const SIZEOF_TIME_T: usize = 8; + #[cfg(not(target_os = "windows"))] + #[pyattr(name = "SIZEOF_TIME_T")] + pub const SIZEOF_TIME_T: usize = 4; + + #[pyattr(name = "CTYPES_MAX_ARGCOUNT")] + pub const CTYPES_MAX_ARGCOUNT: usize = 1024; + + #[pyattr] + pub const FUNCFLAG_STDCALL: u32 = 0x0; + #[pyattr] + pub const FUNCFLAG_CDECL: u32 = 0x1; + #[pyattr] + pub const FUNCFLAG_HRESULT: u32 = 0x2; + #[pyattr] + pub const FUNCFLAG_PYTHONAPI: u32 = 0x4; + #[pyattr] + pub const FUNCFLAG_USE_ERRNO: u32 = 0x8; + #[pyattr] + pub const FUNCFLAG_USE_LASTERROR: u32 = 0x10; + + #[pyattr] + pub const TYPEFLAG_ISPOINTER: u32 = 0x100; + #[pyattr] + pub const TYPEFLAG_HASPOINTER: u32 = 0x200; + + #[pyattr] + pub const DICTFLAG_FINAL: u32 = 0x1000; + + #[pyattr(once)] + fn error(vm: &VirtualMachine) -> PyTypeRef { + vm.ctx.new_exception_type( + "_ctypes", + "ArgumentError", + Some(vec![vm.ctx.exceptions.exception_type.to_owned()]), + ) + } + pub fn get_size(ty: &str) -> usize { match ty { "u" => mem::size_of::(), @@ -68,7 +143,7 @@ pub(crate) mod _ctypes { } else { Ok(PyCSimple { _type_: tp_str, - _value: AtomicCell::new(vm.ctx.none()), + value: AtomicCell::new(vm.ctx.none()), }) } } else { @@ -95,6 +170,41 @@ pub(crate) mod _ctypes { } } + #[pyfunction(name = "LoadLibrary")] + fn load_library(name: String, vm: &VirtualMachine) -> PyResult { + // TODO: audit functions first + let cache = library::libcache(); + let mut cache_write = cache.write(); + let lib_ref = cache_write.get_or_insert_lib(&name, vm).unwrap(); + Ok(lib_ref.get_pointer()) + } + + #[pyfunction(name = "FreeLibrary")] + fn free_library(handle: usize) -> PyResult<()> { + let cache = library::libcache(); + let mut cache_write = cache.write(); + cache_write.drop_lib(handle); + Ok(()) + } + + #[pyfunction(name = "POINTER")] + pub fn pointer(_cls: PyTypeRef) {} + + #[pyfunction] + pub fn pointer_fn(_inst: PyObjectRef) {} + + #[cfg(target_os = "windows")] + #[pyfunction(name = "_check_HRESULT")] + pub fn check_hresult(_self: PyObjectRef, hr: i32, _vm: &VirtualMachine) -> PyResult { + // TODO: fixme + if hr < 0 { + // vm.ctx.new_windows_error(hr) + todo!(); + } else { + Ok(hr) + } + } + #[pyfunction] fn get_errno() -> i32 { errno::errno().0 diff --git a/vm/src/stdlib/ctypes/array.rs b/vm/src/stdlib/ctypes/array.rs new file mode 100644 index 0000000000..8b023582c9 --- /dev/null +++ b/vm/src/stdlib/ctypes/array.rs @@ -0,0 +1,5 @@ +#[pyclass(name = "Array", module = "_ctypes")] +pub struct PyCArray {} + +#[pyclass(flags(BASETYPE, IMMUTABLETYPE))] +impl PyCArray {} diff --git a/vm/src/stdlib/ctypes/base.rs b/vm/src/stdlib/ctypes/base.rs index 0142c45345..5d0316352c 100644 --- a/vm/src/stdlib/ctypes/base.rs +++ b/vm/src/stdlib/ctypes/base.rs @@ -1,6 +1,10 @@ -use crate::builtins::{PyBytes, PyFloat, PyInt, PyModule, PyNone, PyStr}; -use crate::class::PyClassImpl; -use crate::{Py, PyObjectRef, PyResult, TryFromObject, VirtualMachine}; +use crate::builtins::PyType; +use crate::builtins::{PyBytes, PyFloat, PyInt, PyNone, PyStr, PyTypeRef}; +use crate::convert::ToPyObject; +use crate::function::{Either, OptionalArg}; +use crate::stdlib::ctypes::_ctypes::new_simple_type; +use crate::types::Constructor; +use crate::{AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine}; use crossbeam_utils::atomic::AtomicCell; use num_traits::ToPrimitive; use rustpython_common::lock::PyRwLock; @@ -129,16 +133,32 @@ pub struct PyCData { #[pyclass] impl PyCData {} +#[pyclass(module = "_ctypes", name = "PyCSimpleType", base = "PyType")] +pub struct PySimpleMeta {} + +#[pyclass(flags(BASETYPE))] +impl PySimpleMeta { + #[allow(clippy::new_ret_no_self)] + #[pymethod] + fn new(cls: PyTypeRef, _: OptionalArg, vm: &VirtualMachine) -> PyResult { + Ok(PyObjectRef::from( + new_simple_type(Either::B(&cls), vm)? + .into_ref_with_type(vm, cls)? + .clone(), + )) + } +} + #[pyclass( name = "_SimpleCData", base = "PyCData", - module = "_ctypes" - // TODO: metaclass + module = "_ctypes", + metaclass = "PySimpleMeta" )] #[derive(PyPayload)] pub struct PyCSimple { pub _type_: String, - pub _value: AtomicCell, + pub value: AtomicCell, } impl Debug for PyCSimple { @@ -149,13 +169,56 @@ impl Debug for PyCSimple { } } -#[pyclass(flags(BASETYPE))] -impl PyCSimple {} +impl Constructor for PyCSimple { + type Args = (OptionalArg,); -pub fn extend_module_nodes(vm: &VirtualMachine, module: &Py) { - let ctx = &vm.ctx; - extend_module!(vm, module, { - "_CData" => PyCData::make_class(ctx), - "_SimpleCData" => PyCSimple::make_class(ctx), - }) + fn py_new(cls: PyTypeRef, args: Self::Args, vm: &VirtualMachine) -> PyResult { + let attributes = cls.get_attributes(); + let _type_ = attributes + .iter() + .find(|(&k, _)| k.to_object().str(vm).unwrap().to_string() == *"_type_") + .unwrap() + .1 + .str(vm)? + .to_string(); + let value = if let Some(ref v) = args.0.into_option() { + set_primitive(_type_.as_str(), v, vm)? + } else { + match _type_.as_str() { + "c" | "u" => PyObjectRef::from(vm.ctx.new_bytes(vec![0])), + "b" | "B" | "h" | "H" | "i" | "I" | "l" | "q" | "L" | "Q" => { + PyObjectRef::from(vm.ctx.new_int(0)) + } + "f" | "d" | "g" => PyObjectRef::from(vm.ctx.new_float(0.0)), + "?" => PyObjectRef::from(vm.ctx.new_bool(false)), + _ => vm.ctx.none(), // "z" | "Z" | "P" + } + }; + Ok(PyCSimple { + _type_, + value: AtomicCell::new(value), + } + .to_pyobject(vm)) + } +} + +#[pyclass(flags(BASETYPE), with(Constructor))] +impl PyCSimple { + #[pygetset(name = "value")] + pub fn value(instance: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let zelf: &Py = instance + .downcast_ref() + .ok_or_else(|| vm.new_type_error("cannot get value of instance".to_string()))?; + Ok(unsafe { (*zelf.value.as_ptr()).clone() }) + } + + #[pygetset(name = "value", setter)] + fn set_value(instance: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let zelf: PyRef = instance + .downcast() + .map_err(|_| vm.new_type_error("cannot set value of instance".to_string()))?; + let content = set_primitive(zelf._type_.as_str(), &value, vm)?; + zelf.value.store(content); + Ok(()) + } } diff --git a/vm/src/stdlib/ctypes/function.rs b/vm/src/stdlib/ctypes/function.rs new file mode 100644 index 0000000000..bf7f0b9adf --- /dev/null +++ b/vm/src/stdlib/ctypes/function.rs @@ -0,0 +1,24 @@ +use crate::stdlib::ctypes::PyCData; +use crate::PyObjectRef; +use crossbeam_utils::atomic::AtomicCell; +use rustpython_common::lock::PyRwLock; +use std::ffi::c_void; + +#[derive(Debug)] +pub struct Function { + _pointer: *mut c_void, + _arguments: Vec<()>, + _return_type: Box<()>, +} + +#[pyclass(module = "_ctypes", name = "CFuncPtr", base = "PyCData")] +pub struct PyCFuncPtr { + pub _name_: String, + pub _argtypes_: AtomicCell>, + pub _restype_: AtomicCell, + _handle: PyObjectRef, + _f: PyRwLock, +} + +#[pyclass] +impl PyCFuncPtr {} diff --git a/vm/src/stdlib/ctypes/library.rs b/vm/src/stdlib/ctypes/library.rs new file mode 100644 index 0000000000..94b6327440 --- /dev/null +++ b/vm/src/stdlib/ctypes/library.rs @@ -0,0 +1,115 @@ +use crate::VirtualMachine; +use crossbeam_utils::atomic::AtomicCell; +use libloading::Library; +use rustpython_common::lock::PyRwLock; +use std::collections::HashMap; +use std::ffi::c_void; +use std::fmt; +use std::ptr::null; + +pub struct SharedLibrary { + lib: AtomicCell>, +} + +impl fmt::Debug for SharedLibrary { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "SharedLibrary") + } +} + +impl SharedLibrary { + pub fn new(name: &str) -> Result { + Ok(SharedLibrary { + lib: AtomicCell::new(Some(unsafe { Library::new(name)? })), + }) + } + + #[allow(dead_code)] + pub fn get_sym(&self, name: &str) -> Result<*mut c_void, String> { + if let Some(inner) = unsafe { &*self.lib.as_ptr() } { + unsafe { + inner + .get(name.as_bytes()) + .map(|f: libloading::Symbol<*mut c_void>| *f) + .map_err(|err| err.to_string()) + } + } else { + Err("The library has been closed".to_string()) + } + } + + pub fn get_pointer(&self) -> usize { + if let Some(l) = unsafe { &*self.lib.as_ptr() } { + l as *const Library as usize + } else { + null::() as usize + } + } + + pub fn is_closed(&self) -> bool { + unsafe { &*self.lib.as_ptr() }.is_none() + } + + pub fn close(&self) { + let old = self.lib.take(); + self.lib.store(None); + drop(old); + } +} + +impl Drop for SharedLibrary { + fn drop(&mut self) { + self.close(); + } +} + +pub struct ExternalLibs { + libraries: HashMap, +} + +impl ExternalLibs { + pub fn new() -> Self { + Self { + libraries: HashMap::new(), + } + } + + #[allow(dead_code)] + pub fn get_lib(&self, key: usize) -> Option<&SharedLibrary> { + self.libraries.get(&key) + } + + pub fn get_or_insert_lib( + &mut self, + library_path: &str, + _vm: &VirtualMachine, + ) -> Result<&SharedLibrary, libloading::Error> { + let nlib = SharedLibrary::new(library_path)?; + let key = nlib.get_pointer(); + + match self.libraries.get(&key) { + Some(l) => { + if l.is_closed() { + self.libraries.insert(key, nlib); + } + } + _ => { + self.libraries.insert(key, nlib); + } + }; + + Ok(self.libraries.get(&key).unwrap()) + } + + pub fn drop_lib(&mut self, key: usize) { + self.libraries.remove(&key); + } +} + +rustpython_common::static_cell! { + static LIBCACHE: PyRwLock; +} + +pub fn libcache() -> &'static PyRwLock { + LIBCACHE.get_or_init(|| PyRwLock::new(ExternalLibs::new())) +} diff --git a/vm/src/stdlib/ctypes/pointer.rs b/vm/src/stdlib/ctypes/pointer.rs new file mode 100644 index 0000000000..d1360f9862 --- /dev/null +++ b/vm/src/stdlib/ctypes/pointer.rs @@ -0,0 +1,5 @@ +#[pyclass(name = "Pointer", module = "_ctypes")] +pub struct PyCPointer {} + +#[pyclass(flags(BASETYPE, IMMUTABLETYPE))] +impl PyCPointer {} diff --git a/vm/src/stdlib/ctypes/structure.rs b/vm/src/stdlib/ctypes/structure.rs new file mode 100644 index 0000000000..13cca6c260 --- /dev/null +++ b/vm/src/stdlib/ctypes/structure.rs @@ -0,0 +1,5 @@ +#[pyclass(name = "Structure", module = "_ctypes")] +pub struct PyCStructure {} + +#[pyclass(flags(BASETYPE, IMMUTABLETYPE))] +impl PyCStructure {} diff --git a/vm/src/stdlib/ctypes/union.rs b/vm/src/stdlib/ctypes/union.rs new file mode 100644 index 0000000000..5a39d9062e --- /dev/null +++ b/vm/src/stdlib/ctypes/union.rs @@ -0,0 +1,5 @@ +#[pyclass(name = "Union", module = "_ctypes")] +pub struct PyCUnion {} + +#[pyclass(flags(BASETYPE, IMMUTABLETYPE))] +impl PyCUnion {} From 65dcf1ce1c0999a5db9c1bc322ee3bf69b3d509f Mon Sep 17 00:00:00 2001 From: Hanif Ariffin Date: Thu, 20 Feb 2025 10:21:12 +0800 Subject: [PATCH 024/295] Updating test_math.py to CPython 3.12.9 (#5507) * Fixed implementation against CPython 3.12.9 Lib/test/test_math.py tests --------- Signed-off-by: Hanif Ariffin Co-authored-by: Jeong YunWon --- Lib/test/ieee754.txt | 183 ++++++++++++++++++ Lib/test/test_math.py | 402 +++++++++++++++++++++++++++++++++++++-- common/src/float_ops.rs | 63 ++++++ stdlib/src/math.rs | 121 ++++++++---- vm/src/builtins/float.rs | 4 +- 5 files changed, 727 insertions(+), 46 deletions(-) create mode 100644 Lib/test/ieee754.txt diff --git a/Lib/test/ieee754.txt b/Lib/test/ieee754.txt new file mode 100644 index 0000000000..3e986cdb10 --- /dev/null +++ b/Lib/test/ieee754.txt @@ -0,0 +1,183 @@ +====================================== +Python IEEE 754 floating point support +====================================== + +>>> from sys import float_info as FI +>>> from math import * +>>> PI = pi +>>> E = e + +You must never compare two floats with == because you are not going to get +what you expect. We treat two floats as equal if the difference between them +is small than epsilon. +>>> EPS = 1E-15 +>>> def equal(x, y): +... """Almost equal helper for floats""" +... return abs(x - y) < EPS + + +NaNs and INFs +============= + +In Python 2.6 and newer NaNs (not a number) and infinity can be constructed +from the strings 'inf' and 'nan'. + +>>> INF = float('inf') +>>> NINF = float('-inf') +>>> NAN = float('nan') + +>>> INF +inf +>>> NINF +-inf +>>> NAN +nan + +The math module's ``isnan`` and ``isinf`` functions can be used to detect INF +and NAN: +>>> isinf(INF), isinf(NINF), isnan(NAN) +(True, True, True) +>>> INF == -NINF +True + +Infinity +-------- + +Ambiguous operations like ``0 * inf`` or ``inf - inf`` result in NaN. +>>> INF * 0 +nan +>>> INF - INF +nan +>>> INF / INF +nan + +However unambiguous operations with inf return inf: +>>> INF * INF +inf +>>> 1.5 * INF +inf +>>> 0.5 * INF +inf +>>> INF / 1000 +inf + +Not a Number +------------ + +NaNs are never equal to another number, even itself +>>> NAN == NAN +False +>>> NAN < 0 +False +>>> NAN >= 0 +False + +All operations involving a NaN return a NaN except for nan**0 and 1**nan. +>>> 1 + NAN +nan +>>> 1 * NAN +nan +>>> 0 * NAN +nan +>>> 1 ** NAN +1.0 +>>> NAN ** 0 +1.0 +>>> 0 ** NAN +nan +>>> (1.0 + FI.epsilon) * NAN +nan + +Misc Functions +============== + +The power of 1 raised to x is always 1.0, even for special values like 0, +infinity and NaN. + +>>> pow(1, 0) +1.0 +>>> pow(1, INF) +1.0 +>>> pow(1, -INF) +1.0 +>>> pow(1, NAN) +1.0 + +The power of 0 raised to x is defined as 0, if x is positive. Negative +finite values are a domain error or zero division error and NaN result in a +silent NaN. + +>>> pow(0, 0) +1.0 +>>> pow(0, INF) +0.0 +>>> pow(0, -INF) +inf +>>> 0 ** -1 +Traceback (most recent call last): +... +ZeroDivisionError: 0.0 cannot be raised to a negative power +>>> pow(0, NAN) +nan + + +Trigonometric Functions +======================= + +>>> sin(INF) +Traceback (most recent call last): +... +ValueError: math domain error +>>> sin(NINF) +Traceback (most recent call last): +... +ValueError: math domain error +>>> sin(NAN) +nan +>>> cos(INF) +Traceback (most recent call last): +... +ValueError: math domain error +>>> cos(NINF) +Traceback (most recent call last): +... +ValueError: math domain error +>>> cos(NAN) +nan +>>> tan(INF) +Traceback (most recent call last): +... +ValueError: math domain error +>>> tan(NINF) +Traceback (most recent call last): +... +ValueError: math domain error +>>> tan(NAN) +nan + +Neither pi nor tan are exact, but you can assume that tan(pi/2) is a large value +and tan(pi) is a very small value: +>>> tan(PI/2) > 1E10 +True +>>> -tan(-PI/2) > 1E10 +True +>>> tan(PI) < 1E-15 +True + +>>> asin(NAN), acos(NAN), atan(NAN) +(nan, nan, nan) +>>> asin(INF), asin(NINF) +Traceback (most recent call last): +... +ValueError: math domain error +>>> acos(INF), acos(NINF) +Traceback (most recent call last): +... +ValueError: math domain error +>>> equal(atan(INF), PI/2), equal(atan(NINF), -PI/2) +(True, True) + + +Hyberbolic Functions +==================== + diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py index 2f64180652..fa79456ed4 100644 --- a/Lib/test/test_math.py +++ b/Lib/test/test_math.py @@ -4,6 +4,7 @@ from test.support import verbose, requires_IEEE_754 from test import support import unittest +import fractions import itertools import decimal import math @@ -186,6 +187,9 @@ def result_check(expected, got, ulp_tol=5, abs_tol=0.0): # Check exactly equal (applies also to strings representing exceptions) if got == expected: + if not got and not expected: + if math.copysign(1, got) != math.copysign(1, expected): + return f"expected {expected}, got {got} (zero has wrong sign)" return None failure = "not equal" @@ -234,6 +238,10 @@ def __init__(self, value): def __index__(self): return self.value +class BadDescr: + def __get__(self, obj, objtype=None): + raise ValueError + class MathTests(unittest.TestCase): def ftest(self, name, got, expected, ulp_tol=5, abs_tol=0.0): @@ -323,6 +331,7 @@ def testAtan2(self): self.ftest('atan2(0, 1)', math.atan2(0, 1), 0) self.ftest('atan2(1, 1)', math.atan2(1, 1), math.pi/4) self.ftest('atan2(1, 0)', math.atan2(1, 0), math.pi/2) + self.ftest('atan2(1, -1)', math.atan2(1, -1), 3*math.pi/4) # math.atan2(0, x) self.ftest('atan2(0., -inf)', math.atan2(0., NINF), math.pi) @@ -416,16 +425,22 @@ def __ceil__(self): return 42 class TestNoCeil: pass + class TestBadCeil: + __ceil__ = BadDescr() self.assertEqual(math.ceil(TestCeil()), 42) self.assertEqual(math.ceil(FloatCeil()), 42) self.assertEqual(math.ceil(FloatLike(42.5)), 43) self.assertRaises(TypeError, math.ceil, TestNoCeil()) + self.assertRaises(ValueError, math.ceil, TestBadCeil()) t = TestNoCeil() t.__ceil__ = lambda *args: args self.assertRaises(TypeError, math.ceil, t) self.assertRaises(TypeError, math.ceil, t, 0) + self.assertEqual(math.ceil(FloatLike(+1.0)), +1.0) + self.assertEqual(math.ceil(FloatLike(-1.0)), -1.0) + @requires_IEEE_754 def testCopysign(self): self.assertEqual(math.copysign(1, 42), 1.0) @@ -566,16 +581,22 @@ def __floor__(self): return 42 class TestNoFloor: pass + class TestBadFloor: + __floor__ = BadDescr() self.assertEqual(math.floor(TestFloor()), 42) self.assertEqual(math.floor(FloatFloor()), 42) self.assertEqual(math.floor(FloatLike(41.9)), 41) self.assertRaises(TypeError, math.floor, TestNoFloor()) + self.assertRaises(ValueError, math.floor, TestBadFloor()) t = TestNoFloor() t.__floor__ = lambda *args: args self.assertRaises(TypeError, math.floor, t) self.assertRaises(TypeError, math.floor, t, 0) + self.assertEqual(math.floor(FloatLike(+1.0)), +1.0) + self.assertEqual(math.floor(FloatLike(-1.0)), -1.0) + def testFmod(self): self.assertRaises(TypeError, math.fmod) self.ftest('fmod(10, 1)', math.fmod(10, 1), 0.0) @@ -597,6 +618,7 @@ def testFmod(self): self.assertEqual(math.fmod(-3.0, NINF), -3.0) self.assertEqual(math.fmod(0.0, 3.0), 0.0) self.assertEqual(math.fmod(0.0, NINF), 0.0) + self.assertRaises(ValueError, math.fmod, INF, INF) def testFrexp(self): self.assertRaises(TypeError, math.frexp) @@ -638,7 +660,7 @@ def testFsum(self): def msum(iterable): """Full precision summation. Compute sum(iterable) without any intermediate accumulation of error. Based on the 'lsum' function - at http://code.activestate.com/recipes/393090/ + at https://code.activestate.com/recipes/393090-binary-floating-point-summation-accurate-to-full-p/ """ tmant, texp = 0, 0 @@ -666,6 +688,7 @@ def msum(iterable): ([], 0.0), ([0.0], 0.0), ([1e100, 1.0, -1e100, 1e-100, 1e50, -1.0, -1e50], 1e-100), + ([1e100, 1.0, -1e100, 1e-100, 1e50, -1, -1e50], 1e-100), ([2.0**53, -0.5, -2.0**-54], 2.0**53-1.0), ([2.0**53, 1.0, 2.0**-100], 2.0**53+2.0), ([2.0**53+10.0, 1.0, 2.0**-100], 2.0**53+12.0), @@ -713,6 +736,22 @@ def msum(iterable): s = msum(vals) self.assertEqual(msum(vals), math.fsum(vals)) + self.assertEqual(math.fsum([1.0, math.inf]), math.inf) + self.assertTrue(math.isnan(math.fsum([math.nan, 1.0]))) + self.assertEqual(math.fsum([1e100, FloatLike(1.0), -1e100, 1e-100, + 1e50, FloatLike(-1.0), -1e50]), 1e-100) + self.assertRaises(OverflowError, math.fsum, [1e+308, 1e+308]) + self.assertRaises(ValueError, math.fsum, [math.inf, -math.inf]) + self.assertRaises(TypeError, math.fsum, ['spam']) + self.assertRaises(TypeError, math.fsum, 1) + self.assertRaises(OverflowError, math.fsum, [10**1000]) + + def bad_iter(): + yield 1.0 + raise ZeroDivisionError + + self.assertRaises(ZeroDivisionError, math.fsum, bad_iter()) + def testGcd(self): gcd = math.gcd self.assertEqual(gcd(0, 0), 0) @@ -773,9 +812,13 @@ def testHypot(self): # Test allowable types (those with __float__) self.assertEqual(hypot(12.0, 5.0), 13.0) self.assertEqual(hypot(12, 5), 13) + self.assertEqual(hypot(0.75, -1), 1.25) + self.assertEqual(hypot(-1, 0.75), 1.25) + self.assertEqual(hypot(0.75, FloatLike(-1.)), 1.25) + self.assertEqual(hypot(FloatLike(-1.), 0.75), 1.25) self.assertEqual(hypot(Decimal(12), Decimal(5)), 13) self.assertEqual(hypot(Fraction(12, 32), Fraction(5, 32)), Fraction(13, 32)) - self.assertEqual(hypot(bool(1), bool(0), bool(1), bool(1)), math.sqrt(3)) + self.assertEqual(hypot(True, False, True, True, True), 2.0) # Test corner cases self.assertEqual(hypot(0.0, 0.0), 0.0) # Max input is zero @@ -830,6 +873,8 @@ def testHypot(self): scale = FLOAT_MIN / 2.0 ** exp self.assertEqual(math.hypot(4*scale, 3*scale), 5*scale) + self.assertRaises(TypeError, math.hypot, *([1.0]*18), 'spam') + @requires_IEEE_754 @unittest.skipIf(HAVE_DOUBLE_ROUNDING, "hypot() loses accuracy on machines with double rounding") @@ -922,12 +967,16 @@ def testDist(self): # Test allowable types (those with __float__) self.assertEqual(dist((14.0, 1.0), (2.0, -4.0)), 13.0) self.assertEqual(dist((14, 1), (2, -4)), 13) + self.assertEqual(dist((FloatLike(14.), 1), (2, -4)), 13) + self.assertEqual(dist((11, 1), (FloatLike(-1.), -4)), 13) + self.assertEqual(dist((14, FloatLike(-1.)), (2, -6)), 13) + self.assertEqual(dist((14, -1), (2, -6)), 13) self.assertEqual(dist((D(14), D(1)), (D(2), D(-4))), D(13)) self.assertEqual(dist((F(14, 32), F(1, 32)), (F(2, 32), F(-4, 32))), F(13, 32)) - self.assertEqual(dist((True, True, False, True, False), - (True, False, True, True, False)), - sqrt(2.0)) + self.assertEqual(dist((True, True, False, False, True, True), + (True, False, True, False, False, False)), + 2.0) # Test corner cases self.assertEqual(dist((13.25, 12.5, -3.25), @@ -965,6 +1014,8 @@ class T(tuple): dist((1, 2, 3, 4), (5, 6, 7)) with self.assertRaises(ValueError): # Check dimension agree dist((1, 2, 3), (4, 5, 6, 7)) + with self.assertRaises(TypeError): + dist((1,)*17 + ("spam",), (1,)*18) with self.assertRaises(TypeError): # Rejects invalid types dist("abc", "xyz") int_too_big_for_float = 10 ** (sys.float_info.max_10_exp + 5) @@ -972,6 +1023,16 @@ class T(tuple): dist((1, int_too_big_for_float), (2, 3)) with self.assertRaises((ValueError, OverflowError)): dist((2, 3), (1, int_too_big_for_float)) + with self.assertRaises(TypeError): + dist((1,), 2) + with self.assertRaises(TypeError): + dist([1], 2) + + class BadFloat: + __float__ = BadDescr() + + with self.assertRaises(ValueError): + dist([1], [BadFloat()]) # Verify that the one dimensional case is equivalent to abs() for i in range(20): @@ -1110,6 +1171,7 @@ def test_lcm(self): def testLdexp(self): self.assertRaises(TypeError, math.ldexp) + self.assertRaises(TypeError, math.ldexp, 2.0, 1.1) self.ftest('ldexp(0,1)', math.ldexp(0,1), 0) self.ftest('ldexp(1,1)', math.ldexp(1,1), 2) self.ftest('ldexp(1,-1)', math.ldexp(1,-1), 0.5) @@ -1142,6 +1204,7 @@ def testLdexp(self): def testLog(self): self.assertRaises(TypeError, math.log) + self.assertRaises(TypeError, math.log, 1, 2, 3) self.ftest('log(1/e)', math.log(1/math.e), -1) self.ftest('log(1)', math.log(1), 0) self.ftest('log(e)', math.log(math.e), 1) @@ -1152,6 +1215,7 @@ def testLog(self): 2302.5850929940457) self.assertRaises(ValueError, math.log, -1.5) self.assertRaises(ValueError, math.log, -10**1000) + self.assertRaises(ValueError, math.log, 10, -10) self.assertRaises(ValueError, math.log, NINF) self.assertEqual(math.log(INF), INF) self.assertTrue(math.isnan(math.log(NAN))) @@ -1202,6 +1266,277 @@ def testLog10(self): self.assertEqual(math.log(INF), INF) self.assertTrue(math.isnan(math.log10(NAN))) + def testSumProd(self): + sumprod = math.sumprod + Decimal = decimal.Decimal + Fraction = fractions.Fraction + + # Core functionality + self.assertEqual(sumprod(iter([10, 20, 30]), (1, 2, 3)), 140) + self.assertEqual(sumprod([1.5, 2.5], [3.5, 4.5]), 16.5) + self.assertEqual(sumprod([], []), 0) + self.assertEqual(sumprod([-1], [1.]), -1) + self.assertEqual(sumprod([1.], [-1]), -1) + + # Type preservation and coercion + for v in [ + (10, 20, 30), + (1.5, -2.5), + (Fraction(3, 5), Fraction(4, 5)), + (Decimal(3.5), Decimal(4.5)), + (2.5, 10), # float/int + (2.5, Fraction(3, 5)), # float/fraction + (25, Fraction(3, 5)), # int/fraction + (25, Decimal(4.5)), # int/decimal + ]: + for p, q in [(v, v), (v, v[::-1])]: + with self.subTest(p=p, q=q): + expected = sum(p_i * q_i for p_i, q_i in zip(p, q, strict=True)) + actual = sumprod(p, q) + self.assertEqual(expected, actual) + self.assertEqual(type(expected), type(actual)) + + # Bad arguments + self.assertRaises(TypeError, sumprod) # No args + self.assertRaises(TypeError, sumprod, []) # One arg + self.assertRaises(TypeError, sumprod, [], [], []) # Three args + self.assertRaises(TypeError, sumprod, None, [10]) # Non-iterable + self.assertRaises(TypeError, sumprod, [10], None) # Non-iterable + self.assertRaises(TypeError, sumprod, ['x'], [1.0]) + + # Uneven lengths + self.assertRaises(ValueError, sumprod, [10, 20], [30]) + self.assertRaises(ValueError, sumprod, [10], [20, 30]) + + # Overflows + self.assertEqual(sumprod([10**20], [1]), 10**20) + self.assertEqual(sumprod([1], [10**20]), 10**20) + self.assertEqual(sumprod([10**10], [10**10]), 10**20) + self.assertEqual(sumprod([10**7]*10**5, [10**7]*10**5), 10**19) + self.assertRaises(OverflowError, sumprod, [10**1000], [1.0]) + self.assertRaises(OverflowError, sumprod, [1.0], [10**1000]) + + # Error in iterator + def raise_after(n): + for i in range(n): + yield i + raise RuntimeError + with self.assertRaises(RuntimeError): + sumprod(range(10), raise_after(5)) + with self.assertRaises(RuntimeError): + sumprod(raise_after(5), range(10)) + + from test.test_iter import BasicIterClass + + self.assertEqual(sumprod(BasicIterClass(1), [1]), 0) + self.assertEqual(sumprod([1], BasicIterClass(1)), 0) + + # Error in multiplication + class BadMultiply: + def __mul__(self, other): + raise RuntimeError + def __rmul__(self, other): + raise RuntimeError + with self.assertRaises(RuntimeError): + sumprod([10, BadMultiply(), 30], [1, 2, 3]) + with self.assertRaises(RuntimeError): + sumprod([1, 2, 3], [10, BadMultiply(), 30]) + + # Error in addition + with self.assertRaises(TypeError): + sumprod(['abc', 3], [5, 10]) + with self.assertRaises(TypeError): + sumprod([5, 10], ['abc', 3]) + + # Special values should give the same as the pure python recipe + self.assertEqual(sumprod([10.1, math.inf], [20.2, 30.3]), math.inf) + self.assertEqual(sumprod([10.1, math.inf], [math.inf, 30.3]), math.inf) + self.assertEqual(sumprod([10.1, math.inf], [math.inf, math.inf]), math.inf) + self.assertEqual(sumprod([10.1, -math.inf], [20.2, 30.3]), -math.inf) + self.assertTrue(math.isnan(sumprod([10.1, math.inf], [-math.inf, math.inf]))) + self.assertTrue(math.isnan(sumprod([10.1, math.nan], [20.2, 30.3]))) + self.assertTrue(math.isnan(sumprod([10.1, math.inf], [math.nan, 30.3]))) + self.assertTrue(math.isnan(sumprod([10.1, math.inf], [20.3, math.nan]))) + + # Error cases that arose during development + args = ((-5, -5, 10), (1.5, 4611686018427387904, 2305843009213693952)) + self.assertEqual(sumprod(*args), 0.0) + + + @requires_IEEE_754 + @unittest.skipIf(HAVE_DOUBLE_ROUNDING, + "sumprod() accuracy not guaranteed on machines with double rounding") + @support.cpython_only # Other implementations may choose a different algorithm + def test_sumprod_accuracy(self): + sumprod = math.sumprod + self.assertEqual(sumprod([0.1] * 10, [1]*10), 1.0) + self.assertEqual(sumprod([0.1] * 20, [True, False] * 10), 1.0) + self.assertEqual(sumprod([True, False] * 10, [0.1] * 20), 1.0) + self.assertEqual(sumprod([1.0, 10E100, 1.0, -10E100], [1.0]*4), 2.0) + + @support.requires_resource('cpu') + def test_sumprod_stress(self): + sumprod = math.sumprod + product = itertools.product + Decimal = decimal.Decimal + Fraction = fractions.Fraction + + class Int(int): + def __add__(self, other): + return Int(int(self) + int(other)) + def __mul__(self, other): + return Int(int(self) * int(other)) + __radd__ = __add__ + __rmul__ = __mul__ + def __repr__(self): + return f'Int({int(self)})' + + class Flt(float): + def __add__(self, other): + return Int(int(self) + int(other)) + def __mul__(self, other): + return Int(int(self) * int(other)) + __radd__ = __add__ + __rmul__ = __mul__ + def __repr__(self): + return f'Flt({int(self)})' + + def baseline_sumprod(p, q): + """This defines the target behavior including exceptions and special values. + However, it is subject to rounding errors, so float inputs should be exactly + representable with only a few bits. + """ + total = 0 + for p_i, q_i in zip(p, q, strict=True): + total += p_i * q_i + return total + + def run(func, *args): + "Make comparing functions easier. Returns error status, type, and result." + try: + result = func(*args) + except (AssertionError, NameError): + raise + except Exception as e: + return type(e), None, 'None' + return None, type(result), repr(result) + + pools = [ + (-5, 10, -2**20, 2**31, 2**40, 2**61, 2**62, 2**80, 1.5, Int(7)), + (5.25, -3.5, 4.75, 11.25, 400.5, 0.046875, 0.25, -1.0, -0.078125), + (-19.0*2**500, 11*2**1000, -3*2**1500, 17*2*333, + 5.25, -3.25, -3.0*2**(-333), 3, 2**513), + (3.75, 2.5, -1.5, float('inf'), -float('inf'), float('NaN'), 14, + 9, 3+4j, Flt(13), 0.0), + (13.25, -4.25, Decimal('10.5'), Decimal('-2.25'), Fraction(13, 8), + Fraction(-11, 16), 4.75 + 0.125j, 97, -41, Int(3)), + (Decimal('6.125'), Decimal('12.375'), Decimal('-2.75'), Decimal(0), + Decimal('Inf'), -Decimal('Inf'), Decimal('NaN'), 12, 13.5), + (-2.0 ** -1000, 11*2**1000, 3, 7, -37*2**32, -2*2**-537, -2*2**-538, + 2*2**-513), + (-7 * 2.0 ** -510, 5 * 2.0 ** -520, 17, -19.0, -6.25), + (11.25, -3.75, -0.625, 23.375, True, False, 7, Int(5)), + ] + + for pool in pools: + for size in range(4): + for args1 in product(pool, repeat=size): + for args2 in product(pool, repeat=size): + args = (args1, args2) + self.assertEqual( + run(baseline_sumprod, *args), + run(sumprod, *args), + args, + ) + + @requires_IEEE_754 + @unittest.skipIf(HAVE_DOUBLE_ROUNDING, + "sumprod() accuracy not guaranteed on machines with double rounding") + @support.cpython_only # Other implementations may choose a different algorithm + @support.requires_resource('cpu') + def test_sumprod_extended_precision_accuracy(self): + import operator + from fractions import Fraction + from itertools import starmap + from collections import namedtuple + from math import log2, exp2, fabs + from random import choices, uniform, shuffle + from statistics import median + + DotExample = namedtuple('DotExample', ('x', 'y', 'target_sumprod', 'condition')) + + def DotExact(x, y): + vec1 = map(Fraction, x) + vec2 = map(Fraction, y) + return sum(starmap(operator.mul, zip(vec1, vec2, strict=True))) + + def Condition(x, y): + return 2.0 * DotExact(map(abs, x), map(abs, y)) / abs(DotExact(x, y)) + + def linspace(lo, hi, n): + width = (hi - lo) / (n - 1) + return [lo + width * i for i in range(n)] + + def GenDot(n, c): + """ Algorithm 6.1 (GenDot) works as follows. The condition number (5.7) of + the dot product xT y is proportional to the degree of cancellation. In + order to achieve a prescribed cancellation, we generate the first half of + the vectors x and y randomly within a large exponent range. This range is + chosen according to the anticipated condition number. The second half of x + and y is then constructed choosing xi randomly with decreasing exponent, + and calculating yi such that some cancellation occurs. Finally, we permute + the vectors x, y randomly and calculate the achieved condition number. + """ + + assert n >= 6 + n2 = n // 2 + x = [0.0] * n + y = [0.0] * n + b = log2(c) + + # First half with exponents from 0 to |_b/2_| and random ints in between + e = choices(range(int(b/2)), k=n2) + e[0] = int(b / 2) + 1 + e[-1] = 0.0 + + x[:n2] = [uniform(-1.0, 1.0) * exp2(p) for p in e] + y[:n2] = [uniform(-1.0, 1.0) * exp2(p) for p in e] + + # Second half + e = list(map(round, linspace(b/2, 0.0 , n-n2))) + for i in range(n2, n): + x[i] = uniform(-1.0, 1.0) * exp2(e[i - n2]) + y[i] = (uniform(-1.0, 1.0) * exp2(e[i - n2]) - DotExact(x, y)) / x[i] + + # Shuffle + pairs = list(zip(x, y)) + shuffle(pairs) + x, y = zip(*pairs) + + return DotExample(x, y, DotExact(x, y), Condition(x, y)) + + def RelativeError(res, ex): + x, y, target_sumprod, condition = ex + n = DotExact(list(x) + [-res], list(y) + [1]) + return fabs(n / target_sumprod) + + def Trial(dotfunc, c, n): + ex = GenDot(10, c) + res = dotfunc(ex.x, ex.y) + return RelativeError(res, ex) + + times = 1000 # Number of trials + n = 20 # Length of vectors + c = 1e30 # Target condition number + + # If the following test fails, it means that the C math library + # implementation of fma() is not compliant with the C99 standard + # and is inaccurate. To solve this problem, make a new build + # with the symbol UNRELIABLE_FMA defined. That will enable a + # slower but accurate code path that avoids the fma() call. + relative_err = median(Trial(math.sumprod, c, n) for i in range(times)) + self.assertLess(relative_err, 1e-16) + def testModf(self): self.assertRaises(TypeError, math.modf) @@ -1235,6 +1570,7 @@ def testPow(self): self.assertTrue(math.isnan(math.pow(2, NAN))) self.assertTrue(math.isnan(math.pow(0, NAN))) self.assertEqual(math.pow(1, NAN), 1) + self.assertRaises(OverflowError, math.pow, 1e+100, 1e+100) # pow(0., x) self.assertEqual(math.pow(0., INF), 0.) @@ -1550,7 +1886,7 @@ def testTan(self): try: self.assertTrue(math.isnan(math.tan(INF))) self.assertTrue(math.isnan(math.tan(NINF))) - except: + except ValueError: self.assertRaises(ValueError, math.tan, INF) self.assertRaises(ValueError, math.tan, NINF) self.assertTrue(math.isnan(math.tan(NAN))) @@ -1591,6 +1927,8 @@ def __trunc__(self): return 23 class TestNoTrunc: pass + class TestBadTrunc: + __trunc__ = BadDescr() self.assertEqual(math.trunc(TestTrunc()), 23) self.assertEqual(math.trunc(FloatTrunc()), 23) @@ -1599,6 +1937,7 @@ class TestNoTrunc: self.assertRaises(TypeError, math.trunc, 1, 2) self.assertRaises(TypeError, math.trunc, FloatLike(23.5)) self.assertRaises(TypeError, math.trunc, TestNoTrunc()) + self.assertRaises(ValueError, math.trunc, TestBadTrunc()) def testIsfinite(self): self.assertTrue(math.isfinite(0.0)) @@ -1626,11 +1965,11 @@ def testIsinf(self): self.assertFalse(math.isinf(0.)) self.assertFalse(math.isinf(1.)) - @requires_IEEE_754 def test_nan_constant(self): + # `math.nan` must be a quiet NaN with positive sign bit self.assertTrue(math.isnan(math.nan)) + self.assertEqual(math.copysign(1., math.nan), 1.) - @requires_IEEE_754 def test_inf_constant(self): self.assertTrue(math.isinf(math.inf)) self.assertGreater(math.inf, 0.0) @@ -1719,6 +2058,13 @@ def test_testfile(self): except OverflowError: result = 'OverflowError' + # C99+ says for math.h's sqrt: If the argument is +∞ or ±0, it is + # returned, unmodified. On another hand, for csqrt: If z is ±0+0i, + # the result is +0+0i. Lets correct zero sign of er to follow + # first convention. + if id in ['sqrt0002', 'sqrt0003', 'sqrt1001', 'sqrt1023']: + er = math.copysign(er, ar) + # Default tolerances ulp_tol, abs_tol = 5, 0.0 @@ -1802,6 +2148,8 @@ def test_mtestfile(self): '\n '.join(failures)) def test_prod(self): + from fractions import Fraction as F + prod = math.prod self.assertEqual(prod([]), 1) self.assertEqual(prod([], start=5), 5) @@ -1813,6 +2161,14 @@ def test_prod(self): self.assertEqual(prod([1.0, 2.0, 3.0, 4.0, 5.0]), 120.0) self.assertEqual(prod([1, 2, 3, 4.0, 5.0]), 120.0) self.assertEqual(prod([1.0, 2.0, 3.0, 4, 5]), 120.0) + self.assertEqual(prod([1., F(3, 2)]), 1.5) + + # Error in multiplication + class BadMultiply: + def __rmul__(self, other): + raise RuntimeError + with self.assertRaises(RuntimeError): + prod([10., BadMultiply()]) # Test overflow in fast-path for integers self.assertEqual(prod([1, 1, 2**32, 1, 1]), 2**32) @@ -2044,11 +2400,20 @@ def test_nextafter(self): float.fromhex('0x1.fffffffffffffp-1')) self.assertEqual(math.nextafter(1.0, INF), float.fromhex('0x1.0000000000001p+0')) + self.assertEqual(math.nextafter(1.0, -INF, steps=1), + float.fromhex('0x1.fffffffffffffp-1')) + self.assertEqual(math.nextafter(1.0, INF, steps=1), + float.fromhex('0x1.0000000000001p+0')) + self.assertEqual(math.nextafter(1.0, -INF, steps=3), + float.fromhex('0x1.ffffffffffffdp-1')) + self.assertEqual(math.nextafter(1.0, INF, steps=3), + float.fromhex('0x1.0000000000003p+0')) # x == y: y is returned - self.assertEqual(math.nextafter(2.0, 2.0), 2.0) - self.assertEqualSign(math.nextafter(-0.0, +0.0), +0.0) - self.assertEqualSign(math.nextafter(+0.0, -0.0), -0.0) + for steps in range(1, 5): + self.assertEqual(math.nextafter(2.0, 2.0, steps=steps), 2.0) + self.assertEqualSign(math.nextafter(-0.0, +0.0, steps=steps), +0.0) + self.assertEqualSign(math.nextafter(+0.0, -0.0, steps=steps), -0.0) # around 0.0 smallest_subnormal = sys.float_info.min * sys.float_info.epsilon @@ -2073,6 +2438,11 @@ def test_nextafter(self): self.assertIsNaN(math.nextafter(1.0, NAN)) self.assertIsNaN(math.nextafter(NAN, NAN)) + self.assertEqual(1.0, math.nextafter(1.0, INF, steps=0)) + with self.assertRaises(ValueError): + math.nextafter(1.0, INF, steps=-1) + + @requires_IEEE_754 def test_ulp(self): self.assertEqual(math.ulp(1.0), sys.float_info.epsilon) @@ -2112,6 +2482,14 @@ def __float__(self): # argument to a float. self.assertFalse(getattr(y, "converted", False)) + def test_input_exceptions(self): + self.assertRaises(TypeError, math.exp, "spam") + self.assertRaises(TypeError, math.erf, "spam") + self.assertRaises(TypeError, math.atan2, "spam", 1.0) + self.assertRaises(TypeError, math.atan2, 1.0, "spam") + self.assertRaises(TypeError, math.atan2, 1.0) + self.assertRaises(TypeError, math.atan2, 1.0, 2.0, 3.0) + # Custom assertions. def assertIsNaN(self, value): @@ -2252,7 +2630,7 @@ def test_fractions(self): def load_tests(loader, tests, pattern): from doctest import DocFileSuite - # tests.addTest(DocFileSuite("ieee754.txt")) + tests.addTest(DocFileSuite("ieee754.txt")) return tests if __name__ == '__main__': diff --git a/common/src/float_ops.rs b/common/src/float_ops.rs index 69ae8833a2..8e16e71c14 100644 --- a/common/src/float_ops.rs +++ b/common/src/float_ops.rs @@ -133,6 +133,69 @@ pub fn nextafter(x: f64, y: f64) -> f64 { } } +#[allow(clippy::float_cmp)] +pub fn nextafter_with_steps(x: f64, y: f64, steps: u64) -> f64 { + if x == y { + y + } else if x.is_nan() || y.is_nan() { + f64::NAN + } else if x >= f64::INFINITY { + f64::MAX + } else if x <= f64::NEG_INFINITY { + f64::MIN + } else if x == 0.0 { + f64::from_bits(1).copysign(y) + } else { + if steps == 0 { + return x; + } + + if x.is_nan() { + return x; + } + + if y.is_nan() { + return y; + } + + let sign_bit: u64 = 1 << 63; + + let mut ux = x.to_bits(); + let uy = y.to_bits(); + + let ax = ux & !sign_bit; + let ay = uy & !sign_bit; + + // If signs are different + if ((ux ^ uy) & sign_bit) != 0 { + return if ax + ay <= steps { + f64::from_bits(uy) + } else if ax < steps { + let result = (uy & sign_bit) | (steps - ax); + f64::from_bits(result) + } else { + ux -= steps; + f64::from_bits(ux) + }; + } + + // If signs are the same + if ax > ay { + if ax - ay >= steps { + ux -= steps; + f64::from_bits(ux) + } else { + f64::from_bits(uy) + } + } else if ay - ax >= steps { + ux += steps; + f64::from_bits(ux) + } else { + f64::from_bits(uy) + } + } +} + pub fn ulp(x: f64) -> f64 { if x.is_nan() { return x; diff --git a/stdlib/src/math.rs b/stdlib/src/math.rs index 21451203b5..68930cfd0f 100644 --- a/stdlib/src/math.rs +++ b/stdlib/src/math.rs @@ -629,7 +629,7 @@ mod math { #[pyfunction] fn fsum(seq: ArgIterable, vm: &VirtualMachine) -> PyResult { - let mut partials = vec![]; + let mut partials = Vec::with_capacity(32); let mut special_sum = 0.0; let mut inf_sum = 0.0; @@ -637,11 +637,11 @@ mod math { let mut x = *obj?; let xsave = x; - let mut j = 0; + let mut i = 0; // This inner loop applies `hi`/`lo` summation to each // partial so that the list of partial sums remains exact. - for i in 0..partials.len() { - let mut y: f64 = partials[i]; + for j in 0..partials.len() { + let mut y: f64 = partials[j]; if x.abs() < y.abs() { std::mem::swap(&mut x, &mut y); } @@ -650,33 +650,33 @@ mod math { let hi = x + y; let lo = y - (hi - x); if lo != 0.0 { - partials[j] = lo; - j += 1; + partials[i] = lo; + i += 1; } x = hi; } - if !x.is_finite() { - // a nonfinite x could arise either as - // a result of intermediate overflow, or - // as a result of a nan or inf in the - // summands - if xsave.is_finite() { - return Err(vm.new_overflow_error("intermediate overflow in fsum".to_owned())); - } - if xsave.is_infinite() { - inf_sum += xsave; + partials.truncate(i); + if x != 0.0 { + if !x.is_finite() { + // a nonfinite x could arise either as + // a result of intermediate overflow, or + // as a result of a nan or inf in the + // summands + if xsave.is_finite() { + return Err( + vm.new_overflow_error("intermediate overflow in fsum".to_owned()) + ); + } + if xsave.is_infinite() { + inf_sum += xsave; + } + special_sum += xsave; + // reset partials + partials.clear(); + } else { + partials.push(x); } - special_sum += xsave; - // reset partials - partials.clear(); - } - - if j >= partials.len() { - partials.push(x); - } else { - partials[j] = x; - partials.truncate(j + 1); } } if special_sum != 0.0 { @@ -831,9 +831,38 @@ mod math { (x.fract(), x.trunc()) } - #[pyfunction] - fn nextafter(x: ArgIntoFloat, y: ArgIntoFloat) -> f64 { - float_ops::nextafter(*x, *y) + #[derive(FromArgs)] + struct NextAfterArgs { + #[pyarg(positional)] + x: ArgIntoFloat, + #[pyarg(positional)] + y: ArgIntoFloat, + #[pyarg(named, optional)] + steps: OptionalArg, + } + + #[pyfunction] + fn nextafter(arg: NextAfterArgs, vm: &VirtualMachine) -> PyResult { + let steps: Option = arg + .steps + .map(|v| v.try_to_primitive(vm)) + .transpose()? + .into_option(); + match steps { + Some(steps) => { + if steps < 0 { + return Err( + vm.new_value_error("steps must be a non-negative integer".to_string()) + ); + } + Ok(float_ops::nextafter_with_steps( + *arg.x, + *arg.y, + steps as u64, + )) + } + None => Ok(float_ops::nextafter(*arg.x, *arg.y)), + } } #[pyfunction] @@ -917,10 +946,38 @@ mod math { // refer: https://github.com/python/cpython/blob/main/Modules/mathmodule.c#L3093-L3193 for obj in iter.iter(vm)? { let obj = obj?; + result = vm._mul(&result, &obj)?; + } + + Ok(result) + } - result = vm - ._mul(&result, &obj) - .map_err(|_| vm.new_type_error("math type error".to_owned()))?; + #[pyfunction] + fn sumprod( + p: ArgIterable, + q: ArgIterable, + vm: &VirtualMachine, + ) -> PyResult { + let mut p_iter = p.iter(vm)?; + let mut q_iter = q.iter(vm)?; + // We cannot just create a float because the iterator may contain + // anything as long as it supports __add__ and __mul__. + let mut result = vm.new_pyobj(0); + loop { + let m_p = p_iter.next(); + let m_q = q_iter.next(); + match (m_p, m_q) { + (Some(r_p), Some(r_q)) => { + let p = r_p?; + let q = r_q?; + let tmp = vm._mul(&p, &q)?; + result = vm._add(&result, &tmp)?; + } + (None, None) => break, + _ => { + return Err(vm.new_value_error("Inputs are not the same length".to_string())); + } + } } Ok(result) diff --git a/vm/src/builtins/float.rs b/vm/src/builtins/float.rs index 1cd041b7b9..27fdff12ee 100644 --- a/vm/src/builtins/float.rs +++ b/vm/src/builtins/float.rs @@ -124,8 +124,8 @@ fn inner_divmod(v1: f64, v2: f64, vm: &VirtualMachine) -> PyResult<(f64, f64)> { pub fn float_pow(v1: f64, v2: f64, vm: &VirtualMachine) -> PyResult { if v1.is_zero() && v2.is_sign_negative() { - let msg = format!("{v1} cannot be raised to a negative power"); - Err(vm.new_zero_division_error(msg)) + let msg = "0.0 cannot be raised to a negative power"; + Err(vm.new_zero_division_error(msg.to_owned())) } else if v1.is_sign_negative() && (v2.floor() - v2).abs() > f64::EPSILON { let v1 = Complex64::new(v1, 0.); let v2 = Complex64::new(v2, 0.); From a9331bb34df1ce1b61b156044087886e2a182bf3 Mon Sep 17 00:00:00 2001 From: Noa Date: Mon, 10 Feb 2025 23:47:27 -0600 Subject: [PATCH 025/295] Fix warnings for rust 1.85 --- Cargo.lock | 2 +- Cargo.toml | 2 +- common/src/hash.rs | 4 +- compiler/core/src/bytecode.rs | 2 +- src/shell/helper.rs | 2 +- stdlib/src/binascii.rs | 14 ++-- vm/src/builtins/object.rs | 2 +- vm/src/builtins/str.rs | 148 +++++++++++++++++----------------- vm/src/codecs.rs | 4 +- vm/src/stdlib/io.rs | 2 +- vm/src/vm/vm_ops.rs | 4 +- vm/sre_engine/src/string.rs | 4 +- 12 files changed, 95 insertions(+), 95 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 11992d8d8c..edf28dc7f4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1322,7 +1322,7 @@ version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df7151a832e54d2d6b2c827a20e5bcdd80359281cd2c354e725d4b82e7c471de" dependencies = [ - "rand_core 0.9.0", + "rand_core 0.9.1", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index bda7286afd..83007606c6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -104,7 +104,7 @@ members = [ version = "0.4.0" authors = ["RustPython Team"] edition = "2021" -rust-version = "1.83.0" +rust-version = "1.85.0" repository = "https://github.com/RustPython/RustPython" license = "MIT" diff --git a/common/src/hash.rs b/common/src/hash.rs index 700d2ceef5..5b5ac003cb 100644 --- a/common/src/hash.rs +++ b/common/src/hash.rs @@ -114,7 +114,7 @@ pub fn hash_float(value: f64) -> Option { let mut e = frexp.1; let mut x: PyUHash = 0; while m != 0.0 { - x = ((x << 28) & MODULUS) | x >> (BITS - 28); + x = ((x << 28) & MODULUS) | (x >> (BITS - 28)); m *= 268_435_456.0; // 2**28 e -= 28; let y = m as PyUHash; // pull out integer part @@ -132,7 +132,7 @@ pub fn hash_float(value: f64) -> Option { } else { BITS32 - 1 - ((-1 - e) % BITS32) }; - x = ((x << e) & MODULUS) | x >> (BITS32 - e); + x = ((x << e) & MODULUS) | (x >> (BITS32 - e)); Some(fix_sentinel(x as PyHash * value.signum() as PyHash)) } diff --git a/compiler/core/src/bytecode.rs b/compiler/core/src/bytecode.rs index 46839c2695..21d342b541 100644 --- a/compiler/core/src/bytecode.rs +++ b/compiler/core/src/bytecode.rs @@ -197,7 +197,7 @@ impl OpArgState { } #[inline(always)] pub fn extend(&mut self, arg: OpArgByte) -> OpArg { - self.state = self.state << 8 | u32::from(arg.0); + self.state = (self.state << 8) | u32::from(arg.0); OpArg(self.state) } #[inline(always)] diff --git a/src/shell/helper.rs b/src/shell/helper.rs index c228dbdf65..4d663b9424 100644 --- a/src/shell/helper.rs +++ b/src/shell/helper.rs @@ -107,7 +107,7 @@ impl<'vm> ShellHelper<'vm> { .filter(|res| { res.as_ref() .ok() - .map_or(true, |s| s.as_str().starts_with(word_start)) + .is_none_or(|s| s.as_str().starts_with(word_start)) }) .collect::, _>>() .ok()?; diff --git a/stdlib/src/binascii.rs b/stdlib/src/binascii.rs index d348049f4c..eb11645879 100644 --- a/stdlib/src/binascii.rs +++ b/stdlib/src/binascii.rs @@ -76,7 +76,7 @@ mod decl { let mut unhex = Vec::::with_capacity(hex_bytes.len() / 2); for (n1, n2) in hex_bytes.iter().tuples() { if let (Some(n1), Some(n2)) = (unhex_nibble(*n1), unhex_nibble(*n2)) { - unhex.push(n1 << 4 | n2); + unhex.push((n1 << 4) | n2); } else { return Err(super::new_binascii_error( "Non-hexadecimal digit found".to_owned(), @@ -343,7 +343,7 @@ mod decl { if let (Some(ch1), Some(ch2)) = (unhex_nibble(buffer[idx]), unhex_nibble(buffer[idx + 1])) { - out_data.push(ch1 << 4 | ch2); + out_data.push((ch1 << 4) | ch2); } idx += 2; } else { @@ -661,19 +661,19 @@ mod decl { }; if res.len() < length { - res.push(char_a << 2 | char_b >> 4); + res.push((char_a << 2) | (char_b >> 4)); } else if char_a != 0 || char_b != 0 { return trailing_garbage_error(); } if res.len() < length { - res.push((char_b & 0xf) << 4 | char_c >> 2); + res.push(((char_b & 0xf) << 4) | (char_c >> 2)); } else if char_c != 0 { return trailing_garbage_error(); } if res.len() < length { - res.push((char_c & 0x3) << 6 | char_d); + res.push(((char_c & 0x3) << 6) | char_d); } else if char_d != 0 { return trailing_garbage_error(); } @@ -725,8 +725,8 @@ mod decl { let char_c = *chunk.get(2).unwrap_or(&0); res.push(uu_b2a(char_a >> 2, backtick)); - res.push(uu_b2a((char_a & 0x3) << 4 | char_b >> 4, backtick)); - res.push(uu_b2a((char_b & 0xf) << 2 | char_c >> 6, backtick)); + res.push(uu_b2a(((char_a & 0x3) << 4) | (char_b >> 4), backtick)); + res.push(uu_b2a(((char_b & 0xf) << 2) | (char_c >> 6), backtick)); res.push(uu_b2a(char_c & 0x3f, backtick)); } diff --git a/vm/src/builtins/object.rs b/vm/src/builtins/object.rs index f783ee017c..aeb9e93cc6 100644 --- a/vm/src/builtins/object.rs +++ b/vm/src/builtins/object.rs @@ -113,7 +113,7 @@ fn object_getstate_default(obj: &PyObject, required: bool, vm: &VirtualMachine) // )); // } - let state = if obj.dict().map_or(true, |d| d.is_empty()) { + let state = if obj.dict().is_none_or(|d| d.is_empty()) { vm.ctx.none() } else { // let state = object_get_dict(obj.clone(), obj.ctx()).unwrap(); diff --git a/vm/src/builtins/str.rs b/vm/src/builtins/str.rs index 52112f371d..f5a340836c 100644 --- a/vm/src/builtins/str.rs +++ b/vm/src/builtins/str.rs @@ -1565,80 +1565,6 @@ impl AsRef for PyExact { } } -#[cfg(test)] -mod tests { - use super::*; - use crate::Interpreter; - - #[test] - fn str_title() { - let tests = vec![ - (" Hello ", " hello "), - ("Hello ", "hello "), - ("Hello ", "Hello "), - ("Format This As Title String", "fOrMaT thIs aS titLe String"), - ("Format,This-As*Title;String", "fOrMaT,thIs-aS*titLe;String"), - ("Getint", "getInt"), - ("Greek Ωppercases ...", "greek ωppercases ..."), - ("Greek ῼitlecases ...", "greek ῳitlecases ..."), - ]; - for (title, input) in tests { - assert_eq!(PyStr::from(input).title().as_str(), title); - } - } - - #[test] - fn str_istitle() { - let pos = vec![ - "A", - "A Titlecased Line", - "A\nTitlecased Line", - "A Titlecased, Line", - "Greek Ωppercases ...", - "Greek ῼitlecases ...", - ]; - - for s in pos { - assert!(PyStr::from(s).istitle()); - } - - let neg = vec![ - "", - "a", - "\n", - "Not a capitalized String", - "Not\ta Titlecase String", - "Not--a Titlecase String", - "NOT", - ]; - for s in neg { - assert!(!PyStr::from(s).istitle()); - } - } - - #[test] - fn str_maketrans_and_translate() { - Interpreter::without_stdlib(Default::default()).enter(|vm| { - let table = vm.ctx.new_dict(); - table - .set_item("a", vm.ctx.new_str("🎅").into(), vm) - .unwrap(); - table.set_item("b", vm.ctx.none(), vm).unwrap(); - table - .set_item("c", vm.ctx.new_str(ascii!("xda")).into(), vm) - .unwrap(); - let translated = - PyStr::maketrans(table.into(), OptionalArg::Missing, OptionalArg::Missing, vm) - .unwrap(); - let text = PyStr::from("abc"); - let translated = text.translate(translated, vm).unwrap(); - assert_eq!(translated, "🎅xda".to_owned()); - let translated = text.translate(vm.ctx.new_int(3).into(), vm); - assert_eq!("TypeError", &*translated.unwrap_err().class().name(),); - }) - } -} - impl AnyStrWrapper for PyStrRef { type Str = str; fn as_ref(&self) -> &str { @@ -1780,3 +1706,77 @@ impl AsRef for PyStrInterned { self.as_str() } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::Interpreter; + + #[test] + fn str_title() { + let tests = vec![ + (" Hello ", " hello "), + ("Hello ", "hello "), + ("Hello ", "Hello "), + ("Format This As Title String", "fOrMaT thIs aS titLe String"), + ("Format,This-As*Title;String", "fOrMaT,thIs-aS*titLe;String"), + ("Getint", "getInt"), + ("Greek Ωppercases ...", "greek ωppercases ..."), + ("Greek ῼitlecases ...", "greek ῳitlecases ..."), + ]; + for (title, input) in tests { + assert_eq!(PyStr::from(input).title().as_str(), title); + } + } + + #[test] + fn str_istitle() { + let pos = vec![ + "A", + "A Titlecased Line", + "A\nTitlecased Line", + "A Titlecased, Line", + "Greek Ωppercases ...", + "Greek ῼitlecases ...", + ]; + + for s in pos { + assert!(PyStr::from(s).istitle()); + } + + let neg = vec![ + "", + "a", + "\n", + "Not a capitalized String", + "Not\ta Titlecase String", + "Not--a Titlecase String", + "NOT", + ]; + for s in neg { + assert!(!PyStr::from(s).istitle()); + } + } + + #[test] + fn str_maketrans_and_translate() { + Interpreter::without_stdlib(Default::default()).enter(|vm| { + let table = vm.ctx.new_dict(); + table + .set_item("a", vm.ctx.new_str("🎅").into(), vm) + .unwrap(); + table.set_item("b", vm.ctx.none(), vm).unwrap(); + table + .set_item("c", vm.ctx.new_str(ascii!("xda")).into(), vm) + .unwrap(); + let translated = + PyStr::maketrans(table.into(), OptionalArg::Missing, OptionalArg::Missing, vm) + .unwrap(); + let text = PyStr::from("abc"); + let translated = text.translate(translated, vm).unwrap(); + assert_eq!(translated, "🎅xda".to_owned()); + let translated = text.translate(vm.ctx.new_int(3).into(), vm); + assert_eq!("TypeError", &*translated.unwrap_err().class().name(),); + }) + } +} diff --git a/vm/src/codecs.rs b/vm/src/codecs.rs index 61be9f9176..898198f3d3 100644 --- a/vm/src/codecs.rs +++ b/vm/src/codecs.rs @@ -637,10 +637,10 @@ fn surrogatepass_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<(PyOb } } StandardEncoding::Utf16Le => { - c = (p[1] as u32) << 8 | p[0] as u32; + c = ((p[1] as u32) << 8) | p[0] as u32; } StandardEncoding::Utf16Be => { - c = (p[0] as u32) << 8 | p[1] as u32; + c = ((p[0] as u32) << 8) | p[1] as u32; } StandardEncoding::Utf32Le => { c = ((p[3] as u32) << 24) diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index 3c35b05e1a..c4f67cd6a6 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -482,7 +482,7 @@ mod _io { let size = size.to_usize(); let read = instance.get_attr("read", vm)?; let mut res = Vec::new(); - while size.map_or(true, |s| res.len() < s) { + while size.is_none_or(|s| res.len() < s) { let read_res = ArgBytesLike::try_from_object(vm, read.call((1,), vm)?)?; if read_res.with_ref(|b| b.is_empty()) { break; diff --git a/vm/src/vm/vm_ops.rs b/vm/src/vm/vm_ops.rs index 56c32d2927..3aa4144190 100644 --- a/vm/src/vm/vm_ops.rs +++ b/vm/src/vm/vm_ops.rs @@ -298,8 +298,8 @@ impl VirtualMachine { } if let Some(slot_c) = class_c.slots.as_number.left_ternary_op(op_slot) { - if slot_a.is_some_and(|slot_a| slot_a != slot_c) - && slot_b.is_some_and(|slot_b| slot_b != slot_c) + if slot_a.is_some_and(|slot_a| !std::ptr::fn_addr_eq(slot_a, slot_c)) + && slot_b.is_some_and(|slot_b| !std::ptr::fn_addr_eq(slot_b, slot_c)) { let ret = slot_c(a, b, c, self)?; if !ret.is(&self.ctx.not_implemented) { diff --git a/vm/sre_engine/src/string.rs b/vm/sre_engine/src/string.rs index ca8b3179dc..b5a2470f52 100644 --- a/vm/sre_engine/src/string.rs +++ b/vm/sre_engine/src/string.rs @@ -181,7 +181,7 @@ unsafe fn next_code_point(ptr: &mut *const u8) -> u32 { let z = **ptr; *ptr = ptr.offset(1); let y_z = utf8_acc_cont_byte((y & CONT_MASK) as u32, z); - ch = init << 12 | y_z; + ch = (init << 12) | y_z; if x >= 0xF0 { // [x y z w] case // use only the lower 3 bits of `init` @@ -189,7 +189,7 @@ unsafe fn next_code_point(ptr: &mut *const u8) -> u32 { // so the iterator must produce a value here. let w = **ptr; *ptr = ptr.offset(1); - ch = (init & 7) << 18 | utf8_acc_cont_byte(y_z, w); + ch = ((init & 7) << 18) | utf8_acc_cont_byte(y_z, w); } } From 23236aa8c7b0b0227919581cc86d0c9d02798246 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Thu, 20 Feb 2025 10:50:50 -0800 Subject: [PATCH 026/295] test_datetime now works on windows Signed-off-by: Ashwin Naren --- .github/workflows/ci.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index dc2dcc6c7e..9c0f736928 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -18,7 +18,6 @@ env: CARGO_ARGS: --no-default-features --features stdlib,zlib,importlib,encodings,sqlite,ssl # Skip additional tests on Windows. They are checked on Linux and MacOS. WINDOWS_SKIPS: >- - test_datetime test_glob test_importlib test_io From d698b28ce5af90b7c033e07f5c8b5130dad2799b Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sat, 22 Feb 2025 00:22:31 -0800 Subject: [PATCH 027/295] Ensure pymethod cannot be both magic and named simultaneously + macro documentation (#5538) Signed-off-by: Ashwin Naren --- derive-impl/src/pyclass.rs | 7 ++ derive/src/lib.rs | 183 +++++++++++++++++++++++++++++++++++++ 2 files changed, 190 insertions(+) diff --git a/derive-impl/src/pyclass.rs b/derive-impl/src/pyclass.rs index dedb3eb77a..c42069c31c 100644 --- a/derive-impl/src/pyclass.rs +++ b/derive-impl/src/pyclass.rs @@ -1229,6 +1229,13 @@ impl MethodItemMeta { let inner = self.inner(); let name = inner._optional_str("name")?; let magic = inner._bool("magic")?; + if magic && name.is_some() { + bail_span!( + &inner.meta_ident, + "A #[{}] method cannot be magic and have a specified name, choose one.", + inner.meta_name() + ); + } Ok(if let Some(name) = name { name } else { diff --git a/derive/src/lib.rs b/derive/src/lib.rs index 76b7e23488..e59db12f90 100644 --- a/derive/src/lib.rs +++ b/derive/src/lib.rs @@ -12,6 +12,124 @@ pub fn derive_from_args(input: TokenStream) -> TokenStream { derive_impl::derive_from_args(input).into() } +/// The attribute can be applied either to a struct, trait, or impl. +/// # Struct +/// This implements `MaybeTraverse`, `PyClassDef`, and `StaticType` for the struct. +/// Consider deriving `Traverse` to implement it. +/// ## Arguments +/// - `module`: the module which contains the class -- can be omitted if in a `#[pymodule]`. +/// - `name`: the name of the Python class, by default it is the name of the struct. +/// - `base`: the base class of the Python class. +/// This does not cause inheritance of functions or attributes that must be done by a separate trait. +/// # Impl +/// This part implements `PyClassImpl` for the struct. +/// This includes methods, getters/setters, etc.; only annotated methods will be included. +/// Common functions and abilities like instantiation and `__call__` are often implemented by +/// traits rather than in the `impl` itself; see `Constructor` and `Callable` respectively for those. +/// ## Arguments +/// - `name`: the name of the Python class, when no name is provided the struct name is used. +/// - `flags`: the flags of the class, see `PyTypeFlags`. +/// - `BASETYPE`: allows the class to be inheritable. +/// - `IMMUTABLETYPE`: class attributes are immutable. +/// - `with`: which trait implementations are to be included in the python class. +/// ```rust, ignore +/// #[pyclass(module = "mymodule", name = "MyClass", base = "BaseClass")] +/// struct MyStruct { +/// x: i32, +/// } +/// +/// impl Constructor for MyStruct { +/// ... +/// } +/// +/// #[pyclass(with(Constructor))] +/// impl MyStruct { +/// ... +/// } +/// ``` +/// ## Inner markers +/// ### pymethod/pyclassmethod/pystaticmethod +/// `pymethod` is used to mark a method of the Python class. +/// `pyclassmethod` is used to mark a class method. +/// `pystaticmethod` is used to mark a static method. +/// #### Method signature +/// The first parameter can be either `&self` or `: PyRef` for `pymethod`. +/// The first parameter can be `cls: PyTypeRef` for `pyclassmethod`. +/// There is no mandatory parameter for `pystaticmethod`. +/// Both are valid and essentially the same, but the latter can yield more control. +/// The last parameter can optionally be of the type `&VirtualMachine` to access the VM. +/// All other values must implement `IntoPyResult`. +/// Numeric types, `String`, `bool`, and `PyObjectRef` implement this trait, +/// but so does any object that implements `PyValue`. +/// Consider using `OptionalArg` for optional arguments. +/// #### Arguments +/// - `magic`: marks the method as a magic method: the method name is surrounded with double underscores. +/// ```rust, ignore +/// #[pyclass] +/// impl MyStruct { +/// // This will be called as the `__add__` method in Python. +/// #[pymethod(magic)] +/// fn add(&self, other: &Self) -> PyResult { +/// ... +/// } +/// } +/// ``` +/// - `name`: the name of the method in Python, +/// by default it is the same as the Rust method, or surrounded by double underscores if magic is present. +/// This overrides `magic` and the default name and cannot be used with `magic` to prevent ambiguity. +/// ### pygetset +/// This is used to mark a getter/setter pair. +/// #### Arguments +/// - `setter`: marks the method as a setter, it acts as a getter by default. +/// Setter method names should be prefixed with `set_`. +/// - `name`: the name of the attribute in Python, by default it is the same as the Rust method. +/// - `magic`: marks the method as a magic method: the method name is surrounded with double underscores. +/// This cannot be used with `name` to prevent ambiguity. +/// +/// Ensure both the getter and setter are marked with `name` and `magic` in the same manner. +/// #### Examples +/// ```rust, ignore +/// #[pyclass] +/// impl MyStruct { +/// #[pygetset] +/// fn x(&self) -> PyResult { +/// Ok(self.x.lock()) +/// } +/// #[pygetset(setter)] +/// fn set_x(&mut self, value: i32) -> PyResult<()> { +/// self.x.set(value); +/// Ok(()) +/// } +/// } +/// ``` +/// ### pyslot +/// This is used to mark a slot method it should be marked by prefixing the method in rust with `slot_`. +/// #### Arguments +/// - name: the name of the slot method. +/// ### pyattr +/// ### extend_class +/// This helps inherit attributes from a parent class. +/// The method this is applied on should be called `extend_class_with_fields`. +/// #### Examples +/// ```rust, ignore +/// #[extend_class] +/// fn extend_class_with_fields(ctx: &Context, class: &'static Py) { +/// class.set_attr( +/// identifier!(ctx, _fields), +/// ctx.new_tuple(vec![ +/// ctx.new_str(ascii!("body")).into(), +/// ctx.new_str(ascii!("type_ignores")).into(), +/// ]) +/// .into(), +/// ); +/// class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); +/// } +/// ``` +/// ### pymember +/// # Trait +/// `#[pyclass]` on traits functions a lot like `#[pyclass]` on `impl` blocks. +/// Note that associated functions that are annotated with `#[pymethod]` or similar **must** +/// have a body, abstract functions should be wrapped before applying an annotation. #[proc_macro_attribute] pub fn pyclass(attr: TokenStream, item: TokenStream) -> TokenStream { let attr = parse_macro_input!(attr); @@ -34,6 +152,71 @@ pub fn pyexception(attr: TokenStream, item: TokenStream) -> TokenStream { derive_impl::pyexception(attr, item).into() } +/// This attribute must be applied to an inline module. +/// It defines a Python module in the form a `make_module` function in the module; +/// this has to be used in a `get_module_inits` to properly register the module. +/// Additionally, this macro defines 'MODULE_NAME' and 'DOC' in the module. +/// # Arguments +/// - `name`: the name of the python module, +/// by default, it is the name of the module, but this can be configured. +/// ```rust, ignore +/// // This will create a module named `mymodule` +/// #[pymodule(name = "mymodule")] +/// mod module { +/// } +/// ``` +/// - `sub`: declare the module as a submodule of another module. +/// ```rust, ignore +/// #[pymodule(sub)] +/// mod submodule { +/// } +/// +/// #[pymodule(with(submodule))] +/// mod mymodule { +/// } +/// ``` +/// - `with`: declare the list of submodules that this module contains (see `sub` for example). +/// ## Inner markers +/// ### pyattr +/// `pyattr` is a multipurpose marker that can be used in a pymodule. +/// The most common use is to mark a function or class as a part of the module. +/// This can be done by applying it to a function or struct prior to the `#[pyfunction]` or `#[pyclass]` macro. +/// If applied to a constant, it will be added to the module as an attribute. +/// If applied to a function not marked with `pyfunction`, +/// it will also be added to the module as an attribute but the value is the result of the function. +/// If `#[pyattr(once)]` is used in this case, the function will be called once +/// and the result will be stored using a `static_cell`. +/// #### Examples +/// ```rust, ignore +/// #[pymodule] +/// mod mymodule { +/// #[pyattr] +/// const MY_CONSTANT: i32 = 42; +/// #[pyattr] +/// fn another_constant() -> PyResult { +/// Ok(42) +/// } +/// #[pyattr(once)] +/// fn once() -> PyResult { +/// // This will only be called once and the result will be stored. +/// Ok(2 ** 24) +/// } +/// +/// #[pyattr] +/// #[pyfunction] +/// fn my_function(vm: &VirtualMachine) -> PyResult<()> { +/// ... +/// } +/// } +/// ``` +/// ### pyfunction +/// This is used to create a python function. +/// #### Function signature +/// The last argument can optionally be of the type `&VirtualMachine` to access the VM. +/// Refer to the `pymethod` documentation (located in the `pyclass` macro documentation) +/// for more information on what regular argument types are permitted. +/// #### Arguments +/// - `name`: the name of the function in Python, by default it is the same as the associated Rust function. #[proc_macro_attribute] pub fn pymodule(attr: TokenStream, item: TokenStream) -> TokenStream { let attr = parse_macro_input!(attr); From 331a3c108f1fe44806961607ae96c6798677f44b Mon Sep 17 00:00:00 2001 From: Noa Date: Thu, 20 Feb 2025 14:57:40 -0600 Subject: [PATCH 028/295] Switch to criterion in sre_engine benchmarks --- Cargo.lock | 1 + Cargo.toml | 3 +- vm/sre_engine/Cargo.toml | 7 +++ vm/sre_engine/benches/benches.rs | 83 +++++++++++++++++--------------- vm/sre_engine/generate_tests.py | 31 +++++++++--- vm/sre_engine/tests/tests.rs | 34 +++++++------ 6 files changed, 97 insertions(+), 62 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index edf28dc7f4..3711b6f16c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2178,6 +2178,7 @@ name = "rustpython-sre_engine" version = "0.4.0" dependencies = [ "bitflags 2.8.0", + "criterion", "num_enum", "optional", ] diff --git a/Cargo.toml b/Cargo.toml index 83007606c6..446733d515 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,7 +47,7 @@ libc = { workspace = true } rustyline = { workspace = true } [dev-dependencies] -criterion = { version = "0.3.5", features = ["html_reports"] } +criterion = { workspace = true } pyo3 = { version = "0.22", features = ["auto-initialize"] } [[bench]] @@ -144,6 +144,7 @@ bitflags = "2.4.2" bstr = "1" cfg-if = "1.0" chrono = "0.4.39" +criterion = { version = "0.3.5", features = ["html_reports"] } crossbeam-utils = "0.8.21" flame = "0.2.2" getrandom = "0.3" diff --git a/vm/sre_engine/Cargo.toml b/vm/sre_engine/Cargo.toml index 28f98a3212..ac0e7b5f16 100644 --- a/vm/sre_engine/Cargo.toml +++ b/vm/sre_engine/Cargo.toml @@ -10,7 +10,14 @@ rust-version.workspace = true repository.workspace = true license.workspace = true +[[bench]] +name = "benches" +harness = false + [dependencies] num_enum = { workspace = true } bitflags = { workspace = true } optional = "0.5" + +[dev-dependencies] +criterion = { workspace = true } diff --git a/vm/sre_engine/benches/benches.rs b/vm/sre_engine/benches/benches.rs index e31c73b0d0..915d568333 100644 --- a/vm/sre_engine/benches/benches.rs +++ b/vm/sre_engine/benches/benches.rs @@ -1,11 +1,9 @@ -#![feature(test)] - -extern crate test; -use test::Bencher; - use rustpython_sre_engine::{Request, State, StrDrive}; +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion}; + struct Pattern { + pattern: &'static str, code: &'static [u32], } @@ -25,52 +23,51 @@ impl Pattern { } } -#[bench] -fn benchmarks(b: &mut Bencher) { +fn basic(c: &mut Criterion) { // # test common prefix // pattern p1 = re.compile('Python|Perl') # , 'Perl'), # Alternation // START GENERATED by generate_tests.py - #[rustfmt::skip] let p1 = Pattern { code: &[14, 8, 1, 4, 6, 1, 1, 80, 0, 16, 80, 7, 13, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 15, 11, 9, 16, 101, 16, 114, 16, 108, 15, 2, 0, 1] }; + #[rustfmt::skip] let p1 = Pattern { pattern: "Python|Perl", code: &[14, 8, 1, 4, 6, 1, 1, 80, 0, 16, 80, 7, 13, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 15, 11, 9, 16, 101, 16, 114, 16, 108, 15, 2, 0, 1] }; // END GENERATED // pattern p2 = re.compile('(Python|Perl)') #, 'Perl'), # Grouped alternation // START GENERATED by generate_tests.py - #[rustfmt::skip] let p2 = Pattern { code: &[14, 8, 1, 4, 6, 1, 0, 80, 0, 17, 0, 16, 80, 7, 13, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 15, 11, 9, 16, 101, 16, 114, 16, 108, 15, 2, 0, 17, 1, 1] }; + #[rustfmt::skip] let p2 = Pattern { pattern: "(Python|Perl)", code: &[14, 8, 1, 4, 6, 1, 0, 80, 0, 17, 0, 16, 80, 7, 13, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 15, 11, 9, 16, 101, 16, 114, 16, 108, 15, 2, 0, 17, 1, 1] }; // END GENERATED // pattern p3 = re.compile('Python|Perl|Tcl') #, 'Perl'), # Alternation // START GENERATED by generate_tests.py - #[rustfmt::skip] let p3 = Pattern { code: &[14, 9, 4, 3, 6, 16, 80, 16, 84, 0, 7, 15, 16, 80, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 15, 22, 11, 16, 80, 16, 101, 16, 114, 16, 108, 15, 11, 9, 16, 84, 16, 99, 16, 108, 15, 2, 0, 1] }; + #[rustfmt::skip] let p3 = Pattern { pattern: "Python|Perl|Tcl", code: &[14, 9, 4, 3, 6, 16, 80, 16, 84, 0, 7, 15, 16, 80, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 15, 22, 11, 16, 80, 16, 101, 16, 114, 16, 108, 15, 11, 9, 16, 84, 16, 99, 16, 108, 15, 2, 0, 1] }; // END GENERATED // pattern p4 = re.compile('(Python|Perl|Tcl)') #, 'Perl'), # Grouped alternation // START GENERATED by generate_tests.py - #[rustfmt::skip] let p4 = Pattern { code: &[14, 9, 4, 3, 6, 16, 80, 16, 84, 0, 17, 0, 7, 15, 16, 80, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 15, 22, 11, 16, 80, 16, 101, 16, 114, 16, 108, 15, 11, 9, 16, 84, 16, 99, 16, 108, 15, 2, 0, 17, 1, 1] }; + #[rustfmt::skip] let p4 = Pattern { pattern: "(Python|Perl|Tcl)", code: &[14, 9, 4, 3, 6, 16, 80, 16, 84, 0, 17, 0, 7, 15, 16, 80, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 15, 22, 11, 16, 80, 16, 101, 16, 114, 16, 108, 15, 11, 9, 16, 84, 16, 99, 16, 108, 15, 2, 0, 17, 1, 1] }; // END GENERATED // pattern p5 = re.compile('(Python)\\1') #, 'PythonPython'), # Backreference // START GENERATED by generate_tests.py - #[rustfmt::skip] let p5 = Pattern { code: &[14, 18, 1, 12, 12, 6, 0, 80, 121, 116, 104, 111, 110, 0, 0, 0, 0, 0, 0, 17, 0, 16, 80, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 17, 1, 11, 0, 1] }; + #[rustfmt::skip] let p5 = Pattern { pattern: "(Python)\\1", code: &[14, 18, 1, 12, 12, 6, 0, 80, 121, 116, 104, 111, 110, 0, 0, 0, 0, 0, 0, 17, 0, 16, 80, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 17, 1, 11, 0, 1] }; // END GENERATED // pattern p6 = re.compile('([0a-z][a-z0-9]*,)+') #, 'a5,b7,c9,'), # Disable the fastmap optimization // START GENERATED by generate_tests.py - #[rustfmt::skip] let p6 = Pattern { code: &[14, 4, 0, 2, 4294967295, 23, 31, 1, 4294967295, 17, 0, 13, 7, 16, 48, 22, 97, 122, 0, 24, 13, 0, 4294967295, 13, 8, 22, 97, 122, 22, 48, 57, 0, 1, 16, 44, 17, 1, 18, 1] }; + #[rustfmt::skip] let p6 = Pattern { pattern: "([0a-z][a-z0-9]*,)+", code: &[14, 4, 0, 2, 4294967295, 23, 31, 1, 4294967295, 17, 0, 13, 7, 16, 48, 22, 97, 122, 0, 24, 13, 0, 4294967295, 13, 8, 22, 97, 122, 22, 48, 57, 0, 1, 16, 44, 17, 1, 18, 1] }; // END GENERATED // pattern p7 = re.compile('([a-z][a-z0-9]*,)+') #, 'a5,b7,c9,'), # A few sets // START GENERATED by generate_tests.py - #[rustfmt::skip] let p7 = Pattern { code: &[14, 4, 0, 2, 4294967295, 23, 29, 1, 4294967295, 17, 0, 13, 5, 22, 97, 122, 0, 24, 13, 0, 4294967295, 13, 8, 22, 97, 122, 22, 48, 57, 0, 1, 16, 44, 17, 1, 18, 1] }; + #[rustfmt::skip] let p7 = Pattern { pattern: "([a-z][a-z0-9]*,)+", code: &[14, 4, 0, 2, 4294967295, 23, 29, 1, 4294967295, 17, 0, 13, 5, 22, 97, 122, 0, 24, 13, 0, 4294967295, 13, 8, 22, 97, 122, 22, 48, 57, 0, 1, 16, 44, 17, 1, 18, 1] }; // END GENERATED // pattern p8 = re.compile('Python') #, 'Python'), # Simple text literal // START GENERATED by generate_tests.py - #[rustfmt::skip] let p8 = Pattern { code: &[14, 18, 3, 6, 6, 6, 6, 80, 121, 116, 104, 111, 110, 0, 0, 0, 0, 0, 0, 16, 80, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 1] }; + #[rustfmt::skip] let p8 = Pattern { pattern: "Python", code: &[14, 18, 3, 6, 6, 6, 6, 80, 121, 116, 104, 111, 110, 0, 0, 0, 0, 0, 0, 16, 80, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 1] }; // END GENERATED // pattern p9 = re.compile('.*Python') #, 'Python'), # Bad text literal // START GENERATED by generate_tests.py - #[rustfmt::skip] let p9 = Pattern { code: &[14, 4, 0, 6, 4294967295, 24, 5, 0, 4294967295, 2, 1, 16, 80, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 1] }; + #[rustfmt::skip] let p9 = Pattern { pattern: ".*Python", code: &[14, 4, 0, 6, 4294967295, 24, 5, 0, 4294967295, 2, 1, 16, 80, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 1] }; // END GENERATED // pattern p10 = re.compile('.*Python.*') #, 'Python'), # Worse text literal // START GENERATED by generate_tests.py - #[rustfmt::skip] let p10 = Pattern { code: &[14, 4, 0, 6, 4294967295, 24, 5, 0, 4294967295, 2, 1, 16, 80, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 24, 5, 0, 4294967295, 2, 1, 1] }; + #[rustfmt::skip] let p10 = Pattern { pattern: ".*Python.*", code: &[14, 4, 0, 6, 4294967295, 24, 5, 0, 4294967295, 2, 1, 16, 80, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 24, 5, 0, 4294967295, 2, 1, 1] }; // END GENERATED // pattern p11 = re.compile('.*(Python)') #, 'Python'), # Bad text literal with grouping // START GENERATED by generate_tests.py - #[rustfmt::skip] let p11 = Pattern { code: &[14, 4, 0, 6, 4294967295, 24, 5, 0, 4294967295, 2, 1, 17, 0, 16, 80, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 17, 1, 1] }; + #[rustfmt::skip] let p11 = Pattern { pattern: ".*(Python)", code: &[14, 4, 0, 6, 4294967295, 24, 5, 0, 4294967295, 2, 1, 17, 0, 16, 80, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 17, 1, 1] }; // END GENERATED let tests = [ @@ -87,25 +84,33 @@ fn benchmarks(b: &mut Bencher) { (p11, "Python"), ]; - b.iter(move || { - for (p, s) in &tests { - let (req, mut state) = p.state(s.clone()); - assert!(state.search(req)); - let (req, mut state) = p.state(s.clone()); - assert!(state.pymatch(&req)); - let (mut req, mut state) = p.state(s.clone()); - req.match_all = true; - assert!(state.pymatch(&req)); - let s2 = format!("{}{}{}", " ".repeat(10000), s, " ".repeat(10000)); - let (req, mut state) = p.state_range(s2.as_str(), 0..usize::MAX); - assert!(state.search(req)); - let (req, mut state) = p.state_range(s2.as_str(), 10000..usize::MAX); - assert!(state.pymatch(&req)); - let (req, mut state) = p.state_range(s2.as_str(), 10000..10000 + s.len()); - assert!(state.pymatch(&req)); - let (mut req, mut state) = p.state_range(s2.as_str(), 10000..10000 + s.len()); - req.match_all = true; - assert!(state.pymatch(&req)); - } - }) + let mut group = c.benchmark_group("basic"); + + for (p, s) in tests { + group.bench_with_input(BenchmarkId::new(p.pattern, s), s, |b, s| { + b.iter(|| { + let (req, mut state) = p.state(s); + assert!(state.search(req)); + let (req, mut state) = p.state(s); + assert!(state.pymatch(&req)); + let (mut req, mut state) = p.state(s); + req.match_all = true; + assert!(state.pymatch(&req)); + let s2 = format!("{}{}{}", " ".repeat(10000), s, " ".repeat(10000)); + let (req, mut state) = p.state_range(s2.as_str(), 0..usize::MAX); + assert!(state.search(req)); + let (req, mut state) = p.state_range(s2.as_str(), 10000..usize::MAX); + assert!(state.pymatch(&req)); + let (req, mut state) = p.state_range(s2.as_str(), 10000..10000 + s.len()); + assert!(state.pymatch(&req)); + let (mut req, mut state) = p.state_range(s2.as_str(), 10000..10000 + s.len()); + req.match_all = true; + assert!(state.pymatch(&req)); + }); + }); + } } + +criterion_group!(benches, basic); + +criterion_main!(benches); diff --git a/vm/sre_engine/generate_tests.py b/vm/sre_engine/generate_tests.py index 8adf043f29..6621c56813 100644 --- a/vm/sre_engine/generate_tests.py +++ b/vm/sre_engine/generate_tests.py @@ -1,9 +1,6 @@ import os from pathlib import Path import re -import sre_constants -import sre_compile -import sre_parse import json from itertools import chain @@ -11,13 +8,13 @@ sre_engine_magic = int(m.group(1)) del m -assert sre_constants.MAGIC == sre_engine_magic +assert re._constants.MAGIC == sre_engine_magic class CompiledPattern: @classmethod def compile(cls, pattern, flags=0): - p = sre_parse.parse(pattern) - code = sre_compile._code(p, flags) + p = re._parser.parse(pattern) + code = re._compiler._code(p, flags) self = cls() self.pattern = pattern self.code = code @@ -28,12 +25,32 @@ def compile(cls, pattern, flags=0): setattr(CompiledPattern, k, v) +class EscapeRustStr: + hardcoded = { + ord('\r'): '\\r', + ord('\t'): '\\t', + ord('\r'): '\\r', + ord('\n'): '\\n', + ord('\\'): '\\\\', + ord('\''): '\\\'', + ord('\"'): '\\\"', + } + @classmethod + def __class_getitem__(cls, ch): + if (rpl := cls.hardcoded.get(ch)) is not None: + return rpl + if ch in range(0x20, 0x7f): + return ch + return f"\\u{{{ch:x}}}" +def rust_str(s): + return '"' + s.translate(EscapeRustStr) + '"' + # matches `// pattern {varname} = re.compile(...)` pattern_pattern = re.compile(r"^((\s*)\/\/\s*pattern\s+(\w+)\s+=\s+(.+?))$(?:.+?END GENERATED)?", re.M | re.S) def replace_compiled(m): line, indent, varname, pattern = m.groups() pattern = eval(pattern, {"re": CompiledPattern}) - pattern = f"Pattern {{ code: &{json.dumps(pattern.code)} }}" + pattern = f"Pattern {{ pattern: {rust_str(pattern.pattern)}, code: &{json.dumps(pattern.code)} }}" return f'''{line} {indent}// START GENERATED by generate_tests.py {indent}#[rustfmt::skip] let {varname} = {pattern}; diff --git a/vm/sre_engine/tests/tests.rs b/vm/sre_engine/tests/tests.rs index 53494c5e3d..0946fd64ca 100644 --- a/vm/sre_engine/tests/tests.rs +++ b/vm/sre_engine/tests/tests.rs @@ -1,6 +1,7 @@ use rustpython_sre_engine::{Request, State, StrDrive}; struct Pattern { + pattern: &'static str, code: &'static [u32], } @@ -16,7 +17,7 @@ impl Pattern { fn test_2427() { // pattern lookbehind = re.compile(r'(?x)++x') // START GENERATED by generate_tests.py - #[rustfmt::skip] let p = Pattern { code: &[14, 4, 0, 2, 4294967295, 28, 8, 1, 4294967295, 27, 4, 16, 120, 1, 1, 16, 120, 1] }; + #[rustfmt::skip] let p = Pattern { pattern: "(?>x)++x", code: &[14, 4, 0, 2, 4294967295, 28, 8, 1, 4294967295, 27, 4, 16, 120, 1, 1, 16, 120, 1] }; // END GENERATED let (req, mut state) = p.state("xxx"); assert!(!state.pymatch(&req)); @@ -156,7 +157,7 @@ fn test_possessive_atomic_group() { fn test_bug_20998() { // pattern p = re.compile('[a-c]+', re.I) // START GENERATED by generate_tests.py - #[rustfmt::skip] let p = Pattern { code: &[14, 4, 0, 1, 4294967295, 24, 10, 1, 4294967295, 39, 5, 22, 97, 99, 0, 1, 1] }; + #[rustfmt::skip] let p = Pattern { pattern: "[a-c]+", code: &[14, 4, 0, 1, 4294967295, 24, 10, 1, 4294967295, 39, 5, 22, 97, 99, 0, 1, 1] }; // END GENERATED let (mut req, mut state) = p.state("ABC"); req.match_all = true; @@ -168,7 +169,7 @@ fn test_bug_20998() { fn test_bigcharset() { // pattern p = re.compile('[a-z]*', re.I) // START GENERATED by generate_tests.py - #[rustfmt::skip] let p = Pattern { code: &[14, 4, 0, 0, 4294967295, 24, 97, 0, 4294967295, 39, 92, 10, 3, 33685760, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 0, 0, 0, 134217726, 0, 0, 0, 0, 0, 131072, 0, 2147483648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1] }; + #[rustfmt::skip] let p = Pattern { pattern: "[a-z]*", code: &[14, 4, 0, 0, 4294967295, 24, 97, 0, 4294967295, 39, 92, 10, 3, 33685760, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 0, 0, 0, 134217726, 0, 0, 0, 0, 0, 131072, 0, 2147483648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1] }; // END GENERATED let (req, mut state) = p.state("x "); assert!(state.pymatch(&req)); @@ -178,4 +179,7 @@ fn test_bigcharset() { #[test] fn test_search_nonascii() { // pattern p = re.compile('\xe0+') + // START GENERATED by generate_tests.py + #[rustfmt::skip] let p = Pattern { pattern: "\u{e0}+", code: &[14, 4, 0, 1, 4294967295, 24, 6, 1, 4294967295, 16, 224, 1, 1] }; + // END GENERATED } From b4f0a589ed16db087ca3e5a7cde2159b2a554de9 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sat, 22 Feb 2025 16:48:02 -0800 Subject: [PATCH 029/295] platform-dependent Windows testing (#5536) * disable test_argparse on windows * fix test_exceptions and mark it as platform dependent * test importlib on windows * explain why windows tests fail * mark test_argparse as non platform-independent Signed-off-by: Ashwin Naren --- .github/workflows/ci.yaml | 15 +++++++++++---- Lib/test/test_importlib/frozen/test_loader.py | 4 ++++ Lib/test/test_importlib/resources/test_files.py | 7 +++++++ Lib/test/test_importlib/source/test_finder.py | 1 - vm/src/exceptions.rs | 4 +++- 5 files changed, 25 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9c0f736928..3d1d32d12d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -17,13 +17,23 @@ concurrency: env: CARGO_ARGS: --no-default-features --features stdlib,zlib,importlib,encodings,sqlite,ssl # Skip additional tests on Windows. They are checked on Linux and MacOS. + # test_argparse: UnicodeDecodeError + # test_glob: many failing tests + # test_io: many failing tests + # test_os: many failing tests + # test_pathlib: support.rmtree() failing + # test_posixpath: OSError: (22, 'The filename, directory name, or volume label syntax is incorrect. (os error 123)') + # test_unicode: AttributeError: module '_winapi' has no attribute 'GetACP' + # test_venv: couple of failing tests WINDOWS_SKIPS: >- + test_argparse test_glob - test_importlib test_io test_os + test_rlcompleter test_pathlib test_posixpath + test_unicode test_venv # configparser: https://github.com/RustPython/RustPython/issues/4995#issuecomment-1582397417 # socketserver: seems related to configparser crash. @@ -34,7 +44,6 @@ env: # only run on Linux to speed up the CI. PLATFORM_INDEPENDENT_TESTS: >- test__colorize - test_argparse test_array test_asyncgen test_binop @@ -59,7 +68,6 @@ env: test_dis test_enumerate test_exception_variations - test_exceptions test_float test_format test_fractions @@ -100,7 +108,6 @@ env: test_tuple test_types test_unary - test_unicode test_unpack test_weakref test_yield_from diff --git a/Lib/test/test_importlib/frozen/test_loader.py b/Lib/test/test_importlib/frozen/test_loader.py index 4f1af454b5..b1eb399d93 100644 --- a/Lib/test/test_importlib/frozen/test_loader.py +++ b/Lib/test/test_importlib/frozen/test_loader.py @@ -7,6 +7,7 @@ import contextlib import marshal import os.path +import sys import types import unittest import warnings @@ -77,6 +78,7 @@ def test_module(self): self.assertTrue(hasattr(module, '__spec__')) self.assertEqual(module.__spec__.loader_state.origname, name) + @unittest.skipIf(sys.platform == "win32", "TODO: RUSTPYTHON") def test_package(self): name = '__phello__' module, output = self.exec_module(name) @@ -90,6 +92,7 @@ def test_package(self): self.assertEqual(output, 'Hello world!\n') self.assertEqual(module.__spec__.loader_state.origname, name) + @unittest.skipIf(sys.platform == 'win32', "TODO:RUSTPYTHON Flaky on Windows") def test_lacking_parent(self): name = '__phello__.spam' with util.uncache('__phello__'): @@ -147,6 +150,7 @@ def test_get_source(self): result = self.machinery.FrozenImporter.get_source('__hello__') self.assertIsNone(result) + @unittest.skipIf(sys.platform == "win32", "TODO: RUSTPYTHON") def test_is_package(self): # Should be able to tell what is a package. test_for = (('__hello__', False), ('__phello__', True), diff --git a/Lib/test/test_importlib/resources/test_files.py b/Lib/test/test_importlib/resources/test_files.py index 1450cfb310..1d04cda1a8 100644 --- a/Lib/test/test_importlib/resources/test_files.py +++ b/Lib/test/test_importlib/resources/test_files.py @@ -52,6 +52,10 @@ class OpenDiskTests(FilesTests, unittest.TestCase): def setUp(self): self.data = data01 + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") + def test_read_bytes(self): + super().test_read_bytes() + class OpenZipTests(FilesTests, util.ZipSetup, unittest.TestCase): pass @@ -63,6 +67,9 @@ def setUp(self): self.data = namespacedata01 + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") + def test_read_bytes(self): + super().test_read_bytes() class SiteDir: def setUp(self): diff --git a/Lib/test/test_importlib/source/test_finder.py b/Lib/test/test_importlib/source/test_finder.py index 17d09d4cee..12db7c7d35 100644 --- a/Lib/test/test_importlib/source/test_finder.py +++ b/Lib/test/test_importlib/source/test_finder.py @@ -168,7 +168,6 @@ def test_no_read_directory(self): found = self._find(finder, 'doesnotexist') self.assertEqual(found, self.NOT_FOUND) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_ignore_file(self): # If a directory got changed to a file from underneath us, then don't # worry about looking for submodules. diff --git a/vm/src/exceptions.rs b/vm/src/exceptions.rs index 0b61c90174..94a16f912e 100644 --- a/vm/src/exceptions.rs +++ b/vm/src/exceptions.rs @@ -374,7 +374,9 @@ fn write_traceback_entry( writeln!( output, r##" File "{}", line {}, in {}"##, - filename, tb_entry.lineno, tb_entry.frame.code.obj_name + filename.trim_start_matches(r"\\?\"), + tb_entry.lineno, + tb_entry.frame.code.obj_name )?; print_source_line(output, filename, tb_entry.lineno.to_usize())?; From 429754fd3364492f87b0924b73736c5adc20148c Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sat, 22 Feb 2025 23:07:22 -0800 Subject: [PATCH 030/295] Fix unicode decode bug on surrogate error mode (#5546) * subtract with overflow to check for whether to use surrogate * enable test_argparse for windows on ci ------ Signed-off-by: Ashwin Naren --- .github/workflows/ci.yaml | 2 -- vm/src/codecs.rs | 7 ++++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 3d1d32d12d..1eeecbd2a7 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -17,7 +17,6 @@ concurrency: env: CARGO_ARGS: --no-default-features --features stdlib,zlib,importlib,encodings,sqlite,ssl # Skip additional tests on Windows. They are checked on Linux and MacOS. - # test_argparse: UnicodeDecodeError # test_glob: many failing tests # test_io: many failing tests # test_os: many failing tests @@ -26,7 +25,6 @@ env: # test_unicode: AttributeError: module '_winapi' has no attribute 'GetACP' # test_venv: couple of failing tests WINDOWS_SKIPS: >- - test_argparse test_glob test_io test_os diff --git a/vm/src/codecs.rs b/vm/src/codecs.rs index 898198f3d3..767bd873b6 100644 --- a/vm/src/codecs.rs +++ b/vm/src/codecs.rs @@ -619,11 +619,16 @@ fn surrogatepass_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<(PyOb // Not supported, fail with original exception return Err(err.downcast().unwrap()); } + + debug_assert!(range.start <= 0.max(s.len() - 1)); + debug_assert!(range.end >= 1.min(s.len())); + debug_assert!(range.end <= s.len()); + let mut c: u32 = 0; // Try decoding a single surrogate character. If there are more, // let the codec call us again. let p = &s.as_bytes()[range.start..]; - if p.len().saturating_sub(range.start) >= byte_length { + if p.len().overflowing_sub(range.start).0 >= byte_length { match standard_encoding { StandardEncoding::Utf8 => { if (p[0] as u32 & 0xf0) == 0xe0 From 7fada8b97ea1fc015e98d5d6e51ca64c9777ef74 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sat, 22 Feb 2025 20:41:13 -0800 Subject: [PATCH 031/295] fix _ctypes error names Signed-off-by: Ashwin Naren --- vm/src/stdlib/ctypes.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/vm/src/stdlib/ctypes.rs b/vm/src/stdlib/ctypes.rs index 027a680951..2580939b62 100644 --- a/vm/src/stdlib/ctypes.rs +++ b/vm/src/stdlib/ctypes.rs @@ -90,8 +90,8 @@ pub(crate) mod _ctypes { #[pyattr] pub const DICTFLAG_FINAL: u32 = 0x1000; - #[pyattr(once)] - fn error(vm: &VirtualMachine) -> PyTypeRef { + #[pyattr(name = "ArgumentError", once)] + fn argument_error(vm: &VirtualMachine) -> PyTypeRef { vm.ctx.new_exception_type( "_ctypes", "ArgumentError", @@ -99,6 +99,15 @@ pub(crate) mod _ctypes { ) } + #[pyattr(name = "FormatError", once)] + fn format_error(vm: &VirtualMachine) -> PyTypeRef { + vm.ctx.new_exception_type( + "_ctypes", + "FormatError", + Some(vec![vm.ctx.exceptions.exception_type.to_owned()]), + ) + } + pub fn get_size(ty: &str) -> usize { match ty { "u" => mem::size_of::(), From 31c5c3eb9d62f1895a66245f2ea8c06ecadb48e6 Mon Sep 17 00:00:00 2001 From: Axect Date: Mon, 24 Feb 2025 10:44:57 +0900 Subject: [PATCH 032/295] Update `puruspe` version to `0.4.0` To resolve the issue (#5496) --- stdlib/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/Cargo.toml b/stdlib/Cargo.toml index 31f5bd18d4..b5cfab98b4 100644 --- a/stdlib/Cargo.toml +++ b/stdlib/Cargo.toml @@ -49,7 +49,7 @@ base64 = "0.13.0" csv-core = "0.1.11" dyn-clone = "1.0.10" libz-sys = { version = "1.1", default-features = false, optional = true } -puruspe = "0.3.0" +puruspe = "0.4.0" xml-rs = "0.8.14" # random From 1f3a9672c3599e0f3985aa0d04153a0ae2a96dc0 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sun, 23 Feb 2025 20:21:02 -0800 Subject: [PATCH 033/295] Add _winapi.GetACP and enable test_unicode on windows (#5547) Signed-off-by: Ashwin Naren --- .github/workflows/ci.yaml | 2 -- Lib/test/test_charmapcodec.py | 3 --- Lib/test/test_codecs.py | 11 ----------- Lib/test/test_tokenize.py | 2 -- vm/Cargo.toml | 1 + vm/src/stdlib/winapi.rs | 5 +++++ 6 files changed, 6 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1eeecbd2a7..e96c2f26a6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -22,7 +22,6 @@ env: # test_os: many failing tests # test_pathlib: support.rmtree() failing # test_posixpath: OSError: (22, 'The filename, directory name, or volume label syntax is incorrect. (os error 123)') - # test_unicode: AttributeError: module '_winapi' has no attribute 'GetACP' # test_venv: couple of failing tests WINDOWS_SKIPS: >- test_glob @@ -31,7 +30,6 @@ env: test_rlcompleter test_pathlib test_posixpath - test_unicode test_venv # configparser: https://github.com/RustPython/RustPython/issues/4995#issuecomment-1582397417 # socketserver: seems related to configparser crash. diff --git a/Lib/test/test_charmapcodec.py b/Lib/test/test_charmapcodec.py index e69f1c6e4b..8ea75d9129 100644 --- a/Lib/test/test_charmapcodec.py +++ b/Lib/test/test_charmapcodec.py @@ -26,7 +26,6 @@ def codec_search_function(encoding): codecname = 'testcodec' class CharmapCodecTest(unittest.TestCase): - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_constructorx(self): self.assertEqual(str(b'abc', codecname), 'abc') self.assertEqual(str(b'xdef', codecname), 'abcdef') @@ -43,14 +42,12 @@ def test_encodex(self): self.assertEqual('dxf'.encode(codecname), b'dabcf') self.assertEqual('dxfx'.encode(codecname), b'dabcfabc') - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_constructory(self): self.assertEqual(str(b'ydef', codecname), 'def') self.assertEqual(str(b'defy', codecname), 'def') self.assertEqual(str(b'dyf', codecname), 'df') self.assertEqual(str(b'dyfy', codecname), 'df') - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_maptoundefined(self): self.assertRaises(UnicodeError, str, b'abc\001', codecname) diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 085b800b6d..f29e91e088 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -1827,7 +1827,6 @@ def test_decode(self): self.assertEqual(codecs.decode(b'[\xff]', 'ascii', errors='ignore'), '[]') - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_encode(self): self.assertEqual(codecs.encode('\xe4\xf6\xfc', 'latin-1'), b'\xe4\xf6\xfc') @@ -1846,7 +1845,6 @@ def test_register(self): self.assertRaises(TypeError, codecs.register) self.assertRaises(TypeError, codecs.register, 42) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; AttributeError: module '_winapi' has no attribute 'GetACP'") def test_unregister(self): name = "nonexistent_codec_name" search_function = mock.Mock() @@ -1859,28 +1857,23 @@ def test_unregister(self): self.assertRaises(LookupError, codecs.lookup, name) search_function.assert_not_called() - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_lookup(self): self.assertRaises(TypeError, codecs.lookup) self.assertRaises(LookupError, codecs.lookup, "__spam__") self.assertRaises(LookupError, codecs.lookup, " ") - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_getencoder(self): self.assertRaises(TypeError, codecs.getencoder) self.assertRaises(LookupError, codecs.getencoder, "__spam__") - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_getdecoder(self): self.assertRaises(TypeError, codecs.getdecoder) self.assertRaises(LookupError, codecs.getdecoder, "__spam__") - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_getreader(self): self.assertRaises(TypeError, codecs.getreader) self.assertRaises(LookupError, codecs.getreader, "__spam__") - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_getwriter(self): self.assertRaises(TypeError, codecs.getwriter) self.assertRaises(LookupError, codecs.getwriter, "__spam__") @@ -1939,7 +1932,6 @@ def test_undefined(self): self.assertRaises(UnicodeError, codecs.decode, b'abc', 'undefined', errors) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_file_closes_if_lookup_error_raised(self): mock_open = mock.mock_open() with mock.patch('builtins.open', mock_open) as file: @@ -3287,7 +3279,6 @@ def test_multiple_args(self): self.check_note(RuntimeError('a', 'b', 'c'), msg_re) # http://bugs.python.org/issue19609 - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_codec_lookup_failure(self): msg = "^unknown encoding: {}$".format(self.codec_name) with self.assertRaisesRegex(LookupError, msg): @@ -3523,8 +3514,6 @@ def test_incremental(self): False) self.assertEqual(decoded, ('abc', 3)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_mbcs_alias(self): # Check that looking up our 'default' codepage will return # mbcs when we don't have a more specific one available diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index e2d2f89454..44ef4e2416 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -1237,7 +1237,6 @@ def test_utf8_normalization(self): found, consumed_lines = detect_encoding(rl) self.assertEqual(found, "utf-8") - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_short_files(self): readline = self.get_readline((b'print(something)\n',)) encoding, consumed_lines = detect_encoding(readline) @@ -1316,7 +1315,6 @@ def readline(self): ins = Bunk(lines, path) detect_encoding(ins.readline) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_open_error(self): # Issue #23840: open() must close the binary file on error m = BytesIO(b'#coding:xxx') diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 895b29aa50..acc645bb74 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -125,6 +125,7 @@ features = [ workspace = true features = [ "Win32_Foundation", + "Win32_Globalization", "Win32_Networking_WinSock", "Win32_Security", "Win32_Storage_FileSystem", diff --git a/vm/src/stdlib/winapi.rs b/vm/src/stdlib/winapi.rs index ad6db12474..8feef61b14 100644 --- a/vm/src/stdlib/winapi.rs +++ b/vm/src/stdlib/winapi.rs @@ -119,6 +119,11 @@ mod _winapi { Ok(HANDLE(target)) } + #[pyfunction] + fn GetACP() -> u32 { + unsafe { windows_sys::Win32::Globalization::GetACP() } + } + #[pyfunction] fn GetCurrentProcess() -> HANDLE { unsafe { windows::Win32::System::Threading::GetCurrentProcess() } From d7a72b57553473a46fe2bfcf12a6a585f75838d4 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sun, 23 Feb 2025 20:02:26 -0800 Subject: [PATCH 034/295] add constants and implement functions Signed-off-by: Ashwin Naren --- Cargo.lock | 5 +-- vm/src/stdlib/winapi.rs | 68 +++++++++++++++++++++++++++++++++++++++-- vm/src/stdlib/winreg.rs | 10 ++++-- 3 files changed, 75 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3711b6f16c..2075ea3cbd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1644,11 +1644,12 @@ dependencies = [ [[package]] name = "puruspe" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5666f1226a41ebb7191b702947674f4814c9919da58a9f91a60090412664bbce" +checksum = "d76c522e44709f541a403db419a7e34d6fbbc8e6b208589ae29a030cddeefd96" dependencies = [ "lambert_w", + "num-complex", ] [[package]] diff --git a/vm/src/stdlib/winapi.rs b/vm/src/stdlib/winapi.rs index 8feef61b14..6cf8c45bfe 100644 --- a/vm/src/stdlib/winapi.rs +++ b/vm/src/stdlib/winapi.rs @@ -28,10 +28,20 @@ mod _winapi { ERROR_PIPE_CONNECTED, ERROR_SEM_TIMEOUT, GENERIC_READ, GENERIC_WRITE, STILL_ACTIVE, WAIT_ABANDONED, WAIT_ABANDONED_0, WAIT_OBJECT_0, WAIT_TIMEOUT, }, + Globalization::{ + LCMAP_FULLWIDTH, LCMAP_HALFWIDTH, LCMAP_HIRAGANA, LCMAP_KATAKANA, + LCMAP_LINGUISTIC_CASING, LCMAP_LOWERCASE, LCMAP_SIMPLIFIED_CHINESE, LCMAP_TITLECASE, + LCMAP_TRADITIONAL_CHINESE, LCMAP_UPPERCASE, + }, Storage::FileSystem::{ - FILE_FLAG_FIRST_PIPE_INSTANCE, FILE_FLAG_OVERLAPPED, FILE_GENERIC_READ, - FILE_GENERIC_WRITE, FILE_TYPE_CHAR, FILE_TYPE_DISK, FILE_TYPE_PIPE, FILE_TYPE_REMOTE, - FILE_TYPE_UNKNOWN, OPEN_EXISTING, PIPE_ACCESS_DUPLEX, PIPE_ACCESS_INBOUND, SYNCHRONIZE, + COPYFILE2_CALLBACK_CHUNK_FINISHED, COPYFILE2_CALLBACK_CHUNK_STARTED, + COPYFILE2_CALLBACK_ERROR, COPYFILE2_CALLBACK_POLL_CONTINUE, + COPYFILE2_CALLBACK_STREAM_FINISHED, COPYFILE2_CALLBACK_STREAM_STARTED, + COPYFILE2_PROGRESS_CANCEL, COPYFILE2_PROGRESS_CONTINUE, COPYFILE2_PROGRESS_PAUSE, + COPYFILE2_PROGRESS_QUIET, COPYFILE2_PROGRESS_STOP, FILE_FLAG_FIRST_PIPE_INSTANCE, + FILE_FLAG_OVERLAPPED, FILE_GENERIC_READ, FILE_GENERIC_WRITE, FILE_TYPE_CHAR, + FILE_TYPE_DISK, FILE_TYPE_PIPE, FILE_TYPE_REMOTE, FILE_TYPE_UNKNOWN, OPEN_EXISTING, + PIPE_ACCESS_DUPLEX, PIPE_ACCESS_INBOUND, SYNCHRONIZE, }, System::{ Console::{STD_ERROR_HANDLE, STD_INPUT_HANDLE, STD_OUTPUT_HANDLE}, @@ -53,6 +63,13 @@ mod _winapi { IDLE_PRIORITY_CLASS, INFINITE, NORMAL_PRIORITY_CLASS, PROCESS_DUP_HANDLE, REALTIME_PRIORITY_CLASS, STARTF_USESHOWWINDOW, STARTF_USESTDHANDLES, }, + WindowsProgramming::{ + COPY_FILE_ALLOW_DECRYPTED_DESTINATION, COPY_FILE_COPY_SYMLINK, + COPY_FILE_FAIL_IF_EXISTS, COPY_FILE_NO_BUFFERING, COPY_FILE_NO_OFFLOAD, + COPY_FILE_OPEN_SOURCE_FOR_WRITE, COPY_FILE_REQUEST_COMPRESSED_TRAFFIC, + COPY_FILE_REQUEST_SECURITY_PRIVILEGES, COPY_FILE_RESTARTABLE, + COPY_FILE_RESUME_FROM_PAUSE, + }, }, UI::WindowsAndMessaging::SW_HIDE, }; @@ -142,6 +159,16 @@ mod _winapi { } } + #[pyfunction] + fn GetLastError() -> u32 { + unsafe { windows_sys::Win32::Foundation::GetLastError() } + } + + #[pyfunction] + fn GetVersion() -> u32 { + unsafe { windows_sys::Win32::System::SystemInformation::GetVersion() } + } + #[derive(FromArgs)] struct CreateProcessArgs { #[pyarg(positional)] @@ -254,6 +281,21 @@ mod _winapi { )) } + #[pyfunction] + fn OpenProcess( + desired_access: u32, + inherit_handle: bool, + process_id: u32, + ) -> windows_sys::Win32::Foundation::HANDLE { + unsafe { + windows_sys::Win32::System::Threading::OpenProcess( + desired_access, + BOOL::from(inherit_handle), + process_id, + ) + } + } + #[pyfunction] fn NeedCurrentDirectoryForExePath(exe_name: PyStrRef) -> bool { let exe_name = exe_name.as_str().to_wide_with_nul(); @@ -452,4 +494,24 @@ mod _winapi { let (path, _) = path.split_at(length as usize); Ok(String::from_utf16(path).unwrap()) } + + #[pyfunction] + fn OpenMutexW(desired_access: u32, inherit_handle: bool, name: u16) -> PyResult { + let handle = unsafe { + windows_sys::Win32::System::Threading::OpenMutexW( + desired_access, + BOOL::from(inherit_handle), + windows_sys::core::PCWSTR::from(name as _), + ) + }; + // if handle.is_invalid() { + // return Err(errno_err(vm)); + // } + Ok(handle) + } + + #[pyfunction] + fn ReleaseMutex(handle: isize) -> WindowsSysResult { + WindowsSysResult(unsafe { windows_sys::Win32::System::Threading::ReleaseMutex(handle) }) + } } diff --git a/vm/src/stdlib/winreg.rs b/vm/src/stdlib/winreg.rs index 747c58cfd8..f25ca0b960 100644 --- a/vm/src/stdlib/winreg.rs +++ b/vm/src/stdlib/winreg.rs @@ -47,9 +47,13 @@ mod winreg { // value types #[pyattr] pub use windows_sys::Win32::System::Registry::{ - REG_BINARY, REG_DWORD, REG_DWORD_BIG_ENDIAN, REG_DWORD_LITTLE_ENDIAN, REG_EXPAND_SZ, - REG_FULL_RESOURCE_DESCRIPTOR, REG_LINK, REG_MULTI_SZ, REG_NONE, REG_QWORD, - REG_QWORD_LITTLE_ENDIAN, REG_RESOURCE_LIST, REG_RESOURCE_REQUIREMENTS_LIST, REG_SZ, + REG_BINARY, REG_CREATED_NEW_KEY, REG_DWORD, REG_DWORD_BIG_ENDIAN, REG_DWORD_LITTLE_ENDIAN, + REG_EXPAND_SZ, REG_FULL_RESOURCE_DESCRIPTOR, REG_LINK, REG_MULTI_SZ, REG_NONE, + REG_NOTIFY_CHANGE_ATTRIBUTES, REG_NOTIFY_CHANGE_LAST_SET, REG_NOTIFY_CHANGE_NAME, + REG_NOTIFY_CHANGE_SECURITY, REG_OPENED_EXISTING_KEY, REG_OPTION_BACKUP_RESTORE, + REG_OPTION_CREATE_LINK, REG_OPTION_NON_VOLATILE, REG_OPTION_OPEN_LINK, REG_OPTION_RESERVED, + REG_OPTION_VOLATILE, REG_QWORD, REG_QWORD_LITTLE_ENDIAN, REG_RESOURCE_LIST, + REG_RESOURCE_REQUIREMENTS_LIST, REG_SZ, REG_WHOLE_HIVE_VOLATILE, }; #[pyattr] From b55a55afc77865e48dad7b7c5dd7805585c02536 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sat, 22 Feb 2025 11:59:44 -0800 Subject: [PATCH 035/295] update the csv with the temp data for website what's left Signed-off-by: Ashwin Naren --- .github/workflows/cron-ci.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/cron-ci.yaml b/.github/workflows/cron-ci.yaml index 60a06d80c7..bfc758310e 100644 --- a/.github/workflows/cron-ci.yaml +++ b/.github/workflows/cron-ci.yaml @@ -97,6 +97,9 @@ jobs: cd website [ -f ./_data/whats_left.temp ] && cp ./_data/whats_left.temp ./_data/whats_left_lastrun.temp cp ../whats_left.temp ./_data/whats_left.temp + rm _data/whats_left/modules.csv + cat _data/whats_left.temp | grep "(entire module)" | cut -d ' ' -f 1 | sort >> ../_data/whats_left/modules.csv + git add -A if git -c user.name="Github Actions" -c user.email="actions@github.com" commit -m "Update what is left results" --author="$GITHUB_ACTOR"; then git push From 2721f2de3f0d2599ad2f4302dc0d1878bb2152be Mon Sep 17 00:00:00 2001 From: Noa Date: Mon, 24 Feb 2025 17:41:54 -0600 Subject: [PATCH 036/295] Fix a bunch of `random` tests (#5533) --- .gitattributes | 1 + Lib/test/test_random.py | 25 -------- stdlib/src/random.rs | 130 +++++++++++++++++++--------------------- vm/src/stdlib/os.rs | 5 +- 4 files changed, 63 insertions(+), 98 deletions(-) diff --git a/.gitattributes b/.gitattributes index aa993ad110..0c4ae3d850 100644 --- a/.gitattributes +++ b/.gitattributes @@ -4,3 +4,4 @@ Cargo.lock linguist-generated -merge vm/src/stdlib/ast/gen.rs linguist-generated -merge Lib/*.py text working-tree-encoding=UTF-8 eol=LF **/*.rs text working-tree-encoding=UTF-8 eol=LF +*.pck binary diff --git a/Lib/test/test_random.py b/Lib/test/test_random.py index 5bd92f32c2..2cceaea244 100644 --- a/Lib/test/test_random.py +++ b/Lib/test/test_random.py @@ -22,8 +22,6 @@ def randomlist(self, n): """Helper function to make a list of random numbers""" return [self.gen.random() for i in range(n)] - # TODO: RUSTPYTHON AttributeError: 'super' object has no attribute 'getstate' - @unittest.expectedFailure def test_autoseed(self): self.gen.seed() state1 = self.gen.getstate() @@ -32,8 +30,6 @@ def test_autoseed(self): state2 = self.gen.getstate() self.assertNotEqual(state1, state2) - # TODO: RUSTPYTHON AttributeError: 'super' object has no attribute 'getstate' - @unittest.expectedFailure def test_saverestore(self): N = 1000 self.gen.seed() @@ -60,7 +56,6 @@ def __hash__(self): self.assertRaises(TypeError, self.gen.seed, 1, 2, 3, 4) self.assertRaises(TypeError, type(self.gen), []) - @unittest.skip("TODO: RUSTPYTHON, TypeError: Expected type 'bytes', not 'bytearray'") def test_seed_no_mutate_bug_44018(self): a = bytearray(b'1234') self.gen.seed(a) @@ -386,8 +381,6 @@ def test_getrandbits(self): self.assertRaises(ValueError, self.gen.getrandbits, -1) self.assertRaises(TypeError, self.gen.getrandbits, 10.1) - # TODO: RUSTPYTHON AttributeError: 'super' object has no attribute 'getstate' - @unittest.expectedFailure def test_pickling(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): state = pickle.dumps(self.gen, proto) @@ -396,8 +389,6 @@ def test_pickling(self): restoredseq = [newgen.random() for i in range(10)] self.assertEqual(origseq, restoredseq) - # TODO: RUSTPYTHON AttributeError: 'super' object has no attribute 'getstate' - @unittest.expectedFailure def test_bug_1727780(self): # verify that version-2-pickles can be loaded # fine, whether they are created on 32-bit or 64-bit @@ -600,11 +591,6 @@ def test_bug_42008(self): class MersenneTwister_TestBasicOps(TestBasicOps, unittest.TestCase): gen = random.Random() - # TODO: RUSTPYTHON, TypeError: Expected type 'bytes', not 'bytearray' - @unittest.expectedFailure - def test_seed_no_mutate_bug_44018(self): # TODO: RUSTPYTHON, remove when this passes - super().test_seed_no_mutate_bug_44018() # TODO: RUSTPYTHON, remove when this passes - def test_guaranteed_stable(self): # These sequences are guaranteed to stay the same across versions of python self.gen.seed(3456147, version=1) @@ -675,8 +661,6 @@ def test_bug_31482(self): def test_setstate_first_arg(self): self.assertRaises(ValueError, self.gen.setstate, (1, None, None)) - # TODO: RUSTPYTHON AttributeError: 'super' object has no attribute 'getstate' - @unittest.expectedFailure def test_setstate_middle_arg(self): start_state = self.gen.getstate() # Wrong type, s/b tuple @@ -1282,15 +1266,6 @@ def test_betavariate_return_zero(self, gammavariate_mock): class TestRandomSubclassing(unittest.TestCase): - # TODO: RUSTPYTHON Unexpected keyword argument newarg - @unittest.expectedFailure - def test_random_subclass_with_kwargs(self): - # SF bug #1486663 -- this used to erroneously raise a TypeError - class Subclass(random.Random): - def __init__(self, newarg=None): - random.Random.__init__(self) - Subclass(newarg=1) - def test_subclasses_overriding_methods(self): # Subclasses with an overridden random, but only the original # getrandbits method should not rely on getrandbits in for randrange, diff --git a/stdlib/src/random.rs b/stdlib/src/random.rs index 35cbd94457..0692e3ef9b 100644 --- a/stdlib/src/random.rs +++ b/stdlib/src/random.rs @@ -6,73 +6,37 @@ pub(crate) use _random::make_module; mod _random { use crate::common::lock::PyMutex; use crate::vm::{ - builtins::{PyInt, PyTypeRef}, + builtins::{PyInt, PyTupleRef}, + convert::ToPyException, function::OptionalOption, - types::Constructor, - PyObjectRef, PyPayload, PyResult, VirtualMachine, + types::{Constructor, Initializer}, + PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, }; + use itertools::Itertools; use malachite_bigint::{BigInt, BigUint, Sign}; + use mt19937::MT19937; use num_traits::{Signed, Zero}; - use rand::{rngs::StdRng, RngCore, SeedableRng}; - - #[derive(Debug)] - enum PyRng { - Std(Box), - MT(Box), - } - - impl Default for PyRng { - fn default() -> Self { - PyRng::Std(Box::new(StdRng::from_os_rng())) - } - } - - impl RngCore for PyRng { - fn next_u32(&mut self) -> u32 { - match self { - Self::Std(s) => s.next_u32(), - Self::MT(m) => m.next_u32(), - } - } - fn next_u64(&mut self) -> u64 { - match self { - Self::Std(s) => s.next_u64(), - Self::MT(m) => m.next_u64(), - } - } - fn fill_bytes(&mut self, dest: &mut [u8]) { - match self { - Self::Std(s) => s.fill_bytes(dest), - Self::MT(m) => m.fill_bytes(dest), - } - } - } + use rand::{RngCore, SeedableRng}; + use rustpython_vm::types::DefaultConstructor; #[pyattr] #[pyclass(name = "Random")] - #[derive(Debug, PyPayload)] + #[derive(Debug, PyPayload, Default)] struct PyRandom { - rng: PyMutex, + rng: PyMutex, } - impl Constructor for PyRandom { - type Args = OptionalOption; + impl DefaultConstructor for PyRandom {} - fn py_new( - cls: PyTypeRef, - // TODO: use x as the seed. - _x: Self::Args, - vm: &VirtualMachine, - ) -> PyResult { - PyRandom { - rng: PyMutex::default(), - } - .into_ref_with_type(vm, cls) - .map(Into::into) + impl Initializer for PyRandom { + type Args = OptionalOption; + + fn init(zelf: PyRef, x: Self::Args, vm: &VirtualMachine) -> PyResult<()> { + zelf.seed(x, vm) } } - #[pyclass(flags(BASETYPE), with(Constructor))] + #[pyclass(flags(BASETYPE), with(Constructor, Initializer))] impl PyRandom { #[pymethod] fn random(&self) -> f64 { @@ -82,9 +46,8 @@ mod _random { #[pymethod] fn seed(&self, n: OptionalOption, vm: &VirtualMachine) -> PyResult<()> { - let new_rng = n - .flatten() - .map(|n| { + *self.rng.lock() = match n.flatten() { + Some(n) => { // Fallback to using hash if object isn't Int-like. let (_, mut key) = match n.downcast::() { Ok(n) => n.as_bigint().abs(), @@ -95,27 +58,21 @@ mod _random { key.reverse(); } let key = if key.is_empty() { &[0] } else { key.as_slice() }; - Ok(PyRng::MT(Box::new(mt19937::MT19937::new_with_slice_seed( - key, - )))) - }) - .transpose()? - .unwrap_or_default(); - - *self.rng.lock() = new_rng; + MT19937::new_with_slice_seed(key) + } + None => MT19937::try_from_os_rng() + .map_err(|e| std::io::Error::from(e).to_pyexception(vm))?, + }; Ok(()) } #[pymethod] fn getrandbits(&self, k: isize, vm: &VirtualMachine) -> PyResult { match k { - k if k < 0 => { - Err(vm.new_value_error("number of bits must be non-negative".to_owned())) - } + ..0 => Err(vm.new_value_error("number of bits must be non-negative".to_owned())), 0 => Ok(BigInt::zero()), - _ => { + mut k => { let mut rng = self.rng.lock(); - let mut k = k; let mut gen_u32 = |k| { let r = rng.next_u32(); if k < 32 { @@ -145,5 +102,40 @@ mod _random { } } } + + #[pymethod] + fn getstate(&self, vm: &VirtualMachine) -> PyTupleRef { + let rng = self.rng.lock(); + vm.new_tuple( + rng.get_state() + .iter() + .copied() + .chain([rng.get_index() as u32]) + .map(|i| vm.ctx.new_int(i).into()) + .collect::>(), + ) + } + + #[pymethod] + fn setstate(&self, state: PyTupleRef, vm: &VirtualMachine) -> PyResult<()> { + let state: &[_; mt19937::N + 1] = state + .as_slice() + .try_into() + .map_err(|_| vm.new_value_error("state vector is the wrong size".to_owned()))?; + let (index, state) = state.split_last().unwrap(); + let index: usize = index.try_to_value(vm)?; + if index > mt19937::N { + return Err(vm.new_value_error("invalid state".to_owned())); + } + let state: [u32; mt19937::N] = state + .iter() + .map(|i| i.try_to_value(vm)) + .process_results(|it| it.collect_array())? + .unwrap(); + let mut rng = self.rng.lock(); + rng.set_state(&state); + rng.set_index(index); + Ok(()) + } } } diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 6519af094d..e07dca82ef 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -978,10 +978,7 @@ pub(super) mod _os { return Err(vm.new_value_error("negative argument not allowed".to_owned())); } let mut buf = vec![0u8; size as usize]; - getrandom::fill(&mut buf).map_err(|e| match e.raw_os_error() { - Some(errno) => io::Error::from_raw_os_error(errno).into_pyexception(vm), - None => vm.new_os_error("Getting random failed".to_owned()), - })?; + getrandom::fill(&mut buf).map_err(|e| io::Error::from(e).into_pyexception(vm))?; Ok(buf) } From 52208b3c900edfe14663a163c7a16cc83c44a9ba Mon Sep 17 00:00:00 2001 From: Noa Date: Mon, 24 Feb 2025 20:54:13 -0600 Subject: [PATCH 037/295] Update to syn2 (#5556) --- Cargo.lock | 12 ++- Cargo.toml | 2 +- derive-impl/Cargo.toml | 2 +- derive-impl/src/compile_bytecode.rs | 151 +++++++++++----------------- derive-impl/src/from_args.rs | 114 +++++++++------------ derive-impl/src/lib.rs | 11 +- derive-impl/src/pyclass.rs | 33 +++--- derive-impl/src/pymodule.rs | 5 +- derive-impl/src/pytraverse.rs | 52 +++------- derive-impl/src/util.rs | 40 +++++--- derive/src/lib.rs | 7 +- 11 files changed, 186 insertions(+), 243 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2075ea3cbd..9d486786ba 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2056,7 +2056,7 @@ version = "0.4.0" dependencies = [ "rustpython-compiler", "rustpython-derive-impl", - "syn 1.0.109", + "syn 2.0.98", ] [[package]] @@ -2071,7 +2071,7 @@ dependencies = [ "rustpython-compiler-core", "rustpython-doc", "rustpython-parser-core", - "syn 1.0.109", + "syn 2.0.98", "syn-ext", "textwrap 0.16.1", ] @@ -2614,11 +2614,13 @@ dependencies = [ [[package]] name = "syn-ext" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b86cb2b68c5b3c078cac02588bc23f3c04bb828c5d3aedd17980876ec6a7be6" +checksum = "b126de4ef6c2a628a68609dd00733766c3b015894698a438ebdf374933fc31d1" dependencies = [ - "syn 1.0.109", + "proc-macro2", + "quote", + "syn 2.0.98", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 446733d515..e352032de5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -177,7 +177,7 @@ schannel = "0.1.27" static_assertions = "1.1" strum = "0.27" strum_macros = "0.27" -syn = "1.0.109" +syn = "2" thiserror = "2.0" thread_local = "1.1.8" unicode_names2 = "1.3.0" diff --git a/derive-impl/Cargo.toml b/derive-impl/Cargo.toml index 83a4bc4214..debe58106b 100644 --- a/derive-impl/Cargo.toml +++ b/derive-impl/Cargo.toml @@ -20,7 +20,7 @@ syn = { workspace = true, features = ["full", "extra-traits"] } maplit = "1.0.2" proc-macro2 = "1.0.93" quote = "1.0.38" -syn-ext = { version = "0.4.0", features = ["full"] } +syn-ext = { version = "0.5.0", features = ["full"] } textwrap = { version = "0.16.1", default-features = false } [lints] diff --git a/derive-impl/src/compile_bytecode.rs b/derive-impl/src/compile_bytecode.rs index 6b5baef98c..34d5cb8d9d 100644 --- a/derive-impl/src/compile_bytecode.rs +++ b/derive-impl/src/compile_bytecode.rs @@ -13,7 +13,7 @@ //! ) //! ``` -use crate::{extract_spans, Diagnostic}; +use crate::Diagnostic; use once_cell::sync::Lazy; use proc_macro2::{Span, TokenStream}; use quote::quote; @@ -25,10 +25,9 @@ use std::{ }; use syn::{ self, - parse::{Parse, ParseStream, Result as ParseResult}, - parse2, + parse::{ParseStream, Parser, Result as ParseResult}, spanned::Spanned, - Lit, LitByteStr, LitStr, Macro, Meta, MetaNameValue, Token, + LitByteStr, LitStr, Macro, }; static CARGO_MANIFEST_DIR: Lazy = Lazy::new(|| { @@ -233,23 +232,17 @@ impl CompilationSource { } } -/// This is essentially just a comma-separated list of Meta nodes, aka the inside of a MetaList. -struct PyCompileInput { - span: Span, - metas: Vec, -} - -impl PyCompileInput { - fn parse(&self, allow_dir: bool) -> Result { +impl PyCompileArgs { + fn parse(input: TokenStream, allow_dir: bool) -> Result { let mut module_name = None; let mut mode = None; let mut source: Option = None; let mut crate_name = None; - fn assert_source_empty(source: &Option) -> Result<(), Diagnostic> { + fn assert_source_empty(source: &Option) -> Result<(), syn::Error> { if let Some(source) = source { - Err(Diagnostic::spans_error( - source.span, + Err(syn::Error::new( + source.span.0, "Cannot have more than one source", )) } else { @@ -257,59 +250,58 @@ impl PyCompileInput { } } - for meta in &self.metas { - if let Meta::NameValue(name_value) = meta { - let ident = match name_value.path.get_ident() { - Some(ident) => ident, - None => continue, - }; - let check_str = || match &name_value.lit { - Lit::Str(s) => Ok(s), - _ => Err(err_span!(name_value.lit, "{ident} must be a string")), - }; - if ident == "mode" { - let s = check_str()?; - match s.value().parse() { - Ok(mode_val) => mode = Some(mode_val), - Err(e) => bail_span!(s, "{}", e), - } - } else if ident == "module_name" { - module_name = Some(check_str()?.value()) - } else if ident == "source" { - assert_source_empty(&source)?; - let code = check_str()?.value(); - source = Some(CompilationSource { - kind: CompilationSourceKind::SourceCode(code), - span: extract_spans(&name_value).unwrap(), - }); - } else if ident == "file" { - assert_source_empty(&source)?; - let path = check_str()?.value().into(); - source = Some(CompilationSource { - kind: CompilationSourceKind::File(path), - span: extract_spans(&name_value).unwrap(), - }); - } else if ident == "dir" { - if !allow_dir { - bail_span!(ident, "py_compile doesn't accept dir") - } - - assert_source_empty(&source)?; - let path = check_str()?.value().into(); - source = Some(CompilationSource { - kind: CompilationSourceKind::Dir(path), - span: extract_spans(&name_value).unwrap(), - }); - } else if ident == "crate_name" { - let name = check_str()?.parse()?; - crate_name = Some(name); + syn::meta::parser(|meta| { + let ident = meta + .path + .get_ident() + .ok_or_else(|| meta.error("unknown arg"))?; + let check_str = || meta.value()?.call(parse_str); + if ident == "mode" { + let s = check_str()?; + match s.value().parse() { + Ok(mode_val) => mode = Some(mode_val), + Err(e) => bail_span!(s, "{}", e), + } + } else if ident == "module_name" { + module_name = Some(check_str()?.value()) + } else if ident == "source" { + assert_source_empty(&source)?; + let code = check_str()?.value(); + source = Some(CompilationSource { + kind: CompilationSourceKind::SourceCode(code), + span: (ident.span(), meta.input.cursor().span()), + }); + } else if ident == "file" { + assert_source_empty(&source)?; + let path = check_str()?.value().into(); + source = Some(CompilationSource { + kind: CompilationSourceKind::File(path), + span: (ident.span(), meta.input.cursor().span()), + }); + } else if ident == "dir" { + if !allow_dir { + bail_span!(ident, "py_compile doesn't accept dir") } + + assert_source_empty(&source)?; + let path = check_str()?.value().into(); + source = Some(CompilationSource { + kind: CompilationSourceKind::Dir(path), + span: (ident.span(), meta.input.cursor().span()), + }); + } else if ident == "crate_name" { + let name = check_str()?.parse()?; + crate_name = Some(name); + } else { + return Err(meta.error("unknown attr")); } - } + Ok(()) + }) + .parse2(input)?; let source = source.ok_or_else(|| { syn::Error::new( - self.span, + Span::call_site(), "Must have either file or source in py_compile!()/py_freeze!()", ) })?; @@ -323,38 +315,17 @@ impl PyCompileInput { } } -fn parse_meta(input: ParseStream) -> ParseResult { - let path = input.call(syn::Path::parse_mod_style)?; - let eq_token: Token![=] = input.parse()?; +fn parse_str(input: ParseStream) -> ParseResult { let span = input.span(); if input.peek(LitStr) { - Ok(Meta::NameValue(MetaNameValue { - path, - eq_token, - lit: Lit::Str(input.parse()?), - })) + input.parse() } else if let Ok(mac) = input.parse::() { - Ok(Meta::NameValue(MetaNameValue { - path, - eq_token, - lit: Lit::Str(LitStr::new(&mac.tokens.to_string(), mac.span())), - })) + Ok(LitStr::new(&mac.tokens.to_string(), mac.span())) } else { Err(syn::Error::new(span, "Expected string or stringify macro")) } } -impl Parse for PyCompileInput { - fn parse(input: ParseStream) -> ParseResult { - let span = input.cursor().span(); - let metas = input - .parse_terminated::(parse_meta)? - .into_iter() - .collect(); - Ok(PyCompileInput { span, metas }) - } -} - struct PyCompileArgs { source: CompilationSource, mode: Mode, @@ -366,8 +337,7 @@ pub fn impl_py_compile( input: TokenStream, compiler: &dyn Compiler, ) -> Result { - let input: PyCompileInput = parse2(input)?; - let args = input.parse(false)?; + let args = PyCompileArgs::parse(input, false)?; let crate_name = args.crate_name; let code = args @@ -388,8 +358,7 @@ pub fn impl_py_freeze( input: TokenStream, compiler: &dyn Compiler, ) -> Result { - let input: PyCompileInput = parse2(input)?; - let args = input.parse(true)?; + let args = PyCompileArgs::parse(input, true)?; let crate_name = args.crate_name; let code_map = args.source.compile(args.mode, args.module_name, compiler)?; diff --git a/derive-impl/src/from_args.rs b/derive-impl/src/from_args.rs index 7b5b684213..47b0530d88 100644 --- a/derive-impl/src/from_args.rs +++ b/derive-impl/src/from_args.rs @@ -1,9 +1,8 @@ use proc_macro2::TokenStream; use quote::{quote, ToTokens}; use syn::ext::IdentExt; -use syn::{ - parse_quote, Attribute, Data, DeriveInput, Expr, Field, Ident, Lit, Meta, NestedMeta, Result, -}; +use syn::meta::ParseNestedMeta; +use syn::{parse_quote, Attribute, Data, DeriveInput, Expr, Field, Ident, Result, Token}; /// The kind of the python parameter, this corresponds to the value of Parameter.kind /// (https://docs.python.org/3/library/inspect.html#inspect.Parameter.kind) @@ -36,84 +35,61 @@ type DefaultValue = Option; impl ArgAttribute { fn from_attribute(attr: &Attribute) -> Option> { - if !attr.path.is_ident("pyarg") { + if !attr.path().is_ident("pyarg") { return None; } let inner = move || { - let Meta::List(list) = attr.parse_meta()? else { - bail_span!(attr, "pyarg must be a list, like #[pyarg(...)]") - }; - let mut iter = list.nested.iter(); - let first_arg = iter.next().ok_or_else(|| { - err_span!(list, "There must be at least one argument to #[pyarg()]") + let mut arg_attr = None; + attr.parse_nested_meta(|meta| { + let Some(arg_attr) = &mut arg_attr else { + let kind = meta + .path + .get_ident() + .and_then(ParameterKind::from_ident) + .ok_or_else(|| { + meta.error( + "The first argument to #[pyarg()] must be the parameter type, \ + either 'positional', 'any', 'named', or 'flatten'.", + ) + })?; + arg_attr = Some(ArgAttribute { + name: None, + kind, + default: None, + }); + return Ok(()); + }; + arg_attr.parse_argument(meta) })?; - let kind = match first_arg { - NestedMeta::Meta(Meta::Path(path)) => { - path.get_ident().and_then(ParameterKind::from_ident) - } - _ => None, - }; - let kind = kind.ok_or_else(|| { - err_span!( - first_arg, - "The first argument to #[pyarg()] must be the parameter type, either \ - 'positional', 'any', 'named', or 'flatten'." - ) - })?; - - let mut attribute = ArgAttribute { - name: None, - kind, - default: None, - }; - - for arg in iter { - attribute.parse_argument(arg)?; - } - - Ok(attribute) + arg_attr + .ok_or_else(|| err_span!(attr, "There must be at least one argument to #[pyarg()]")) }; Some(inner()) } - fn parse_argument(&mut self, arg: &NestedMeta) -> Result<()> { + fn parse_argument(&mut self, meta: ParseNestedMeta<'_>) -> Result<()> { if let ParameterKind::Flatten = self.kind { - bail_span!(arg, "can't put additional arguments on a flatten arg") + return Err(meta.error("can't put additional arguments on a flatten arg")); } - match arg { - NestedMeta::Meta(Meta::Path(path)) => { - if path.is_ident("default") || path.is_ident("optional") { - if self.default.is_none() { - self.default = Some(None); - } - } else { - bail_span!(path, "Unrecognized pyarg attribute"); - } + if meta.path.is_ident("default") && meta.input.peek(Token![=]) { + if matches!(self.default, Some(Some(_))) { + return Err(meta.error("Default already set")); } - NestedMeta::Meta(Meta::NameValue(name_value)) => { - if name_value.path.is_ident("default") { - if matches!(self.default, Some(Some(_))) { - bail_span!(name_value, "Default already set"); - } - - match name_value.lit { - Lit::Str(ref val) => self.default = Some(Some(val.parse()?)), - _ => bail_span!(name_value, "Expected string value for default argument"), - } - } else if name_value.path.is_ident("name") { - if self.name.is_some() { - bail_span!(name_value, "already have a name") - } - - match &name_value.lit { - Lit::Str(val) => self.name = Some(val.value()), - _ => bail_span!(name_value, "Expected string value for name argument"), - } - } else { - bail_span!(name_value, "Unrecognized pyarg attribute"); - } + let val = meta.value()?; + let val = val.parse::()?; + self.default = Some(Some(val.parse()?)) + } else if meta.path.is_ident("default") || meta.path.is_ident("optional") { + if self.default.is_none() { + self.default = Some(None); + } + } else if meta.path.is_ident("name") { + if self.name.is_some() { + return Err(meta.error("already have a name")); } - _ => bail_span!(arg, "Unrecognized pyarg attribute"), + let val = meta.value()?.parse::()?; + self.name = Some(val.value()) + } else { + return Err(meta.error("Unrecognized pyarg attribute")); } Ok(()) diff --git a/derive-impl/src/lib.rs b/derive-impl/src/lib.rs index 35292e7de0..a1f97c96b0 100644 --- a/derive-impl/src/lib.rs +++ b/derive-impl/src/lib.rs @@ -20,11 +20,12 @@ mod pypayload; mod pystructseq; mod pytraverse; -use error::{extract_spans, Diagnostic}; +use error::Diagnostic; use proc_macro2::TokenStream; use quote::ToTokens; use rustpython_doc as doc; -use syn::{AttributeArgs, DeriveInput, Item}; +use syn::{DeriveInput, Item}; +use syn_ext::types::PunctuatedNestedMeta; pub use compile_bytecode::Compiler; @@ -38,7 +39,7 @@ pub fn derive_from_args(input: DeriveInput) -> TokenStream { result_to_tokens(from_args::impl_from_args(input)) } -pub fn pyclass(attr: AttributeArgs, item: Item) -> TokenStream { +pub fn pyclass(attr: PunctuatedNestedMeta, item: Item) -> TokenStream { if matches!(item, syn::Item::Impl(_) | syn::Item::Trait(_)) { result_to_tokens(pyclass::impl_pyclass_impl(attr, item)) } else { @@ -46,7 +47,7 @@ pub fn pyclass(attr: AttributeArgs, item: Item) -> TokenStream { } } -pub fn pyexception(attr: AttributeArgs, item: Item) -> TokenStream { +pub fn pyexception(attr: PunctuatedNestedMeta, item: Item) -> TokenStream { if matches!(item, syn::Item::Impl(_)) { result_to_tokens(pyclass::impl_pyexception_impl(attr, item)) } else { @@ -54,7 +55,7 @@ pub fn pyexception(attr: AttributeArgs, item: Item) -> TokenStream { } } -pub fn pymodule(attr: AttributeArgs, item: Item) -> TokenStream { +pub fn pymodule(attr: PunctuatedNestedMeta, item: Item) -> TokenStream { result_to_tokens(pymodule::impl_pymodule(attr, item)) } diff --git a/derive-impl/src/pyclass.rs b/derive-impl/src/pyclass.rs index c42069c31c..c3f148e2eb 100644 --- a/derive-impl/src/pyclass.rs +++ b/derive-impl/src/pyclass.rs @@ -8,10 +8,9 @@ use proc_macro2::{Delimiter, Group, Span, TokenStream, TokenTree}; use quote::{quote, quote_spanned, ToTokens}; use std::collections::{HashMap, HashSet}; use std::str::FromStr; -use syn::{ - parse_quote, spanned::Spanned, Attribute, AttributeArgs, Ident, Item, Meta, NestedMeta, Result, -}; +use syn::{parse_quote, spanned::Spanned, Attribute, Ident, Item, Result}; use syn_ext::ext::*; +use syn_ext::types::*; #[derive(Copy, Clone, Debug)] enum AttrName { @@ -98,7 +97,7 @@ fn extract_items_into_context<'a, Item>( context.errors.ok_or_push(context.member_items.validate()); } -pub(crate) fn impl_pyclass_impl(attr: AttributeArgs, item: Item) -> Result { +pub(crate) fn impl_pyclass_impl(attr: PunctuatedNestedMeta, item: Item) -> Result { let mut context = ImplContext::default(); let mut tokens = match item { Item::Impl(mut imp) => { @@ -235,9 +234,7 @@ pub(crate) fn impl_pyclass_impl(attr: AttributeArgs, item: Item) -> Result { - &method.sig.ident.to_string() == "extend_slots" - } + syn::TraitItem::Fn(item) => item.sig.ident == "extend_slots", _ => false, }; if has { @@ -344,7 +341,7 @@ fn generate_class_def( }; let basicsize = quote!(std::mem::size_of::<#ident>()); let is_pystruct = attrs.iter().any(|attr| { - attr.path.is_ident("derive") + attr.path().is_ident("derive") && if let Ok(Meta::List(l)) = attr.parse_meta() { l.nested .into_iter() @@ -418,7 +415,7 @@ fn generate_class_def( Ok(tokens) } -pub(crate) fn impl_pyclass(attr: AttributeArgs, item: Item) -> Result { +pub(crate) fn impl_pyclass(attr: PunctuatedNestedMeta, item: Item) -> Result { if matches!(item, syn::Item::Use(_)) { return Ok(quote!(#item)); } @@ -534,7 +531,7 @@ pub(crate) fn impl_pyclass(attr: AttributeArgs, item: Item) -> Result Result { +pub(crate) fn impl_pyexception(attr: PunctuatedNestedMeta, item: Item) -> Result { let (ident, _attrs) = pyexception_ident_and_attrs(&item)?; let fake_ident = Ident::new("pyclass", item.span()); let class_meta = ExceptionItemMeta::from_nested(ident.clone(), fake_ident, attr.into_iter())?; @@ -573,7 +570,7 @@ pub(crate) fn impl_pyexception(attr: AttributeArgs, item: Item) -> Result Result { +pub(crate) fn impl_pyexception_impl(attr: PunctuatedNestedMeta, item: Item) -> Result { let Item::Impl(imp) = item else { return Ok(item.into_token_stream()); }; @@ -1454,7 +1451,7 @@ struct ExtractedImplAttrs { with_slots: TokenStream, } -fn extract_impl_attrs(attr: AttributeArgs, item: &Ident) -> Result { +fn extract_impl_attrs(attr: PunctuatedNestedMeta, item: &Ident) -> Result { let mut withs = Vec::new(); let mut with_method_defs = Vec::new(); let mut with_slots = Vec::new(); @@ -1474,7 +1471,7 @@ fn extract_impl_attrs(attr: AttributeArgs, item: &Ident) -> Result { + NestedMeta::Meta(Meta::List(MetaList { path, nested, .. })) => { if path.is_ident("with") { for meta in nested { let NestedMeta::Meta(Meta::Path(path)) = &meta else { @@ -1530,12 +1527,16 @@ fn extract_impl_attrs(attr: AttributeArgs, item: &Ident) -> Result { + NestedMeta::Meta(Meta::NameValue(syn::MetaNameValue { path, value, .. })) => { if path.is_ident("payload") { - if let syn::Lit::Str(lit) = lit { + if let syn::Expr::Lit(syn::ExprLit { + lit: syn::Lit::Str(lit), + .. + }) = value + { payload = Some(Ident::new(&lit.value(), lit.span())); } else { - bail_span!(lit, "payload must be a string literal") + bail_span!(value, "payload must be a string literal") } } else { bail_span!(path, "Unknown pyimpl attribute") diff --git a/derive-impl/src/pymodule.rs b/derive-impl/src/pymodule.rs index b9a59c8280..db65e24e21 100644 --- a/derive-impl/src/pymodule.rs +++ b/derive-impl/src/pymodule.rs @@ -7,8 +7,9 @@ use crate::util::{ use proc_macro2::{Delimiter, Group, TokenStream, TokenTree}; use quote::{quote, quote_spanned, ToTokens}; use std::{collections::HashSet, str::FromStr}; -use syn::{parse_quote, spanned::Spanned, Attribute, AttributeArgs, Ident, Item, Result}; +use syn::{parse_quote, spanned::Spanned, Attribute, Ident, Item, Result}; use syn_ext::ext::*; +use syn_ext::types::PunctuatedNestedMeta; #[derive(Clone, Copy, Eq, PartialEq)] enum AttrName { @@ -51,7 +52,7 @@ struct ModuleContext { errors: Vec, } -pub fn impl_pymodule(attr: AttributeArgs, module_item: Item) -> Result { +pub fn impl_pymodule(attr: PunctuatedNestedMeta, module_item: Item) -> Result { let (doc, mut module_item) = match module_item { Item::Mod(m) => (m.attrs.doc(), m), other => bail_span!(other, "#[pymodule] can only be on a full module"), diff --git a/derive-impl/src/pytraverse.rs b/derive-impl/src/pytraverse.rs index 93aa233a18..d2fe2dac81 100644 --- a/derive-impl/src/pytraverse.rs +++ b/derive-impl/src/pytraverse.rs @@ -1,6 +1,6 @@ use proc_macro2::TokenStream; use quote::quote; -use syn::{Attribute, DeriveInput, Field, Meta, MetaList, NestedMeta, Result}; +use syn::{Attribute, DeriveInput, Field, Result}; struct TraverseAttr { /// set to `true` if the attribute is `#[pytraverse(skip)]` @@ -9,47 +9,25 @@ struct TraverseAttr { const ATTR_TRAVERSE: &str = "pytraverse"; -/// get the `#[pytraverse(..)]` attribute from the struct -fn valid_get_traverse_attr_from_meta_list(list: &MetaList) -> Result { - let find_skip_and_only_skip = || { - let len = list.nested.len(); - if len != 1 { - return None; - } - let mut iter = list.nested.iter(); - // we have checked the length, so unwrap is safe - let first_arg = iter.next().unwrap(); - let skip = match first_arg { - NestedMeta::Meta(Meta::Path(path)) => match path.is_ident("skip") { - true => true, - false => return None, - }, - _ => return None, - }; - Some(skip) - }; - let skip = find_skip_and_only_skip().ok_or_else(|| { - err_span!( - list, - "only support attr is #[pytraverse(skip)], got arguments: {:?}", - list.nested - ) - })?; - Ok(TraverseAttr { skip }) -} - /// only accept `#[pytraverse(skip)]` for now fn pytraverse_arg(attr: &Attribute) -> Option> { - if !attr.path.is_ident(ATTR_TRAVERSE) { + if !attr.path().is_ident(ATTR_TRAVERSE) { return None; } let ret = || { - let parsed = attr.parse_meta()?; - if let Meta::List(list) = parsed { - valid_get_traverse_attr_from_meta_list(&list) - } else { - bail_span!(attr, "pytraverse must be a list, like #[pytraverse(skip)]") - } + let mut skip = false; + attr.parse_nested_meta(|meta| { + if meta.path.is_ident("skip") { + if skip { + return Err(meta.error("already specified skip")); + } + skip = true; + } else { + return Err(meta.error("unknown attr")); + } + Ok(()) + })?; + Ok(TraverseAttr { skip }) }; Some(ret()) } diff --git a/derive-impl/src/util.rs b/derive-impl/src/util.rs index f016b0d1e9..bd568032c3 100644 --- a/derive-impl/src/util.rs +++ b/derive-impl/src/util.rs @@ -2,12 +2,10 @@ use itertools::Itertools; use proc_macro2::{Span, TokenStream}; use quote::{quote, ToTokens}; use std::collections::{HashMap, HashSet}; -use syn::{ - spanned::Spanned, Attribute, Ident, Meta, MetaList, NestedMeta, Result, Signature, UseTree, -}; +use syn::{spanned::Spanned, Attribute, Ident, Result, Signature, UseTree}; use syn_ext::{ ext::{AttributeExt as SynAttributeExt, *}, - types::PunctuatedNestedMeta, + types::*, }; pub(crate) const ALL_ALLOWED_NAMES: &[&str] = &[ @@ -167,7 +165,11 @@ impl ItemMetaInner { pub fn _optional_str(&self, key: &str) -> Result> { let value = if let Some((_, meta)) = self.meta_map.get(key) { let Meta::NameValue(syn::MetaNameValue { - lit: syn::Lit::Str(lit), + value: + syn::Expr::Lit(syn::ExprLit { + lit: syn::Lit::Str(lit), + .. + }), .. }) = meta else { @@ -193,7 +195,11 @@ impl ItemMetaInner { let value = if let Some((_, meta)) = self.meta_map.get(key) { match meta { Meta::NameValue(syn::MetaNameValue { - lit: syn::Lit::Bool(lit), + value: + syn::Expr::Lit(syn::ExprLit { + lit: syn::Lit::Bool(lit), + .. + }), .. }) => lit.value, Meta::Path(_) => true, @@ -210,7 +216,7 @@ impl ItemMetaInner { key: &str, ) -> Result>> { let value = if let Some((_, meta)) = self.meta_map.get(key) { - let Meta::List(syn::MetaList { + let Meta::List(MetaList { path: _, nested, .. }) = meta else { @@ -350,7 +356,11 @@ impl ClassItemMeta { if let Some((_, meta)) = inner.meta_map.get(KEY) { match meta { Meta::NameValue(syn::MetaNameValue { - lit: syn::Lit::Str(lit), + value: + syn::Expr::Lit(syn::ExprLit { + lit: syn::Lit::Str(lit), + .. + }), .. }) => return Ok(lit.value()), Meta::Path(_) => return Ok(inner.item_name()), @@ -387,11 +397,11 @@ impl ClassItemMeta { let value = if let Some((_, meta)) = inner.meta_map.get(KEY) { match meta { Meta::NameValue(syn::MetaNameValue { - lit: syn::Lit::Str(lit), + value: syn::Expr::Lit(syn::ExprLit{lit:syn::Lit::Str(lit),..}), .. }) => Ok(Some(lit.value())), Meta::NameValue(syn::MetaNameValue { - lit: syn::Lit::Bool(lit), + value: syn::Expr::Lit(syn::ExprLit{lit:syn::Lit::Bool(lit),..}), .. }) => if lit.value { Err(lit.span()) @@ -448,7 +458,11 @@ impl ExceptionItemMeta { if let Some((_, meta)) = inner.meta_map.get(KEY) { match meta { Meta::NameValue(syn::MetaNameValue { - lit: syn::Lit::Str(lit), + value: + syn::Expr::Lit(syn::ExprLit { + lit: syn::Lit::Str(lit), + .. + }), .. }) => return Ok(lit.value()), Meta::Path(_) => { @@ -489,7 +503,7 @@ impl std::ops::Deref for ExceptionItemMeta { pub(crate) trait AttributeExt: SynAttributeExt { fn promoted_nested(&self) -> Result; fn ident_and_promoted_nested(&self) -> Result<(&Ident, PunctuatedNestedMeta)>; - fn try_remove_name(&mut self, name: &str) -> Result>; + fn try_remove_name(&mut self, name: &str) -> Result>; fn fill_nested_meta(&mut self, name: &str, new_item: F) -> Result<()> where F: Fn() -> NestedMeta; @@ -512,7 +526,7 @@ impl AttributeExt for Attribute { Ok((self.get_ident().unwrap(), self.promoted_nested()?)) } - fn try_remove_name(&mut self, item_name: &str) -> Result> { + fn try_remove_name(&mut self, item_name: &str) -> Result> { self.try_meta_mut(|meta| { let nested = match meta { Meta::List(MetaList { ref mut nested, .. }) => Ok(nested), diff --git a/derive/src/lib.rs b/derive/src/lib.rs index e59db12f90..9e94bf43cb 100644 --- a/derive/src/lib.rs +++ b/derive/src/lib.rs @@ -5,6 +5,7 @@ use proc_macro::TokenStream; use rustpython_derive_impl as derive_impl; use syn::parse_macro_input; +use syn::punctuated::Punctuated; #[proc_macro_derive(FromArgs, attributes(pyarg))] pub fn derive_from_args(input: TokenStream) -> TokenStream { @@ -132,7 +133,7 @@ pub fn derive_from_args(input: TokenStream) -> TokenStream { /// have a body, abstract functions should be wrapped before applying an annotation. #[proc_macro_attribute] pub fn pyclass(attr: TokenStream, item: TokenStream) -> TokenStream { - let attr = parse_macro_input!(attr); + let attr = parse_macro_input!(attr with Punctuated::parse_terminated); let item = parse_macro_input!(item); derive_impl::pyclass(attr, item).into() } @@ -147,7 +148,7 @@ pub fn pyclass(attr: TokenStream, item: TokenStream) -> TokenStream { /// #[proc_macro_attribute] pub fn pyexception(attr: TokenStream, item: TokenStream) -> TokenStream { - let attr = parse_macro_input!(attr); + let attr = parse_macro_input!(attr with Punctuated::parse_terminated); let item = parse_macro_input!(item); derive_impl::pyexception(attr, item).into() } @@ -219,7 +220,7 @@ pub fn pyexception(attr: TokenStream, item: TokenStream) -> TokenStream { /// - `name`: the name of the function in Python, by default it is the same as the associated Rust function. #[proc_macro_attribute] pub fn pymodule(attr: TokenStream, item: TokenStream) -> TokenStream { - let attr = parse_macro_input!(attr); + let attr = parse_macro_input!(attr with Punctuated::parse_terminated); let item = parse_macro_input!(item); derive_impl::pymodule(attr, item).into() } From 1c3b198a170b057d190f546a93a9212f8c980218 Mon Sep 17 00:00:00 2001 From: Noa Date: Thu, 20 Feb 2025 18:14:47 -0600 Subject: [PATCH 038/295] Enable unsafe_op_in_unsafe_fn lint --- Cargo.toml | 1 + common/src/boxvec.rs | 13 +++++--- common/src/encodings.rs | 4 +-- common/src/linked_list.rs | 54 +++++++++++++++++---------------- common/src/lock/cell_lock.rs | 4 +-- common/src/lock/thread_mutex.rs | 2 +- compiler/core/src/bytecode.rs | 6 ++-- jit/src/lib.rs | 24 +++++++++------ stdlib/src/locale.rs | 12 +++++--- stdlib/src/select.rs | 34 +++++++++++---------- stdlib/src/socket.rs | 6 ++-- stdlib/src/sqlite.rs | 40 ++++++++++++------------ stdlib/src/ssl.rs | 5 +-- vm/src/builtins/str.rs | 2 +- vm/src/builtins/type.rs | 2 +- vm/src/function/method.rs | 2 +- vm/src/intern.rs | 2 +- vm/src/object/core.rs | 30 ++++++++++-------- vm/src/object/ext.rs | 14 +++++---- vm/src/protocol/buffer.rs | 7 +++-- vm/src/stdlib/posix.rs | 2 +- vm/src/stdlib/time.rs | 4 ++- vm/src/vm/context.rs | 2 +- 23 files changed, 148 insertions(+), 124 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index e352032de5..7ae2ea202e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -189,6 +189,7 @@ wasm-bindgen = "0.2.100" [workspace.lints.rust] unsafe_code = "allow" +unsafe_op_in_unsafe_fn = "deny" [workspace.lints.clippy] perf = "warn" diff --git a/common/src/boxvec.rs b/common/src/boxvec.rs index 9b73fa0103..3b1f7e90a0 100644 --- a/common/src/boxvec.rs +++ b/common/src/boxvec.rs @@ -87,13 +87,16 @@ impl BoxVec { pub unsafe fn push_unchecked(&mut self, element: T) { let len = self.len(); debug_assert!(len < self.capacity()); - ptr::write(self.get_unchecked_ptr(len), element); - self.set_len(len + 1); + // SAFETY: len < capacity + unsafe { + ptr::write(self.get_unchecked_ptr(len), element); + self.set_len(len + 1); + } } /// Get pointer to where element at `index` would be unsafe fn get_unchecked_ptr(&mut self, index: usize) -> *mut T { - self.xs.as_mut_ptr().add(index).cast() + unsafe { self.xs.as_mut_ptr().add(index).cast() } } pub fn insert(&mut self, index: usize, element: T) { @@ -568,7 +571,7 @@ unsafe fn raw_ptr_add(ptr: *mut T, offset: usize) -> *mut T { // Special case for ZST (ptr as usize).wrapping_add(offset) as _ } else { - ptr.add(offset) + unsafe { ptr.add(offset) } } } @@ -576,7 +579,7 @@ unsafe fn raw_ptr_write(ptr: *mut T, value: T) { if mem::size_of::() == 0 { /* nothing */ } else { - ptr::write(ptr, value) + unsafe { ptr::write(ptr, value) } } } diff --git a/common/src/encodings.rs b/common/src/encodings.rs index 4e0c1de56a..858d3b8c6b 100644 --- a/common/src/encodings.rs +++ b/common/src/encodings.rs @@ -42,8 +42,8 @@ struct DecodeError<'a> { /// # Safety /// `v[..valid_up_to]` must be valid utf8 unsafe fn make_decode_err(v: &[u8], valid_up_to: usize, err_len: Option) -> DecodeError<'_> { - let valid_prefix = core::str::from_utf8_unchecked(v.get_unchecked(..valid_up_to)); - let rest = v.get_unchecked(valid_up_to..); + let (valid_prefix, rest) = unsafe { v.split_at_unchecked(valid_up_to) }; + let valid_prefix = unsafe { core::str::from_utf8_unchecked(valid_prefix) }; DecodeError { valid_prefix, rest, diff --git a/common/src/linked_list.rs b/common/src/linked_list.rs index 3040bab0b9..7f55d727fb 100644 --- a/common/src/linked_list.rs +++ b/common/src/linked_list.rs @@ -208,37 +208,39 @@ impl LinkedList { /// The caller **must** ensure that `node` is currently contained by /// `self` or not contained by any other list. pub unsafe fn remove(&mut self, node: NonNull) -> Option { - if let Some(prev) = L::pointers(node).as_ref().get_prev() { - debug_assert_eq!(L::pointers(prev).as_ref().get_next(), Some(node)); - L::pointers(prev) - .as_mut() - .set_next(L::pointers(node).as_ref().get_next()); - } else { - if self.head != Some(node) { - return None; + unsafe { + if let Some(prev) = L::pointers(node).as_ref().get_prev() { + debug_assert_eq!(L::pointers(prev).as_ref().get_next(), Some(node)); + L::pointers(prev) + .as_mut() + .set_next(L::pointers(node).as_ref().get_next()); + } else { + if self.head != Some(node) { + return None; + } + + self.head = L::pointers(node).as_ref().get_next(); } - self.head = L::pointers(node).as_ref().get_next(); - } + if let Some(next) = L::pointers(node).as_ref().get_next() { + debug_assert_eq!(L::pointers(next).as_ref().get_prev(), Some(node)); + L::pointers(next) + .as_mut() + .set_prev(L::pointers(node).as_ref().get_prev()); + } else { + // // This might be the last item in the list + // if self.tail != Some(node) { + // return None; + // } + + // self.tail = L::pointers(node).as_ref().get_prev(); + } - if let Some(next) = L::pointers(node).as_ref().get_next() { - debug_assert_eq!(L::pointers(next).as_ref().get_prev(), Some(node)); - L::pointers(next) - .as_mut() - .set_prev(L::pointers(node).as_ref().get_prev()); - } else { - // // This might be the last item in the list - // if self.tail != Some(node) { - // return None; - // } + L::pointers(node).as_mut().set_next(None); + L::pointers(node).as_mut().set_prev(None); - // self.tail = L::pointers(node).as_ref().get_prev(); + Some(L::from_raw(node)) } - - L::pointers(node).as_mut().set_next(None); - L::pointers(node).as_mut().set_prev(None); - - Some(L::from_raw(node)) } // pub fn last(&self) -> Option<&L::Target> { diff --git a/common/src/lock/cell_lock.rs b/common/src/lock/cell_lock.rs index b10101f269..1edd622a20 100644 --- a/common/src/lock/cell_lock.rs +++ b/common/src/lock/cell_lock.rs @@ -140,12 +140,12 @@ unsafe impl RawRwLockUpgrade for RawCellRwLock { #[inline] unsafe fn unlock_upgradable(&self) { - self.unlock_shared() + unsafe { self.unlock_shared() } } #[inline] unsafe fn upgrade(&self) { - if !self.try_upgrade() { + if !unsafe { self.try_upgrade() } { deadlock("upgrade ", "RwLock") } } diff --git a/common/src/lock/thread_mutex.rs b/common/src/lock/thread_mutex.rs index ba36898780..35b0b9ac6d 100644 --- a/common/src/lock/thread_mutex.rs +++ b/common/src/lock/thread_mutex.rs @@ -65,7 +65,7 @@ impl RawThreadMutex { /// This method may only be called if the mutex is held by the current thread. pub unsafe fn unlock(&self) { self.owner.store(0, Ordering::Relaxed); - self.mutex.unlock(); + unsafe { self.mutex.unlock() }; } } diff --git a/compiler/core/src/bytecode.rs b/compiler/core/src/bytecode.rs index 21d342b541..11e49a47db 100644 --- a/compiler/core/src/bytecode.rs +++ b/compiler/core/src/bytecode.rs @@ -293,10 +293,8 @@ impl Arg { /// # Safety /// T::from_op_arg(self) must succeed pub unsafe fn get_unchecked(self, arg: OpArg) -> T { - match T::from_op_arg(arg.0) { - Some(t) => t, - None => std::hint::unreachable_unchecked(), - } + // SAFETY: requirements forwarded from caller + unsafe { T::from_op_arg(arg.0).unwrap_unchecked() } } } diff --git a/jit/src/lib.rs b/jit/src/lib.rs index 99bfb45c78..78619eaf62 100644 --- a/jit/src/lib.rs +++ b/jit/src/lib.rs @@ -152,12 +152,14 @@ impl CompiledCode { } unsafe fn invoke_raw(&self, cif_args: &[libffi::middle::Arg]) -> Option { - let cif = self.sig.to_cif(); - let value = cif.call::( - libffi::middle::CodePtr::from_ptr(self.code as *const _), - cif_args, - ); - self.sig.ret.as_ref().map(|ty| value.to_typed(ty)) + unsafe { + let cif = self.sig.to_cif(); + let value = cif.call::( + libffi::middle::CodePtr::from_ptr(self.code as *const _), + cif_args, + ); + self.sig.ret.as_ref().map(|ty| value.to_typed(ty)) + } } } @@ -290,10 +292,12 @@ union UnTypedAbiValue { impl UnTypedAbiValue { unsafe fn to_typed(self, ty: &JitType) -> AbiValue { - match ty { - JitType::Int => AbiValue::Int(self.int), - JitType::Float => AbiValue::Float(self.float), - JitType::Bool => AbiValue::Bool(self.boolean != 0), + unsafe { + match ty { + JitType::Int => AbiValue::Int(self.int), + JitType::Float => AbiValue::Float(self.float), + JitType::Bool => AbiValue::Bool(self.boolean != 0), + } } } } diff --git a/stdlib/src/locale.rs b/stdlib/src/locale.rs index bbe1008e53..7390561af7 100644 --- a/stdlib/src/locale.rs +++ b/stdlib/src/locale.rs @@ -78,11 +78,13 @@ mod _locale { return vm.ctx.new_list(group_vec); } - let mut ptr = group; - while ![0, libc::c_char::MAX].contains(&*ptr) { - let val = vm.ctx.new_int(*ptr); - group_vec.push(val.into()); - ptr = ptr.add(1); + unsafe { + let mut ptr = group; + while ![0, libc::c_char::MAX].contains(&*ptr) { + let val = vm.ctx.new_int(*ptr); + group_vec.push(val.into()); + ptr = ptr.add(1); + } } // https://github.com/python/cpython/blob/677320348728ce058fa3579017e985af74a236d4/Modules/_localemodule.c#L80 if !group_vec.is_empty() { diff --git a/stdlib/src/select.rs b/stdlib/src/select.rs index af76d86c8a..dfc974943a 100644 --- a/stdlib/src/select.rs +++ b/stdlib/src/select.rs @@ -36,28 +36,30 @@ mod platform { // based off winsock2.h: https://gist.github.com/piscisaureus/906386#file-winsock2-h-L128-L141 pub unsafe fn FD_SET(fd: RawFd, set: *mut fd_set) { - let mut slot = (&raw mut (*set).fd_array).cast::(); - let fd_count = (*set).fd_count; - for _ in 0..fd_count { - if *slot == fd { - return; + unsafe { + let mut slot = (&raw mut (*set).fd_array).cast::(); + let fd_count = (*set).fd_count; + for _ in 0..fd_count { + if *slot == fd { + return; + } + slot = slot.add(1); + } + // slot == &fd_array[fd_count] at this point + if fd_count < FD_SETSIZE { + *slot = fd as RawFd; + (*set).fd_count += 1; } - slot = slot.add(1); - } - // slot == &fd_array[fd_count] at this point - if fd_count < FD_SETSIZE { - *slot = fd as RawFd; - (*set).fd_count += 1; } } pub unsafe fn FD_ZERO(set: *mut fd_set) { - (*set).fd_count = 0; + unsafe { (*set).fd_count = 0 }; } pub unsafe fn FD_ISSET(fd: RawFd, set: *mut fd_set) -> bool { use WinSock::__WSAFDIsSet; - __WSAFDIsSet(fd as _, set) != 0 + unsafe { __WSAFDIsSet(fd as _, set) != 0 } } pub fn check_err(x: i32) -> bool { @@ -82,7 +84,7 @@ mod platform { #[allow(non_snake_case)] pub unsafe fn FD_ISSET(fd: RawFd, set: *const fd_set) -> bool { - let set = &*set; + let set = unsafe { &*set }; let n = set.__nfds; for p in &set.__fds[..n] { if *p == fd { @@ -94,7 +96,7 @@ mod platform { #[allow(non_snake_case)] pub unsafe fn FD_SET(fd: RawFd, set: *mut fd_set) { - let set = &mut *set; + let set = unsafe { &mut *set }; let n = set.__nfds; for p in &set.__fds[..n] { if *p == fd { @@ -107,7 +109,7 @@ mod platform { #[allow(non_snake_case)] pub unsafe fn FD_ZERO(set: *mut fd_set) { - let set = &mut *set; + let set = unsafe { &mut *set }; set.__nfds = 0; } diff --git a/stdlib/src/socket.rs b/stdlib/src/socket.rs index bf2f5ecd30..4172c5bb35 100644 --- a/stdlib/src/socket.rs +++ b/stdlib/src/socket.rs @@ -1788,7 +1788,7 @@ mod _socket { } unsafe fn slice_as_uninit(v: &mut [T]) -> &mut [MaybeUninit] { - &mut *(v as *mut [T] as *mut [MaybeUninit]) + unsafe { &mut *(v as *mut [T] as *mut [MaybeUninit]) } } enum IoOrPyException { @@ -2312,12 +2312,12 @@ mod _socket { #[cfg(unix)] { use std::os::unix::io::FromRawFd; - Socket::from_raw_fd(fileno) + unsafe { Socket::from_raw_fd(fileno) } } #[cfg(windows)] { use std::os::windows::io::FromRawSocket; - Socket::from_raw_socket(fileno) + unsafe { Socket::from_raw_socket(fileno) } } } pub(super) fn sock_fileno(sock: &Socket) -> RawSocket { diff --git a/stdlib/src/sqlite.rs b/stdlib/src/sqlite.rs index 6cb3deae7d..1d2c6928e7 100644 --- a/stdlib/src/sqlite.rs +++ b/stdlib/src/sqlite.rs @@ -398,7 +398,7 @@ mod _sqlite { } unsafe extern "C" fn destructor(data: *mut c_void) { - drop(Box::from_raw(data.cast::())); + drop(unsafe { Box::from_raw(data.cast::()) }); } unsafe extern "C" fn func_callback( @@ -407,8 +407,8 @@ mod _sqlite { argv: *mut *mut sqlite3_value, ) { let context = SqliteContext::from(context); - let (func, vm) = (*context.user_data::()).retrieve(); - let args = std::slice::from_raw_parts(argv, argc as usize); + let (func, vm) = unsafe { (*context.user_data::()).retrieve() }; + let args = unsafe { std::slice::from_raw_parts(argv, argc as usize) }; let f = || -> PyResult<()> { let db = context.db_handle(); @@ -434,12 +434,12 @@ mod _sqlite { argv: *mut *mut sqlite3_value, ) { let context = SqliteContext::from(context); - let (cls, vm) = (*context.user_data::()).retrieve(); - let args = std::slice::from_raw_parts(argv, argc as usize); + let (cls, vm) = unsafe { (*context.user_data::()).retrieve() }; + let args = unsafe { std::slice::from_raw_parts(argv, argc as usize) }; let instance = context.aggregate_context::<*const PyObject>(); - if (*instance).is_null() { + if unsafe { (*instance).is_null() } { match cls.call((), vm) { - Ok(obj) => *instance = obj.into_raw(), + Ok(obj) => unsafe { *instance = obj.into_raw() }, Err(exc) => { return context.result_exception( vm, @@ -449,16 +449,16 @@ mod _sqlite { } } } - let instance = &**instance; + let instance = unsafe { &**instance }; Self::call_method_with_args(context, instance, "step", args, vm); } unsafe extern "C" fn finalize_callback(context: *mut sqlite3_context) { let context = SqliteContext::from(context); - let (_, vm) = (*context.user_data::()).retrieve(); + let (_, vm) = unsafe { (*context.user_data::()).retrieve() }; let instance = context.aggregate_context::<*const PyObject>(); - let Some(instance) = (*instance).as_ref() else { + let Some(instance) = (unsafe { (*instance).as_ref() }) else { return; }; @@ -472,7 +472,7 @@ mod _sqlite { b_len: c_int, b_ptr: *const c_void, ) -> c_int { - let (callable, vm) = (*data.cast::()).retrieve(); + let (callable, vm) = unsafe { (*data.cast::()).retrieve() }; let f = || -> PyResult { let text1 = ptr_to_string(a_ptr.cast(), a_len, null_mut(), vm)?; @@ -499,9 +499,9 @@ mod _sqlite { unsafe extern "C" fn value_callback(context: *mut sqlite3_context) { let context = SqliteContext::from(context); - let (_, vm) = (*context.user_data::()).retrieve(); + let (_, vm) = unsafe { (*context.user_data::()).retrieve() }; let instance = context.aggregate_context::<*const PyObject>(); - let instance = &**instance; + let instance = unsafe { &**instance }; Self::callback_result_from_method(context, instance, "value", vm); } @@ -512,10 +512,10 @@ mod _sqlite { argv: *mut *mut sqlite3_value, ) { let context = SqliteContext::from(context); - let (_, vm) = (*context.user_data::()).retrieve(); - let args = std::slice::from_raw_parts(argv, argc as usize); + let (_, vm) = unsafe { (*context.user_data::()).retrieve() }; + let args = unsafe { std::slice::from_raw_parts(argv, argc as usize) }; let instance = context.aggregate_context::<*const PyObject>(); - let instance = &**instance; + let instance = unsafe { &**instance }; Self::call_method_with_args(context, instance, "inverse", args, vm); } @@ -528,7 +528,7 @@ mod _sqlite { db_name: *const libc::c_char, access: *const libc::c_char, ) -> c_int { - let (callable, vm) = (*data.cast::()).retrieve(); + let (callable, vm) = unsafe { (*data.cast::()).retrieve() }; let f = || -> PyResult { let arg1 = ptr_to_str(arg1, vm)?; let arg2 = ptr_to_str(arg2, vm)?; @@ -551,8 +551,8 @@ mod _sqlite { stmt: *mut c_void, sql: *mut c_void, ) -> c_int { - let (callable, vm) = (*data.cast::()).retrieve(); - let expanded = sqlite3_expanded_sql(stmt.cast()); + let (callable, vm) = unsafe { (*data.cast::()).retrieve() }; + let expanded = unsafe { sqlite3_expanded_sql(stmt.cast()) }; let f = || -> PyResult<()> { let stmt = ptr_to_str(expanded, vm).or_else(|_| ptr_to_str(sql.cast(), vm))?; callable.call((stmt,), vm)?; @@ -563,7 +563,7 @@ mod _sqlite { } unsafe extern "C" fn progress_callback(data: *mut c_void) -> c_int { - let (callable, vm) = (*data.cast::()).retrieve(); + let (callable, vm) = unsafe { (*data.cast::()).retrieve() }; if let Ok(val) = callable.call((), vm) { if let Ok(val) = val.is_true(vm) { return val as c_int; diff --git a/stdlib/src/ssl.rs b/stdlib/src/ssl.rs index 3622b74d1a..eb757f9a71 100644 --- a/stdlib/src/ssl.rs +++ b/stdlib/src/ssl.rs @@ -4,7 +4,8 @@ pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { // if openssl is vendored, it doesn't know the locations of system certificates #[cfg(feature = "ssl-vendor")] if let None | Some("0") = option_env!("OPENSSL_NO_VENDOR") { - openssl_probe::init_ssl_cert_env_vars(); + // TODO: use openssl_probe::probe() instead + unsafe { openssl_probe::init_openssl_env_vars() }; } openssl::init(); _ssl::make_module(vm) @@ -283,7 +284,7 @@ mod _ssl { if ptr.is_null() { None } else { - Some(Asn1Object::from_ptr(ptr)) + Some(unsafe { Asn1Object::from_ptr(ptr) }) } } diff --git a/vm/src/builtins/str.rs b/vm/src/builtins/str.rs index f5a340836c..f37a004297 100644 --- a/vm/src/builtins/str.rs +++ b/vm/src/builtins/str.rs @@ -313,7 +313,7 @@ impl PyStr { /// # Safety /// Given `bytes` must be ascii pub unsafe fn new_ascii_unchecked(bytes: Vec) -> Self { - Self::new_str_unchecked(bytes, PyStrKind::Ascii) + unsafe { Self::new_str_unchecked(bytes, PyStrKind::Ascii) } } pub fn new_ref(zelf: impl Into, ctx: &Context) -> PyRef { diff --git a/vm/src/builtins/type.rs b/vm/src/builtins/type.rs index 15df5ff3c5..c3fc28cd60 100644 --- a/vm/src/builtins/type.rs +++ b/vm/src/builtins/type.rs @@ -69,7 +69,7 @@ pub struct PointerSlot(NonNull); impl PointerSlot { pub unsafe fn borrow_static(&self) -> &'static T { - self.0.as_ref() + unsafe { self.0.as_ref() } } } diff --git a/vm/src/function/method.rs b/vm/src/function/method.rs index c8652ffb50..adf53d7d93 100644 --- a/vm/src/function/method.rs +++ b/vm/src/function/method.rs @@ -274,7 +274,7 @@ impl HeapMethodDef { impl Py { pub(crate) unsafe fn method(&self) -> &'static PyMethodDef { - &*(&self.method as *const _) + unsafe { &*(&self.method as *const _) } } pub fn build_function(&self, vm: &VirtualMachine) -> PyRef { diff --git a/vm/src/intern.rs b/vm/src/intern.rs index 45bd45d965..5bb657f222 100644 --- a/vm/src/intern.rs +++ b/vm/src/intern.rs @@ -117,7 +117,7 @@ impl CachedPyStrRef { /// the given cache must be alive while returned reference is alive #[inline] unsafe fn as_interned_str(&self) -> &'static PyStrInterned { - std::mem::transmute_copy(self) + unsafe { std::mem::transmute_copy(self) } } #[inline] diff --git a/vm/src/object/core.rs b/vm/src/object/core.rs index b326935464..f8a5ceef93 100644 --- a/vm/src/object/core.rs +++ b/vm/src/object/core.rs @@ -77,19 +77,19 @@ use std::{ pub(super) struct Erased; pub(super) unsafe fn drop_dealloc_obj(x: *mut PyObject) { - drop(Box::from_raw(x as *mut PyInner)); + drop(unsafe { Box::from_raw(x as *mut PyInner) }); } pub(super) unsafe fn debug_obj( x: &PyObject, f: &mut fmt::Formatter, ) -> fmt::Result { - let x = &*(x as *const PyObject as *const PyInner); + let x = unsafe { &*(x as *const PyObject as *const PyInner) }; fmt::Debug::fmt(x, f) } /// Call `try_trace` on payload pub(super) unsafe fn try_trace_obj(x: &PyObject, tracer_fn: &mut TraverseFn) { - let x = &*(x as *const PyObject as *const PyInner); + let x = unsafe { &*(x as *const PyObject as *const PyInner) }; let payload = &x.payload; payload.try_traverse(tracer_fn) } @@ -278,7 +278,7 @@ impl WeakRefList { } unsafe fn dealloc(ptr: NonNull>) { - drop(Box::from_raw(ptr.as_ptr())); + drop(unsafe { Box::from_raw(ptr.as_ptr()) }); } fn get_weak_references(&self) -> Vec> { @@ -317,12 +317,14 @@ unsafe impl Link for WeakLink { #[inline(always)] unsafe fn from_raw(ptr: NonNull) -> Self::Handle { - PyRef::from_raw(ptr.as_ptr()) + // SAFETY: requirements forwarded from caller + unsafe { PyRef::from_raw(ptr.as_ptr()) } } #[inline(always)] unsafe fn pointers(target: NonNull) -> NonNull> { - NonNull::new_unchecked(&raw mut (*target.as_ptr()).0.payload.pointers) + // SAFETY: requirements forwarded from caller + unsafe { NonNull::new_unchecked(&raw mut (*target.as_ptr()).0.payload.pointers) } } } @@ -520,7 +522,7 @@ impl PyObjectRef { #[inline(always)] pub unsafe fn from_raw(ptr: *const PyObject) -> Self { Self { - ptr: NonNull::new_unchecked(ptr as *mut PyObject), + ptr: unsafe { NonNull::new_unchecked(ptr as *mut PyObject) }, } } @@ -567,7 +569,8 @@ impl PyObjectRef { #[inline(always)] pub unsafe fn downcast_unchecked_ref(&self) -> &Py { debug_assert!(self.payload_is::()); - &*(self as *const PyObjectRef as *const PyRef) + // SAFETY: requirements forwarded from caller + unsafe { &*(self as *const PyObjectRef as *const PyRef) } } // ideally we'd be able to define these in pyobject.rs, but method visibility rules are weird @@ -752,7 +755,8 @@ impl PyObject { #[inline(always)] pub unsafe fn downcast_unchecked_ref(&self) -> &Py { debug_assert!(self.payload_is::()); - &*(self as *const PyObject as *const Py) + // SAFETY: requirements forwarded from caller + unsafe { &*(self as *const PyObject as *const Py) } } #[inline(always)] @@ -814,13 +818,13 @@ impl PyObject { /// Can only be called when ref_count has dropped to zero. `ptr` must be valid #[inline(never)] unsafe fn drop_slow(ptr: NonNull) { - if let Err(()) = ptr.as_ref().drop_slow_inner() { + if let Err(()) = unsafe { ptr.as_ref().drop_slow_inner() } { // abort drop for whatever reason return; } - let drop_dealloc = ptr.as_ref().0.vtable.drop_dealloc; + let drop_dealloc = unsafe { ptr.as_ref().0.vtable.drop_dealloc }; // call drop only when there are no references in scope - stacked borrows stuff - drop_dealloc(ptr.as_ptr()) + unsafe { drop_dealloc(ptr.as_ptr()) } } /// # Safety @@ -1022,7 +1026,7 @@ impl PyRef { #[inline(always)] pub(crate) unsafe fn from_raw(raw: *const Py) -> Self { Self { - ptr: NonNull::new_unchecked(raw as *mut _), + ptr: unsafe { NonNull::new_unchecked(raw as *mut _) }, } } diff --git a/vm/src/object/ext.rs b/vm/src/object/ext.rs index f7247bc5e0..59650443b0 100644 --- a/vm/src/object/ext.rs +++ b/vm/src/object/ext.rs @@ -60,7 +60,7 @@ impl PyExact { /// Given reference must be exact type of payload T #[inline(always)] pub unsafe fn ref_unchecked(r: &Py) -> &Self { - &*(r as *const _ as *const Self) + unsafe { &*(r as *const _ as *const Self) } } } @@ -294,7 +294,7 @@ impl PyAtomicRef { pub unsafe fn swap(&self, pyref: PyRef) -> PyRef { let py = PyRef::leak(pyref) as *const Py as *mut _; let old = Radium::swap(&self.inner, py, Ordering::AcqRel); - PyRef::from_raw(old.cast()) + unsafe { PyRef::from_raw(old.cast()) } } pub fn swap_to_temporary_refs(&self, pyref: PyRef, vm: &VirtualMachine) { @@ -380,7 +380,7 @@ impl PyAtomicRef { pub unsafe fn swap(&self, obj: PyObjectRef) -> PyObjectRef { let obj = obj.into_raw(); let old = Radium::swap(&self.inner, obj as *mut _, Ordering::AcqRel); - PyObjectRef::from_raw(old as _) + unsafe { PyObjectRef::from_raw(old as _) } } pub fn swap_to_temporary_refs(&self, obj: PyObjectRef, vm: &VirtualMachine) { @@ -422,9 +422,11 @@ impl PyAtomicRef> { pub unsafe fn swap(&self, obj: Option) -> Option { let val = obj.map(|x| x.into_raw() as *mut _).unwrap_or(null_mut()); let old = Radium::swap(&self.inner, val, Ordering::AcqRel); - old.cast::() - .as_ref() - .map(|x| PyObjectRef::from_raw(x)) + unsafe { + old.cast::() + .as_ref() + .map(|x| PyObjectRef::from_raw(x)) + } } pub fn swap_to_temporary_refs(&self, obj: Option, vm: &VirtualMachine) { diff --git a/vm/src/protocol/buffer.rs b/vm/src/protocol/buffer.rs index 3783ccf4b6..0dfe3f0027 100644 --- a/vm/src/protocol/buffer.rs +++ b/vm/src/protocol/buffer.rs @@ -133,8 +133,11 @@ impl PyBuffer { // after this function, the owner should use forget() // or wrap PyBuffer in the ManaullyDrop to prevent drop() pub(crate) unsafe fn drop_without_release(&mut self) { - std::ptr::drop_in_place(&mut self.obj); - std::ptr::drop_in_place(&mut self.desc); + // SAFETY: requirements forwarded from caller + unsafe { + std::ptr::drop_in_place(&mut self.obj); + std::ptr::drop_in_place(&mut self.desc); + } } } diff --git a/vm/src/stdlib/posix.rs b/vm/src/stdlib/posix.rs index 1b843bab60..c913ca0e5c 100644 --- a/vm/src/stdlib/posix.rs +++ b/vm/src/stdlib/posix.rs @@ -2299,7 +2299,7 @@ pub mod module { #[cfg(target_os = "linux")] unsafe fn sys_getrandom(buf: *mut libc::c_void, buflen: usize, flags: u32) -> isize { - libc::syscall(libc::SYS_getrandom, buf, buflen, flags as usize) as _ + unsafe { libc::syscall(libc::SYS_getrandom, buf, buflen, flags as usize) as _ } } #[cfg(target_os = "linux")] diff --git a/vm/src/stdlib/time.rs b/vm/src/stdlib/time.rs index 5a493c3472..73915ab7dc 100644 --- a/vm/src/stdlib/time.rs +++ b/vm/src/stdlib/time.rs @@ -209,7 +209,9 @@ mod decl { use crate::builtins::tuple::IntoPyTuple; unsafe fn to_str(s: *const std::ffi::c_char) -> String { - std::ffi::CStr::from_ptr(s).to_string_lossy().into_owned() + unsafe { std::ffi::CStr::from_ptr(s) } + .to_string_lossy() + .into_owned() } unsafe { (to_str(super::c_tzname[0]), to_str(super::c_tzname[1])) }.into_pytuple(vm) } diff --git a/vm/src/vm/context.rs b/vm/src/vm/context.rs index f67035d0f3..9f031f01ce 100644 --- a/vm/src/vm/context.rs +++ b/vm/src/vm/context.rs @@ -62,7 +62,7 @@ macro_rules! declare_const_name { impl ConstName { unsafe fn new(pool: &StringPool, typ: &PyTypeRef) -> Self { Self { - $($name: pool.intern(stringify!($name), typ.clone()),)* + $($name: unsafe { pool.intern(stringify!($name), typ.clone()) },)* } } } From 0a8b0406f57542e0afb31096ddb0301e8ac9131e Mon Sep 17 00:00:00 2001 From: Noa Date: Mon, 24 Feb 2025 21:11:25 -0600 Subject: [PATCH 039/295] Enable missing_unsafe_on_extern lint --- Cargo.toml | 2 ++ common/src/crt_fd.rs | 4 ++-- common/src/fileutils.rs | 2 +- common/src/macros.rs | 2 +- common/src/os.rs | 4 ++-- src/lib.rs | 2 +- stdlib/src/locale.rs | 2 +- stdlib/src/select.rs | 2 +- vm/src/stdlib/msvcrt.rs | 6 +++--- vm/src/stdlib/nt.rs | 4 ++-- vm/src/stdlib/os.rs | 2 +- vm/src/stdlib/posix.rs | 4 ++-- vm/src/stdlib/signal.rs | 2 +- vm/src/stdlib/time.rs | 2 +- 14 files changed, 21 insertions(+), 19 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 7ae2ea202e..a667b737f8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -190,6 +190,8 @@ wasm-bindgen = "0.2.100" [workspace.lints.rust] unsafe_code = "allow" unsafe_op_in_unsafe_fn = "deny" +missing_unsafe_on_extern = "deny" +unsafe_attr_outside_unsafe = "deny" [workspace.lints.clippy] perf = "warn" diff --git a/common/src/crt_fd.rs b/common/src/crt_fd.rs index 14f61b8059..64d4df98a5 100644 --- a/common/src/crt_fd.rs +++ b/common/src/crt_fd.rs @@ -6,7 +6,7 @@ use std::{cmp, ffi, io}; #[cfg(windows)] use libc::commit as fsync; #[cfg(windows)] -extern "C" { +unsafe extern "C" { #[link_name = "_chsize_s"] fn ftruncate(fd: i32, len: i64) -> i32; } @@ -74,7 +74,7 @@ impl Fd { #[cfg(windows)] pub fn to_raw_handle(&self) -> io::Result { - extern "C" { + unsafe extern "C" { fn _get_osfhandle(fd: i32) -> libc::intptr_t; } let handle = unsafe { suppress_iph!(_get_osfhandle(self.0)) }; diff --git a/common/src/fileutils.rs b/common/src/fileutils.rs index dcb78675d8..8713e11229 100644 --- a/common/src/fileutils.rs +++ b/common/src/fileutils.rs @@ -94,7 +94,7 @@ pub mod windows { } } - extern "C" { + unsafe extern "C" { fn _get_osfhandle(fd: i32) -> libc::intptr_t; } diff --git a/common/src/macros.rs b/common/src/macros.rs index 318ab06986..08d00e592d 100644 --- a/common/src/macros.rs +++ b/common/src/macros.rs @@ -41,7 +41,7 @@ pub mod __macro_private { libc::uintptr_t, ); #[cfg(target_env = "msvc")] - extern "C" { + unsafe extern "C" { pub fn _set_thread_local_invalid_parameter_handler( pNew: InvalidParamHandler, ) -> InvalidParamHandler; diff --git a/common/src/os.rs b/common/src/os.rs index c16ec014c5..8a832270bc 100644 --- a/common/src/os.rs +++ b/common/src/os.rs @@ -23,7 +23,7 @@ pub fn last_os_error() -> io::Error { let err = io::Error::last_os_error(); // FIXME: probably not ideal, we need a bigger dichotomy between GetLastError and errno if err.raw_os_error() == Some(0) { - extern "C" { + unsafe extern "C" { fn _get_errno(pValue: *mut i32) -> i32; } let mut errno = 0; @@ -44,7 +44,7 @@ pub fn last_os_error() -> io::Error { pub fn last_posix_errno() -> i32 { let err = io::Error::last_os_error(); if err.raw_os_error() == Some(0) { - extern "C" { + unsafe extern "C" { fn _get_errno(pValue: *mut i32) -> i32; } let mut errno = 0; diff --git a/src/lib.rs b/src/lib.rs index 0e35a6ad83..9fa38b968c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -78,7 +78,7 @@ pub fn run(init: impl FnOnce(&mut VirtualMachine) + 'static) -> ExitCode { // don't translate newlines (\r\n <=> \n) #[cfg(windows)] { - extern "C" { + unsafe extern "C" { fn _setmode(fd: i32, flags: i32) -> i32; } unsafe { diff --git a/stdlib/src/locale.rs b/stdlib/src/locale.rs index 7390561af7..8e44508fb6 100644 --- a/stdlib/src/locale.rs +++ b/stdlib/src/locale.rs @@ -30,7 +30,7 @@ struct lconv { } #[cfg(windows)] -extern "C" { +unsafe extern "C" { fn localeconv() -> *mut lconv; } diff --git a/stdlib/src/select.rs b/stdlib/src/select.rs index dfc974943a..6554064d53 100644 --- a/stdlib/src/select.rs +++ b/stdlib/src/select.rs @@ -113,7 +113,7 @@ mod platform { set.__nfds = 0; } - extern "C" { + unsafe extern "C" { pub fn select( nfds: libc::c_int, readfds: *mut fd_set, diff --git a/vm/src/stdlib/msvcrt.rs b/vm/src/stdlib/msvcrt.rs index 03ddb44f22..2572494671 100644 --- a/vm/src/stdlib/msvcrt.rs +++ b/vm/src/stdlib/msvcrt.rs @@ -24,7 +24,7 @@ mod msvcrt { unsafe { suppress_iph!(_setmode(fd, libc::O_BINARY)) }; } - extern "C" { + unsafe extern "C" { fn _getch() -> i32; fn _getwch() -> u32; fn _getche() -> i32; @@ -70,7 +70,7 @@ mod msvcrt { Ok(()) } - extern "C" { + unsafe extern "C" { fn _setmode(fd: i32, flags: i32) -> i32; } @@ -84,7 +84,7 @@ mod msvcrt { } } - extern "C" { + unsafe extern "C" { fn _open_osfhandle(osfhandle: isize, flags: i32) -> i32; fn _get_osfhandle(fd: i32) -> libc::intptr_t; } diff --git a/vm/src/stdlib/nt.rs b/vm/src/stdlib/nt.rs index d2785f42dd..09d26ba279 100644 --- a/vm/src/stdlib/nt.rs +++ b/vm/src/stdlib/nt.rs @@ -116,7 +116,7 @@ pub(crate) mod module { // cwait is available on MSVC only (according to CPython) #[cfg(target_env = "msvc")] - extern "C" { + unsafe extern "C" { fn _cwait(termstat: *mut i32, procHandle: intptr_t, action: i32) -> intptr_t; } @@ -194,7 +194,7 @@ pub(crate) mod module { } #[cfg(target_env = "msvc")] - extern "C" { + unsafe extern "C" { fn _wexecv(cmdname: *const u16, argv: *const *const u16) -> intptr_t; } diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index e07dca82ef..61b54f18fd 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -966,7 +966,7 @@ pub(super) mod _os { #[pyfunction] fn abort() { - extern "C" { + unsafe extern "C" { fn abort(); } unsafe { abort() } diff --git a/vm/src/stdlib/posix.rs b/vm/src/stdlib/posix.rs index c913ca0e5c..77ebf9e620 100644 --- a/vm/src/stdlib/posix.rs +++ b/vm/src/stdlib/posix.rs @@ -971,7 +971,7 @@ pub mod module { #[cfg(any(target_os = "macos", target_os = "freebsd", target_os = "netbsd",))] #[pyfunction] fn lchmod(path: OsPath, mode: u32, vm: &VirtualMachine) -> PyResult<()> { - extern "C" { + unsafe extern "C" { fn lchmod(path: *const libc::c_char, mode: libc::mode_t) -> libc::c_int; } let c_path = path.clone().into_cstring(vm)?; @@ -1605,7 +1605,7 @@ pub mod module { // from libstd: // https://github.com/rust-lang/rust/blob/daecab3a784f28082df90cebb204998051f3557d/src/libstd/sys/unix/fs.rs#L1251 #[cfg(target_os = "macos")] - extern "C" { + unsafe extern "C" { fn fcopyfile( in_fd: libc::c_int, out_fd: libc::c_int, diff --git a/vm/src/stdlib/signal.rs b/vm/src/stdlib/signal.rs index 8c5db53d17..da7c934552 100644 --- a/vm/src/stdlib/signal.rs +++ b/vm/src/stdlib/signal.rs @@ -78,7 +78,7 @@ pub(crate) mod _signal { pub const SIG_ERR: sighandler_t = -1 as _; #[cfg(all(unix, not(target_os = "redox")))] - extern "C" { + unsafe extern "C" { fn siginterrupt(sig: i32, flag: i32) -> i32; } diff --git a/vm/src/stdlib/time.rs b/vm/src/stdlib/time.rs index 73915ab7dc..3c87550f93 100644 --- a/vm/src/stdlib/time.rs +++ b/vm/src/stdlib/time.rs @@ -17,7 +17,7 @@ pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { #[cfg(not(target_env = "msvc"))] #[cfg(not(target_arch = "wasm32"))] -extern "C" { +unsafe extern "C" { #[cfg(not(target_os = "freebsd"))] #[link_name = "daylight"] static c_daylight: std::ffi::c_int; From 92e02a7f798a62200766701ba7b3d76b072b7525 Mon Sep 17 00:00:00 2001 From: Noa Date: Thu, 20 Feb 2025 18:22:13 -0600 Subject: [PATCH 040/295] Make PyObjectRef::{from,into}_raw() use NonNull --- stdlib/src/sqlite.rs | 8 ++++---- vm/src/object/core.rs | 12 +++++------- vm/src/object/ext.rs | 28 +++++++++++++++++----------- 3 files changed, 26 insertions(+), 22 deletions(-) diff --git a/stdlib/src/sqlite.rs b/stdlib/src/sqlite.rs index 1d2c6928e7..9225ccbc50 100644 --- a/stdlib/src/sqlite.rs +++ b/stdlib/src/sqlite.rs @@ -76,7 +76,7 @@ mod _sqlite { ffi::{c_int, c_longlong, c_uint, c_void, CStr}, fmt::Debug, ops::Deref, - ptr::{null, null_mut}, + ptr::{null, null_mut, NonNull}, thread::ThreadId, }; @@ -381,7 +381,7 @@ mod _sqlite { } struct CallbackData { - obj: *const PyObject, + obj: NonNull, vm: *const VirtualMachine, } @@ -394,7 +394,7 @@ mod _sqlite { } fn retrieve(&self) -> (&PyObject, &VirtualMachine) { - unsafe { (&*self.obj, &*self.vm) } + unsafe { (self.obj.as_ref(), &*self.vm) } } unsafe extern "C" fn destructor(data: *mut c_void) { @@ -439,7 +439,7 @@ mod _sqlite { let instance = context.aggregate_context::<*const PyObject>(); if unsafe { (*instance).is_null() } { match cls.call((), vm) { - Ok(obj) => unsafe { *instance = obj.into_raw() }, + Ok(obj) => unsafe { *instance = obj.into_raw().as_ptr() }, Err(exc) => { return context.result_exception( vm, diff --git a/vm/src/object/core.rs b/vm/src/object/core.rs index f8a5ceef93..61c31a02e3 100644 --- a/vm/src/object/core.rs +++ b/vm/src/object/core.rs @@ -354,7 +354,7 @@ impl PyWeak { if !obj_ptr.as_ref().0.ref_count.safe_inc() { return None; } - Some(PyObjectRef::from_raw(obj_ptr.as_ptr())) + Some(PyObjectRef::from_raw(obj_ptr)) } } @@ -508,8 +508,8 @@ impl ToOwned for PyObject { impl PyObjectRef { #[inline(always)] - pub fn into_raw(self) -> *const PyObject { - let ptr = self.as_raw(); + pub fn into_raw(self) -> NonNull { + let ptr = self.ptr; std::mem::forget(self); ptr } @@ -520,10 +520,8 @@ impl PyObjectRef { /// dropped more than once due to mishandling the reference count by calling this function /// too many times. #[inline(always)] - pub unsafe fn from_raw(ptr: *const PyObject) -> Self { - Self { - ptr: unsafe { NonNull::new_unchecked(ptr as *mut PyObject) }, - } + pub unsafe fn from_raw(ptr: NonNull) -> Self { + Self { ptr } } /// Attempt to downcast this reference to a subclass. diff --git a/vm/src/object/ext.rs b/vm/src/object/ext.rs index 59650443b0..129a6a79bb 100644 --- a/vm/src/object/ext.rs +++ b/vm/src/object/ext.rs @@ -12,7 +12,13 @@ use crate::{ vm::Context, VirtualMachine, }; -use std::{borrow::Borrow, fmt, marker::PhantomData, ops::Deref, ptr::null_mut}; +use std::{ + borrow::Borrow, + fmt, + marker::PhantomData, + ops::Deref, + ptr::{null_mut, NonNull}, +}; /* Python objects and references. @@ -352,7 +358,7 @@ impl From for PyAtomicRef { fn from(obj: PyObjectRef) -> Self { let obj = obj.into_raw(); Self { - inner: Radium::new(obj as *mut _), + inner: Radium::new(obj.cast().as_ptr()), _phantom: Default::default(), } } @@ -379,8 +385,8 @@ impl PyAtomicRef { #[must_use] pub unsafe fn swap(&self, obj: PyObjectRef) -> PyObjectRef { let obj = obj.into_raw(); - let old = Radium::swap(&self.inner, obj as *mut _, Ordering::AcqRel); - unsafe { PyObjectRef::from_raw(old as _) } + let old = Radium::swap(&self.inner, obj.cast().as_ptr(), Ordering::AcqRel); + unsafe { PyObjectRef::from_raw(NonNull::new_unchecked(old.cast())) } } pub fn swap_to_temporary_refs(&self, obj: PyObjectRef, vm: &VirtualMachine) { @@ -393,7 +399,9 @@ impl PyAtomicRef { impl From> for PyAtomicRef> { fn from(obj: Option) -> Self { - let val = obj.map(|x| x.into_raw() as *mut _).unwrap_or(null_mut()); + let val = obj + .map(|x| x.into_raw().as_ptr().cast()) + .unwrap_or(null_mut()); Self { inner: Radium::new(val), _phantom: Default::default(), @@ -420,13 +428,11 @@ impl PyAtomicRef> { /// until no more reference can be used via PyAtomicRef::deref() #[must_use] pub unsafe fn swap(&self, obj: Option) -> Option { - let val = obj.map(|x| x.into_raw() as *mut _).unwrap_or(null_mut()); + let val = obj + .map(|x| x.into_raw().as_ptr().cast()) + .unwrap_or(null_mut()); let old = Radium::swap(&self.inner, val, Ordering::AcqRel); - unsafe { - old.cast::() - .as_ref() - .map(|x| PyObjectRef::from_raw(x)) - } + unsafe { NonNull::new(old.cast::()).map(|x| PyObjectRef::from_raw(x)) } } pub fn swap_to_temporary_refs(&self, obj: Option, vm: &VirtualMachine) { From 4881f61be647f0b30e207bca9225f4d537d79dc8 Mon Sep 17 00:00:00 2001 From: Noa Date: Mon, 24 Feb 2025 23:02:50 -0600 Subject: [PATCH 041/295] Mark env::{set,remove}_var() unsafe --- Cargo.toml | 1 + vm/src/stdlib/os.rs | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index a667b737f8..19de548286 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -192,6 +192,7 @@ unsafe_code = "allow" unsafe_op_in_unsafe_fn = "deny" missing_unsafe_on_extern = "deny" unsafe_attr_outside_unsafe = "deny" +deprecated_safe_2024 = "deny" [workspace.lints.clippy] perf = "warn" diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 61b54f18fd..380887656b 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -401,7 +401,8 @@ pub(super) mod _os { } let key = super::bytes_as_osstr(key, vm)?; let value = super::bytes_as_osstr(value, vm)?; - env::set_var(key, value); + // SAFETY: requirements forwarded from the caller + unsafe { env::set_var(key, value) }; Ok(()) } @@ -421,7 +422,8 @@ pub(super) mod _os { )); } let key = super::bytes_as_osstr(key, vm)?; - env::remove_var(key); + // SAFETY: requirements forwarded from the caller + unsafe { env::remove_var(key) }; Ok(()) } From 085ac8843892da5e66c4f7e467e58c717422d6cb Mon Sep 17 00:00:00 2001 From: Noa Date: Thu, 20 Feb 2025 22:47:26 -0600 Subject: [PATCH 042/295] Use non-env-var methods from openssl_probe --- stdlib/Cargo.toml | 4 +- stdlib/build.rs | 6 +++ stdlib/src/ssl.rs | 132 +++++++++++++++++++++++++++++----------------- vm/src/utils.rs | 23 ++++---- 4 files changed, 103 insertions(+), 62 deletions(-) diff --git a/stdlib/Cargo.toml b/stdlib/Cargo.toml index b5cfab98b4..37e17e2fc8 100644 --- a/stdlib/Cargo.toml +++ b/stdlib/Cargo.toml @@ -17,8 +17,8 @@ threading = ["rustpython-common/threading", "rustpython-vm/threading"] zlib = ["libz-sys", "flate2/zlib"] bz2 = ["bzip2"] sqlite = ["dep:libsqlite3-sys"] -ssl = ["openssl", "openssl-sys", "foreign-types-shared"] -ssl-vendor = ["ssl", "openssl/vendored", "openssl-probe"] +ssl = ["openssl", "openssl-sys", "foreign-types-shared", "openssl-probe"] +ssl-vendor = ["ssl", "openssl/vendored"] [dependencies] # rustpython crates diff --git a/stdlib/build.rs b/stdlib/build.rs index 0109c633d7..3eb8a2d6b6 100644 --- a/stdlib/build.rs +++ b/stdlib/build.rs @@ -1,5 +1,6 @@ fn main() { println!(r#"cargo::rustc-check-cfg=cfg(osslconf, values("OPENSSL_NO_COMP"))"#); + println!(r#"cargo::rustc-check-cfg=cfg(openssl_vendored)"#); #[allow(clippy::unusual_byte_groupings)] let ossl_vers = [ @@ -36,4 +37,9 @@ fn main() { println!("cargo:rustc-cfg=osslconf=\"{conf}\""); } } + // it's possible for openssl-sys to link against the system openssl under certain conditions, + // so let the ssl module know to only perform a probe if we're actually vendored + if std::env::var("DEP_OPENSSL_VENDORED").is_ok_and(|s| s == "1") { + println!("cargo::rustc-cfg=openssl_vendored") + } } diff --git a/stdlib/src/ssl.rs b/stdlib/src/ssl.rs index eb757f9a71..e90f08f2ef 100644 --- a/stdlib/src/ssl.rs +++ b/stdlib/src/ssl.rs @@ -1,20 +1,32 @@ use crate::vm::{builtins::PyModule, PyRef, VirtualMachine}; +use openssl_probe::ProbeResult; pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { - // if openssl is vendored, it doesn't know the locations of system certificates - #[cfg(feature = "ssl-vendor")] - if let None | Some("0") = option_env!("OPENSSL_NO_VENDOR") { - // TODO: use openssl_probe::probe() instead - unsafe { openssl_probe::init_openssl_env_vars() }; - } - openssl::init(); + // if openssl is vendored, it doesn't know the locations + // of system certificates - cache the probe result now. + #[cfg(openssl_vendored)] + LazyLock::force(&PROBE); _ssl::make_module(vm) } +// define our own copy of ProbeResult so we can handle the vendor case +// easily, without having to have a bunch of cfgs +cfg_if::cfg_if! { + if #[cfg(openssl_vendored)] { + use std::sync::LazyLock; + static PROBE: LazyLock = LazyLock::new(openssl_probe::probe); + fn probe() -> &'static ProbeResult { &PROBE } + } else { + fn probe() -> &'static ProbeResult { + &ProbeResult { cert_file: None, cert_dir: None } + } + } +} + #[allow(non_upper_case_globals)] #[pymodule(with(ossl101, windows))] mod _ssl { - use super::bio; + use super::{bio, probe}; use crate::{ common::{ ascii, @@ -46,10 +58,12 @@ mod _ssl { x509::{self, X509Ref, X509}, }; use openssl_sys as sys; + use rustpython_vm::ospath::OsPath; use std::{ ffi::CStr, fmt, io::{Read, Write}, + path::Path, time::Instant, }; @@ -355,21 +369,43 @@ mod _ssl { .ok_or_else(|| vm.new_value_error(format!("unknown NID {nid}"))) } - #[pyfunction] - fn get_default_verify_paths() -> (String, String, String, String) { - macro_rules! convert { - ($f:ident) => { - CStr::from_ptr(sys::$f()).to_string_lossy().into_owned() - }; - } - unsafe { - ( - convert!(X509_get_default_cert_file_env), - convert!(X509_get_default_cert_file), - convert!(X509_get_default_cert_dir_env), - convert!(X509_get_default_cert_dir), - ) + fn get_cert_file_dir() -> (&'static Path, &'static Path) { + let probe = probe(); + // on windows, these should be utf8 strings + fn path_from_bytes(c: &CStr) -> &Path { + #[cfg(unix)] + { + use std::os::unix::ffi::OsStrExt; + std::ffi::OsStr::from_bytes(c.to_bytes()).as_ref() + } + #[cfg(windows)] + { + c.to_str().unwrap().as_ref() + } } + let cert_file = probe.cert_file.as_deref().unwrap_or_else(|| { + path_from_bytes(unsafe { CStr::from_ptr(sys::X509_get_default_cert_file()) }) + }); + let cert_dir = probe.cert_dir.as_deref().unwrap_or_else(|| { + path_from_bytes(unsafe { CStr::from_ptr(sys::X509_get_default_cert_dir()) }) + }); + (cert_file, cert_dir) + } + + #[pyfunction] + fn get_default_verify_paths( + vm: &VirtualMachine, + ) -> PyResult<(&'static str, PyObjectRef, &'static str, PyObjectRef)> { + let cert_file_env = unsafe { CStr::from_ptr(sys::X509_get_default_cert_file_env()) } + .to_str() + .unwrap(); + let cert_dir_env = unsafe { CStr::from_ptr(sys::X509_get_default_cert_dir_env()) } + .to_str() + .unwrap(); + let (cert_file, cert_dir) = get_cert_file_dir(); + let cert_file = OsPath::new_str(cert_file).filename(vm)?; + let cert_dir = OsPath::new_str(cert_dir).filename(vm)?; + Ok((cert_file_env, cert_file, cert_dir_env, cert_dir)) } #[pyfunction(name = "RAND_status")] @@ -591,9 +627,18 @@ mod _ssl { #[pymethod] fn set_default_verify_paths(&self, vm: &VirtualMachine) -> PyResult<()> { - self.builder() - .set_default_verify_paths() - .map_err(|e| convert_openssl_error(vm, e)) + cfg_if::cfg_if! { + if #[cfg(openssl_vendored)] { + let (cert_file, cert_dir) = get_cert_file_dir(); + self.builder() + .load_verify_locations(Some(cert_file), Some(cert_dir)) + .map_err(|e| convert_openssl_error(vm, e)) + } else { + self.builder() + .set_default_verify_paths() + .map_err(|e| convert_openssl_error(vm, e)) + } + } } #[pymethod] @@ -640,6 +685,12 @@ mod _ssl { vm.new_type_error("cafile, capath and cadata cannot be all omitted".to_owned()) ); } + if let Some(cafile) = &args.cafile { + cafile.ensure_no_nul(vm)? + } + if let Some(capath) = &args.capath { + capath.ensure_no_nul(vm)? + } #[cold] fn invalid_cadata(vm: &VirtualMachine) -> PyBaseExceptionRef { @@ -648,6 +699,8 @@ mod _ssl { ) } + let mut ctx = self.builder(); + // validate cadata type and load cadata if let Some(cadata) = args.cadata { let certs = match cadata { @@ -660,7 +713,6 @@ mod _ssl { Either::B(b) => b.with_ref(x509_stack_from_der), }; let certs = certs.map_err(|e| convert_openssl_error(vm, e))?; - let mut ctx = self.builder(); let store = ctx.cert_store_mut(); for cert in certs { store @@ -670,29 +722,11 @@ mod _ssl { } if args.cafile.is_some() || args.capath.is_some() { - let cafile = args.cafile.map(|s| s.to_cstring(vm)).transpose()?; - let capath = args.capath.map(|s| s.to_cstring(vm)).transpose()?; - let ret = unsafe { - let ctx = self.ctx.write(); - sys::SSL_CTX_load_verify_locations( - ctx.as_ptr(), - cafile - .as_ref() - .map_or_else(std::ptr::null, |cs| cs.as_ptr()), - capath - .as_ref() - .map_or_else(std::ptr::null, |cs| cs.as_ptr()), - ) - }; - if ret != 1 { - let errno = crate::common::os::last_posix_errno(); - let err = if errno != 0 { - crate::vm::stdlib::os::errno_err(vm) - } else { - convert_openssl_error(vm, ErrorStack::get()) - }; - return Err(err); - } + ctx.load_verify_locations( + args.cafile.as_ref().map(|s| s.as_str().as_ref()), + args.capath.as_ref().map(|s| s.as_str().as_ref()), + ) + .map_err(|e| convert_openssl_error(vm, e))?; } Ok(()) diff --git a/vm/src/utils.rs b/vm/src/utils.rs index 2c5ff79d3f..c7ab7148d1 100644 --- a/vm/src/utils.rs +++ b/vm/src/utils.rs @@ -1,6 +1,7 @@ use crate::{ builtins::PyStr, convert::{ToPyException, ToPyObject}, + exceptions::cstring_error, PyObjectRef, PyResult, VirtualMachine, }; @@ -17,22 +18,22 @@ impl ToPyObject for std::convert::Infallible { } } -pub trait ToCString { - fn to_cstring(&self, vm: &VirtualMachine) -> PyResult; -} - -impl ToCString for &str { - fn to_cstring(&self, vm: &VirtualMachine) -> PyResult { - std::ffi::CString::new(*self).map_err(|err| err.to_pyexception(vm)) - } -} - -impl ToCString for PyStr { +pub trait ToCString: AsRef { fn to_cstring(&self, vm: &VirtualMachine) -> PyResult { std::ffi::CString::new(self.as_ref()).map_err(|err| err.to_pyexception(vm)) } + fn ensure_no_nul(&self, vm: &VirtualMachine) -> PyResult<()> { + if self.as_ref().as_bytes().contains(&b'\0') { + Err(cstring_error(vm)) + } else { + Ok(()) + } + } } +impl ToCString for &str {} +impl ToCString for PyStr {} + pub(crate) fn collection_repr<'a, I>( class_name: Option<&str>, prefix: &str, From 864e8598f82ceca9915bd55fa05d273f11d2d6b8 Mon Sep 17 00:00:00 2001 From: Noa Date: Mon, 24 Feb 2025 23:12:30 -0600 Subject: [PATCH 043/295] Enable rust2024-incompatible pat and keyword-ident lints --- Cargo.toml | 4 ++++ common/src/int.rs | 2 +- compiler/codegen/src/compile.rs | 8 ++++---- derive-impl/src/pytraverse.rs | 2 +- derive-impl/src/util.rs | 2 +- jit/src/lib.rs | 6 +++--- stdlib/src/csv.rs | 20 +++++++------------- stdlib/src/json/machinery.rs | 2 +- stdlib/src/mmap.rs | 4 ++-- stdlib/src/socket.rs | 2 +- stdlib/src/syslog.rs | 2 +- vm/src/builtins/iter.rs | 2 +- vm/src/builtins/mappingproxy.rs | 4 ++-- vm/src/coroutine.rs | 32 ++++++++++++++++---------------- vm/src/frame.rs | 18 +++++++++--------- vm/src/function/argument.rs | 2 +- vm/src/stdlib/ast.rs | 4 ++-- vm/src/stdlib/ctypes/base.rs | 2 +- vm/src/stdlib/os.rs | 4 ++-- wasm/lib/src/vm_class.rs | 4 ++-- 20 files changed, 62 insertions(+), 64 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 19de548286..a649a7ed0f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -190,9 +190,13 @@ wasm-bindgen = "0.2.100" [workspace.lints.rust] unsafe_code = "allow" unsafe_op_in_unsafe_fn = "deny" + +# rust_2024_compatibility missing_unsafe_on_extern = "deny" unsafe_attr_outside_unsafe = "deny" deprecated_safe_2024 = "deny" +rust_2024_incompatible_pat = "deny" +keyword_idents_2024 = "deny" [workspace.lints.clippy] perf = "warn" diff --git a/common/src/int.rs b/common/src/int.rs index ca449ac708..00b5231dff 100644 --- a/common/src/int.rs +++ b/common/src/int.rs @@ -50,7 +50,7 @@ pub fn bytes_to_int(lit: &[u8], mut base: u32) -> Option { base = parsed; true } else { - if let [_first, ref others @ .., last] = lit { + if let [_first, others @ .., last] = lit { let is_zero = others.iter().all(|&c| c == b'0' || c == b'_') && *last == b'0'; if !is_zero { diff --git a/compiler/codegen/src/compile.rs b/compiler/codegen/src/compile.rs index d650e998ea..7f8696baf2 100644 --- a/compiler/codegen/src/compile.rs +++ b/compiler/codegen/src/compile.rs @@ -3302,13 +3302,13 @@ impl Compiler { elt, generators, .. }) => { Self::contains_await(elt) - || generators.iter().any(|gen| Self::contains_await(&gen.iter)) + || generators.iter().any(|jen| Self::contains_await(&jen.iter)) } Expr::SetComp(located_ast::ExprSetComp { elt, generators, .. }) => { Self::contains_await(elt) - || generators.iter().any(|gen| Self::contains_await(&gen.iter)) + || generators.iter().any(|jen| Self::contains_await(&jen.iter)) } Expr::DictComp(located_ast::ExprDictComp { key, @@ -3318,13 +3318,13 @@ impl Compiler { }) => { Self::contains_await(key) || Self::contains_await(value) - || generators.iter().any(|gen| Self::contains_await(&gen.iter)) + || generators.iter().any(|jen| Self::contains_await(&jen.iter)) } Expr::GeneratorExp(located_ast::ExprGeneratorExp { elt, generators, .. }) => { Self::contains_await(elt) - || generators.iter().any(|gen| Self::contains_await(&gen.iter)) + || generators.iter().any(|jen| Self::contains_await(&jen.iter)) } Expr::Starred(expr) => Self::contains_await(&expr.value), Expr::IfExp(located_ast::ExprIfExp { diff --git a/derive-impl/src/pytraverse.rs b/derive-impl/src/pytraverse.rs index d2fe2dac81..728722b83a 100644 --- a/derive-impl/src/pytraverse.rs +++ b/derive-impl/src/pytraverse.rs @@ -70,7 +70,7 @@ fn gen_trace_code(item: &mut DeriveInput) -> Result { syn::Data::Struct(s) => { let fields = &mut s.fields; match fields { - syn::Fields::Named(ref mut fields) => { + syn::Fields::Named(fields) => { let res: Vec = fields .named .iter_mut() diff --git a/derive-impl/src/util.rs b/derive-impl/src/util.rs index bd568032c3..cfe54a5c72 100644 --- a/derive-impl/src/util.rs +++ b/derive-impl/src/util.rs @@ -529,7 +529,7 @@ impl AttributeExt for Attribute { fn try_remove_name(&mut self, item_name: &str) -> Result> { self.try_meta_mut(|meta| { let nested = match meta { - Meta::List(MetaList { ref mut nested, .. }) => Ok(nested), + Meta::List(MetaList { nested, .. }) => Ok(nested), other => Err(syn::Error::new( other.span(), format!( diff --git a/jit/src/lib.rs b/jit/src/lib.rs index 78619eaf62..37f1f2a3dd 100644 --- a/jit/src/lib.rs +++ b/jit/src/lib.rs @@ -215,9 +215,9 @@ pub enum AbiValue { impl AbiValue { fn to_libffi_arg(&self) -> libffi::middle::Arg { match self { - AbiValue::Int(ref i) => libffi::middle::Arg::new(i), - AbiValue::Float(ref f) => libffi::middle::Arg::new(f), - AbiValue::Bool(ref b) => libffi::middle::Arg::new(b), + AbiValue::Int(i) => libffi::middle::Arg::new(i), + AbiValue::Float(f) => libffi::middle::Arg::new(f), + AbiValue::Bool(b) => libffi::middle::Arg::new(b), } } } diff --git a/stdlib/src/csv.rs b/stdlib/src/csv.rs index 2dd7f2ae12..7b3448e6b5 100644 --- a/stdlib/src/csv.rs +++ b/stdlib/src/csv.rs @@ -272,12 +272,12 @@ mod _csv { let Some(name) = name.payload_if_subclass::(vm) else { return Err(vm.new_type_error("argument 0 must be a string".to_string())); }; - let mut dialect = match dialect { + let dialect = match dialect { OptionalArg::Present(d) => PyDialect::try_from_object(vm, d) .map_err(|_| vm.new_type_error("argument 1 must be a dialect object".to_owned()))?, OptionalArg::Missing => opts.result(vm)?, }; - opts.update_pydialect(&mut dialect); + let dialect = opts.update_pydialect(dialect); GLOBAL_HASHMAP .lock() .insert(name.as_str().to_owned(), dialect); @@ -665,7 +665,7 @@ mod _csv { } impl FormatOptions { - fn update_pydialect<'b>(&self, res: &'b mut PyDialect) -> &'b mut PyDialect { + fn update_pydialect(&self, mut res: PyDialect) -> PyDialect { macro_rules! check_and_fill { ($res:ident, $e:ident) => {{ if let Some(t) = self.$e { @@ -699,24 +699,18 @@ mod _csv { DialectItem::Str(name) => { let g = GLOBAL_HASHMAP.lock(); if let Some(dialect) = g.get(name) { - let mut dialect = *dialect; - self.update_pydialect(&mut dialect); - Ok(dialect) + Ok(self.update_pydialect(*dialect)) } else { Err(new_csv_error(vm, format!("{} is not registed.", name))) } // TODO // Maybe need to update the obj from HashMap } - DialectItem::Obj(mut o) => { - self.update_pydialect(&mut o); - Ok(o) - } + DialectItem::Obj(o) => Ok(self.update_pydialect(*o)), DialectItem::None => { let g = GLOBAL_HASHMAP.lock(); - let mut res = *g.get("excel").unwrap(); - self.update_pydialect(&mut res); - Ok(res) + let res = *g.get("excel").unwrap(); + Ok(self.update_pydialect(res)) } } } diff --git a/stdlib/src/json/machinery.rs b/stdlib/src/json/machinery.rs index fc6b530866..0614314f4f 100644 --- a/stdlib/src/json/machinery.rs +++ b/stdlib/src/json/machinery.rs @@ -135,7 +135,7 @@ pub fn scanstring<'a>( }; let unterminated_err = || DecodeError::new("Unterminated string starting at", end - 1); let mut chars = s.char_indices().enumerate().skip(end).peekable(); - let (_, (mut chunk_start, _)) = chars.peek().ok_or_else(unterminated_err)?; + let &(_, (mut chunk_start, _)) = chars.peek().ok_or_else(unterminated_err)?; while let Some((char_i, (byte_i, c))) = chars.next() { match c { '"' => { diff --git a/stdlib/src/mmap.rs b/stdlib/src/mmap.rs index 8b657532c4..aed665222e 100644 --- a/stdlib/src/mmap.rs +++ b/stdlib/src/mmap.rs @@ -497,8 +497,8 @@ mod mmap { fn as_bytes(&self) -> BorrowedValue<[u8]> { PyMutexGuard::map_immutable(self.mmap.lock(), |m| { match m.as_ref().expect("mmap closed or invalid") { - MmapObj::Read(ref mmap) => &mmap[..], - MmapObj::Write(ref mmap) => &mmap[..], + MmapObj::Read(mmap) => &mmap[..], + MmapObj::Write(mmap) => &mmap[..], } }) .into() diff --git a/stdlib/src/socket.rs b/stdlib/src/socket.rs index 4172c5bb35..95f03ee4f8 100644 --- a/stdlib/src/socket.rs +++ b/stdlib/src/socket.rs @@ -1924,7 +1924,7 @@ mod _socket { let host = opts.host.as_ref().map(|s| s.as_str()); let port = opts.port.as_ref().map(|p| -> std::borrow::Cow { match p { - Either::A(ref s) => s.as_str().into(), + Either::A(s) => s.as_str().into(), Either::B(i) => i.to_string().into(), } }); diff --git a/stdlib/src/syslog.rs b/stdlib/src/syslog.rs index 9879f3ffaf..5fc6e30a56 100644 --- a/stdlib/src/syslog.rs +++ b/stdlib/src/syslog.rs @@ -49,7 +49,7 @@ mod syslog { impl GlobalIdent { fn as_ptr(&self) -> *const c_char { match self { - GlobalIdent::Explicit(ref cstr) => cstr.as_ptr(), + GlobalIdent::Explicit(cstr) => cstr.as_ptr(), GlobalIdent::Implicit => std::ptr::null(), } } diff --git a/vm/src/builtins/iter.rs b/vm/src/builtins/iter.rs index 2fed23a5c4..dbe4fc6597 100644 --- a/vm/src/builtins/iter.rs +++ b/vm/src/builtins/iter.rs @@ -28,7 +28,7 @@ pub enum IterStatus { unsafe impl Traverse for IterStatus { fn traverse(&self, tracer_fn: &mut TraverseFn) { match self { - IterStatus::Active(ref r) => r.traverse(tracer_fn), + IterStatus::Active(r) => r.traverse(tracer_fn), IterStatus::Exhausted => (), } } diff --git a/vm/src/builtins/mappingproxy.rs b/vm/src/builtins/mappingproxy.rs index 7b2386a39b..297ddd0a77 100644 --- a/vm/src/builtins/mappingproxy.rs +++ b/vm/src/builtins/mappingproxy.rs @@ -29,8 +29,8 @@ enum MappingProxyInner { unsafe impl Traverse for MappingProxyInner { fn traverse(&self, tracer_fn: &mut TraverseFn) { match self { - MappingProxyInner::Class(ref r) => r.traverse(tracer_fn), - MappingProxyInner::Mapping(ref arg) => arg.traverse(tracer_fn), + MappingProxyInner::Class(r) => r.traverse(tracer_fn), + MappingProxyInner::Mapping(arg) => arg.traverse(tracer_fn), } } } diff --git a/vm/src/coroutine.rs b/vm/src/coroutine.rs index 6d6b743310..5dd34a799d 100644 --- a/vm/src/coroutine.rs +++ b/vm/src/coroutine.rs @@ -36,8 +36,8 @@ pub struct Coro { exception: PyMutex>, // exc_state } -fn gen_name(gen: &PyObject, vm: &VirtualMachine) -> &'static str { - let typ = gen.class(); +fn gen_name(jen: &PyObject, vm: &VirtualMachine) -> &'static str { + let typ = jen.class(); if typ.is(vm.ctx.types.coroutine_type) { "coroutine" } else if typ.is(vm.ctx.types.async_generator) { @@ -67,7 +67,7 @@ impl Coro { fn run_with_context( &self, - gen: &PyObject, + jen: &PyObject, vm: &VirtualMachine, func: F, ) -> PyResult @@ -75,7 +75,7 @@ impl Coro { F: FnOnce(FrameRef) -> PyResult, { if self.running.compare_exchange(false, true).is_err() { - return Err(vm.new_value_error(format!("{} already executing", gen_name(gen, vm)))); + return Err(vm.new_value_error(format!("{} already executing", gen_name(jen, vm)))); } vm.push_exception(self.exception.lock().take()); @@ -90,7 +90,7 @@ impl Coro { pub fn send( &self, - gen: &PyObject, + jen: &PyObject, value: PyObjectRef, vm: &VirtualMachine, ) -> PyResult { @@ -102,22 +102,22 @@ impl Coro { } else if !vm.is_none(&value) { return Err(vm.new_type_error(format!( "can't send non-None value to a just-started {}", - gen_name(gen, vm), + gen_name(jen, vm), ))); } else { None }; - let result = self.run_with_context(gen, vm, |f| f.resume(value, vm)); + let result = self.run_with_context(jen, vm, |f| f.resume(value, vm)); self.maybe_close(&result); match result { Ok(exec_res) => Ok(exec_res.into_iter_return(vm)), Err(e) => { if e.fast_isinstance(vm.ctx.exceptions.stop_iteration) { let err = - vm.new_runtime_error(format!("{} raised StopIteration", gen_name(gen, vm))); + vm.new_runtime_error(format!("{} raised StopIteration", gen_name(jen, vm))); err.set_cause(Some(e)); Err(err) - } else if gen.class().is(vm.ctx.types.async_generator) + } else if jen.class().is(vm.ctx.types.async_generator) && e.fast_isinstance(vm.ctx.exceptions.stop_async_iteration) { let err = vm @@ -132,7 +132,7 @@ impl Coro { } pub fn throw( &self, - gen: &PyObject, + jen: &PyObject, exc_type: PyObjectRef, exc_val: PyObjectRef, exc_tb: PyObjectRef, @@ -141,16 +141,16 @@ impl Coro { if self.closed.load() { return Err(vm.normalize_exception(exc_type, exc_val, exc_tb)?); } - let result = self.run_with_context(gen, vm, |f| f.gen_throw(vm, exc_type, exc_val, exc_tb)); + let result = self.run_with_context(jen, vm, |f| f.gen_throw(vm, exc_type, exc_val, exc_tb)); self.maybe_close(&result); Ok(result?.into_iter_return(vm)) } - pub fn close(&self, gen: &PyObject, vm: &VirtualMachine) -> PyResult<()> { + pub fn close(&self, jen: &PyObject, vm: &VirtualMachine) -> PyResult<()> { if self.closed.load() { return Ok(()); } - let result = self.run_with_context(gen, vm, |f| { + let result = self.run_with_context(jen, vm, |f| { f.gen_throw( vm, vm.ctx.exceptions.generator_exit.to_owned().into(), @@ -161,7 +161,7 @@ impl Coro { self.closed.store(true); match result { Ok(ExecutionResult::Yield(_)) => { - Err(vm.new_runtime_error(format!("{} ignored GeneratorExit", gen_name(gen, vm)))) + Err(vm.new_runtime_error(format!("{} ignored GeneratorExit", gen_name(jen, vm)))) } Err(e) if !is_gen_exit(&e, vm) => Err(e), _ => Ok(()), @@ -183,10 +183,10 @@ impl Coro { pub fn set_name(&self, name: PyStrRef) { *self.name.lock() = name; } - pub fn repr(&self, gen: &PyObject, id: usize, vm: &VirtualMachine) -> String { + pub fn repr(&self, jen: &PyObject, id: usize, vm: &VirtualMachine) -> String { format!( "<{} object {} at {:#x}>", - gen_name(gen, vm), + gen_name(jen, vm), self.name.lock(), id ) diff --git a/vm/src/frame.rs b/vm/src/frame.rs index fac6da8df5..afa5c708e7 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -426,19 +426,19 @@ impl ExecutingFrame<'_> { exc_val: PyObjectRef, exc_tb: PyObjectRef, ) -> PyResult { - if let Some(gen) = self.yield_from_target() { + if let Some(jen) = self.yield_from_target() { // borrow checker shenanigans - we only need to use exc_type/val/tb if the following // variable is Some - let thrower = if let Some(coro) = self.builtin_coro(gen) { + let thrower = if let Some(coro) = self.builtin_coro(jen) { Some(Either::A(coro)) } else { - vm.get_attribute_opt(gen.to_owned(), "throw")? + vm.get_attribute_opt(jen.to_owned(), "throw")? .map(Either::B) }; if let Some(thrower) = thrower { let ret = match thrower { Either::A(coro) => coro - .throw(gen, exc_type, exc_val, exc_tb, vm) + .throw(jen, exc_type, exc_val, exc_tb, vm) .to_pyresult(vm), // FIXME: Either::B(meth) => meth.call((exc_type, exc_val, exc_tb), vm), }; @@ -1568,16 +1568,16 @@ impl ExecutingFrame<'_> { fn _send( &self, - gen: &PyObject, + jen: &PyObject, val: PyObjectRef, vm: &VirtualMachine, ) -> PyResult { - match self.builtin_coro(gen) { - Some(coro) => coro.send(gen, val, vm), + match self.builtin_coro(jen) { + Some(coro) => coro.send(jen, val, vm), // FIXME: turn return type to PyResult then ExecutionResult will be simplified - None if vm.is_none(&val) => PyIter::new(gen).next(vm), + None if vm.is_none(&val) => PyIter::new(jen).next(vm), None => { - let meth = gen.get_attr("send", vm)?; + let meth = jen.get_attr("send", vm)?; PyIterReturn::from_pyresult(meth.call((val,), vm), vm) } } diff --git a/vm/src/function/argument.rs b/vm/src/function/argument.rs index bc60cfa253..b0e731ff57 100644 --- a/vm/src/function/argument.rs +++ b/vm/src/function/argument.rs @@ -497,7 +497,7 @@ where { fn traverse(&self, tracer_fn: &mut TraverseFn) { match self { - OptionalArg::Present(ref o) => o.traverse(tracer_fn), + OptionalArg::Present(o) => o.traverse(tracer_fn), OptionalArg::Missing => (), } } diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs index 8b081294b9..616d84368c 100644 --- a/vm/src/stdlib/ast.rs +++ b/vm/src/stdlib/ast.rs @@ -3,7 +3,7 @@ //! This module makes use of the parser logic, and translates all ast nodes //! into python ast.AST objects. -mod gen; +mod r#gen; use crate::{ builtins::{self, PyDict, PyModule, PyStrRef, PyType}, @@ -398,6 +398,6 @@ pub const PY_COMPILE_FLAGS_MASK: i32 = PY_COMPILE_FLAG_AST_ONLY pub fn make_module(vm: &VirtualMachine) -> PyRef { let module = _ast::make_module(vm); - gen::extend_module_nodes(vm, &module); + r#gen::extend_module_nodes(vm, &module); module } diff --git a/vm/src/stdlib/ctypes/base.rs b/vm/src/stdlib/ctypes/base.rs index 5d0316352c..a4147c62b2 100644 --- a/vm/src/stdlib/ctypes/base.rs +++ b/vm/src/stdlib/ctypes/base.rs @@ -176,7 +176,7 @@ impl Constructor for PyCSimple { let attributes = cls.get_attributes(); let _type_ = attributes .iter() - .find(|(&k, _)| k.to_object().str(vm).unwrap().to_string() == *"_type_") + .find(|(k, _)| k.to_object().str(vm).unwrap().to_string() == *"_type_") .unwrap() .1 .str(vm)? diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 380887656b..6a4233f3be 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -380,8 +380,8 @@ pub(super) mod _os { fn env_bytes_as_bytes(obj: &Either) -> &[u8] { match obj { - Either::A(ref s) => s.as_str().as_bytes(), - Either::B(ref b) => b.as_bytes(), + Either::A(s) => s.as_str().as_bytes(), + Either::B(b) => b.as_bytes(), } } diff --git a/wasm/lib/src/vm_class.rs b/wasm/lib/src/vm_class.rs index 0c62bb32e7..4ba5760511 100644 --- a/wasm/lib/src/vm_class.rs +++ b/wasm/lib/src/vm_class.rs @@ -233,7 +233,7 @@ impl WASMVirtualMachine { #[wasm_bindgen(js_name = addToScope)] pub fn add_to_scope(&self, name: String, value: JsValue) -> Result<(), JsValue> { - self.with_vm(move |vm, StoredVirtualMachine { ref scope, .. }| { + self.with_vm(move |vm, StoredVirtualMachine { scope, .. }| { let value = convert::js_to_py(vm, value); scope.globals.set_item(&name, value, vm).into_js(vm) })? @@ -335,7 +335,7 @@ impl WASMVirtualMachine { mode: Mode, source_path: Option, ) -> Result { - self.with_vm(|vm, StoredVirtualMachine { ref scope, .. }| { + self.with_vm(|vm, StoredVirtualMachine { scope, .. }| { let source_path = source_path.unwrap_or_else(|| "".to_owned()); let code = vm.compile(source, mode, source_path); let code = code.map_err(convert::syntax_err)?; From 235adafa0b6e50473e1a90f684e311903a72089b Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sun, 23 Feb 2025 19:20:51 -0800 Subject: [PATCH 044/295] tests Signed-off-by: Ashwin Naren --- extra_tests/snippets/stdlib_math.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/extra_tests/snippets/stdlib_math.py b/extra_tests/snippets/stdlib_math.py index 442bbc97a5..94d8c7347c 100644 --- a/extra_tests/snippets/stdlib_math.py +++ b/extra_tests/snippets/stdlib_math.py @@ -291,3 +291,5 @@ def assertAllNotClose(examples, *args, **kwargs): assert math.fmod(-3.0, NINF) == -3.0 assert math.fmod(0.0, 3.0) == 0.0 assert math.fmod(0.0, NINF) == 0.0 + +assert math.gamma(1) == 1.0 From 4468dcbe341fe24e3ad3b35bdf222fc1864eee23 Mon Sep 17 00:00:00 2001 From: Noa Date: Tue, 25 Feb 2025 20:36:26 -0600 Subject: [PATCH 045/295] Switch to libz-rs-sys for zlib implementation --- .github/workflows/ci.yaml | 2 +- .github/workflows/cron-ci.yaml | 4 +-- .github/workflows/release.yml | 2 +- Cargo.lock | 24 ++++++++------ Cargo.toml | 3 +- stdlib/Cargo.toml | 5 ++- stdlib/src/zlib.rs | 57 +++++----------------------------- 7 files changed, 29 insertions(+), 68 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e96c2f26a6..23f47b574c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -15,7 +15,7 @@ concurrency: cancel-in-progress: true env: - CARGO_ARGS: --no-default-features --features stdlib,zlib,importlib,encodings,sqlite,ssl + CARGO_ARGS: --no-default-features --features stdlib,importlib,encodings,sqlite,ssl # Skip additional tests on Windows. They are checked on Linux and MacOS. # test_glob: many failing tests # test_io: many failing tests diff --git a/.github/workflows/cron-ci.yaml b/.github/workflows/cron-ci.yaml index bfc758310e..4b8e701ced 100644 --- a/.github/workflows/cron-ci.yaml +++ b/.github/workflows/cron-ci.yaml @@ -6,7 +6,7 @@ on: name: Periodic checks/tasks env: - CARGO_ARGS: --no-default-features --features stdlib,zlib,importlib,encodings,ssl,jit + CARGO_ARGS: --no-default-features --features stdlib,importlib,encodings,ssl,jit PYTHON_VERSION: "3.13.1" jobs: @@ -24,7 +24,7 @@ jobs: python-version: ${{ env.PYTHON_VERSION }} - run: sudo apt-get update && sudo apt-get -y install lcov - name: Run cargo-llvm-cov with Rust tests. - run: cargo llvm-cov --no-report --workspace --exclude rustpython_wasm --verbose --no-default-features --features stdlib,zlib,importlib,encodings,ssl,jit + run: cargo llvm-cov --no-report --workspace --exclude rustpython_wasm --verbose --no-default-features --features stdlib,importlib,encodings,ssl,jit - name: Run cargo-llvm-cov with Python snippets. run: python scripts/cargo-llvm-cov.py continue-on-error: true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5554ed0a8b..99035d04fc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -16,7 +16,7 @@ permissions: contents: write env: - CARGO_ARGS: --no-default-features --features stdlib,zlib,importlib,encodings,sqlite,ssl + CARGO_ARGS: --no-default-features --features stdlib,importlib,encodings,sqlite,ssl jobs: build: diff --git a/Cargo.lock b/Cargo.lock index 9d486786ba..67e0813b7b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -720,12 +720,12 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.35" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c936bfdafb507ebbf50b8074c54fa31c5be9a1e7e5f467dd659697041407d07c" +checksum = "11faaf5a5236997af9848be0bef4db95824b1d534ebc64d0f0c6cf3e67bd38dc" dependencies = [ "crc32fast", - "libz-sys", + "libz-rs-sys", "miniz_oxide", ] @@ -1137,14 +1137,12 @@ dependencies = [ ] [[package]] -name = "libz-sys" -version = "1.1.21" +name = "libz-rs-sys" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df9b68e50e6e0b26f672573834882eb57759f6db9b3be2ea3c35c91188bb4eaa" +checksum = "902bc563b5d65ad9bba616b490842ef0651066a1a1dc3ce1087113ffcb873c8d" dependencies = [ - "cc", - "pkg-config", - "vcpkg", + "zlib-rs", ] [[package]] @@ -2210,7 +2208,7 @@ dependencies = [ "junction", "libc", "libsqlite3-sys", - "libz-sys", + "libz-rs-sys", "mac_address", "malachite-bigint", "md-5", @@ -3479,3 +3477,9 @@ dependencies = [ "quote", "syn 2.0.98", ] + +[[package]] +name = "zlib-rs" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b20717f0917c908dc63de2e44e97f1e6b126ca58d0e391cee86d504eb8fbd05" diff --git a/Cargo.toml b/Cargo.toml index a649a7ed0f..6c48714929 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ repository.workspace = true license.workspace = true [features] -default = ["threading", "stdlib", "zlib", "importlib"] +default = ["threading", "stdlib", "importlib"] importlib = ["rustpython-vm/importlib"] encodings = ["rustpython-vm/encodings"] stdlib = ["rustpython-stdlib", "rustpython-pylib", "encodings"] @@ -18,7 +18,6 @@ flame-it = ["rustpython-vm/flame-it", "flame", "flamescope"] freeze-stdlib = ["stdlib", "rustpython-vm/freeze-stdlib", "rustpython-pylib?/freeze-stdlib"] jit = ["rustpython-vm/jit"] threading = ["rustpython-vm/threading", "rustpython-stdlib/threading"] -zlib = ["stdlib", "rustpython-stdlib/zlib"] bz2 = ["stdlib", "rustpython-stdlib/bz2"] sqlite = ["rustpython-stdlib/sqlite"] ssl = ["rustpython-stdlib/ssl"] diff --git a/stdlib/Cargo.toml b/stdlib/Cargo.toml index 37e17e2fc8..1086dea090 100644 --- a/stdlib/Cargo.toml +++ b/stdlib/Cargo.toml @@ -14,7 +14,6 @@ license.workspace = true default = ["compiler"] compiler = ["rustpython-vm/compiler"] threading = ["rustpython-common/threading", "rustpython-vm/threading"] -zlib = ["libz-sys", "flate2/zlib"] bz2 = ["bzip2"] sqlite = ["dep:libsqlite3-sys"] ssl = ["openssl", "openssl-sys", "foreign-types-shared", "openssl-probe"] @@ -48,7 +47,6 @@ memchr = { workspace = true } base64 = "0.13.0" csv-core = "0.1.11" dyn-clone = "1.0.10" -libz-sys = { version = "1.1", default-features = false, optional = true } puruspe = "0.4.0" xml-rs = "0.8.14" @@ -81,7 +79,8 @@ ucd = "0.1.1" # compression adler32 = "1.2.0" crc32fast = "1.3.2" -flate2 = "1.0.28" +flate2 = { version = "1.1", default-features = false, features = ["zlib-rs"] } +libz-sys = { package = "libz-rs-sys", version = "0.4" } bzip2 = { version = "0.4", optional = true } # uuid diff --git a/stdlib/src/zlib.rs b/stdlib/src/zlib.rs index 83a7535c33..a4828552a7 100644 --- a/stdlib/src/zlib.rs +++ b/stdlib/src/zlib.rs @@ -9,6 +9,7 @@ mod zlib { common::lock::PyMutex, convert::TryFromBorrowedObject, function::{ArgBytesLike, ArgPrimitiveIndex, ArgSize, OptionalArg}, + types::Constructor, PyObject, PyPayload, PyResult, VirtualMachine, }; use adler32::RollingAdler32 as Adler32; @@ -19,35 +20,12 @@ mod zlib { }; use std::io::Write; - #[cfg(not(feature = "zlib"))] - mod constants { - pub const Z_NO_COMPRESSION: i32 = 0; - pub const Z_BEST_COMPRESSION: i32 = 9; - pub const Z_BEST_SPEED: i32 = 1; - pub const Z_DEFAULT_COMPRESSION: i32 = -1; - pub const Z_NO_FLUSH: i32 = 0; - pub const Z_PARTIAL_FLUSH: i32 = 1; - pub const Z_SYNC_FLUSH: i32 = 2; - pub const Z_FULL_FLUSH: i32 = 3; - // not sure what the value here means, but it's the only compression method zlibmodule - // supports, so it doesn't really matter - pub const Z_DEFLATED: i32 = 8; - } - #[cfg(feature = "zlib")] - use libz_sys as constants; - - #[pyattr] - use constants::{ - Z_BEST_COMPRESSION, Z_BEST_SPEED, Z_DEFAULT_COMPRESSION, Z_DEFLATED as DEFLATED, - Z_FULL_FLUSH, Z_NO_COMPRESSION, Z_NO_FLUSH, Z_PARTIAL_FLUSH, Z_SYNC_FLUSH, - }; - - #[cfg(feature = "zlib")] #[pyattr] use libz_sys::{ - Z_BLOCK, Z_DEFAULT_STRATEGY, Z_FILTERED, Z_FINISH, Z_FIXED, Z_HUFFMAN_ONLY, Z_RLE, Z_TREES, + Z_BEST_COMPRESSION, Z_BEST_SPEED, Z_BLOCK, Z_DEFAULT_COMPRESSION, Z_DEFAULT_STRATEGY, + Z_DEFLATED as DEFLATED, Z_FILTERED, Z_FINISH, Z_FIXED, Z_FULL_FLUSH, Z_HUFFMAN_ONLY, + Z_NO_COMPRESSION, Z_NO_FLUSH, Z_PARTIAL_FLUSH, Z_RLE, Z_SYNC_FLUSH, Z_TREES, }; - use rustpython_vm::types::Constructor; // copied from zlibmodule.c (commit 530f506ac91338) #[pyattr] @@ -119,11 +97,11 @@ mod zlib { header: bool, // [De]Compress::new_with_window_bits is only enabled for zlib; miniz_oxide doesn't // support wbits (yet?) - #[cfg(feature = "zlib")] wbits: u8, }, - #[cfg(feature = "zlib")] - Gzip { wbits: u8 }, + Gzip { + wbits: u8, + }, } impl InitOptions { @@ -131,12 +109,7 @@ mod zlib { let header = wbits > 0; let wbits = wbits.unsigned_abs(); match wbits { - 9..=15 => Ok(InitOptions::Standard { - header, - #[cfg(feature = "zlib")] - wbits, - }), - #[cfg(feature = "zlib")] + 9..=15 => Ok(InitOptions::Standard { header, wbits }), 25..=31 => Ok(InitOptions::Gzip { wbits: wbits - 16 }), _ => Err(vm.new_value_error("Invalid initialization option".to_owned())), } @@ -144,23 +117,15 @@ mod zlib { fn decompress(self) -> Decompress { match self { - #[cfg(not(feature = "zlib"))] - Self::Standard { header } => Decompress::new(header), - #[cfg(feature = "zlib")] Self::Standard { header, wbits } => Decompress::new_with_window_bits(header, wbits), - #[cfg(feature = "zlib")] Self::Gzip { wbits } => Decompress::new_gzip(wbits), } } fn compress(self, level: Compression) -> Compress { match self { - #[cfg(not(feature = "zlib"))] - Self::Standard { header } => Compress::new(level, header), - #[cfg(feature = "zlib")] Self::Standard { header, wbits } => { Compress::new_with_window_bits(level, header, wbits) } - #[cfg(feature = "zlib")] Self::Gzip { wbits } => Compress::new_gzip(level, wbits), } } @@ -264,7 +229,6 @@ mod zlib { struct DecompressobjArgs { #[pyarg(any, default = "ArgPrimitiveIndex { value: MAX_WBITS }")] wbits: ArgPrimitiveIndex, - #[cfg(feature = "zlib")] #[pyarg(any, optional)] _zdict: OptionalArg, } @@ -273,7 +237,6 @@ mod zlib { fn decompressobj(args: DecompressobjArgs, vm: &VirtualMachine) -> PyResult { #[allow(unused_mut)] let mut decompress = InitOptions::new(args.wbits.value, vm)?.decompress(); - #[cfg(feature = "zlib")] if let OptionalArg::Present(_dict) = args._zdict { // FIXME: always fails // dict.with_ref(|d| decompress.set_dictionary(d)); @@ -426,10 +389,8 @@ mod zlib { wbits: ArgPrimitiveIndex, #[pyarg(any, name = "_memLevel", default = "DEF_MEM_LEVEL")] _mem_level: u8, - #[cfg(feature = "zlib")] #[pyarg(any, default = "Z_DEFAULT_STRATEGY")] _strategy: i32, - #[cfg(feature = "zlib")] #[pyarg(any, optional)] zdict: Option, } @@ -439,7 +400,6 @@ mod zlib { let CompressobjArgs { level, wbits, - #[cfg(feature = "zlib")] zdict, .. } = args; @@ -447,7 +407,6 @@ mod zlib { level.ok_or_else(|| vm.new_value_error("invalid initialization option".to_owned()))?; #[allow(unused_mut)] let mut compress = InitOptions::new(wbits.value, vm)?.compress(level); - #[cfg(feature = "zlib")] if let Some(zdict) = zdict { zdict.with_ref(|zdict| compress.set_dictionary(zdict).unwrap()); } From 8c5602f2fb84aacad2495f6b296ef74ef9e164ac Mon Sep 17 00:00:00 2001 From: Noa Date: Tue, 25 Feb 2025 20:40:23 -0600 Subject: [PATCH 046/295] Fix a bunch of zlib tests --- Lib/test/test_zlib.py | 52 +--- derive-impl/src/pymodule.rs | 2 +- stdlib/src/zlib.rs | 594 ++++++++++++++++++++---------------- 3 files changed, 342 insertions(+), 306 deletions(-) diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py index 686131be74..0a75457ad8 100644 --- a/Lib/test/test_zlib.py +++ b/Lib/test/test_zlib.py @@ -20,18 +20,18 @@ 'requires Decompress.copy()') -# def _zlib_runtime_version_tuple(zlib_version=zlib.ZLIB_RUNTIME_VERSION): -# # Register "1.2.3" as "1.2.3.0" -# # or "1.2.0-linux","1.2.0.f","1.2.0.f-linux" -# v = zlib_version.split('-', 1)[0].split('.') -# if len(v) < 4: -# v.append('0') -# elif not v[-1].isnumeric(): -# v[-1] = '0' -# return tuple(map(int, v)) -# -# -# ZLIB_RUNTIME_VERSION_TUPLE = _zlib_runtime_version_tuple() +def _zlib_runtime_version_tuple(zlib_version=zlib.ZLIB_RUNTIME_VERSION): + # Register "1.2.3" as "1.2.3.0" + # or "1.2.0-linux","1.2.0.f","1.2.0.f-linux" + v = zlib_version.split('-', 1)[0].split('.') + if len(v) < 4: + v.append('0') + elif not v[-1].isnumeric(): + v[-1] = '0' + return tuple(map(int, v)) + + +ZLIB_RUNTIME_VERSION_TUPLE = _zlib_runtime_version_tuple() # bpo-46623: When a hardware accelerator is used (currently only on s390x), @@ -66,8 +66,6 @@ class VersionTestCase(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_library_version(self): # Test that the major version of the actual library in use matches the # major version that we were compiled against. We can't guarantee that @@ -282,8 +280,6 @@ def test_64bit_compress(self, size): class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure # Test compression object def test_pair(self): # straightforward compress/decompress objects @@ -307,8 +303,6 @@ def test_pair(self): self.assertIsInstance(dco.unconsumed_tail, bytes) self.assertIsInstance(dco.unused_data, bytes) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_keywords(self): level = 2 method = zlib.DEFLATED @@ -466,8 +460,6 @@ def test_decompressmaxlen(self, flush=False): def test_decompressmaxlenflush(self): self.test_decompressmaxlen(flush=True) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_maxlenmisc(self): # Misc tests of max_length dco = zlib.decompressobj() @@ -498,7 +490,7 @@ def test_clear_unconsumed_tail(self): ddata += dco.decompress(dco.unconsumed_tail) self.assertEqual(dco.unconsumed_tail, b"") - # TODO: RUSTPYTHON + # TODO: RUSTPYTHON: Z_BLOCK support in flate2 @unittest.expectedFailure def test_flushes(self): # Test flush() with the various options, using all the @@ -560,8 +552,6 @@ def test_empty_flush(self): dco = zlib.decompressobj() self.assertEqual(dco.flush(), b"") # Returns nothing - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_dictionary(self): h = HAMLET_SCENE # Build a simulated dictionary out of the words in HAMLET. @@ -578,8 +568,6 @@ def test_dictionary(self): dco = zlib.decompressobj() self.assertRaises(zlib.error, dco.decompress, cd) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_dictionary_streaming(self): # This simulates the reuse of a compressor object for compressing # several separate data streams. @@ -652,8 +640,6 @@ def test_decompress_unused_data(self): self.assertEqual(dco.unconsumed_tail, b'') self.assertEqual(dco.unused_data, remainder) - # TODO: RUSTPYTHON - @unittest.expectedFailure # issue27164 def test_decompress_raw_with_dictionary(self): zdict = b'abcdefghijklmnopqrstuvwxyz' @@ -829,7 +815,7 @@ def test_large_unconsumed_tail(self, size): finally: comp = uncomp = data = None - # TODO: RUSTPYTHON + # TODO: RUSTPYTHON: wbits=0 support in flate2 @unittest.expectedFailure def test_wbits(self): # wbits=0 only supported since zlib v1.2.3.5 @@ -997,8 +983,6 @@ def testDecompressUnusedData(self): self.assertEqual(text, self.TEXT) self.assertEqual(zlibd.unused_data, unused_data) - # TODO: RUSTPYTHON - @unittest.expectedFailure def testEOFError(self): zlibd = zlib._ZlibDecompressor() text = zlibd.decompress(self.DATA) @@ -1029,8 +1013,6 @@ def testPickle(self): with self.assertRaises(TypeError): pickle.dumps(zlib._ZlibDecompressor(), proto) - # TODO: RUSTPYTHON - @unittest.expectedFailure def testDecompressorChunksMaxsize(self): zlibd = zlib._ZlibDecompressor() max_length = 100 @@ -1062,8 +1044,6 @@ def testDecompressorChunksMaxsize(self): self.assertEqual(out, self.BIG_TEXT) self.assertEqual(zlibd.unused_data, b"") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_decompressor_inputbuf_1(self): # Test reusing input buffer after moving existing # contents to beginning @@ -1086,8 +1066,6 @@ def test_decompressor_inputbuf_1(self): out.append(zlibd.decompress(self.DATA[105:])) self.assertEqual(b''.join(out), self.TEXT) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_decompressor_inputbuf_2(self): # Test reusing input buffer by appending data at the # end right away @@ -1109,8 +1087,6 @@ def test_decompressor_inputbuf_2(self): out.append(zlibd.decompress(self.DATA[300:])) self.assertEqual(b''.join(out), self.TEXT) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_decompressor_inputbuf_3(self): # Test reusing input buffer after extending it diff --git a/derive-impl/src/pymodule.rs b/derive-impl/src/pymodule.rs index db65e24e21..188231a71f 100644 --- a/derive-impl/src/pymodule.rs +++ b/derive-impl/src/pymodule.rs @@ -728,7 +728,7 @@ impl ModuleItem for AttributeItem { ( quote_spanned! { ident.span() => { #let_obj - for name in [(#(#names,)*)] { + for name in [#(#names),*] { vm.__module_set_attr(module, vm.ctx.intern_str(name), obj.clone()).unwrap(); } }}, diff --git a/stdlib/src/zlib.rs b/stdlib/src/zlib.rs index a4828552a7..bcfe93aff5 100644 --- a/stdlib/src/zlib.rs +++ b/stdlib/src/zlib.rs @@ -5,7 +5,7 @@ pub(crate) use zlib::make_module; #[pymodule] mod zlib { use crate::vm::{ - builtins::{PyBaseExceptionRef, PyBytes, PyBytesRef, PyIntRef, PyTypeRef}, + builtins::{PyBaseExceptionRef, PyBytesRef, PyIntRef, PyTypeRef}, common::lock::PyMutex, convert::TryFromBorrowedObject, function::{ArgBytesLike, ArgPrimitiveIndex, ArgSize, OptionalArg}, @@ -13,7 +13,6 @@ mod zlib { PyObject, PyPayload, PyResult, VirtualMachine, }; use adler32::RollingAdler32 as Adler32; - use crossbeam_utils::atomic::AtomicCell; use flate2::{ write::ZlibEncoder, Compress, Compression, Decompress, FlushCompress, FlushDecompress, Status, @@ -27,6 +26,17 @@ mod zlib { Z_NO_COMPRESSION, Z_NO_FLUSH, Z_PARTIAL_FLUSH, Z_RLE, Z_SYNC_FLUSH, Z_TREES, }; + // we're statically linking libz-rs, so the compile-time and runtime + // versions will always be the same + #[pyattr(name = "ZLIB_RUNTIME_VERSION")] + #[pyattr] + const ZLIB_VERSION: &str = unsafe { + match std::ffi::CStr::from_ptr(libz_sys::zlibVersion()).to_str() { + Ok(s) => s, + Err(_) => unreachable!(), + } + }; + // copied from zlibmodule.c (commit 530f506ac91338) #[pyattr] const MAX_WBITS: i8 = 15; @@ -80,15 +90,10 @@ mod zlib { } = args; let level = level.ok_or_else(|| new_zlib_error("Bad compression level", vm))?; - let encoded_bytes = if args.wbits.value == MAX_WBITS { - let mut encoder = ZlibEncoder::new(Vec::new(), level); - data.with_ref(|input_bytes| encoder.write_all(input_bytes).unwrap()); - encoder.finish().unwrap() - } else { - let mut inner = CompressInner::new(InitOptions::new(wbits.value, vm)?.compress(level)); - data.with_ref(|input_bytes| inner.compress(input_bytes, vm))?; - inner.flush(vm)? - }; + let compress = InitOptions::new(wbits.value, vm)?.compress(level); + let mut encoder = ZlibEncoder::new_with_compress(Vec::new(), compress); + data.with_ref(|input_bytes| encoder.write_all(input_bytes).unwrap()); + let encoded_bytes = encoder.finish().unwrap(); Ok(vm.ctx.new_bytes(encoded_bytes)) } @@ -109,6 +114,11 @@ mod zlib { let header = wbits > 0; let wbits = wbits.unsigned_abs(); match wbits { + // TODO: wbits = 0 should be a valid option: + // > windowBits can also be zero to request that inflate use the window size in + // > the zlib header of the compressed stream. + // but flate2 doesn't expose it + // 0 => ... 9..=15 => Ok(InitOptions::Standard { header, wbits }), 25..=31 => Ok(InitOptions::Gzip { wbits: wbits - 16 }), _ => Err(vm.new_value_error("Invalid initialization option".to_owned())), @@ -131,29 +141,81 @@ mod zlib { } } + #[derive(Clone)] + struct Chunker<'a> { + data1: &'a [u8], + data2: &'a [u8], + } + impl<'a> Chunker<'a> { + fn new(data: &'a [u8]) -> Self { + Self { + data1: data, + data2: &[], + } + } + fn chain(data1: &'a [u8], data2: &'a [u8]) -> Self { + if data1.is_empty() { + Self { + data1: data2, + data2: &[], + } + } else { + Self { data1, data2 } + } + } + fn len(&self) -> usize { + self.data1.len() + self.data2.len() + } + fn is_empty(&self) -> bool { + self.data1.is_empty() + } + fn to_vec(&self) -> Vec { + [self.data1, self.data2].concat() + } + fn chunk(&self) -> &'a [u8] { + self.data1.get(..CHUNKSIZE).unwrap_or(self.data1) + } + fn advance(&mut self, consumed: usize) { + self.data1 = &self.data1[consumed..]; + if self.data1.is_empty() { + self.data1 = std::mem::take(&mut self.data2); + } + } + } + fn _decompress( - mut data: &[u8], + data: &[u8], d: &mut Decompress, bufsize: usize, max_length: Option, is_flush: bool, + zdict: Option<&ArgBytesLike>, + vm: &VirtualMachine, + ) -> PyResult<(Vec, bool)> { + let mut data = Chunker::new(data); + _decompress_chunks(&mut data, d, bufsize, max_length, is_flush, zdict, vm) + } + + fn _decompress_chunks( + data: &mut Chunker<'_>, + d: &mut Decompress, + bufsize: usize, + max_length: Option, + is_flush: bool, + zdict: Option<&ArgBytesLike>, vm: &VirtualMachine, ) -> PyResult<(Vec, bool)> { if data.is_empty() { return Ok((Vec::new(), true)); } + let max_length = max_length.unwrap_or(usize::MAX); let mut buf = Vec::new(); - loop { - let final_chunk = data.len() <= CHUNKSIZE; - let chunk = if final_chunk { - data - } else { - &data[..CHUNKSIZE] - }; - // if this is the final chunk, finish it + 'outer: loop { + let chunk = data.chunk(); let flush = if is_flush { - if final_chunk { + // if this is the final chunk, finish it + if chunk.len() == data.len() { FlushDecompress::Finish } else { FlushDecompress::None @@ -162,34 +224,43 @@ mod zlib { FlushDecompress::Sync }; loop { - let additional = if let Some(max_length) = max_length { - std::cmp::min(bufsize, max_length - buf.capacity()) - } else { - bufsize - }; + let additional = std::cmp::min(bufsize, max_length - buf.capacity()); if additional == 0 { return Ok((buf, false)); } - buf.reserve_exact(additional); + let prev_in = d.total_in(); - let status = d - .decompress_vec(chunk, &mut buf, flush) - .map_err(|_| new_zlib_error("invalid input data", vm))?; + let res = d.decompress_vec(chunk, &mut buf, flush); let consumed = d.total_in() - prev_in; - data = &data[consumed as usize..]; - let stream_end = status == Status::StreamEnd; - if stream_end || data.is_empty() { - // we've reached the end of the stream, we're done - buf.shrink_to_fit(); - return Ok((buf, stream_end)); - } else if !chunk.is_empty() && consumed == 0 { - // we're gonna need a bigger buffer - continue; - } else { - // next chunk - break; - } + + data.advance(consumed as usize); + + match res { + Ok(status) => { + let stream_end = status == Status::StreamEnd; + if stream_end || data.is_empty() { + // we've reached the end of the stream, we're done + buf.shrink_to_fit(); + return Ok((buf, stream_end)); + } else if !chunk.is_empty() && consumed == 0 { + // we're gonna need a bigger buffer + continue; + } else { + // next chunk + continue 'outer; + } + } + Err(e) => { + let Some(zdict) = e.needs_dictionary().and(zdict) else { + return Err(new_zlib_error(&e.to_string(), vm)); + }; + d.set_dictionary(&zdict.borrow_buf()) + .map_err(|_| new_zlib_error("failed to set dictionary", vm))?; + // now try the next chunk + continue 'outer; + } + }; } } } @@ -214,7 +285,8 @@ mod zlib { } = args; data.with_ref(|data| { let mut d = InitOptions::new(wbits.value, vm)?.decompress(); - let (buf, stream_end) = _decompress(data, &mut d, bufsize.value, None, false, vm)?; + let (buf, stream_end) = + _decompress(data, &mut d, bufsize.value, None, false, None, vm)?; if !stream_end { return Err(new_zlib_error( "Error -5 while decompressing data: incomplete or truncated stream", @@ -230,101 +302,116 @@ mod zlib { #[pyarg(any, default = "ArgPrimitiveIndex { value: MAX_WBITS }")] wbits: ArgPrimitiveIndex, #[pyarg(any, optional)] - _zdict: OptionalArg, + zdict: OptionalArg, } #[pyfunction] fn decompressobj(args: DecompressobjArgs, vm: &VirtualMachine) -> PyResult { - #[allow(unused_mut)] let mut decompress = InitOptions::new(args.wbits.value, vm)?.decompress(); - if let OptionalArg::Present(_dict) = args._zdict { - // FIXME: always fails - // dict.with_ref(|d| decompress.set_dictionary(d)); + let zdict = args.zdict.into_option(); + if let Some(dict) = &zdict { + if args.wbits.value < 0 { + dict.with_ref(|d| decompress.set_dictionary(d)) + .map_err(|_| new_zlib_error("failed to set dictionary", vm))?; + } } + let inner = PyDecompressInner { + decompress: Some(decompress), + eof: false, + zdict, + unused_data: vm.ctx.empty_bytes.clone(), + unconsumed_tail: vm.ctx.empty_bytes.clone(), + }; Ok(PyDecompress { - decompress: PyMutex::new(decompress), - eof: AtomicCell::new(false), - unused_data: PyMutex::new(PyBytes::from(vec![]).into_ref(&vm.ctx)), - unconsumed_tail: PyMutex::new(PyBytes::from(vec![]).into_ref(&vm.ctx)), + inner: PyMutex::new(inner), }) } + + #[derive(Debug)] + struct PyDecompressInner { + decompress: Option, + zdict: Option, + eof: bool, + unused_data: PyBytesRef, + unconsumed_tail: PyBytesRef, + } + #[pyattr] #[pyclass(name = "Decompress")] #[derive(Debug, PyPayload)] struct PyDecompress { - decompress: PyMutex, - eof: AtomicCell, - unused_data: PyMutex, - unconsumed_tail: PyMutex, + inner: PyMutex, } + #[pyclass] impl PyDecompress { #[pygetset] fn eof(&self) -> bool { - self.eof.load() + self.inner.lock().eof } #[pygetset] fn unused_data(&self) -> PyBytesRef { - self.unused_data.lock().clone() + self.inner.lock().unused_data.clone() } #[pygetset] fn unconsumed_tail(&self) -> PyBytesRef { - self.unconsumed_tail.lock().clone() + self.inner.lock().unconsumed_tail.clone() } - fn save_unused_input( - &self, - d: &Decompress, + fn decompress_inner( + inner: &mut PyDecompressInner, data: &[u8], - stream_end: bool, - orig_in: u64, + bufsize: usize, + max_length: Option, + is_flush: bool, vm: &VirtualMachine, - ) { - let leftover = &data[(d.total_in() - orig_in) as usize..]; - - if stream_end && !leftover.is_empty() { - let mut unused_data = self.unused_data.lock(); - let unused: Vec<_> = unused_data - .as_bytes() - .iter() - .chain(leftover) - .copied() - .collect(); - *unused_data = vm.ctx.new_pyref(unused); + ) -> PyResult<(PyResult>, bool)> { + let Some(d) = &mut inner.decompress else { + return Err(new_zlib_error(USE_AFTER_FINISH_ERR, vm)); + }; + + let zdict = if is_flush { None } else { inner.zdict.as_ref() }; + + let prev_in = d.total_in(); + let (ret, stream_end) = + match _decompress(data, d, bufsize, max_length, is_flush, zdict, vm) { + Ok((buf, stream_end)) => (Ok(buf), stream_end), + Err(err) => (Err(err), false), + }; + let consumed = (d.total_in() - prev_in) as usize; + + // save unused input + let unconsumed = &data[consumed..]; + if !unconsumed.is_empty() { + if stream_end { + let unused = [inner.unused_data.as_bytes(), unconsumed].concat(); + inner.unused_data = vm.ctx.new_pyref(unused); + } else { + inner.unconsumed_tail = vm.ctx.new_bytes(unconsumed.to_vec()); + } + } else if !inner.unconsumed_tail.is_empty() { + inner.unconsumed_tail = vm.ctx.empty_bytes.clone(); } + + Ok((ret, stream_end)) } #[pymethod] fn decompress(&self, args: DecompressArgs, vm: &VirtualMachine) -> PyResult> { - let max_length = args.max_length.value; + let max_length: usize = args + .max_length + .map_or(0, |x| x.value) + .try_into() + .map_err(|_| vm.new_value_error("must be non-negative".to_owned()))?; let max_length = (max_length != 0).then_some(max_length); - let data = args.data.borrow_buf(); - let data = &*data; + let data = &*args.data.borrow_buf(); - let mut d = self.decompress.lock(); - let orig_in = d.total_in(); + let inner = &mut *self.inner.lock(); let (ret, stream_end) = - match _decompress(data, &mut d, DEF_BUF_SIZE, max_length, false, vm) { - Ok((buf, true)) => { - self.eof.store(true); - (Ok(buf), true) - } - Ok((buf, false)) => (Ok(buf), false), - Err(err) => (Err(err), false), - }; - self.save_unused_input(&d, data, stream_end, orig_in, vm); - - let leftover = if stream_end { - b"" - } else { - &data[(d.total_in() - orig_in) as usize..] - }; + Self::decompress_inner(inner, data, DEF_BUF_SIZE, max_length, false, vm)?; - let mut unconsumed_tail = self.unconsumed_tail.lock(); - if !leftover.is_empty() || !unconsumed_tail.is_empty() { - *unconsumed_tail = PyBytes::from(leftover.to_owned()).into_ref(&vm.ctx); - } + inner.eof |= stream_end; ret } @@ -332,36 +419,22 @@ mod zlib { #[pymethod] fn flush(&self, length: OptionalArg, vm: &VirtualMachine) -> PyResult> { let length = match length { - OptionalArg::Present(l) => { - let l: isize = l.into(); - if l <= 0 { - return Err( - vm.new_value_error("length must be greater than zero".to_owned()) - ); - } else { - l as usize - } + OptionalArg::Present(ArgSize { value }) if value <= 0 => { + return Err(vm.new_value_error("length must be greater than zero".to_owned())) } + OptionalArg::Present(ArgSize { value }) => value as usize, OptionalArg::Missing => DEF_BUF_SIZE, }; - let mut data = self.unconsumed_tail.lock(); - let mut d = self.decompress.lock(); + let inner = &mut *self.inner.lock(); + let data = std::mem::replace(&mut inner.unconsumed_tail, vm.ctx.empty_bytes.clone()); - let orig_in = d.total_in(); - - let (ret, stream_end) = match _decompress(&data, &mut d, length, None, true, vm) { - Ok((buf, stream_end)) => (Ok(buf), stream_end), - Err(err) => (Err(err), false), - }; - self.save_unused_input(&d, &data, stream_end, orig_in, vm); + let (ret, _) = Self::decompress_inner(inner, &data, length, None, true, vm)?; - *data = PyBytes::from(Vec::new()).into_ref(&vm.ctx); + if inner.eof { + inner.decompress = None; + } - // TODO: drop the inner decompressor, somehow - // if stream_end { - // - // } ret } } @@ -370,11 +443,8 @@ mod zlib { struct DecompressArgs { #[pyarg(positional)] data: ArgBytesLike, - #[pyarg( - any, - default = "rustpython_vm::function::ArgPrimitiveIndex { value: 0 }" - )] - max_length: ArgPrimitiveIndex, + #[pyarg(any, optional)] + max_length: OptionalArg, } #[derive(FromArgs)] @@ -384,13 +454,13 @@ mod zlib { level: Level, // only DEFLATED is valid right now, it's w/e #[pyarg(any, default = "DEFLATED")] - _method: i32, + method: i32, #[pyarg(any, default = "ArgPrimitiveIndex { value: MAX_WBITS }")] wbits: ArgPrimitiveIndex, - #[pyarg(any, name = "_memLevel", default = "DEF_MEM_LEVEL")] - _mem_level: u8, + #[pyarg(any, name = "memLevel", default = "DEF_MEM_LEVEL")] + mem_level: u8, #[pyarg(any, default = "Z_DEFAULT_STRATEGY")] - _strategy: i32, + strategy: i32, #[pyarg(any, optional)] zdict: Option, } @@ -417,8 +487,7 @@ mod zlib { #[derive(Debug)] struct CompressInner { - compress: Compress, - unconsumed: Vec, + compress: Option, } #[pyattr] @@ -436,10 +505,17 @@ mod zlib { data.with_ref(|b| inner.compress(b, vm)) } - // TODO: mode argument isn't used #[pymethod] - fn flush(&self, _mode: OptionalArg, vm: &VirtualMachine) -> PyResult> { - self.inner.lock().flush(vm) + fn flush(&self, mode: OptionalArg, vm: &VirtualMachine) -> PyResult> { + let mode = match mode.unwrap_or(Z_FINISH) { + Z_NO_FLUSH => return Ok(vec![]), + Z_PARTIAL_FLUSH => FlushCompress::Partial, + Z_SYNC_FLUSH => FlushCompress::Sync, + Z_FULL_FLUSH => FlushCompress::Full, + Z_FINISH => FlushCompress::Finish, + _ => return Err(new_zlib_error("invalid mode", vm)), + }; + self.inner.lock().flush(mode, vm) } // TODO: This is an optional feature of Compress @@ -456,59 +532,55 @@ mod zlib { impl CompressInner { fn new(compress: Compress) -> Self { Self { - compress, - unconsumed: Vec::new(), + compress: Some(compress), } } + + fn get_compress(&mut self, vm: &VirtualMachine) -> PyResult<&mut Compress> { + self.compress + .as_mut() + .ok_or_else(|| new_zlib_error(USE_AFTER_FINISH_ERR, vm)) + } + fn compress(&mut self, data: &[u8], vm: &VirtualMachine) -> PyResult> { - let orig_in = self.compress.total_in() as usize; - let mut cur_in = 0; - let unconsumed = std::mem::take(&mut self.unconsumed); + let c = self.get_compress(vm)?; let mut buf = Vec::new(); - 'outer: for chunk in unconsumed.chunks(CHUNKSIZE).chain(data.chunks(CHUNKSIZE)) { - while cur_in < chunk.len() { + for mut chunk in data.chunks(CHUNKSIZE) { + while !chunk.is_empty() { buf.reserve(DEF_BUF_SIZE); - let status = self - .compress - .compress_vec(&chunk[cur_in..], &mut buf, FlushCompress::None) - .map_err(|_| { - self.unconsumed.extend_from_slice(&data[cur_in..]); - new_zlib_error("error while compressing", vm) - })?; - cur_in = (self.compress.total_in() as usize) - orig_in; - match status { - Status::Ok => continue, - Status::StreamEnd => break 'outer, - _ => break, - } + let prev_in = c.total_in(); + c.compress_vec(chunk, &mut buf, FlushCompress::None) + .map_err(|_| new_zlib_error("error while compressing", vm))?; + let consumed = c.total_in() - prev_in; + chunk = &chunk[consumed as usize..]; } } - self.unconsumed.extend_from_slice(&data[cur_in..]); buf.shrink_to_fit(); Ok(buf) } - // TODO: flush mode (FlushDecompress) parameter - fn flush(&mut self, vm: &VirtualMachine) -> PyResult> { - let data = std::mem::take(&mut self.unconsumed); - let mut data_it = data.chunks(CHUNKSIZE); + fn flush(&mut self, mode: FlushCompress, vm: &VirtualMachine) -> PyResult> { + let c = self.get_compress(vm)?; let mut buf = Vec::new(); - loop { - let chunk = data_it.next().unwrap_or(&[]); + let status = loop { if buf.len() == buf.capacity() { buf.reserve(DEF_BUF_SIZE); } - let status = self - .compress - .compress_vec(chunk, &mut buf, FlushCompress::Finish) + let status = c + .compress_vec(&[], &mut buf, mode) .map_err(|_| new_zlib_error("error while compressing", vm))?; - match status { - Status::StreamEnd => break, - _ => continue, + if buf.len() != buf.capacity() { + break status; } + }; + + match status { + Status::Ok | Status::BufError => {} + Status::StreamEnd if mode == FlushCompress::Finish => self.compress = None, + Status::StreamEnd => return Err(new_zlib_error("unexpected eof", vm)), } buf.shrink_to_fit(); @@ -520,6 +592,8 @@ mod zlib { vm.new_exception_msg(vm.class("zlib", "error"), message.to_owned()) } + const USE_AFTER_FINISH_ERR: &str = "Error -2: inconsistent stream state"; + struct Level(Option); impl Level { @@ -551,130 +625,116 @@ mod zlib { #[pyattr] #[pyclass(name = "_ZlibDecompressor")] #[derive(Debug, PyPayload)] - pub struct ZlibDecompressor { - decompress: PyMutex, - unused_data: PyMutex, - unconsumed_tail: PyMutex, + struct ZlibDecompressor { + inner: PyMutex, + } + + #[derive(Debug)] + struct ZlibDecompressorInner { + decompress: Decompress, + unused_data: PyBytesRef, + input_buffer: Vec, + zdict: Option, + eof: bool, + needs_input: bool, } impl Constructor for ZlibDecompressor { - type Args = (); - - fn py_new(cls: PyTypeRef, _args: Self::Args, vm: &VirtualMachine) -> PyResult { - let decompress = Decompress::new(true); - let zlib_decompressor = Self { - decompress: PyMutex::new(decompress), - unused_data: PyMutex::new(PyBytes::from(vec![]).into_ref(&vm.ctx)), - unconsumed_tail: PyMutex::new(PyBytes::from(vec![]).into_ref(&vm.ctx)), + type Args = DecompressobjArgs; + + fn py_new(cls: PyTypeRef, args: Self::Args, vm: &VirtualMachine) -> PyResult { + let mut decompress = InitOptions::new(args.wbits.value, vm)?.decompress(); + let zdict = args.zdict.into_option(); + if let Some(dict) = &zdict { + if args.wbits.value < 0 { + dict.with_ref(|d| decompress.set_dictionary(d)) + .map_err(|_| new_zlib_error("failed to set dictionary", vm))?; + } + } + let inner = ZlibDecompressorInner { + decompress, + unused_data: vm.ctx.empty_bytes.clone(), + input_buffer: Vec::new(), + zdict, + eof: false, + needs_input: true, }; - zlib_decompressor - .into_ref_with_type(vm, cls) - .map(Into::into) + Self { + inner: PyMutex::new(inner), + } + .into_ref_with_type(vm, cls) + .map(Into::into) } } #[pyclass(with(Constructor))] impl ZlibDecompressor { #[pygetset] - fn unused_data(&self) -> PyBytesRef { - self.unused_data.lock().clone() + fn eof(&self) -> bool { + self.inner.lock().eof } #[pygetset] - fn unconsumed_tail(&self) -> PyBytesRef { - self.unconsumed_tail.lock().clone() + fn unused_data(&self) -> PyBytesRef { + self.inner.lock().unused_data.clone() } - fn save_unused_input( - &self, - d: &Decompress, - data: &[u8], - stream_end: bool, - orig_in: u64, - vm: &VirtualMachine, - ) { - let leftover = &data[(d.total_in() - orig_in) as usize..]; - - if stream_end && !leftover.is_empty() { - let mut unused_data = self.unused_data.lock(); - let unused: Vec<_> = unused_data - .as_bytes() - .iter() - .chain(leftover) - .copied() - .collect(); - *unused_data = vm.ctx.new_pyref(unused); - } + #[pygetset] + fn needs_input(&self) -> bool { + self.inner.lock().needs_input } #[pymethod] - fn decompress(&self, args: PyBytesRef, vm: &VirtualMachine) -> PyResult> { - // let max_length = args.max_length.value; - // let max_length = (max_length != 0).then_some(max_length); - let max_length = None; - let data = args.as_bytes(); - - let mut d = self.decompress.lock(); - let orig_in = d.total_in(); - - let (ret, stream_end) = - match _decompress(data, &mut d, DEF_BUF_SIZE, max_length, false, vm) { - Ok((buf, true)) => { - // Eof is true - (Ok(buf), true) - } - Ok((buf, false)) => (Ok(buf), false), - Err(err) => (Err(err), false), - }; - self.save_unused_input(&d, data, stream_end, orig_in, vm); + fn decompress(&self, args: DecompressArgs, vm: &VirtualMachine) -> PyResult> { + let max_length = args + .max_length + .into_option() + .and_then(|ArgSize { value }| usize::try_from(value).ok()); + let data = &*args.data.borrow_buf(); - let leftover = if stream_end { - b"" - } else { - &data[(d.total_in() - orig_in) as usize..] - }; + let inner = &mut *self.inner.lock(); - let mut unconsumed_tail = self.unconsumed_tail.lock(); - if !leftover.is_empty() || !unconsumed_tail.is_empty() { - *unconsumed_tail = PyBytes::from(leftover.to_owned()).into_ref(&vm.ctx); + if inner.eof { + return Err(vm.new_eof_error("End of stream already reached".to_owned())); } - ret - } + let input_buffer = &mut inner.input_buffer; + let d = &mut inner.decompress; - #[pymethod] - fn flush(&self, length: OptionalArg, vm: &VirtualMachine) -> PyResult> { - let length = match length { - OptionalArg::Present(l) => { - let l: isize = l.into(); - if l <= 0 { - return Err( - vm.new_value_error("length must be greater than zero".to_owned()) - ); - } else { - l as usize - } - } - OptionalArg::Missing => DEF_BUF_SIZE, - }; + let mut chunks = Chunker::chain(input_buffer, data); - let mut data = self.unconsumed_tail.lock(); - let mut d = self.decompress.lock(); + let zdict = inner.zdict.as_ref(); + let bufsize = DEF_BUF_SIZE; - let orig_in = d.total_in(); + let prev_len = chunks.len(); + let (ret, stream_end) = + match _decompress_chunks(&mut chunks, d, bufsize, max_length, false, zdict, vm) { + Ok((buf, stream_end)) => (Ok(buf), stream_end), + Err(err) => (Err(err), false), + }; + let consumed = prev_len - chunks.len(); - let (ret, stream_end) = match _decompress(&data, &mut d, length, None, true, vm) { - Ok((buf, stream_end)) => (Ok(buf), stream_end), - Err(err) => (Err(err), false), - }; - self.save_unused_input(&d, &data, stream_end, orig_in, vm); + inner.eof |= stream_end; - *data = PyBytes::from(Vec::new()).into_ref(&vm.ctx); + if inner.eof { + inner.needs_input = false; + if !chunks.is_empty() { + inner.unused_data = vm.ctx.new_bytes(chunks.to_vec()); + } + } else if chunks.is_empty() { + input_buffer.clear(); + inner.needs_input = true; + } else { + inner.needs_input = false; + if let Some(n_consumed_from_data) = consumed.checked_sub(input_buffer.len()) { + input_buffer.clear(); + input_buffer.extend_from_slice(&data[n_consumed_from_data..]); + } else { + input_buffer.drain(..consumed); + input_buffer.extend_from_slice(data); + } + } - // TODO: drop the inner decompressor, somehow - // if stream_end { - // - // } ret } From aba3d5c0823b7fc7de1828c9e4ebcd1f87fe320a Mon Sep 17 00:00:00 2001 From: CPython Developers <> Date: Wed, 26 Feb 2025 00:09:49 -0600 Subject: [PATCH 047/295] Update gzip,test_gzip from CPython 3.13 --- Lib/gzip.py | 130 ++++++++++---------- Lib/test/test_gzip.py | 270 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 325 insertions(+), 75 deletions(-) diff --git a/Lib/gzip.py b/Lib/gzip.py index 5b20e5ba69..1a3c82ce7e 100644 --- a/Lib/gzip.py +++ b/Lib/gzip.py @@ -15,12 +15,16 @@ FTEXT, FHCRC, FEXTRA, FNAME, FCOMMENT = 1, 2, 4, 8, 16 -READ, WRITE = 1, 2 +READ = 'rb' +WRITE = 'wb' _COMPRESS_LEVEL_FAST = 1 _COMPRESS_LEVEL_TRADEOFF = 6 _COMPRESS_LEVEL_BEST = 9 +READ_BUFFER_SIZE = 128 * 1024 +_WRITE_BUFFER_SIZE = 4 * io.DEFAULT_BUFFER_SIZE + def open(filename, mode="rb", compresslevel=_COMPRESS_LEVEL_BEST, encoding=None, errors=None, newline=None): @@ -118,6 +122,21 @@ class BadGzipFile(OSError): """Exception raised in some cases for invalid gzip files.""" +class _WriteBufferStream(io.RawIOBase): + """Minimal object to pass WriteBuffer flushes into GzipFile""" + def __init__(self, gzip_file): + self.gzip_file = gzip_file + + def write(self, data): + return self.gzip_file._write_raw(data) + + def seekable(self): + return False + + def writable(self): + return True + + class GzipFile(_compression.BaseStream): """The GzipFile class simulates most of the methods of a file object with the exception of the truncate() method. @@ -160,9 +179,10 @@ def __init__(self, filename=None, mode=None, and 9 is slowest and produces the most compression. 0 is no compression at all. The default is 9. - The mtime argument is an optional numeric timestamp to be written - to the last modification time field in the stream when compressing. - If omitted or None, the current time is used. + The optional mtime argument is the timestamp requested by gzip. The time + is in Unix format, i.e., seconds since 00:00:00 UTC, January 1, 1970. + If mtime is omitted or None, the current time is used. Use mtime = 0 + to generate a compressed stream that does not depend on creation time. """ @@ -182,6 +202,7 @@ def __init__(self, filename=None, mode=None, if mode is None: mode = getattr(fileobj, 'mode', 'rb') + if mode.startswith('r'): self.mode = READ raw = _GzipReader(fileobj) @@ -204,6 +225,9 @@ def __init__(self, filename=None, mode=None, zlib.DEF_MEM_LEVEL, 0) self._write_mtime = mtime + self._buffer_size = _WRITE_BUFFER_SIZE + self._buffer = io.BufferedWriter(_WriteBufferStream(self), + buffer_size=self._buffer_size) else: raise ValueError("Invalid mode: {!r}".format(mode)) @@ -212,14 +236,6 @@ def __init__(self, filename=None, mode=None, if self.mode == WRITE: self._write_gzip_header(compresslevel) - @property - def filename(self): - import warnings - warnings.warn("use the name attribute", DeprecationWarning, 2) - if self.mode == WRITE and self.name[-3:] != ".gz": - return self.name + ".gz" - return self.name - @property def mtime(self): """Last modification time read from stream, or None""" @@ -237,6 +253,11 @@ def _init_write(self, filename): self.bufsize = 0 self.offset = 0 # Current file offset for seek(), tell(), etc + def tell(self): + self._check_not_closed() + self._buffer.flush() + return super().tell() + def _write_gzip_header(self, compresslevel): self.fileobj.write(b'\037\213') # magic header self.fileobj.write(b'\010') # compression method @@ -278,6 +299,10 @@ def write(self,data): if self.fileobj is None: raise ValueError("write() on closed GzipFile object") + return self._buffer.write(data) + + def _write_raw(self, data): + # Called by our self._buffer underlying WriteBufferStream. if isinstance(data, (bytes, bytearray)): length = len(data) else: @@ -326,11 +351,11 @@ def closed(self): def close(self): fileobj = self.fileobj - if fileobj is None: + if fileobj is None or self._buffer.closed: return - self.fileobj = None try: if self.mode == WRITE: + self._buffer.flush() fileobj.write(self.compress.flush()) write32u(fileobj, self.crc) # self.size may exceed 2 GiB, or even 4 GiB @@ -338,6 +363,7 @@ def close(self): elif self.mode == READ: self._buffer.close() finally: + self.fileobj = None myfileobj = self.myfileobj if myfileobj: self.myfileobj = None @@ -346,6 +372,7 @@ def close(self): def flush(self,zlib_mode=zlib.Z_SYNC_FLUSH): self._check_not_closed() if self.mode == WRITE: + self._buffer.flush() # Ensure the compressor's buffer is flushed self.fileobj.write(self.compress.flush(zlib_mode)) self.fileobj.flush() @@ -376,6 +403,9 @@ def seekable(self): def seek(self, offset, whence=io.SEEK_SET): if self.mode == WRITE: + self._check_not_closed() + # Flush buffer to ensure validity of self.offset + self._buffer.flush() if whence != io.SEEK_SET: if whence == io.SEEK_CUR: offset = self.offset + offset @@ -384,10 +414,10 @@ def seek(self, offset, whence=io.SEEK_SET): if offset < self.offset: raise OSError('Negative seek in write mode') count = offset - self.offset - chunk = b'\0' * 1024 - for i in range(count // 1024): + chunk = b'\0' * self._buffer_size + for i in range(count // self._buffer_size): self.write(chunk) - self.write(b'\0' * (count % 1024)) + self.write(b'\0' * (count % self._buffer_size)) elif self.mode == READ: self._check_not_closed() return self._buffer.seek(offset, whence) @@ -454,7 +484,7 @@ def _read_gzip_header(fp): class _GzipReader(_compression.DecompressReader): def __init__(self, fp): - super().__init__(_PaddedFile(fp), zlib.decompressobj, + super().__init__(_PaddedFile(fp), zlib._ZlibDecompressor, wbits=-zlib.MAX_WBITS) # Set flag indicating start of a new member self._new_member = True @@ -502,12 +532,13 @@ def read(self, size=-1): self._new_member = False # Read a chunk of data from the file - buf = self._fp.read(io.DEFAULT_BUFFER_SIZE) + if self._decompressor.needs_input: + buf = self._fp.read(READ_BUFFER_SIZE) + uncompress = self._decompressor.decompress(buf, size) + else: + uncompress = self._decompressor.decompress(b"", size) - uncompress = self._decompressor.decompress(buf, size) - if self._decompressor.unconsumed_tail != b"": - self._fp.prepend(self._decompressor.unconsumed_tail) - elif self._decompressor.unused_data != b"": + if self._decompressor.unused_data != b"": # Prepend the already read bytes to the fileobj so they can # be seen by _read_eof() and _read_gzip_header() self._fp.prepend(self._decompressor.unused_data) @@ -518,14 +549,11 @@ def read(self, size=-1): raise EOFError("Compressed file ended before the " "end-of-stream marker was reached") - self._add_read_data( uncompress ) + self._crc = zlib.crc32(uncompress, self._crc) + self._stream_size += len(uncompress) self._pos += len(uncompress) return uncompress - def _add_read_data(self, data): - self._crc = zlib.crc32(data, self._crc) - self._stream_size = self._stream_size + len(data) - def _read_eof(self): # We've read to the end of the file # We check that the computed CRC and size of the @@ -552,43 +580,21 @@ def _rewind(self): self._new_member = True -def _create_simple_gzip_header(compresslevel: int, - mtime = None) -> bytes: - """ - Write a simple gzip header with no extra fields. - :param compresslevel: Compresslevel used to determine the xfl bytes. - :param mtime: The mtime (must support conversion to a 32-bit integer). - :return: A bytes object representing the gzip header. - """ - if mtime is None: - mtime = time.time() - if compresslevel == _COMPRESS_LEVEL_BEST: - xfl = 2 - elif compresslevel == _COMPRESS_LEVEL_FAST: - xfl = 4 - else: - xfl = 0 - # Pack ID1 and ID2 magic bytes, method (8=deflate), header flags (no extra - # fields added to header), mtime, xfl and os (255 for unknown OS). - return struct.pack(" Date: Tue, 25 Feb 2025 15:16:04 -0800 Subject: [PATCH 048/295] migrate to the 2024 edition Signed-off-by: Ashwin Naren --- Cargo.toml | 2 +- vm/src/suggestion.rs | 6 ++---- vm/sre_engine/src/string.rs | 32 ++++++++++++++++---------------- 3 files changed, 19 insertions(+), 21 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 6c48714929..55cc120335 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -102,7 +102,7 @@ members = [ [workspace.package] version = "0.4.0" authors = ["RustPython Team"] -edition = "2021" +edition = "2024" rust-version = "1.85.0" repository = "https://github.com/RustPython/RustPython" license = "MIT" diff --git a/vm/src/suggestion.rs b/vm/src/suggestion.rs index d46630a651..bc0556ef8b 100644 --- a/vm/src/suggestion.rs +++ b/vm/src/suggestion.rs @@ -52,10 +52,8 @@ pub fn offer_suggestions(exc: &PyBaseExceptionRef, vm: &VirtualMachine) -> Optio calculate_suggestions(vm.dir(Some(obj)).ok()?.borrow_vec().iter(), &name) } else if exc.class().is(vm.ctx.exceptions.name_error) { let name = exc.as_object().get_attr("name", vm).unwrap(); - let mut tb = exc.traceback()?; - for traceback in tb.iter() { - tb = traceback; - } + let tb = exc.traceback()?; + let tb = tb.iter().last().unwrap_or(tb); let varnames = tb.frame.code.clone().co_varnames(vm); if let Some(suggestions) = calculate_suggestions(varnames.iter(), &name) { diff --git a/vm/sre_engine/src/string.rs b/vm/sre_engine/src/string.rs index b5a2470f52..a32d74603a 100644 --- a/vm/sre_engine/src/string.rs +++ b/vm/sre_engine/src/string.rs @@ -157,8 +157,8 @@ impl StrDrive for &str { #[inline] unsafe fn next_code_point(ptr: &mut *const u8) -> u32 { // Decode UTF-8 - let x = **ptr; - *ptr = ptr.offset(1); + let x = unsafe { **ptr}; + *ptr = unsafe { ptr.offset(1) }; if x < 128 { return x as u32; @@ -170,16 +170,16 @@ unsafe fn next_code_point(ptr: &mut *const u8) -> u32 { let init = utf8_first_byte(x, 2); // SAFETY: `bytes` produces an UTF-8-like string, // so the iterator must produce a value here. - let y = **ptr; - *ptr = ptr.offset(1); + let y = unsafe { **ptr }; + *ptr = unsafe { ptr.offset(1) }; let mut ch = utf8_acc_cont_byte(init, y); if x >= 0xE0 { // [[x y z] w] case // 5th bit in 0xE0 .. 0xEF is always clear, so `init` is still valid // SAFETY: `bytes` produces an UTF-8-like string, // so the iterator must produce a value here. - let z = **ptr; - *ptr = ptr.offset(1); + let z = unsafe { **ptr }; + *ptr = unsafe { ptr.offset(1) }; let y_z = utf8_acc_cont_byte((y & CONT_MASK) as u32, z); ch = (init << 12) | y_z; if x >= 0xF0 { @@ -187,8 +187,8 @@ unsafe fn next_code_point(ptr: &mut *const u8) -> u32 { // use only the lower 3 bits of `init` // SAFETY: `bytes` produces an UTF-8-like string, // so the iterator must produce a value here. - let w = **ptr; - *ptr = ptr.offset(1); + let w = unsafe { **ptr}; + *ptr = unsafe { ptr.offset(1) }; ch = ((init & 7) << 18) | utf8_acc_cont_byte(y_z, w); } } @@ -205,8 +205,8 @@ unsafe fn next_code_point(ptr: &mut *const u8) -> u32 { #[inline] unsafe fn next_code_point_reverse(ptr: &mut *const u8) -> u32 { // Decode UTF-8 - *ptr = ptr.offset(-1); - let w = match **ptr { + *ptr = unsafe { ptr.offset(-1) }; + let w = match unsafe { **ptr } { next_byte if next_byte < 128 => return next_byte as u32, back_byte => back_byte, }; @@ -216,20 +216,20 @@ unsafe fn next_code_point_reverse(ptr: &mut *const u8) -> u32 { let mut ch; // SAFETY: `bytes` produces an UTF-8-like string, // so the iterator must produce a value here. - *ptr = ptr.offset(-1); - let z = **ptr; + *ptr = unsafe { ptr.offset(-1) }; + let z = unsafe { **ptr }; ch = utf8_first_byte(z, 2); if utf8_is_cont_byte(z) { // SAFETY: `bytes` produces an UTF-8-like string, // so the iterator must produce a value here. - *ptr = ptr.offset(-1); - let y = **ptr; + *ptr = unsafe { ptr.offset(-1) }; + let y = unsafe { **ptr }; ch = utf8_first_byte(y, 3); if utf8_is_cont_byte(y) { // SAFETY: `bytes` produces an UTF-8-like string, // so the iterator must produce a value here. - *ptr = ptr.offset(-1); - let x = **ptr; + *ptr = unsafe { ptr.offset(-1) }; + let x = unsafe { **ptr }; ch = utf8_first_byte(x, 4); ch = utf8_acc_cont_byte(ch, y); } From b870b0e1b510a5e3910f2585ef4f20eeb03c092b Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Wed, 26 Feb 2025 04:42:38 +0000 Subject: [PATCH 049/295] 2024 edition formatting Signed-off-by: Ashwin Naren --- benches/execution.rs | 6 +- benches/microbenchmarks.rs | 4 +- common/src/fileutils.rs | 14 ++-- common/src/float_ops.rs | 12 +-- common/src/hash.rs | 6 +- common/src/str.rs | 6 +- compiler/codegen/src/compile.rs | 14 ++-- compiler/codegen/src/ir.rs | 6 +- compiler/codegen/src/symboltable.rs | 9 +- compiler/src/lib.rs | 6 +- derive-impl/src/compile_bytecode.rs | 17 ++-- derive-impl/src/from_args.rs | 4 +- derive-impl/src/pyclass.rs | 33 ++++---- derive-impl/src/pymodule.rs | 12 +-- derive-impl/src/util.rs | 16 ++-- derive/src/lib.rs | 2 +- examples/call_between_rust_and_python.rs | 4 +- examples/generator.rs | 2 +- examples/package_embed.rs | 2 +- examples/parse_folder.rs | 2 +- rustfmt.toml | 2 +- src/interpreter.rs | 2 +- src/lib.rs | 4 +- src/settings.rs | 4 +- src/shell.rs | 4 +- src/shell/helper.rs | 3 +- stdlib/src/array.rs | 6 +- stdlib/src/binascii.rs | 9 +- stdlib/src/bisect.rs | 2 +- stdlib/src/blake2.rs | 2 +- stdlib/src/bz2.rs | 6 +- stdlib/src/cmath.rs | 2 +- stdlib/src/contextvars.rs | 5 +- stdlib/src/csv.rs | 6 +- stdlib/src/dis.rs | 2 +- stdlib/src/faulthandler.rs | 2 +- stdlib/src/fcntl.rs | 6 +- stdlib/src/gc.rs | 2 +- stdlib/src/grp.rs | 2 +- stdlib/src/hashlib.rs | 6 +- stdlib/src/json.rs | 6 +- stdlib/src/locale.rs | 18 ++-- stdlib/src/math.rs | 17 ++-- stdlib/src/md5.rs | 2 +- stdlib/src/mmap.rs | 5 +- stdlib/src/multiprocessing.rs | 2 +- stdlib/src/overlapped.rs | 2 +- stdlib/src/posixsubprocess.rs | 2 +- stdlib/src/pyexpat.rs | 4 +- stdlib/src/pystruct.rs | 4 +- stdlib/src/random.rs | 8 +- stdlib/src/resource.rs | 6 +- stdlib/src/scproxy.rs | 2 +- stdlib/src/select.rs | 25 +++--- stdlib/src/sha1.rs | 2 +- stdlib/src/sha256.rs | 4 +- stdlib/src/sha3.rs | 4 +- stdlib/src/sha512.rs | 4 +- stdlib/src/socket.rs | 100 +++++++++++------------ stdlib/src/sqlite.rs | 65 ++++++++------- stdlib/src/ssl.rs | 42 +++++----- stdlib/src/statistics.rs | 2 +- stdlib/src/syslog.rs | 2 +- stdlib/src/termios.rs | 35 ++++---- stdlib/src/unicodedata.rs | 10 +-- stdlib/src/uuid.rs | 2 +- stdlib/src/zlib.rs | 8 +- vm/src/anystr.rs | 8 +- vm/src/buffer.rs | 4 +- vm/src/builtins/asyncgenerator.rs | 4 +- vm/src/builtins/bool.rs | 4 +- vm/src/builtins/builtin_func.rs | 4 +- vm/src/builtins/bytearray.rs | 8 +- vm/src/builtins/bytes.rs | 8 +- vm/src/builtins/classmethod.rs | 2 +- vm/src/builtins/code.rs | 2 +- vm/src/builtins/complex.rs | 4 +- vm/src/builtins/coroutine.rs | 2 +- vm/src/builtins/descriptor.rs | 4 +- vm/src/builtins/dict.rs | 11 ++- vm/src/builtins/enumerate.rs | 2 +- vm/src/builtins/filter.rs | 2 +- vm/src/builtins/float.rs | 6 +- vm/src/builtins/frame.rs | 2 +- vm/src/builtins/function.rs | 6 +- vm/src/builtins/function/jitfunc.rs | 4 +- vm/src/builtins/generator.rs | 2 +- vm/src/builtins/genericalias.rs | 5 +- vm/src/builtins/getset.rs | 2 +- vm/src/builtins/int.rs | 20 ++--- vm/src/builtins/iter.rs | 2 +- vm/src/builtins/list.rs | 2 +- vm/src/builtins/map.rs | 2 +- vm/src/builtins/mappingproxy.rs | 2 +- vm/src/builtins/memory.rs | 17 ++-- vm/src/builtins/module.rs | 15 ++-- vm/src/builtins/namespace.rs | 4 +- vm/src/builtins/object.rs | 2 +- vm/src/builtins/property.rs | 2 +- vm/src/builtins/range.rs | 7 +- vm/src/builtins/set.rs | 6 +- vm/src/builtins/singletons.rs | 2 +- vm/src/builtins/slice.rs | 8 +- vm/src/builtins/staticmethod.rs | 2 +- vm/src/builtins/str.rs | 14 ++-- vm/src/builtins/super.rs | 2 +- vm/src/builtins/traceback.rs | 2 +- vm/src/builtins/tuple.rs | 2 +- vm/src/builtins/type.rs | 12 +-- vm/src/builtins/union.rs | 2 +- vm/src/builtins/weakproxy.rs | 3 +- vm/src/builtins/weakref.rs | 2 +- vm/src/builtins/zip.rs | 2 +- vm/src/bytesinner.rs | 6 +- vm/src/cformat.rs | 4 +- vm/src/class.rs | 2 +- vm/src/codecs.rs | 2 +- vm/src/convert/to_pyobject.rs | 2 +- vm/src/convert/try_from.rs | 2 +- vm/src/coroutine.rs | 2 +- vm/src/dictdatatype.rs | 4 +- vm/src/eval.rs | 2 +- vm/src/exceptions.rs | 8 +- vm/src/format.rs | 2 +- vm/src/frame.rs | 28 ++++--- vm/src/function/argument.rs | 2 +- vm/src/function/arithmetic.rs | 2 +- vm/src/function/buffer.rs | 4 +- vm/src/function/builtin.rs | 4 +- vm/src/function/either.rs | 2 +- vm/src/function/fspath.rs | 2 +- vm/src/function/getset.rs | 2 +- vm/src/function/method.rs | 4 +- vm/src/function/mod.rs | 4 +- vm/src/function/number.rs | 2 +- vm/src/function/protocol.rs | 4 +- vm/src/import.rs | 6 +- vm/src/intern.rs | 2 +- vm/src/iter.rs | 2 +- vm/src/object/core.rs | 2 +- vm/src/object/ext.rs | 4 +- vm/src/object/payload.rs | 2 +- vm/src/object/traverse.rs | 2 +- vm/src/object/traverse_object.rs | 4 +- vm/src/ospath.rs | 2 +- vm/src/protocol/buffer.rs | 2 +- vm/src/protocol/iter.rs | 2 +- vm/src/protocol/mapping.rs | 4 +- vm/src/protocol/number.rs | 6 +- vm/src/protocol/object.rs | 6 +- vm/src/protocol/sequence.rs | 4 +- vm/src/py_io.rs | 2 +- vm/src/py_serde.rs | 2 +- vm/src/scope.rs | 2 +- vm/src/sequence.rs | 4 +- vm/src/signal.rs | 2 +- vm/src/sliceable.rs | 11 +-- vm/src/stdlib/ast.rs | 8 +- vm/src/stdlib/atexit.rs | 2 +- vm/src/stdlib/builtins.rs | 11 ++- vm/src/stdlib/codecs.rs | 2 +- vm/src/stdlib/collections.rs | 2 +- vm/src/stdlib/ctypes/function.rs | 2 +- vm/src/stdlib/errno.rs | 8 +- vm/src/stdlib/functools.rs | 2 +- vm/src/stdlib/imp.rs | 11 ++- vm/src/stdlib/io.rs | 24 +++--- vm/src/stdlib/itertools.rs | 8 +- vm/src/stdlib/marshal.rs | 2 +- vm/src/stdlib/mod.rs | 2 +- vm/src/stdlib/msvcrt.rs | 2 +- vm/src/stdlib/nt.rs | 6 +- vm/src/stdlib/operator.rs | 4 +- vm/src/stdlib/os.rs | 26 ++---- vm/src/stdlib/posix.rs | 32 ++------ vm/src/stdlib/posix_compat.rs | 6 +- vm/src/stdlib/pwd.rs | 2 +- vm/src/stdlib/signal.rs | 6 +- vm/src/stdlib/sre.rs | 7 +- vm/src/stdlib/string.rs | 2 +- vm/src/stdlib/symtable.rs | 2 +- vm/src/stdlib/sys.rs | 20 ++--- vm/src/stdlib/sysconfigdata.rs | 2 +- vm/src/stdlib/thread.rs | 6 +- vm/src/stdlib/time.rs | 14 ++-- vm/src/stdlib/typing.rs | 4 +- vm/src/stdlib/warnings.rs | 4 +- vm/src/stdlib/weakref.rs | 2 +- vm/src/stdlib/winapi.rs | 4 +- vm/src/stdlib/winreg.rs | 8 +- vm/src/suggestion.rs | 4 +- vm/src/types/slot.rs | 4 +- vm/src/types/structseq.rs | 2 +- vm/src/types/zoo.rs | 2 +- vm/src/utils.rs | 2 +- vm/src/version.rs | 2 +- vm/src/vm/compile.rs | 2 +- vm/src/vm/context.rs | 9 +- vm/src/vm/interpreter.rs | 6 +- vm/src/vm/mod.rs | 12 +-- vm/src/vm/vm_new.rs | 4 +- vm/src/vm/vm_object.rs | 8 +- vm/src/vm/vm_ops.rs | 2 +- vm/src/warn.rs | 9 +- vm/src/windows.rs | 6 +- vm/sre_engine/benches/benches.rs | 2 +- vm/sre_engine/src/engine.rs | 2 +- vm/sre_engine/src/lib.rs | 2 +- vm/sre_engine/src/string.rs | 4 +- wasm/lib/src/browser_module.rs | 4 +- wasm/lib/src/convert.rs | 9 +- wasm/lib/src/js_module.rs | 8 +- wasm/lib/src/vm_class.rs | 2 +- wasm/lib/src/wasm_builtins.rs | 2 +- 214 files changed, 644 insertions(+), 744 deletions(-) diff --git a/benches/execution.rs b/benches/execution.rs index 6ec9b89f2a..57529bbb3a 100644 --- a/benches/execution.rs +++ b/benches/execution.rs @@ -1,11 +1,11 @@ use criterion::measurement::WallTime; use criterion::{ - black_box, criterion_group, criterion_main, Bencher, BenchmarkGroup, BenchmarkId, Criterion, - Throughput, + Bencher, BenchmarkGroup, BenchmarkId, Criterion, Throughput, black_box, criterion_group, + criterion_main, }; use rustpython_compiler::Mode; -use rustpython_parser::ast; use rustpython_parser::Parse; +use rustpython_parser::ast; use rustpython_vm::{Interpreter, PyResult, Settings}; use std::collections::HashMap; use std::path::Path; diff --git a/benches/microbenchmarks.rs b/benches/microbenchmarks.rs index b742b959b7..6f41f00d6c 100644 --- a/benches/microbenchmarks.rs +++ b/benches/microbenchmarks.rs @@ -1,6 +1,6 @@ use criterion::{ - criterion_group, criterion_main, measurement::WallTime, BatchSize, BenchmarkGroup, BenchmarkId, - Criterion, Throughput, + BatchSize, BenchmarkGroup, BenchmarkId, Criterion, Throughput, criterion_group, criterion_main, + measurement::WallTime, }; use pyo3::types::PyAnyMethods; use rustpython_compiler::Mode; diff --git a/common/src/fileutils.rs b/common/src/fileutils.rs index 8713e11229..7d5ff01942 100644 --- a/common/src/fileutils.rs +++ b/common/src/fileutils.rs @@ -5,7 +5,7 @@ pub use libc::stat as StatStruct; #[cfg(windows)] -pub use windows::{fstat, StatStruct}; +pub use windows::{StatStruct, fstat}; #[cfg(not(windows))] pub fn fstat(fd: libc::c_int) -> std::io::Result { @@ -28,19 +28,19 @@ pub mod windows { use std::ffi::{CString, OsStr, OsString}; use std::os::windows::ffi::OsStrExt; use std::sync::OnceLock; - use windows_sys::core::PCWSTR; use windows_sys::Win32::Foundation::{ - FreeLibrary, SetLastError, BOOL, ERROR_INVALID_HANDLE, ERROR_NOT_SUPPORTED, FILETIME, - HANDLE, INVALID_HANDLE_VALUE, + BOOL, ERROR_INVALID_HANDLE, ERROR_NOT_SUPPORTED, FILETIME, FreeLibrary, HANDLE, + INVALID_HANDLE_VALUE, SetLastError, }; use windows_sys::Win32::Storage::FileSystem::{ - FileBasicInfo, FileIdInfo, GetFileInformationByHandle, GetFileInformationByHandleEx, - GetFileType, BY_HANDLE_FILE_INFORMATION, FILE_ATTRIBUTE_DIRECTORY, FILE_ATTRIBUTE_READONLY, + BY_HANDLE_FILE_INFORMATION, FILE_ATTRIBUTE_DIRECTORY, FILE_ATTRIBUTE_READONLY, FILE_ATTRIBUTE_REPARSE_POINT, FILE_BASIC_INFO, FILE_ID_INFO, FILE_TYPE_CHAR, - FILE_TYPE_DISK, FILE_TYPE_PIPE, FILE_TYPE_UNKNOWN, + FILE_TYPE_DISK, FILE_TYPE_PIPE, FILE_TYPE_UNKNOWN, FileBasicInfo, FileIdInfo, + GetFileInformationByHandle, GetFileInformationByHandleEx, GetFileType, }; use windows_sys::Win32::System::LibraryLoader::{GetProcAddress, LoadLibraryW}; use windows_sys::Win32::System::SystemServices::IO_REPARSE_TAG_SYMLINK; + use windows_sys::core::PCWSTR; pub const S_IFIFO: libc::c_int = 0o010000; pub const S_IFLNK: libc::c_int = 0o120000; diff --git a/common/src/float_ops.rs b/common/src/float_ops.rs index 8e16e71c14..46e2d57067 100644 --- a/common/src/float_ops.rs +++ b/common/src/float_ops.rs @@ -64,11 +64,7 @@ pub fn gt_int(value: f64, other_int: &BigInt) -> bool { } pub fn div(v1: f64, v2: f64) -> Option { - if v2 != 0.0 { - Some(v1 / v2) - } else { - None - } + if v2 != 0.0 { Some(v1 / v2) } else { None } } pub fn mod_(v1: f64, v2: f64) -> Option { @@ -125,11 +121,7 @@ pub fn nextafter(x: f64, y: f64) -> f64 { let b = x.to_bits(); let bits = if (y > x) == (x > 0.0) { b + 1 } else { b - 1 }; let ret = f64::from_bits(bits); - if ret == 0.0 { - ret.copysign(x) - } else { - ret - } + if ret == 0.0 { ret.copysign(x) } else { ret } } } diff --git a/common/src/hash.rs b/common/src/hash.rs index 5b5ac003cb..bbc30b1fe1 100644 --- a/common/src/hash.rs +++ b/common/src/hash.rs @@ -150,11 +150,7 @@ pub fn hash_bigint(value: &BigInt) -> PyHash { #[inline(always)] pub fn fix_sentinel(x: PyHash) -> PyHash { - if x == SENTINEL { - -2 - } else { - x - } + if x == SENTINEL { -2 } else { x } } #[inline] diff --git a/common/src/str.rs b/common/src/str.rs index 65b1e376cb..b4f7a1a636 100644 --- a/common/src/str.rs +++ b/common/src/str.rs @@ -241,11 +241,7 @@ pub mod levenshtein { if b.is_ascii_uppercase() { b += b'a' - b'A'; } - if a == b { - CASE_COST - } else { - MOVE_COST - } + if a == b { CASE_COST } else { MOVE_COST } } pub fn levenshtein_distance(a: &str, b: &str, max_cost: usize) -> usize { diff --git a/compiler/codegen/src/compile.rs b/compiler/codegen/src/compile.rs index 7f8696baf2..6b1d15c720 100644 --- a/compiler/codegen/src/compile.rs +++ b/compiler/codegen/src/compile.rs @@ -8,21 +8,21 @@ #![deny(clippy::cast_possible_truncation)] use crate::{ + IndexSet, error::{CodegenError, CodegenErrorType}, ir, symboltable::{self, SymbolFlags, SymbolScope, SymbolTable}, - IndexSet, }; use itertools::Itertools; use num_complex::Complex64; use num_traits::ToPrimitive; use rustpython_ast::located::{self as located_ast, Located}; use rustpython_compiler_core::{ + Mode, bytecode::{ self, Arg as OpArgMarker, CodeObject, ComparisonOperator, ConstantData, Instruction, OpArg, OpArgType, }, - Mode, }; use rustpython_parser_core::source_code::{LineNumber, SourceLocation}; use std::borrow::Cow; @@ -975,7 +975,7 @@ impl Compiler { } } located_ast::Expr::BinOp(_) | located_ast::Expr::UnaryOp(_) => { - return Err(self.error(CodegenErrorType::Delete("expression"))) + return Err(self.error(CodegenErrorType::Delete("expression"))); } _ => return Err(self.error(CodegenErrorType::Delete(expression.python_name()))), } @@ -1213,7 +1213,7 @@ impl Compiler { if !finalbody.is_empty() { emit!(self, Instruction::PopBlock); // pop excepthandler block - // We enter the finally block, without exception. + // We enter the finally block, without exception. emit!(self, Instruction::EnterFinally); } @@ -3124,7 +3124,9 @@ impl Compiler { | "with_statement" | "print_function" | "unicode_literals" | "generator_stop" => {} "annotations" => self.future_annotations = true, other => { - return Err(self.error(CodegenErrorType::InvalidFutureFeature(other.to_owned()))) + return Err( + self.error(CodegenErrorType::InvalidFutureFeature(other.to_owned())) + ); } } } @@ -3477,8 +3479,8 @@ impl ToU32 for usize { #[cfg(test)] mod tests { use super::*; - use rustpython_parser::ast::Suite; use rustpython_parser::Parse; + use rustpython_parser::ast::Suite; use rustpython_parser_core::source_code::LinearLocator; fn compile_exec(source: &str) -> CodeObject { diff --git a/compiler/codegen/src/ir.rs b/compiler/codegen/src/ir.rs index 9f1a86e51d..08e68d283a 100644 --- a/compiler/codegen/src/ir.rs +++ b/compiler/codegen/src/ir.rs @@ -199,11 +199,7 @@ impl CodeInfo { }) .collect::>(); - if found_cellarg { - Some(cell2arg) - } else { - None - } + if found_cellarg { Some(cell2arg) } else { None } } fn dce(&mut self) { diff --git a/compiler/codegen/src/symboltable.rs b/compiler/codegen/src/symboltable.rs index 80c6e389fc..7db5c4edbe 100644 --- a/compiler/codegen/src/symboltable.rs +++ b/compiler/codegen/src/symboltable.rs @@ -8,8 +8,8 @@ Inspirational file: https://github.com/python/cpython/blob/main/Python/symtable. */ use crate::{ - error::{CodegenError, CodegenErrorType}, IndexMap, + error::{CodegenError, CodegenErrorType}, }; use bitflags::bitflags; use rustpython_ast::{self as ast, located::Located}; @@ -505,7 +505,10 @@ impl SymbolTableAnalyzer { // check if assignee is an iterator in top scope if parent_symbol.flags.contains(SymbolFlags::ITER) { return Err(SymbolTableError { - error: format!("assignment expression cannot rebind comprehension iteration variable {}", symbol.name), + error: format!( + "assignment expression cannot rebind comprehension iteration variable {}", + symbol.name + ), location: None, }); } @@ -1408,7 +1411,7 @@ impl SymbolTableBuilder { return Err(SymbolTableError { error: format!("cannot define nonlocal '{name}' at top level."), location, - }) + }); } _ => { // Ok! diff --git a/compiler/src/lib.rs b/compiler/src/lib.rs index c281058555..7d226cd1ce 100644 --- a/compiler/src/lib.rs +++ b/compiler/src/lib.rs @@ -1,9 +1,9 @@ use rustpython_codegen::{compile, symboltable}; -use rustpython_parser::ast::{self as ast, fold::Fold, ConstantOptimizer}; +use rustpython_parser::ast::{self as ast, ConstantOptimizer, fold::Fold}; pub use rustpython_codegen::compile::CompileOpts; -pub use rustpython_compiler_core::{bytecode::CodeObject, Mode}; -pub use rustpython_parser::{source_code::LinearLocator, Parse}; +pub use rustpython_compiler_core::{Mode, bytecode::CodeObject}; +pub use rustpython_parser::{Parse, source_code::LinearLocator}; // these modules are out of repository. re-exporting them here for convenience. pub use rustpython_codegen as codegen; diff --git a/derive-impl/src/compile_bytecode.rs b/derive-impl/src/compile_bytecode.rs index 34d5cb8d9d..2a57134197 100644 --- a/derive-impl/src/compile_bytecode.rs +++ b/derive-impl/src/compile_bytecode.rs @@ -17,17 +17,16 @@ use crate::Diagnostic; use once_cell::sync::Lazy; use proc_macro2::{Span, TokenStream}; use quote::quote; -use rustpython_compiler_core::{bytecode::CodeObject, frozen, Mode}; +use rustpython_compiler_core::{Mode, bytecode::CodeObject, frozen}; use std::{ collections::HashMap, env, fs, path::{Path, PathBuf}, }; use syn::{ - self, + self, LitByteStr, LitStr, Macro, parse::{ParseStream, Parser, Result as ParseResult}, spanned::Spanned, - LitByteStr, LitStr, Macro, }; static CARGO_MANIFEST_DIR: Lazy = Lazy::new(|| { @@ -118,11 +117,13 @@ impl CompilationSource { })?; self.compile_string(&source, mode, module_name, compiler, || rel_path.display()) } - CompilationSourceKind::SourceCode(code) => { - self.compile_string(&textwrap::dedent(code), mode, module_name, compiler, || { - "string literal" - }) - } + CompilationSourceKind::SourceCode(code) => self.compile_string( + &textwrap::dedent(code), + mode, + module_name, + compiler, + || "string literal", + ), CompilationSourceKind::Dir(_) => { unreachable!("Can't use compile_single with directory source") } diff --git a/derive-impl/src/from_args.rs b/derive-impl/src/from_args.rs index 47b0530d88..2273046ed4 100644 --- a/derive-impl/src/from_args.rs +++ b/derive-impl/src/from_args.rs @@ -1,8 +1,8 @@ use proc_macro2::TokenStream; -use quote::{quote, ToTokens}; +use quote::{ToTokens, quote}; use syn::ext::IdentExt; use syn::meta::ParseNestedMeta; -use syn::{parse_quote, Attribute, Data, DeriveInput, Expr, Field, Ident, Result, Token}; +use syn::{Attribute, Data, DeriveInput, Expr, Field, Ident, Result, Token, parse_quote}; /// The kind of the python parameter, this corresponds to the value of Parameter.kind /// (https://docs.python.org/3/library/inspect.html#inspect.Parameter.kind) diff --git a/derive-impl/src/pyclass.rs b/derive-impl/src/pyclass.rs index c3f148e2eb..077bd36bb8 100644 --- a/derive-impl/src/pyclass.rs +++ b/derive-impl/src/pyclass.rs @@ -1,14 +1,14 @@ use super::Diagnostic; use crate::util::{ - format_doc, pyclass_ident_and_attrs, pyexception_ident_and_attrs, text_signature, - ClassItemMeta, ContentItem, ContentItemInner, ErrorVec, ExceptionItemMeta, ItemMeta, - ItemMetaInner, ItemNursery, SimpleItemMeta, ALL_ALLOWED_NAMES, + ALL_ALLOWED_NAMES, ClassItemMeta, ContentItem, ContentItemInner, ErrorVec, ExceptionItemMeta, + ItemMeta, ItemMetaInner, ItemNursery, SimpleItemMeta, format_doc, pyclass_ident_and_attrs, + pyexception_ident_and_attrs, text_signature, }; use proc_macro2::{Delimiter, Group, Span, TokenStream, TokenTree}; -use quote::{quote, quote_spanned, ToTokens}; +use quote::{ToTokens, quote, quote_spanned}; use std::collections::{HashMap, HashSet}; use std::str::FromStr; -use syn::{parse_quote, spanned::Spanned, Attribute, Ident, Item, Result}; +use syn::{Attribute, Ident, Item, Result, parse_quote, spanned::Spanned}; use syn_ext::ext::*; use syn_ext::types::*; @@ -126,7 +126,7 @@ pub(crate) fn impl_pyclass_impl(attr: PunctuatedNestedMeta, item: Item) -> Resul return Err(syn::Error::new_spanned( segment, "Py{Ref} is expected but Py{Ref} is found", - )) + )); } } } @@ -134,7 +134,7 @@ pub(crate) fn impl_pyclass_impl(attr: PunctuatedNestedMeta, item: Item) -> Resul return Err(syn::Error::new_spanned( segment, "Py{Ref} is expected but Py{Ref}? is found", - )) + )); } } } else { @@ -152,7 +152,7 @@ pub(crate) fn impl_pyclass_impl(attr: PunctuatedNestedMeta, item: Item) -> Resul return Err(syn::Error::new_spanned( imp.self_ty, "PyImpl can only be implemented for Py{Ref} or T", - )) + )); } }; @@ -1237,11 +1237,7 @@ impl MethodItemMeta { name } else { let name = inner.item_name(); - if magic { - format!("__{name}__") - } else { - name - } + if magic { format!("__{name}__") } else { name } }) } } @@ -1308,11 +1304,7 @@ impl GetSetItemMeta { GetSetItemKind::Set => extract_prefix_name("set_", "setter")?, GetSetItemKind::Delete => extract_prefix_name("del_", "deleter")?, }; - if magic { - format!("__{name}__") - } else { - name - } + if magic { format!("__{name}__") } else { name } }; Ok((py_name, kind)) } @@ -1488,7 +1480,10 @@ fn extract_impl_attrs(attr: PunctuatedNestedMeta, item: &Ident) -> Result { return Err( self.new_syn_error(other.span(), "can only be on a function, const and use") - ) + ); } }; diff --git a/derive-impl/src/util.rs b/derive-impl/src/util.rs index cfe54a5c72..7e0eb96fb0 100644 --- a/derive-impl/src/util.rs +++ b/derive-impl/src/util.rs @@ -1,8 +1,8 @@ use itertools::Itertools; use proc_macro2::{Span, TokenStream}; -use quote::{quote, ToTokens}; +use quote::{ToTokens, quote}; use std::collections::{HashMap, HashSet}; -use syn::{spanned::Spanned, Attribute, Ident, Result, Signature, UseTree}; +use syn::{Attribute, Ident, Result, Signature, UseTree, spanned::Spanned}; use syn_ext::{ ext::{AttributeExt as SynAttributeExt, *}, types::*, @@ -75,7 +75,7 @@ impl ItemNursery { impl ToTokens for ValidatedItemNursery { fn to_tokens(&self, tokens: &mut TokenStream) { - let mut sorted = self.0 .0.clone(); + let mut sorted = self.0.0.clone(); sorted.sort_by(|a, b| a.sort_order.cmp(&b.sort_order)); tokens.extend(sorted.iter().map(|item| { let cfgs = &item.cfgs; @@ -447,7 +447,7 @@ impl ItemMeta for ExceptionItemMeta { Self(ClassItemMeta(inner)) } fn inner(&self) -> &ItemMetaInner { - &self.0 .0 + &self.0.0 } } @@ -470,12 +470,12 @@ impl ExceptionItemMeta { let type_name = inner.item_name(); let Some(py_name) = type_name.as_str().strip_prefix("Py") else { bail_span!( - inner.item_ident, - "#[pyexception] expects its underlying type to be named `Py` prefixed" - ) + inner.item_ident, + "#[pyexception] expects its underlying type to be named `Py` prefixed" + ) }; py_name.to_string() - }) + }); } _ => {} } diff --git a/derive/src/lib.rs b/derive/src/lib.rs index 9e94bf43cb..97ca026e76 100644 --- a/derive/src/lib.rs +++ b/derive/src/lib.rs @@ -245,7 +245,7 @@ impl derive_impl::Compiler for Compiler { mode: rustpython_compiler::Mode, module_name: String, ) -> Result> { - use rustpython_compiler::{compile, CompileOpts}; + use rustpython_compiler::{CompileOpts, compile}; Ok(compile(source, mode, module_name, CompileOpts::default())?) } } diff --git a/examples/call_between_rust_and_python.rs b/examples/call_between_rust_and_python.rs index 78eef62200..576390d059 100644 --- a/examples/call_between_rust_and_python.rs +++ b/examples/call_between_rust_and_python.rs @@ -1,5 +1,5 @@ use rustpython::vm::{ - pyclass, pymodule, PyObject, PyPayload, PyResult, TryFromBorrowedObject, VirtualMachine, + PyObject, PyPayload, PyResult, TryFromBorrowedObject, VirtualMachine, pyclass, pymodule, }; pub fn main() { @@ -31,7 +31,7 @@ pub fn main() { #[pymodule] mod rust_py_module { use super::*; - use rustpython::vm::{builtins::PyList, convert::ToPyObject, PyObjectRef}; + use rustpython::vm::{PyObjectRef, builtins::PyList, convert::ToPyObject}; #[pyfunction] fn rust_function( diff --git a/examples/generator.rs b/examples/generator.rs index 010ccd3797..937687ab8f 100644 --- a/examples/generator.rs +++ b/examples/generator.rs @@ -1,9 +1,9 @@ use rustpython_vm as vm; use std::process::ExitCode; use vm::{ + Interpreter, PyResult, builtins::PyIntRef, protocol::{PyIter, PyIterReturn}, - Interpreter, PyResult, }; fn py_main(interp: &Interpreter) -> vm::PyResult<()> { diff --git a/examples/package_embed.rs b/examples/package_embed.rs index fffa98623a..975e734593 100644 --- a/examples/package_embed.rs +++ b/examples/package_embed.rs @@ -1,6 +1,6 @@ use rustpython_vm as vm; use std::process::ExitCode; -use vm::{builtins::PyStrRef, Interpreter}; +use vm::{Interpreter, builtins::PyStrRef}; fn py_main(interp: &Interpreter) -> vm::PyResult { interp.enter(|vm| { diff --git a/examples/parse_folder.rs b/examples/parse_folder.rs index c10450e018..f54be635c8 100644 --- a/examples/parse_folder.rs +++ b/examples/parse_folder.rs @@ -12,7 +12,7 @@ extern crate env_logger; extern crate log; use clap::{App, Arg}; -use rustpython_parser::{ast, Parse}; +use rustpython_parser::{Parse, ast}; use std::{ path::Path, time::{Duration, Instant}, diff --git a/rustfmt.toml b/rustfmt.toml index 3a26366d4d..f216078d96 100644 --- a/rustfmt.toml +++ b/rustfmt.toml @@ -1 +1 @@ -edition = "2021" +edition = "2024" diff --git a/src/interpreter.rs b/src/interpreter.rs index 9d4dc4ee98..35710ae829 100644 --- a/src/interpreter.rs +++ b/src/interpreter.rs @@ -1,4 +1,4 @@ -use rustpython_vm::{builtins::PyModule, Interpreter, PyRef, Settings, VirtualMachine}; +use rustpython_vm::{Interpreter, PyRef, Settings, VirtualMachine, builtins::PyModule}; pub type InitHook = Box; diff --git a/src/lib.rs b/src/lib.rs index 9fa38b968c..b0a176acf2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -50,14 +50,14 @@ mod interpreter; mod settings; mod shell; -use rustpython_vm::{scope::Scope, PyResult, VirtualMachine}; +use rustpython_vm::{PyResult, VirtualMachine, scope::Scope}; use std::env; use std::io::IsTerminal; use std::process::ExitCode; pub use interpreter::InterpreterConfig; pub use rustpython_vm as vm; -pub use settings::{opts_with_clap, InstallPipMode, RunMode}; +pub use settings::{InstallPipMode, RunMode, opts_with_clap}; pub use shell::run_shell; /// The main cli of the `rustpython` interpreter. This function will return `std::process::ExitCode` diff --git a/src/settings.rs b/src/settings.rs index f6b7f21f25..35114374c8 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -238,7 +238,9 @@ fn settings_from(matches: &ArgMatches) -> (Settings, RunMode) { settings.int_max_str_digits = match env::var("PYTHONINTMAXSTRDIGITS").unwrap().parse() { Ok(digits @ (0 | 640..)) => digits, _ => { - error!("Fatal Python error: config_init_int_max_str_digits: PYTHONINTMAXSTRDIGITS: invalid limit; must be >= 640 or 0 for unlimited.\nPython runtime state: preinitialized"); + error!( + "Fatal Python error: config_init_int_max_str_digits: PYTHONINTMAXSTRDIGITS: invalid limit; must be >= 640 or 0 for unlimited.\nPython runtime state: preinitialized" + ); std::process::exit(1); } }; diff --git a/src/shell.rs b/src/shell.rs index 91becb0bbf..00b6710061 100644 --- a/src/shell.rs +++ b/src/shell.rs @@ -1,12 +1,12 @@ mod helper; -use rustpython_parser::{lexer::LexicalErrorType, ParseErrorType, Tok}; +use rustpython_parser::{ParseErrorType, Tok, lexer::LexicalErrorType}; use rustpython_vm::{ + AsObject, PyResult, VirtualMachine, builtins::PyBaseExceptionRef, compiler::{self, CompileError, CompileErrorType}, readline::{Readline, ReadlineResult}, scope::Scope, - AsObject, PyResult, VirtualMachine, }; enum ShellExecResult { diff --git a/src/shell/helper.rs b/src/shell/helper.rs index 4d663b9424..0fe2f5ca93 100644 --- a/src/shell/helper.rs +++ b/src/shell/helper.rs @@ -1,8 +1,9 @@ #![cfg_attr(target_arch = "wasm32", allow(dead_code))] use rustpython_vm::{ + AsObject, PyResult, TryFromObject, VirtualMachine, builtins::{PyDictRef, PyStrRef}, function::ArgIterable, - identifier, AsObject, PyResult, TryFromObject, VirtualMachine, + identifier, }; pub struct ShellHelper<'vm> { diff --git a/stdlib/src/array.rs b/stdlib/src/array.rs index 9bd58f8043..494e98dc1b 100644 --- a/stdlib/src/array.rs +++ b/stdlib/src/array.rs @@ -1,6 +1,6 @@ // spell-checker:ignore typecode tofile tolist fromfile -use rustpython_vm::{builtins::PyModule, PyRef, VirtualMachine}; +use rustpython_vm::{PyRef, VirtualMachine, builtins::PyModule}; pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { let module = array::make_module(vm); @@ -41,6 +41,7 @@ mod array { str::wchar_t, }, vm::{ + AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, atomic_func, builtins::{ PositionIterInternal, PyByteArray, PyBytes, PyBytesRef, PyDictRef, PyFloat, PyInt, @@ -64,7 +65,6 @@ mod array { AsBuffer, AsMapping, AsSequence, Comparable, Constructor, IterNext, Iterable, PyComparisonOp, Representable, SelfIter, }, - AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, }, }; use itertools::Itertools; @@ -1528,7 +1528,7 @@ mod array { 2 => Some(Self::Utf16 { big_endian }), 4 => Some(Self::Utf32 { big_endian }), _ => None, - } + }; } 'f' => { // Copied from CPython diff --git a/stdlib/src/binascii.rs b/stdlib/src/binascii.rs index eb11645879..ce3f9febd5 100644 --- a/stdlib/src/binascii.rs +++ b/stdlib/src/binascii.rs @@ -2,7 +2,7 @@ pub(super) use decl::crc32; pub(crate) use decl::make_module; -use rustpython_vm::{builtins::PyBaseExceptionRef, convert::ToPyException, VirtualMachine}; +use rustpython_vm::{VirtualMachine, builtins::PyBaseExceptionRef, convert::ToPyException}; const PAD: u8 = 61u8; const MAXLINESIZE: usize = 76; // Excluding the CRLF @@ -11,10 +11,10 @@ const MAXLINESIZE: usize = 76; // Excluding the CRLF mod decl { use super::{MAXLINESIZE, PAD}; use crate::vm::{ + PyResult, VirtualMachine, builtins::{PyIntRef, PyTypeRef}, convert::ToPyException, function::{ArgAsciiBuffer, ArgBytesLike, OptionalArg}, - PyResult, VirtualMachine, }; use itertools::Itertools; @@ -751,7 +751,10 @@ impl ToPyException for Base64DecodeError { InvalidByte(_, _) => "Only base64 data is allowed".to_owned(), InvalidLastSymbol(_, PAD) => "Excess data after padding".to_owned(), InvalidLastSymbol(length, _) => { - format!("Invalid base64-encoded string: number of data characters {} cannot be 1 more than a multiple of 4", length) + format!( + "Invalid base64-encoded string: number of data characters {} cannot be 1 more than a multiple of 4", + length + ) } InvalidLength => "Incorrect padding".to_owned(), }; diff --git a/stdlib/src/bisect.rs b/stdlib/src/bisect.rs index aaab65d788..4d67ee50b9 100644 --- a/stdlib/src/bisect.rs +++ b/stdlib/src/bisect.rs @@ -3,9 +3,9 @@ pub(crate) use _bisect::make_module; #[pymodule] mod _bisect { use crate::vm::{ + PyObjectRef, PyResult, VirtualMachine, function::{ArgIndex, OptionalArg}, types::PyComparisonOp, - PyObjectRef, PyResult, VirtualMachine, }; #[derive(FromArgs)] diff --git a/stdlib/src/blake2.rs b/stdlib/src/blake2.rs index 9b7da3327c..4209c966e8 100644 --- a/stdlib/src/blake2.rs +++ b/stdlib/src/blake2.rs @@ -4,7 +4,7 @@ pub(crate) use _blake2::make_module; #[pymodule] mod _blake2 { - use crate::hashlib::_hashlib::{local_blake2b, local_blake2s, BlakeHashArgs}; + use crate::hashlib::_hashlib::{BlakeHashArgs, local_blake2b, local_blake2s}; use crate::vm::{PyPayload, PyResult, VirtualMachine}; #[pyfunction] diff --git a/stdlib/src/bz2.rs b/stdlib/src/bz2.rs index f150b06eb8..ba74a38db1 100644 --- a/stdlib/src/bz2.rs +++ b/stdlib/src/bz2.rs @@ -6,13 +6,13 @@ pub(crate) use _bz2::make_module; mod _bz2 { use crate::common::lock::PyMutex; use crate::vm::{ + VirtualMachine, builtins::{PyBytesRef, PyTypeRef}, function::{ArgBytesLike, OptionalArg}, object::{PyPayload, PyResult}, types::Constructor, - VirtualMachine, }; - use bzip2::{write::BzEncoder, Decompress, Status}; + use bzip2::{Decompress, Status, write::BzEncoder}; use std::{fmt, io::Write}; // const BUFSIZ: i32 = 8192; @@ -196,7 +196,7 @@ mod _bz2 { _ => { return Err( vm.new_value_error("compresslevel must be between 1 and 9".to_owned()) - ) + ); } }; diff --git a/stdlib/src/cmath.rs b/stdlib/src/cmath.rs index c5badcf72a..4611ea344e 100644 --- a/stdlib/src/cmath.rs +++ b/stdlib/src/cmath.rs @@ -4,8 +4,8 @@ pub(crate) use cmath::make_module; #[pymodule] mod cmath { use crate::vm::{ - function::{ArgIntoComplex, ArgIntoFloat, OptionalArg}, PyResult, VirtualMachine, + function::{ArgIntoComplex, ArgIntoFloat, OptionalArg}, }; use num_complex::Complex64; diff --git a/stdlib/src/contextvars.rs b/stdlib/src/contextvars.rs index 40a59050b3..1e27b8b9e5 100644 --- a/stdlib/src/contextvars.rs +++ b/stdlib/src/contextvars.rs @@ -1,4 +1,4 @@ -use crate::vm::{builtins::PyModule, class::StaticType, PyRef, VirtualMachine}; +use crate::vm::{PyRef, VirtualMachine, builtins::PyModule, class::StaticType}; use _contextvars::PyContext; use std::cell::RefCell; @@ -23,14 +23,13 @@ thread_local! { #[pymodule] mod _contextvars { use crate::vm::{ - atomic_func, + AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, atomic_func, builtins::{PyStrRef, PyTypeRef}, class::StaticType, common::hash::PyHash, function::{ArgCallable, FuncArgs, OptionalArg}, protocol::{PyMappingMethods, PySequenceMethods}, types::{AsMapping, AsSequence, Constructor, Hashable, Representable}, - AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, }; use crossbeam_utils::atomic::AtomicCell; use indexmap::IndexMap; diff --git a/stdlib/src/csv.rs b/stdlib/src/csv.rs index 7b3448e6b5..03a5429ba4 100644 --- a/stdlib/src/csv.rs +++ b/stdlib/src/csv.rs @@ -4,11 +4,11 @@ pub(crate) use _csv::make_module; mod _csv { use crate::common::lock::PyMutex; use crate::vm::{ + AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, builtins::{PyBaseExceptionRef, PyInt, PyNone, PyStr, PyType, PyTypeError, PyTypeRef}, function::{ArgIterable, ArgumentError, FromArgs, FuncArgs, OptionalArg}, protocol::{PyIter, PyIterReturn}, types::{Constructor, IterNext, Iterable, SelfIter}, - AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, }; use csv_core::Terminator; use itertools::{self, Itertools}; @@ -396,7 +396,7 @@ mod _csv { Some(write_meth) => write_meth, None if file.is_callable() => file, None => { - return Err(vm.new_type_error("argument 1 must have a \"write\" method".to_owned())) + return Err(vm.new_type_error("argument 1 must have a \"write\" method".to_owned())); } }; @@ -995,7 +995,7 @@ mod _csv { csv_core::ReadRecordResult::OutputEndsFull => resize_buf(output_ends), csv_core::ReadRecordResult::Record => break, csv_core::ReadRecordResult::End => { - return Ok(PyIterReturn::StopIteration(None)) + return Ok(PyIterReturn::StopIteration(None)); } } } diff --git a/stdlib/src/dis.rs b/stdlib/src/dis.rs index 12c2ea75df..69767ffbba 100644 --- a/stdlib/src/dis.rs +++ b/stdlib/src/dis.rs @@ -3,9 +3,9 @@ pub(crate) use decl::make_module; #[pymodule(name = "dis")] mod decl { use crate::vm::{ + PyObjectRef, PyRef, PyResult, TryFromObject, VirtualMachine, builtins::{PyCode, PyDictRef, PyStrRef}, bytecode::CodeFlags, - PyObjectRef, PyRef, PyResult, TryFromObject, VirtualMachine, }; #[pyfunction] diff --git a/stdlib/src/faulthandler.rs b/stdlib/src/faulthandler.rs index 2fb93ecc88..9ffd931291 100644 --- a/stdlib/src/faulthandler.rs +++ b/stdlib/src/faulthandler.rs @@ -2,7 +2,7 @@ pub(crate) use decl::make_module; #[pymodule(name = "faulthandler")] mod decl { - use crate::vm::{frame::Frame, function::OptionalArg, stdlib::sys::PyStderr, VirtualMachine}; + use crate::vm::{VirtualMachine, frame::Frame, function::OptionalArg, stdlib::sys::PyStderr}; fn dump_frame(frame: &Frame, vm: &VirtualMachine) { let stderr = PyStderr(vm); diff --git a/stdlib/src/fcntl.rs b/stdlib/src/fcntl.rs index ee73e50397..307d6e4351 100644 --- a/stdlib/src/fcntl.rs +++ b/stdlib/src/fcntl.rs @@ -3,10 +3,10 @@ pub(crate) use fcntl::make_module; #[pymodule] mod fcntl { use crate::vm::{ + PyResult, VirtualMachine, builtins::PyIntRef, function::{ArgMemoryBuffer, ArgStrOrBytesLike, Either, OptionalArg}, stdlib::{io, os}, - PyResult, VirtualMachine, }; // TODO: supply these from (please file an issue/PR upstream): @@ -20,7 +20,7 @@ mod fcntl { // I_LINK, I_UNLINK, I_PLINK, I_PUNLINK #[pyattr] - use libc::{FD_CLOEXEC, F_GETFD, F_GETFL, F_SETFD, F_SETFL}; + use libc::{F_GETFD, F_GETFL, F_SETFD, F_SETFL, FD_CLOEXEC}; #[cfg(not(target_os = "wasi"))] #[pyattr] @@ -45,7 +45,7 @@ mod fcntl { #[cfg(any(target_os = "android", target_os = "fuchsia", target_os = "linux"))] #[pyattr] use libc::{ - F_ADD_SEALS, F_GETLEASE, F_GETPIPE_SZ, F_GET_SEALS, F_NOTIFY, F_SEAL_GROW, F_SEAL_SEAL, + F_ADD_SEALS, F_GET_SEALS, F_GETLEASE, F_GETPIPE_SZ, F_NOTIFY, F_SEAL_GROW, F_SEAL_SEAL, F_SEAL_SHRINK, F_SEAL_WRITE, F_SETLEASE, F_SETPIPE_SZ, }; diff --git a/stdlib/src/gc.rs b/stdlib/src/gc.rs index c78eea9c29..6e906ebab2 100644 --- a/stdlib/src/gc.rs +++ b/stdlib/src/gc.rs @@ -2,7 +2,7 @@ pub(crate) use gc::make_module; #[pymodule] mod gc { - use crate::vm::{function::FuncArgs, PyResult, VirtualMachine}; + use crate::vm::{PyResult, VirtualMachine, function::FuncArgs}; #[pyfunction] fn collect(_args: FuncArgs, _vm: &VirtualMachine) -> i32 { diff --git a/stdlib/src/grp.rs b/stdlib/src/grp.rs index d3eb0848bb..2cdad56588 100644 --- a/stdlib/src/grp.rs +++ b/stdlib/src/grp.rs @@ -3,11 +3,11 @@ pub(crate) use grp::make_module; #[pymodule] mod grp { use crate::vm::{ + PyObjectRef, PyResult, VirtualMachine, builtins::{PyIntRef, PyListRef, PyStrRef}, convert::{IntoPyException, ToPyObject}, exceptions, types::PyStructSequence, - PyObjectRef, PyResult, VirtualMachine, }; use nix::unistd; use std::ptr::NonNull; diff --git a/stdlib/src/hashlib.rs b/stdlib/src/hashlib.rs index 6944c37f9d..6124b6d242 100644 --- a/stdlib/src/hashlib.rs +++ b/stdlib/src/hashlib.rs @@ -6,16 +6,16 @@ pub(crate) use _hashlib::make_module; pub mod _hashlib { use crate::common::lock::PyRwLock; use crate::vm::{ + PyObjectRef, PyPayload, PyResult, VirtualMachine, builtins::{PyBytes, PyStrRef, PyTypeRef}, convert::ToPyObject, function::{ArgBytesLike, ArgStrOrBytesLike, FuncArgs, OptionalArg}, protocol::PyBuffer, - PyObjectRef, PyPayload, PyResult, VirtualMachine, }; use blake2::{Blake2b512, Blake2s256}; - use digest::{core_api::BlockSizeUser, DynDigest}; + use digest::{DynDigest, core_api::BlockSizeUser}; use digest::{ExtendableOutput, Update}; - use dyn_clone::{clone_trait_object, DynClone}; + use dyn_clone::{DynClone, clone_trait_object}; use md5::Md5; use sha1::Sha1; use sha2::{Sha224, Sha256, Sha384, Sha512}; diff --git a/stdlib/src/json.rs b/stdlib/src/json.rs index 921e545e5d..aaac0b8bef 100644 --- a/stdlib/src/json.rs +++ b/stdlib/src/json.rs @@ -5,12 +5,12 @@ mod machinery; mod _json { use super::machinery; use crate::vm::{ + AsObject, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine, builtins::{PyBaseExceptionRef, PyStrRef, PyType, PyTypeRef}, convert::{ToPyObject, ToPyResult}, function::{IntoFuncArgs, OptionalArg}, protocol::PyIterReturn, types::{Callable, Constructor}, - AsObject, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine, }; use malachite_bigint::BigInt; use std::str::FromStr; @@ -80,14 +80,14 @@ mod _json { None => { return Ok(PyIterReturn::StopIteration(Some( vm.ctx.new_int(idx).into(), - ))) + ))); } }; let next_idx = idx + c.len_utf8(); match c { '"' => { return scanstring(pystr, next_idx, OptionalArg::Present(self.strict), vm) - .map(|x| PyIterReturn::Return(x.to_pyobject(vm))) + .map(|x| PyIterReturn::Return(x.to_pyobject(vm))); } '{' => { // TODO: parse the object in rust diff --git a/stdlib/src/locale.rs b/stdlib/src/locale.rs index 8e44508fb6..9ca71a0957 100644 --- a/stdlib/src/locale.rs +++ b/stdlib/src/locale.rs @@ -40,10 +40,10 @@ use libc::localeconv; #[pymodule] mod _locale { use rustpython_vm::{ + PyObjectRef, PyResult, VirtualMachine, builtins::{PyDictRef, PyIntRef, PyListRef, PyStrRef, PyTypeRef}, convert::ToPyException, function::OptionalArg, - PyObjectRef, PyResult, VirtualMachine, }; use std::{ ffi::{CStr, CString}, @@ -56,12 +56,12 @@ mod _locale { ))] #[pyattr] use libc::{ - ABDAY_1, ABDAY_2, ABDAY_3, ABDAY_4, ABDAY_5, ABDAY_6, ABDAY_7, ABMON_1, ABMON_10, ABMON_11, - ABMON_12, ABMON_2, ABMON_3, ABMON_4, ABMON_5, ABMON_6, ABMON_7, ABMON_8, ABMON_9, - ALT_DIGITS, AM_STR, CODESET, CRNCYSTR, DAY_1, DAY_2, DAY_3, DAY_4, DAY_5, DAY_6, DAY_7, - D_FMT, D_T_FMT, ERA, ERA_D_FMT, ERA_D_T_FMT, ERA_T_FMT, LC_MESSAGES, MON_1, MON_10, MON_11, - MON_12, MON_2, MON_3, MON_4, MON_5, MON_6, MON_7, MON_8, MON_9, NOEXPR, PM_STR, RADIXCHAR, - THOUSEP, T_FMT, T_FMT_AMPM, YESEXPR, + ABDAY_1, ABDAY_2, ABDAY_3, ABDAY_4, ABDAY_5, ABDAY_6, ABDAY_7, ABMON_1, ABMON_2, ABMON_3, + ABMON_4, ABMON_5, ABMON_6, ABMON_7, ABMON_8, ABMON_9, ABMON_10, ABMON_11, ABMON_12, + ALT_DIGITS, AM_STR, CODESET, CRNCYSTR, D_FMT, D_T_FMT, DAY_1, DAY_2, DAY_3, DAY_4, DAY_5, + DAY_6, DAY_7, ERA, ERA_D_FMT, ERA_D_T_FMT, ERA_T_FMT, LC_MESSAGES, MON_1, MON_2, MON_3, + MON_4, MON_5, MON_6, MON_7, MON_8, MON_9, MON_10, MON_11, MON_12, NOEXPR, PM_STR, + RADIXCHAR, T_FMT, T_FMT_AMPM, THOUSEP, YESEXPR, }; #[pyattr] @@ -148,9 +148,7 @@ mod _locale { } macro_rules! set_int_field { - ($lc:expr, $field:ident) => {{ - result.set_item(stringify!($field), vm.new_pyobj((*$lc).$field), vm)? - }}; + ($lc:expr, $field:ident) => {{ result.set_item(stringify!($field), vm.new_pyobj((*$lc).$field), vm)? }}; } macro_rules! set_group_field { diff --git a/stdlib/src/math.rs b/stdlib/src/math.rs index 68930cfd0f..f86ebb591e 100644 --- a/stdlib/src/math.rs +++ b/stdlib/src/math.rs @@ -3,9 +3,10 @@ pub(crate) use math::make_module; #[pymodule] mod math { use crate::vm::{ - builtins::{try_bigint_to_f64, try_f64_to_bigint, PyFloat, PyInt, PyIntRef, PyStrInterned}, + PyObject, PyObjectRef, PyRef, PyResult, VirtualMachine, + builtins::{PyFloat, PyInt, PyIntRef, PyStrInterned, try_bigint_to_f64, try_f64_to_bigint}, function::{ArgIndex, ArgIntoFloat, ArgIterable, Either, OptionalArg, PosArgs}, - identifier, PyObject, PyObjectRef, PyRef, PyResult, VirtualMachine, + identifier, }; use itertools::Itertools; use malachite_bigint::BigInt; @@ -475,21 +476,13 @@ mod math { #[pyfunction] fn erf(x: ArgIntoFloat) -> f64 { let x = *x; - if x.is_nan() { - x - } else { - puruspe::erf(x) - } + if x.is_nan() { x } else { puruspe::erf(x) } } #[pyfunction] fn erfc(x: ArgIntoFloat) -> f64 { let x = *x; - if x.is_nan() { - x - } else { - puruspe::erfc(x) - } + if x.is_nan() { x } else { puruspe::erfc(x) } } #[pyfunction] diff --git a/stdlib/src/md5.rs b/stdlib/src/md5.rs index 833d217f5b..dca48242bb 100644 --- a/stdlib/src/md5.rs +++ b/stdlib/src/md5.rs @@ -2,7 +2,7 @@ pub(crate) use _md5::make_module; #[pymodule] mod _md5 { - use crate::hashlib::_hashlib::{local_md5, HashArgs}; + use crate::hashlib::_hashlib::{HashArgs, local_md5}; use crate::vm::{PyPayload, PyResult, VirtualMachine}; #[pyfunction] diff --git a/stdlib/src/mmap.rs b/stdlib/src/mmap.rs index aed665222e..e96339c370 100644 --- a/stdlib/src/mmap.rs +++ b/stdlib/src/mmap.rs @@ -8,7 +8,8 @@ mod mmap { lock::{MapImmutable, PyMutex, PyMutexGuard}, }; use crate::vm::{ - atomic_func, + AsObject, FromArgs, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, + TryFromBorrowedObject, VirtualMachine, atomic_func, builtins::{PyBytes, PyBytesRef, PyInt, PyIntRef, PyTypeRef}, byte::{bytes_from_object, value_from_object}, convert::ToPyException, @@ -18,8 +19,6 @@ mod mmap { }, sliceable::{SaturatedSlice, SequenceIndex, SequenceIndexOp}, types::{AsBuffer, AsMapping, AsSequence, Constructor, Representable}, - AsObject, FromArgs, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, - TryFromBorrowedObject, VirtualMachine, }; use crossbeam_utils::atomic::AtomicCell; use memmap2::{Advice, Mmap, MmapMut, MmapOptions}; diff --git a/stdlib/src/multiprocessing.rs b/stdlib/src/multiprocessing.rs index a6d902eb63..2db922e16b 100644 --- a/stdlib/src/multiprocessing.rs +++ b/stdlib/src/multiprocessing.rs @@ -3,7 +3,7 @@ pub(crate) use _multiprocessing::make_module; #[cfg(windows)] #[pymodule] mod _multiprocessing { - use crate::vm::{function::ArgBytesLike, stdlib::os, PyResult, VirtualMachine}; + use crate::vm::{PyResult, VirtualMachine, function::ArgBytesLike, stdlib::os}; use windows_sys::Win32::Networking::WinSock::{self, SOCKET}; #[pyfunction] diff --git a/stdlib/src/overlapped.rs b/stdlib/src/overlapped.rs index 9d08d88bcd..45eac5f51b 100644 --- a/stdlib/src/overlapped.rs +++ b/stdlib/src/overlapped.rs @@ -6,13 +6,13 @@ mod _overlapped { // straight-forward port of Modules/overlapped.c use crate::vm::{ + Py, PyObjectRef, PyPayload, PyResult, VirtualMachine, builtins::{PyBaseExceptionRef, PyBytesRef, PyTypeRef}, common::lock::PyMutex, convert::{ToPyException, ToPyObject}, protocol::PyBuffer, stdlib::os::errno_err, types::Constructor, - Py, PyObjectRef, PyPayload, PyResult, VirtualMachine, }; use windows_sys::Win32::{ Foundation::{self, GetLastError, HANDLE}, diff --git a/stdlib/src/posixsubprocess.rs b/stdlib/src/posixsubprocess.rs index 64e897b607..cff00a70aa 100644 --- a/stdlib/src/posixsubprocess.rs +++ b/stdlib/src/posixsubprocess.rs @@ -28,7 +28,7 @@ mod _posixsubprocess { use rustpython_vm::{AsObject, TryFromBorrowedObject}; use super::*; - use crate::vm::{convert::IntoPyException, PyResult, VirtualMachine}; + use crate::vm::{PyResult, VirtualMachine, convert::IntoPyException}; #[pyfunction] fn fork_exec(args: ForkExecArgs, vm: &VirtualMachine) -> PyResult { diff --git a/stdlib/src/pyexpat.rs b/stdlib/src/pyexpat.rs index 89267d3f7e..3cfe048f17 100644 --- a/stdlib/src/pyexpat.rs +++ b/stdlib/src/pyexpat.rs @@ -3,7 +3,7 @@ * */ -use crate::vm::{builtins::PyModule, extend_module, PyRef, VirtualMachine}; +use crate::vm::{PyRef, VirtualMachine, builtins::PyModule, extend_module}; pub fn make_module(vm: &VirtualMachine) -> PyRef { let module = _pyexpat::make_module(vm); @@ -32,10 +32,10 @@ macro_rules! create_property { #[pymodule(name = "pyexpat")] mod _pyexpat { use crate::vm::{ + Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, builtins::{PyStr, PyStrRef, PyType}, function::ArgBytesLike, function::{IntoFuncArgs, OptionalArg}, - Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, }; use rustpython_common::lock::PyRwLock; use std::io::Cursor; diff --git a/stdlib/src/pystruct.rs b/stdlib/src/pystruct.rs index 2d83e9570d..f8d41414f7 100644 --- a/stdlib/src/pystruct.rs +++ b/stdlib/src/pystruct.rs @@ -10,13 +10,13 @@ pub(crate) use _struct::make_module; #[pymodule] pub(crate) mod _struct { use crate::vm::{ - buffer::{new_struct_error, struct_error_type, FormatSpec}, + AsObject, Py, PyObjectRef, PyPayload, PyResult, TryFromObject, VirtualMachine, + buffer::{FormatSpec, new_struct_error, struct_error_type}, builtins::{PyBytes, PyStr, PyStrRef, PyTupleRef, PyTypeRef}, function::{ArgBytesLike, ArgMemoryBuffer, PosArgs}, match_class, protocol::PyIterReturn, types::{Constructor, IterNext, Iterable, SelfIter}, - AsObject, Py, PyObjectRef, PyPayload, PyResult, TryFromObject, VirtualMachine, }; use crossbeam_utils::atomic::AtomicCell; diff --git a/stdlib/src/random.rs b/stdlib/src/random.rs index 0692e3ef9b..685c0ae8b9 100644 --- a/stdlib/src/random.rs +++ b/stdlib/src/random.rs @@ -6,11 +6,11 @@ pub(crate) use _random::make_module; mod _random { use crate::common::lock::PyMutex; use crate::vm::{ + PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, builtins::{PyInt, PyTupleRef}, convert::ToPyException, function::OptionalOption, types::{Constructor, Initializer}, - PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, }; use itertools::Itertools; use malachite_bigint::{BigInt, BigUint, Sign}; @@ -75,11 +75,7 @@ mod _random { let mut rng = self.rng.lock(); let mut gen_u32 = |k| { let r = rng.next_u32(); - if k < 32 { - r >> (32 - k) - } else { - r - } + if k < 32 { r >> (32 - k) } else { r } }; let words = (k - 1) / 32 + 1; diff --git a/stdlib/src/resource.rs b/stdlib/src/resource.rs index c1b74f2b67..e103cce779 100644 --- a/stdlib/src/resource.rs +++ b/stdlib/src/resource.rs @@ -3,10 +3,10 @@ pub(crate) use resource::make_module; #[pymodule] mod resource { use crate::vm::{ + PyObject, PyObjectRef, PyResult, TryFromBorrowedObject, VirtualMachine, convert::{ToPyException, ToPyObject}, stdlib::os, types::PyStructSequence, - PyObject, PyObjectRef, PyResult, TryFromBorrowedObject, VirtualMachine, }; use std::{io, mem}; @@ -24,8 +24,8 @@ mod resource { // TODO: RLIMIT_OFILE, #[pyattr] use libc::{ - RLIMIT_AS, RLIMIT_CORE, RLIMIT_CPU, RLIMIT_DATA, RLIMIT_FSIZE, RLIMIT_MEMLOCK, - RLIMIT_NOFILE, RLIMIT_NPROC, RLIMIT_RSS, RLIMIT_STACK, RLIM_INFINITY, + RLIM_INFINITY, RLIMIT_AS, RLIMIT_CORE, RLIMIT_CPU, RLIMIT_DATA, RLIMIT_FSIZE, + RLIMIT_MEMLOCK, RLIMIT_NOFILE, RLIMIT_NPROC, RLIMIT_RSS, RLIMIT_STACK, }; #[cfg(any(target_os = "linux", target_os = "android", target_os = "emscripten"))] diff --git a/stdlib/src/scproxy.rs b/stdlib/src/scproxy.rs index 7108f50d8f..9bf29626ab 100644 --- a/stdlib/src/scproxy.rs +++ b/stdlib/src/scproxy.rs @@ -5,9 +5,9 @@ mod _scproxy { // straight-forward port of Modules/_scproxy.c use crate::vm::{ + PyResult, VirtualMachine, builtins::{PyDictRef, PyStr}, convert::ToPyObject, - PyResult, VirtualMachine, }; use system_configuration::core_foundation::{ array::CFArray, diff --git a/stdlib/src/select.rs b/stdlib/src/select.rs index 6554064d53..4003856bb9 100644 --- a/stdlib/src/select.rs +++ b/stdlib/src/select.rs @@ -1,6 +1,6 @@ use crate::vm::{ - builtins::PyListRef, builtins::PyModule, PyObject, PyObjectRef, PyRef, PyResult, TryFromObject, - VirtualMachine, + PyObject, PyObjectRef, PyRef, PyResult, TryFromObject, VirtualMachine, builtins::PyListRef, + builtins::PyModule, }; use std::{io, mem}; @@ -19,7 +19,7 @@ pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { #[cfg(unix)] mod platform { - pub use libc::{fd_set, select, timeval, FD_ISSET, FD_SET, FD_SETSIZE, FD_ZERO}; + pub use libc::{FD_ISSET, FD_SET, FD_SETSIZE, FD_ZERO, fd_set, select, timeval}; pub use std::os::unix::io::RawFd; pub fn check_err(x: i32) -> bool { @@ -30,8 +30,8 @@ mod platform { #[allow(non_snake_case)] #[cfg(windows)] mod platform { + pub use WinSock::{FD_SET as fd_set, FD_SETSIZE, SOCKET as RawFd, TIMEVAL as timeval, select}; use windows_sys::Win32::Networking::WinSock; - pub use WinSock::{select, FD_SET as fd_set, FD_SETSIZE, SOCKET as RawFd, TIMEVAL as timeval}; // based off winsock2.h: https://gist.github.com/piscisaureus/906386#file-winsock2-h-L128-L141 @@ -69,7 +69,7 @@ mod platform { #[cfg(target_os = "wasi")] mod platform { - pub use libc::{timeval, FD_SETSIZE}; + pub use libc::{FD_SETSIZE, timeval}; pub use std::os::wasi::io::RawFd; pub fn check_err(x: i32) -> bool { @@ -124,8 +124,8 @@ mod platform { } } -pub use platform::timeval; use platform::RawFd; +pub use platform::timeval; #[derive(Traverse)] struct Selectable { @@ -218,11 +218,11 @@ fn sec_to_timeval(sec: f64) -> timeval { mod decl { use super::*; use crate::vm::{ + PyObjectRef, PyResult, VirtualMachine, builtins::PyTypeRef, convert::ToPyException, function::{Either, OptionalOption}, stdlib::time, - PyObjectRef, PyResult, VirtualMachine, }; #[pyattr] @@ -327,12 +327,12 @@ mod decl { pub(super) mod poll { use super::*; use crate::vm::{ + AsObject, PyPayload, builtins::PyFloat, common::lock::PyMutex, convert::{IntoPyException, ToPyObject}, function::OptionalArg, stdlib::io::Fildes, - AsObject, PyPayload, }; use libc::pollfd; use num_traits::{Signed, ToPrimitive}; @@ -494,8 +494,9 @@ mod decl { #[cfg(any(target_os = "linux", target_os = "android", target_os = "redox"))] #[pyattr] use libc::{ - EPOLLERR, EPOLLEXCLUSIVE, EPOLLHUP, EPOLLIN, EPOLLMSG, EPOLLONESHOT, EPOLLOUT, EPOLLPRI, - EPOLLRDBAND, EPOLLRDHUP, EPOLLRDNORM, EPOLLWAKEUP, EPOLLWRBAND, EPOLLWRNORM, EPOLL_CLOEXEC, + EPOLL_CLOEXEC, EPOLLERR, EPOLLEXCLUSIVE, EPOLLHUP, EPOLLIN, EPOLLMSG, EPOLLONESHOT, + EPOLLOUT, EPOLLPRI, EPOLLRDBAND, EPOLLRDHUP, EPOLLRDNORM, EPOLLWAKEUP, EPOLLWRBAND, + EPOLLWRNORM, }; #[cfg(any(target_os = "linux", target_os = "android", target_os = "redox"))] #[pyattr] @@ -505,13 +506,13 @@ mod decl { pub(super) mod epoll { use super::*; use crate::vm::{ + PyPayload, builtins::PyTypeRef, common::lock::{PyRwLock, PyRwLockReadGuard}, convert::{IntoPyException, ToPyObject}, function::OptionalArg, stdlib::io::Fildes, types::Constructor, - PyPayload, }; use rustix::event::epoll::{self, EventData, EventFlags}; use std::ops::Deref; @@ -645,7 +646,7 @@ mod decl { ..-1 => { return Err(vm.new_value_error(format!( "maxevents must be greater than 0, got {maxevents}" - ))) + ))); } -1 => libc::FD_SETSIZE - 1, _ => maxevents as usize, diff --git a/stdlib/src/sha1.rs b/stdlib/src/sha1.rs index 3820e7d96a..04845bb76b 100644 --- a/stdlib/src/sha1.rs +++ b/stdlib/src/sha1.rs @@ -2,7 +2,7 @@ pub(crate) use _sha1::make_module; #[pymodule] mod _sha1 { - use crate::hashlib::_hashlib::{local_sha1, HashArgs}; + use crate::hashlib::_hashlib::{HashArgs, local_sha1}; use crate::vm::{PyPayload, PyResult, VirtualMachine}; #[pyfunction] diff --git a/stdlib/src/sha256.rs b/stdlib/src/sha256.rs index cae0172666..5d031968ae 100644 --- a/stdlib/src/sha256.rs +++ b/stdlib/src/sha256.rs @@ -1,4 +1,4 @@ -use crate::vm::{builtins::PyModule, PyRef, VirtualMachine}; +use crate::vm::{PyRef, VirtualMachine, builtins::PyModule}; pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { let _ = vm.import("_hashlib", 0); @@ -7,7 +7,7 @@ pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { #[pymodule] mod _sha256 { - use crate::hashlib::_hashlib::{local_sha224, local_sha256, HashArgs}; + use crate::hashlib::_hashlib::{HashArgs, local_sha224, local_sha256}; use crate::vm::{PyPayload, PyResult, VirtualMachine}; #[pyfunction] diff --git a/stdlib/src/sha3.rs b/stdlib/src/sha3.rs index f0c1c5ef69..07b61d9aed 100644 --- a/stdlib/src/sha3.rs +++ b/stdlib/src/sha3.rs @@ -3,8 +3,8 @@ pub(crate) use _sha3::make_module; #[pymodule] mod _sha3 { use crate::hashlib::_hashlib::{ - local_sha3_224, local_sha3_256, local_sha3_384, local_sha3_512, local_shake_128, - local_shake_256, HashArgs, + HashArgs, local_sha3_224, local_sha3_256, local_sha3_384, local_sha3_512, local_shake_128, + local_shake_256, }; use crate::vm::{PyPayload, PyResult, VirtualMachine}; diff --git a/stdlib/src/sha512.rs b/stdlib/src/sha512.rs index 8c510fb730..baf63fdacf 100644 --- a/stdlib/src/sha512.rs +++ b/stdlib/src/sha512.rs @@ -1,4 +1,4 @@ -use crate::vm::{builtins::PyModule, PyRef, VirtualMachine}; +use crate::vm::{PyRef, VirtualMachine, builtins::PyModule}; pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { let _ = vm.import("_hashlib", 0); @@ -7,7 +7,7 @@ pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { #[pymodule] mod _sha512 { - use crate::hashlib::_hashlib::{local_sha384, local_sha512, HashArgs}; + use crate::hashlib::_hashlib::{HashArgs, local_sha384, local_sha512}; use crate::vm::{PyPayload, PyResult, VirtualMachine}; #[pyfunction] diff --git a/stdlib/src/socket.rs b/stdlib/src/socket.rs index 95f03ee4f8..a38b4f123c 100644 --- a/stdlib/src/socket.rs +++ b/stdlib/src/socket.rs @@ -1,6 +1,6 @@ -use crate::vm::{builtins::PyModule, PyRef, VirtualMachine}; +use crate::vm::{PyRef, VirtualMachine, builtins::PyModule}; #[cfg(feature = "ssl")] -pub(super) use _socket::{sock_select, timeout_error_msg, PySocket, SelectKind}; +pub(super) use _socket::{PySocket, SelectKind, sock_select, timeout_error_msg}; pub fn make_module(vm: &VirtualMachine) -> PyRef { #[cfg(windows)] @@ -12,13 +12,13 @@ pub fn make_module(vm: &VirtualMachine) -> PyRef { mod _socket { use crate::common::lock::{PyMappedRwLockReadGuard, PyRwLock, PyRwLockReadGuard}; use crate::vm::{ + AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, builtins::{PyBaseExceptionRef, PyListRef, PyStrRef, PyTupleRef, PyTypeRef}, common::os::ErrorExt, convert::{IntoPyException, ToPyObject, TryFromBorrowedObject, TryFromObject}, function::{ArgBytesLike, ArgMemoryBuffer, Either, FsPath, OptionalArg, OptionalOption}, types::{Constructor, DefaultConstructor, Initializer, Representable}, utils::ToCString, - AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, }; use crossbeam_utils::atomic::AtomicCell; use num_traits::ToPrimitive; @@ -40,32 +40,33 @@ mod _socket { INADDR_ANY, INADDR_BROADCAST, INADDR_LOOPBACK, INADDR_NONE, }; pub use winapi::um::winsock2::{ - getprotobyname, getservbyname, getservbyport, getsockopt, setsockopt, - SO_EXCLUSIVEADDRUSE, + SO_EXCLUSIVEADDRUSE, getprotobyname, getservbyname, getservbyport, getsockopt, + setsockopt, }; pub use winapi::um::ws2tcpip::{ EAI_AGAIN, EAI_BADFLAGS, EAI_FAIL, EAI_FAMILY, EAI_MEMORY, EAI_NODATA, EAI_NONAME, EAI_SERVICE, EAI_SOCKTYPE, }; pub use windows_sys::Win32::Networking::WinSock::{ - AF_DECnet, AF_APPLETALK, AF_IPX, AF_LINK, AI_ADDRCONFIG, AI_ALL, AI_CANONNAME, - AI_NUMERICSERV, AI_V4MAPPED, IPPORT_RESERVED, IPPROTO_AH, IPPROTO_DSTOPTS, IPPROTO_EGP, - IPPROTO_ESP, IPPROTO_FRAGMENT, IPPROTO_GGP, IPPROTO_HOPOPTS, IPPROTO_ICMP, - IPPROTO_ICMPV6, IPPROTO_IDP, IPPROTO_IGMP, IPPROTO_IP, IPPROTO_IP as IPPROTO_IPIP, - IPPROTO_IPV4, IPPROTO_IPV6, IPPROTO_ND, IPPROTO_NONE, IPPROTO_PIM, IPPROTO_PUP, - IPPROTO_RAW, IPPROTO_ROUTING, IPPROTO_TCP, IPPROTO_UDP, IPV6_CHECKSUM, IPV6_DONTFRAG, - IPV6_HOPLIMIT, IPV6_HOPOPTS, IPV6_JOIN_GROUP, IPV6_LEAVE_GROUP, IPV6_MULTICAST_HOPS, + AF_APPLETALK, AF_DECnet, AF_IPX, AF_LINK, AI_ADDRCONFIG, AI_ALL, AI_CANONNAME, + AI_NUMERICSERV, AI_V4MAPPED, IP_ADD_MEMBERSHIP, IP_DROP_MEMBERSHIP, IP_HDRINCL, + IP_MULTICAST_IF, IP_MULTICAST_LOOP, IP_MULTICAST_TTL, IP_OPTIONS, IP_RECVDSTADDR, + IP_TOS, IP_TTL, IPPORT_RESERVED, IPPROTO_AH, IPPROTO_DSTOPTS, IPPROTO_EGP, IPPROTO_ESP, + IPPROTO_FRAGMENT, IPPROTO_GGP, IPPROTO_HOPOPTS, IPPROTO_ICMP, IPPROTO_ICMPV6, + IPPROTO_IDP, IPPROTO_IGMP, IPPROTO_IP, IPPROTO_IP as IPPROTO_IPIP, IPPROTO_IPV4, + IPPROTO_IPV6, IPPROTO_ND, IPPROTO_NONE, IPPROTO_PIM, IPPROTO_PUP, IPPROTO_RAW, + IPPROTO_ROUTING, IPPROTO_TCP, IPPROTO_UDP, IPV6_CHECKSUM, IPV6_DONTFRAG, IPV6_HOPLIMIT, + IPV6_HOPOPTS, IPV6_JOIN_GROUP, IPV6_LEAVE_GROUP, IPV6_MULTICAST_HOPS, IPV6_MULTICAST_IF, IPV6_MULTICAST_LOOP, IPV6_PKTINFO, IPV6_RECVRTHDR, IPV6_RECVTCLASS, - IPV6_RTHDR, IPV6_TCLASS, IPV6_UNICAST_HOPS, IPV6_V6ONLY, IP_ADD_MEMBERSHIP, - IP_DROP_MEMBERSHIP, IP_HDRINCL, IP_MULTICAST_IF, IP_MULTICAST_LOOP, IP_MULTICAST_TTL, - IP_OPTIONS, IP_RECVDSTADDR, IP_TOS, IP_TTL, MSG_BCAST, MSG_CTRUNC, MSG_DONTROUTE, - MSG_MCAST, MSG_OOB, MSG_PEEK, MSG_TRUNC, MSG_WAITALL, NI_DGRAM, NI_MAXHOST, NI_MAXSERV, - NI_NAMEREQD, NI_NOFQDN, NI_NUMERICHOST, NI_NUMERICSERV, RCVALL_IPLEVEL, RCVALL_OFF, - RCVALL_ON, RCVALL_SOCKETLEVELONLY, SD_BOTH as SHUT_RDWR, SD_RECEIVE as SHUT_RD, - SD_SEND as SHUT_WR, SIO_KEEPALIVE_VALS, SIO_LOOPBACK_FAST_PATH, SIO_RCVALL, SOCK_DGRAM, - SOCK_RAW, SOCK_RDM, SOCK_SEQPACKET, SOCK_STREAM, SOL_SOCKET, SOMAXCONN, SO_BROADCAST, - SO_ERROR, SO_LINGER, SO_OOBINLINE, SO_REUSEADDR, SO_TYPE, SO_USELOOPBACK, TCP_NODELAY, - WSAEBADF, WSAECONNRESET, WSAENOTSOCK, WSAEWOULDBLOCK, + IPV6_RTHDR, IPV6_TCLASS, IPV6_UNICAST_HOPS, IPV6_V6ONLY, MSG_BCAST, MSG_CTRUNC, + MSG_DONTROUTE, MSG_MCAST, MSG_OOB, MSG_PEEK, MSG_TRUNC, MSG_WAITALL, NI_DGRAM, + NI_MAXHOST, NI_MAXSERV, NI_NAMEREQD, NI_NOFQDN, NI_NUMERICHOST, NI_NUMERICSERV, + RCVALL_IPLEVEL, RCVALL_OFF, RCVALL_ON, RCVALL_SOCKETLEVELONLY, SD_BOTH as SHUT_RDWR, + SD_RECEIVE as SHUT_RD, SD_SEND as SHUT_WR, SIO_KEEPALIVE_VALS, SIO_LOOPBACK_FAST_PATH, + SIO_RCVALL, SO_BROADCAST, SO_ERROR, SO_LINGER, SO_OOBINLINE, SO_REUSEADDR, SO_TYPE, + SO_USELOOPBACK, SOCK_DGRAM, SOCK_RAW, SOCK_RDM, SOCK_SEQPACKET, SOCK_STREAM, + SOL_SOCKET, SOMAXCONN, TCP_NODELAY, WSAEBADF, WSAECONNRESET, WSAENOTSOCK, + WSAEWOULDBLOCK, }; pub const IF_NAMESIZE: usize = windows_sys::Win32::NetworkManagement::Ndis::IF_MAX_STRING_SIZE as _; @@ -86,14 +87,14 @@ mod _socket { IPPROTO_ICMPV6, IPPROTO_IP, IPPROTO_IPV6, IPPROTO_TCP, IPPROTO_TCP as SOL_TCP, IPPROTO_UDP, MSG_CTRUNC, MSG_DONTROUTE, MSG_OOB, MSG_PEEK, MSG_TRUNC, MSG_WAITALL, NI_DGRAM, NI_MAXHOST, NI_NAMEREQD, NI_NOFQDN, NI_NUMERICHOST, NI_NUMERICSERV, SHUT_RD, SHUT_RDWR, SHUT_WR, - SOCK_DGRAM, SOCK_STREAM, SOL_SOCKET, SO_BROADCAST, SO_ERROR, SO_LINGER, SO_OOBINLINE, - SO_REUSEADDR, SO_TYPE, TCP_NODELAY, + SO_BROADCAST, SO_ERROR, SO_LINGER, SO_OOBINLINE, SO_REUSEADDR, SO_TYPE, SOCK_DGRAM, + SOCK_STREAM, SOL_SOCKET, TCP_NODELAY, }; #[cfg(not(target_os = "redox"))] #[pyattr] use c::{ - AF_DECnet, AF_APPLETALK, AF_IPX, IPPROTO_AH, IPPROTO_DSTOPTS, IPPROTO_EGP, IPPROTO_ESP, + AF_APPLETALK, AF_DECnet, AF_IPX, IPPROTO_AH, IPPROTO_DSTOPTS, IPPROTO_EGP, IPPROTO_ESP, IPPROTO_FRAGMENT, IPPROTO_HOPOPTS, IPPROTO_IDP, IPPROTO_IGMP, IPPROTO_IPIP, IPPROTO_NONE, IPPROTO_PIM, IPPROTO_PUP, IPPROTO_RAW, IPPROTO_ROUTING, }; @@ -126,8 +127,9 @@ mod _socket { J1939_IDLE_ADDR, J1939_MAX_UNICAST_ADDR, J1939_NLA_BYTES_ACKED, J1939_NLA_PAD, J1939_NO_ADDR, J1939_NO_NAME, J1939_NO_PGN, J1939_PGN_ADDRESS_CLAIMED, J1939_PGN_ADDRESS_COMMANDED, J1939_PGN_MAX, J1939_PGN_PDU1_MAX, J1939_PGN_REQUEST, - SCM_J1939_DEST_ADDR, SCM_J1939_DEST_NAME, SCM_J1939_ERRQUEUE, SCM_J1939_PRIO, SOL_CAN_BASE, - SOL_CAN_RAW, SO_J1939_ERRQUEUE, SO_J1939_FILTER, SO_J1939_PROMISC, SO_J1939_SEND_PRIO, + SCM_J1939_DEST_ADDR, SCM_J1939_DEST_NAME, SCM_J1939_ERRQUEUE, SCM_J1939_PRIO, + SO_J1939_ERRQUEUE, SO_J1939_FILTER, SO_J1939_PROMISC, SO_J1939_SEND_PRIO, SOL_CAN_BASE, + SOL_CAN_RAW, }; #[cfg(all(target_os = "linux", target_env = "gnu"))] @@ -168,11 +170,11 @@ mod _socket { #[pyattr] use c::{ ALG_OP_DECRYPT, ALG_OP_ENCRYPT, ALG_SET_AEAD_ASSOCLEN, ALG_SET_AEAD_AUTHSIZE, ALG_SET_IV, - ALG_SET_KEY, ALG_SET_OP, IPV6_DSTOPTS, IPV6_NEXTHOP, IPV6_PATHMTU, IPV6_RECVDSTOPTS, - IPV6_RECVHOPLIMIT, IPV6_RECVHOPOPTS, IPV6_RECVPATHMTU, IPV6_RTHDRDSTOPTS, - IP_DEFAULT_MULTICAST_LOOP, IP_RECVOPTS, IP_RETOPTS, NETLINK_CRYPTO, NETLINK_DNRTMSG, - NETLINK_FIREWALL, NETLINK_IP6_FW, NETLINK_NFLOG, NETLINK_ROUTE, NETLINK_USERSOCK, - NETLINK_XFRM, SOL_ALG, SO_PASSSEC, SO_PEERSEC, + ALG_SET_KEY, ALG_SET_OP, IP_DEFAULT_MULTICAST_LOOP, IP_RECVOPTS, IP_RETOPTS, IPV6_DSTOPTS, + IPV6_NEXTHOP, IPV6_PATHMTU, IPV6_RECVDSTOPTS, IPV6_RECVHOPLIMIT, IPV6_RECVHOPOPTS, + IPV6_RECVPATHMTU, IPV6_RTHDRDSTOPTS, NETLINK_CRYPTO, NETLINK_DNRTMSG, NETLINK_FIREWALL, + NETLINK_IP6_FW, NETLINK_NFLOG, NETLINK_ROUTE, NETLINK_USERSOCK, NETLINK_XFRM, SO_PASSSEC, + SO_PEERSEC, SOL_ALG, }; #[cfg(any(target_os = "android", target_vendor = "apple"))] @@ -190,9 +192,9 @@ mod _socket { #[cfg(any(unix, target_os = "android", windows))] #[pyattr] use c::{ - INADDR_BROADCAST, IPV6_MULTICAST_HOPS, IPV6_MULTICAST_IF, IPV6_MULTICAST_LOOP, - IPV6_UNICAST_HOPS, IPV6_V6ONLY, IP_ADD_MEMBERSHIP, IP_DROP_MEMBERSHIP, IP_MULTICAST_IF, - IP_MULTICAST_LOOP, IP_MULTICAST_TTL, IP_TTL, + INADDR_BROADCAST, IP_ADD_MEMBERSHIP, IP_DROP_MEMBERSHIP, IP_MULTICAST_IF, + IP_MULTICAST_LOOP, IP_MULTICAST_TTL, IP_TTL, IPV6_MULTICAST_HOPS, IPV6_MULTICAST_IF, + IPV6_MULTICAST_LOOP, IPV6_UNICAST_HOPS, IPV6_V6ONLY, }; #[cfg(any(unix, target_os = "android", windows))] @@ -213,8 +215,8 @@ mod _socket { AF_ALG, AF_ASH, AF_ATMPVC, AF_ATMSVC, AF_AX25, AF_BRIDGE, AF_CAN, AF_ECONET, AF_IRDA, AF_LLC, AF_NETBEUI, AF_NETLINK, AF_NETROM, AF_PACKET, AF_PPPOX, AF_RDS, AF_SECURITY, AF_TIPC, AF_VSOCK, AF_WANPIPE, AF_X25, IP_TRANSPARENT, MSG_CONFIRM, MSG_ERRQUEUE, - MSG_FASTOPEN, MSG_MORE, PF_CAN, PF_PACKET, PF_RDS, SCM_CREDENTIALS, SOL_IP, SOL_TIPC, - SOL_UDP, SO_BINDTODEVICE, SO_MARK, TCP_CORK, TCP_DEFER_ACCEPT, TCP_LINGER2, TCP_QUICKACK, + MSG_FASTOPEN, MSG_MORE, PF_CAN, PF_PACKET, PF_RDS, SCM_CREDENTIALS, SO_BINDTODEVICE, + SO_MARK, SOL_IP, SOL_TIPC, SOL_UDP, TCP_CORK, TCP_DEFER_ACCEPT, TCP_LINGER2, TCP_QUICKACK, TCP_SYNCNT, TCP_WINDOW_CLAMP, }; @@ -271,7 +273,7 @@ mod _socket { #[cfg(any(target_os = "android", target_os = "linux", windows))] #[pyattr] - use c::{IPV6_HOPOPTS, IPV6_RECVRTHDR, IPV6_RTHDR, IP_OPTIONS}; + use c::{IP_OPTIONS, IPV6_HOPOPTS, IPV6_RECVRTHDR, IPV6_RTHDR}; #[cfg(any( target_os = "dragonfly", @@ -525,7 +527,7 @@ mod _socket { ))] #[pyattr] use c::{ - AF_LINK, IPPROTO_GGP, IPV6_JOIN_GROUP, IPV6_LEAVE_GROUP, IP_RECVDSTADDR, SO_USELOOPBACK, + AF_LINK, IP_RECVDSTADDR, IPPROTO_GGP, IPV6_JOIN_GROUP, IPV6_LEAVE_GROUP, SO_USELOOPBACK, }; #[cfg(any( @@ -633,7 +635,7 @@ mod _socket { #[pyattr] use c::{ EAI_AGAIN, EAI_BADFLAGS, EAI_FAIL, EAI_FAMILY, EAI_MEMORY, EAI_NONAME, EAI_SERVICE, - EAI_SOCKTYPE, IPV6_RECVTCLASS, IPV6_TCLASS, IP_HDRINCL, IP_TOS, SOMAXCONN, + EAI_SOCKTYPE, IP_HDRINCL, IP_TOS, IPV6_RECVTCLASS, IPV6_TCLASS, SOMAXCONN, }; #[cfg(not(any( @@ -1473,11 +1475,7 @@ mod _socket { #[pymethod] fn gettimeout(&self) -> Option { let timeout = self.timeout.load(); - if timeout >= 0.0 { - Some(timeout) - } else { - None - } + if timeout >= 0.0 { Some(timeout) } else { None } } #[pymethod] @@ -1601,7 +1599,7 @@ mod _socket { _ => { return Err(vm .new_value_error("`how` must be SHUT_RD, SHUT_WR, or SHUT_RDWR".to_owned()) - .into()) + .into()); } }; Ok(self.sock()?.shutdown(how)?) @@ -2054,7 +2052,7 @@ mod _socket { _ => { return Err(vm .new_type_error("illegal sockaddr argument".to_owned()) - .into()) + .into()); } } let (addr, flowinfo, scopeid) = Address::from_tuple_ipv6(&address, vm)?; @@ -2259,7 +2257,7 @@ mod _socket { _ => { return Err(vm .new_os_error("address family mismatched".to_owned()) - .into()) + .into()); } } return Ok(SocketAddr::V4(net::SocketAddrV4::new( @@ -2433,11 +2431,7 @@ mod _socket { #[pyfunction] fn getdefaulttimeout() -> Option { let timeout = DEFAULT_TIMEOUT.load(); - if timeout >= 0.0 { - Some(timeout) - } else { - None - } + if timeout >= 0.0 { Some(timeout) } else { None } } #[pyfunction] diff --git a/stdlib/src/sqlite.rs b/stdlib/src/sqlite.rs index 9225ccbc50..5487511e20 100644 --- a/stdlib/src/sqlite.rs +++ b/stdlib/src/sqlite.rs @@ -8,7 +8,7 @@ // spell-checker:ignore cantlock commithook foreignkey notnull primarykey gettemppath autoindex convpath // spell-checker:ignore dbmoved vnode nbytes -use rustpython_vm::{builtins::PyModule, AsObject, PyRef, VirtualMachine}; +use rustpython_vm::{AsObject, PyRef, VirtualMachine, builtins::PyModule}; // pub(crate) use _sqlite::make_module; pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { @@ -21,29 +21,29 @@ pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { #[pymodule] mod _sqlite { use libsqlite3_sys::{ - sqlite3, sqlite3_aggregate_context, sqlite3_backup_finish, sqlite3_backup_init, - sqlite3_backup_pagecount, sqlite3_backup_remaining, sqlite3_backup_step, sqlite3_bind_blob, - sqlite3_bind_double, sqlite3_bind_int64, sqlite3_bind_null, sqlite3_bind_parameter_count, - sqlite3_bind_parameter_name, sqlite3_bind_text, sqlite3_blob, sqlite3_blob_bytes, - sqlite3_blob_close, sqlite3_blob_open, sqlite3_blob_read, sqlite3_blob_write, - sqlite3_busy_timeout, sqlite3_changes, sqlite3_close_v2, sqlite3_column_blob, - sqlite3_column_bytes, sqlite3_column_count, sqlite3_column_decltype, sqlite3_column_double, - sqlite3_column_int64, sqlite3_column_name, sqlite3_column_text, sqlite3_column_type, - sqlite3_complete, sqlite3_context, sqlite3_context_db_handle, sqlite3_create_collation_v2, - sqlite3_create_function_v2, sqlite3_create_window_function, sqlite3_data_count, - sqlite3_db_handle, sqlite3_errcode, sqlite3_errmsg, sqlite3_exec, sqlite3_expanded_sql, - sqlite3_extended_errcode, sqlite3_finalize, sqlite3_get_autocommit, sqlite3_interrupt, - sqlite3_last_insert_rowid, sqlite3_libversion, sqlite3_limit, sqlite3_open_v2, - sqlite3_prepare_v2, sqlite3_progress_handler, sqlite3_reset, sqlite3_result_blob, - sqlite3_result_double, sqlite3_result_error, sqlite3_result_error_nomem, - sqlite3_result_error_toobig, sqlite3_result_int64, sqlite3_result_null, - sqlite3_result_text, sqlite3_set_authorizer, sqlite3_sleep, sqlite3_step, sqlite3_stmt, - sqlite3_stmt_busy, sqlite3_stmt_readonly, sqlite3_threadsafe, sqlite3_total_changes, - sqlite3_trace_v2, sqlite3_user_data, sqlite3_value, sqlite3_value_blob, - sqlite3_value_bytes, sqlite3_value_double, sqlite3_value_int64, sqlite3_value_text, - sqlite3_value_type, SQLITE_BLOB, SQLITE_DETERMINISTIC, SQLITE_FLOAT, SQLITE_INTEGER, - SQLITE_NULL, SQLITE_OPEN_CREATE, SQLITE_OPEN_READWRITE, SQLITE_OPEN_URI, SQLITE_TEXT, - SQLITE_TRACE_STMT, SQLITE_TRANSIENT, SQLITE_UTF8, + SQLITE_BLOB, SQLITE_DETERMINISTIC, SQLITE_FLOAT, SQLITE_INTEGER, SQLITE_NULL, + SQLITE_OPEN_CREATE, SQLITE_OPEN_READWRITE, SQLITE_OPEN_URI, SQLITE_TEXT, SQLITE_TRACE_STMT, + SQLITE_TRANSIENT, SQLITE_UTF8, sqlite3, sqlite3_aggregate_context, sqlite3_backup_finish, + sqlite3_backup_init, sqlite3_backup_pagecount, sqlite3_backup_remaining, + sqlite3_backup_step, sqlite3_bind_blob, sqlite3_bind_double, sqlite3_bind_int64, + sqlite3_bind_null, sqlite3_bind_parameter_count, sqlite3_bind_parameter_name, + sqlite3_bind_text, sqlite3_blob, sqlite3_blob_bytes, sqlite3_blob_close, sqlite3_blob_open, + sqlite3_blob_read, sqlite3_blob_write, sqlite3_busy_timeout, sqlite3_changes, + sqlite3_close_v2, sqlite3_column_blob, sqlite3_column_bytes, sqlite3_column_count, + sqlite3_column_decltype, sqlite3_column_double, sqlite3_column_int64, sqlite3_column_name, + sqlite3_column_text, sqlite3_column_type, sqlite3_complete, sqlite3_context, + sqlite3_context_db_handle, sqlite3_create_collation_v2, sqlite3_create_function_v2, + sqlite3_create_window_function, sqlite3_data_count, sqlite3_db_handle, sqlite3_errcode, + sqlite3_errmsg, sqlite3_exec, sqlite3_expanded_sql, sqlite3_extended_errcode, + sqlite3_finalize, sqlite3_get_autocommit, sqlite3_interrupt, sqlite3_last_insert_rowid, + sqlite3_libversion, sqlite3_limit, sqlite3_open_v2, sqlite3_prepare_v2, + sqlite3_progress_handler, sqlite3_reset, sqlite3_result_blob, sqlite3_result_double, + sqlite3_result_error, sqlite3_result_error_nomem, sqlite3_result_error_toobig, + sqlite3_result_int64, sqlite3_result_null, sqlite3_result_text, sqlite3_set_authorizer, + sqlite3_sleep, sqlite3_step, sqlite3_stmt, sqlite3_stmt_busy, sqlite3_stmt_readonly, + sqlite3_threadsafe, sqlite3_total_changes, sqlite3_trace_v2, sqlite3_user_data, + sqlite3_value, sqlite3_value_blob, sqlite3_value_bytes, sqlite3_value_double, + sqlite3_value_int64, sqlite3_value_text, sqlite3_value_type, }; use malachite_bigint::Sign; use rustpython_common::{ @@ -53,13 +53,16 @@ mod _sqlite { static_cell, }; use rustpython_vm::{ - atomic_func, + __exports::paste, + AsObject, Py, PyAtomicRef, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, + TryFromBorrowedObject, VirtualMachine, atomic_func, builtins::{ PyBaseException, PyBaseExceptionRef, PyByteArray, PyBytes, PyDict, PyDictRef, PyFloat, PyInt, PyIntRef, PySlice, PyStr, PyStrRef, PyTuple, PyTupleRef, PyType, PyTypeRef, }, convert::IntoObject, function::{ArgCallable, ArgIterable, FsPath, FuncArgs, OptionalArg, PyComparisonValue}, + object::{Traverse, TraverseFn}, protocol::{PyBuffer, PyIterReturn, PyMappingMethods, PySequence, PySequenceMethods}, sliceable::{SaturatedSliceIter, SliceableSequenceOp}, types::{ @@ -67,16 +70,12 @@ mod _sqlite { PyComparisonOp, SelfIter, }, utils::ToCString, - AsObject, Py, PyAtomicRef, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, - TryFromBorrowedObject, VirtualMachine, - __exports::paste, - object::{Traverse, TraverseFn}, }; use std::{ - ffi::{c_int, c_longlong, c_uint, c_void, CStr}, + ffi::{CStr, c_int, c_longlong, c_uint, c_void}, fmt::Debug, ops::Deref, - ptr::{null, null_mut, NonNull}, + ptr::{NonNull, null, null_mut}, thread::ThreadId, }; @@ -445,7 +444,7 @@ mod _sqlite { vm, exc, "user-defined aggregate's '__init__' method raised error\0", - ) + ); } } } @@ -2082,7 +2081,7 @@ mod _sqlite { _ => { return Err(vm.new_value_error( "'origin' should be os.SEEK_SET, os.SEEK_CUR, or os.SEEK_END".to_owned(), - )) + )); } } diff --git a/stdlib/src/ssl.rs b/stdlib/src/ssl.rs index e90f08f2ef..2b8ffc7d8c 100644 --- a/stdlib/src/ssl.rs +++ b/stdlib/src/ssl.rs @@ -1,4 +1,4 @@ -use crate::vm::{builtins::PyModule, PyRef, VirtualMachine}; +use crate::vm::{PyRef, VirtualMachine, builtins::PyModule}; use openssl_probe::ProbeResult; pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { @@ -36,6 +36,7 @@ mod _ssl { }, socket::{self, PySocket}, vm::{ + PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, builtins::{PyBaseExceptionRef, PyStrRef, PyType, PyTypeRef, PyWeak}, convert::{ToPyException, ToPyObject}, exceptions, @@ -45,7 +46,6 @@ mod _ssl { }, types::Constructor, utils::ToCString, - PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, }, }; use crossbeam_utils::atomic::AtomicCell; @@ -55,7 +55,7 @@ mod _ssl { error::ErrorStack, nid::Nid, ssl::{self, SslContextBuilder, SslOptions, SslVerifyMode}, - x509::{self, X509Ref, X509}, + x509::{self, X509, X509Ref}, }; use openssl_sys as sys; use rustpython_vm::ospath::OsPath; @@ -70,9 +70,6 @@ mod _ssl { // Constants #[pyattr] use sys::{ - SSL_OP_NO_SSLv2 as OP_NO_SSLv2, - SSL_OP_NO_SSLv3 as OP_NO_SSLv3, - SSL_OP_NO_TLSv1 as OP_NO_TLSv1, // TODO: so many more of these SSL_AD_DECODE_ERROR as ALERT_DESCRIPTION_DECODE_ERROR, SSL_AD_ILLEGAL_PARAMETER as ALERT_DESCRIPTION_ILLEGAL_PARAMETER, @@ -92,7 +89,10 @@ mod _ssl { // X509_V_FLAG_X509_STRICT as VERIFY_X509_STRICT, SSL_ERROR_ZERO_RETURN, SSL_OP_CIPHER_SERVER_PREFERENCE as OP_CIPHER_SERVER_PREFERENCE, + SSL_OP_NO_SSLv2 as OP_NO_SSLv2, + SSL_OP_NO_SSLv3 as OP_NO_SSLv3, SSL_OP_NO_TICKET as OP_NO_TICKET, + SSL_OP_NO_TLSv1 as OP_NO_TLSv1, SSL_OP_SINGLE_DH_USE as OP_SINGLE_DH_USE, }; @@ -601,7 +601,7 @@ mod _ssl { return Err(vm.new_value_error( "Cannot set verify_mode to CERT_NONE when check_hostname is enabled." .to_owned(), - )) + )); } CertRequirements::None => SslVerifyMode::NONE, CertRequirements::Optional => SslVerifyMode::PEER, @@ -1006,11 +1006,7 @@ mod _ssl { #[pymethod] fn version(&self) -> Option<&'static str> { let v = self.stream.read().ssl().version_str(); - if v == "unknown" { - None - } else { - Some(v) - } + if v == "unknown" { None } else { Some(v) } } #[pymethod] @@ -1058,7 +1054,7 @@ mod _ssl { return Err(socket::timeout_error_msg( vm, "The handshake operation timed out".to_owned(), - )) + )); } SelectRet::Closed => return Err(socket_closed_error(vm)), SelectRet::Nonblocking => {} @@ -1084,7 +1080,7 @@ mod _ssl { return Err(socket::timeout_error_msg( vm, "The write operation timed out".to_owned(), - )) + )); } SelectRet::Closed => return Err(socket_closed_error(vm)), _ => {} @@ -1100,7 +1096,7 @@ mod _ssl { return Err(socket::timeout_error_msg( vm, "The write operation timed out".to_owned(), - )) + )); } SelectRet::Closed => return Err(socket_closed_error(vm)), SelectRet::Nonblocking => {} @@ -1152,7 +1148,7 @@ mod _ssl { return Err(socket::timeout_error_msg( vm, "The read operation timed out".to_owned(), - )) + )); } SelectRet::Nonblocking => {} _ => { @@ -1385,13 +1381,13 @@ mod _ssl { #[cfg(target_os = "android")] mod android { use super::convert_openssl_error; - use crate::vm::{builtins::PyBaseExceptionRef, VirtualMachine}; + use crate::vm::{VirtualMachine, builtins::PyBaseExceptionRef}; use openssl::{ ssl::SslContextBuilder, - x509::{store::X509StoreBuilder, X509}, + x509::{X509, store::X509StoreBuilder}, }; use std::{ - fs::{read_dir, File}, + fs::{File, read_dir}, io::Read, path::Path, }; @@ -1465,8 +1461,8 @@ mod windows {} mod ossl101 { #[pyattr] use openssl_sys::{ - SSL_OP_NO_TLSv1_1 as OP_NO_TLSv1_1, SSL_OP_NO_TLSv1_2 as OP_NO_TLSv1_2, - SSL_OP_NO_COMPRESSION as OP_NO_COMPRESSION, + SSL_OP_NO_COMPRESSION as OP_NO_COMPRESSION, SSL_OP_NO_TLSv1_1 as OP_NO_TLSv1_1, + SSL_OP_NO_TLSv1_2 as OP_NO_TLSv1_2, }; } @@ -1484,15 +1480,15 @@ mod windows { use crate::{ common::ascii, vm::{ + PyObjectRef, PyPayload, PyResult, VirtualMachine, builtins::{PyFrozenSet, PyStrRef}, convert::ToPyException, - PyObjectRef, PyPayload, PyResult, VirtualMachine, }, }; #[pyfunction] fn enum_certificates(store_name: PyStrRef, vm: &VirtualMachine) -> PyResult> { - use schannel::{cert_context::ValidUses, cert_store::CertStore, RawPointer}; + use schannel::{RawPointer, cert_context::ValidUses, cert_store::CertStore}; use windows_sys::Win32::Security::Cryptography; // TODO: check every store for it, not just 2 of them: diff --git a/stdlib/src/statistics.rs b/stdlib/src/statistics.rs index 356bfd66f9..72e5d129a0 100644 --- a/stdlib/src/statistics.rs +++ b/stdlib/src/statistics.rs @@ -2,7 +2,7 @@ pub(crate) use _statistics::make_module; #[pymodule] mod _statistics { - use crate::vm::{function::ArgIntoFloat, PyResult, VirtualMachine}; + use crate::vm::{PyResult, VirtualMachine, function::ArgIntoFloat}; // See https://github.com/python/cpython/blob/6846d6712a0894f8e1a91716c11dd79f42864216/Modules/_statisticsmodule.c#L28-L120 #[allow(clippy::excessive_precision)] diff --git a/stdlib/src/syslog.rs b/stdlib/src/syslog.rs index 5fc6e30a56..3b36f9ea74 100644 --- a/stdlib/src/syslog.rs +++ b/stdlib/src/syslog.rs @@ -6,10 +6,10 @@ pub(crate) use syslog::make_module; mod syslog { use crate::common::lock::PyRwLock; use crate::vm::{ + PyObjectRef, PyPayload, PyResult, VirtualMachine, builtins::{PyStr, PyStrRef}, function::{OptionalArg, OptionalOption}, utils::ToCString, - PyObjectRef, PyPayload, PyResult, VirtualMachine, }; use std::{ffi::CStr, os::raw::c_char}; diff --git a/stdlib/src/termios.rs b/stdlib/src/termios.rs index 84c12b2b68..5c49d62a3c 100644 --- a/stdlib/src/termios.rs +++ b/stdlib/src/termios.rs @@ -3,10 +3,10 @@ pub(crate) use self::termios::make_module; #[pymodule] mod termios { use crate::vm::{ + PyObjectRef, PyResult, TryFromObject, VirtualMachine, builtins::{PyBaseExceptionRef, PyBytes, PyInt, PyListRef, PyTypeRef}, common::os::ErrorExt, convert::ToPyObject, - PyObjectRef, PyResult, TryFromObject, VirtualMachine, }; use termios::Termios; @@ -55,9 +55,9 @@ mod termios { ))] #[pyattr] use libc::{ - FIONCLEX, FIONREAD, TIOCEXCL, TIOCMBIC, TIOCMBIS, TIOCMGET, TIOCMSET, TIOCM_CAR, TIOCM_CD, - TIOCM_CTS, TIOCM_DSR, TIOCM_DTR, TIOCM_LE, TIOCM_RI, TIOCM_RNG, TIOCM_RTS, TIOCM_SR, - TIOCM_ST, TIOCNXCL, TIOCSCTTY, + FIONCLEX, FIONREAD, TIOCEXCL, TIOCM_CAR, TIOCM_CD, TIOCM_CTS, TIOCM_DSR, TIOCM_DTR, + TIOCM_LE, TIOCM_RI, TIOCM_RNG, TIOCM_RTS, TIOCM_SR, TIOCM_ST, TIOCMBIC, TIOCMBIS, TIOCMGET, + TIOCMSET, TIOCNXCL, TIOCSCTTY, }; #[cfg(any(target_os = "android", target_os = "linux"))] #[pyattr] @@ -100,12 +100,6 @@ mod termios { ))] #[pyattr] use termios::os::target::TCSASOFT; - #[cfg(any(target_os = "android", target_os = "linux"))] - #[pyattr] - use termios::os::target::{ - B1000000, B1152000, B1500000, B2000000, B2500000, B3000000, B3500000, B4000000, B500000, - B576000, CBAUDEX, - }; #[cfg(any( target_os = "android", target_os = "freebsd", @@ -116,6 +110,12 @@ mod termios { ))] #[pyattr] use termios::os::target::{B460800, B921600}; + #[cfg(any(target_os = "android", target_os = "linux"))] + #[pyattr] + use termios::os::target::{ + B500000, B576000, B1000000, B1152000, B1500000, B2000000, B2500000, B3000000, B3500000, + B4000000, CBAUDEX, + }; #[cfg(any( target_os = "android", target_os = "illumos", @@ -154,16 +154,17 @@ mod termios { use termios::os::target::{VSWTCH, VSWTCH as VSWTC}; #[pyattr] use termios::{ + B0, B50, B75, B110, B134, B150, B200, B300, B600, B1200, B1800, B2400, B4800, B9600, + B19200, B38400, BRKINT, CLOCAL, CREAD, CS5, CS6, CS7, CS8, CSIZE, CSTOPB, ECHO, ECHOE, + ECHOK, ECHONL, HUPCL, ICANON, ICRNL, IEXTEN, IGNBRK, IGNCR, IGNPAR, INLCR, INPCK, ISIG, + ISTRIP, IXANY, IXOFF, IXON, NOFLSH, OCRNL, ONLCR, ONLRET, ONOCR, OPOST, PARENB, PARMRK, + PARODD, TCIFLUSH, TCIOFF, TCIOFLUSH, TCION, TCOFLUSH, TCOOFF, TCOON, TCSADRAIN, TCSAFLUSH, + TCSANOW, TOSTOP, VEOF, VEOL, VERASE, VINTR, VKILL, VMIN, VQUIT, VSTART, VSTOP, VSUSP, + VTIME, os::target::{ - B115200, B230400, B57600, CRTSCTS, ECHOCTL, ECHOKE, ECHOPRT, EXTA, EXTB, FLUSHO, + B57600, B115200, B230400, CRTSCTS, ECHOCTL, ECHOKE, ECHOPRT, EXTA, EXTB, FLUSHO, IMAXBEL, NCCS, PENDIN, VDISCARD, VEOL2, VLNEXT, VREPRINT, VWERASE, }, - B0, B110, B1200, B134, B150, B1800, B19200, B200, B2400, B300, B38400, B4800, B50, B600, - B75, B9600, BRKINT, CLOCAL, CREAD, CS5, CS6, CS7, CS8, CSIZE, CSTOPB, ECHO, ECHOE, ECHOK, - ECHONL, HUPCL, ICANON, ICRNL, IEXTEN, IGNBRK, IGNCR, IGNPAR, INLCR, INPCK, ISIG, ISTRIP, - IXANY, IXOFF, IXON, NOFLSH, OCRNL, ONLCR, ONLRET, ONOCR, OPOST, PARENB, PARMRK, PARODD, - TCIFLUSH, TCIOFF, TCIOFLUSH, TCION, TCOFLUSH, TCOOFF, TCOON, TCSADRAIN, TCSAFLUSH, TCSANOW, - TOSTOP, VEOF, VEOL, VERASE, VINTR, VKILL, VMIN, VQUIT, VSTART, VSTOP, VSUSP, VTIME, }; #[pyfunction] diff --git a/stdlib/src/unicodedata.rs b/stdlib/src/unicodedata.rs index 70483073a7..49f3ef6250 100644 --- a/stdlib/src/unicodedata.rs +++ b/stdlib/src/unicodedata.rs @@ -5,8 +5,8 @@ // spell-checker:ignore nfkc unistr unidata use crate::vm::{ - builtins::PyModule, builtins::PyStr, convert::TryFromBorrowedObject, PyObject, PyObjectRef, - PyPayload, PyRef, PyResult, VirtualMachine, + PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, builtins::PyModule, + builtins::PyStr, convert::TryFromBorrowedObject, }; pub fn make_module(vm: &VirtualMachine) -> PyRef { @@ -61,14 +61,14 @@ impl<'a> TryFromBorrowedObject<'a> for NormalizeForm { #[pymodule] mod unicodedata { use crate::vm::{ - builtins::PyStrRef, function::OptionalArg, PyObjectRef, PyPayload, PyRef, PyResult, - VirtualMachine, + PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, builtins::PyStrRef, + function::OptionalArg, }; use itertools::Itertools; use ucd::{Codepoint, EastAsianWidth}; use unic_char_property::EnumeratedCharProperty; use unic_normal::StrNormalForm; - use unic_ucd_age::{Age, UnicodeVersion, UNICODE_VERSION}; + use unic_ucd_age::{Age, UNICODE_VERSION, UnicodeVersion}; use unic_ucd_bidi::BidiClass; use unic_ucd_category::GeneralCategory; diff --git a/stdlib/src/uuid.rs b/stdlib/src/uuid.rs index 4cadbb787b..0bd7d0db8a 100644 --- a/stdlib/src/uuid.rs +++ b/stdlib/src/uuid.rs @@ -5,7 +5,7 @@ mod _uuid { use crate::{builtins::PyNone, vm::VirtualMachine}; use mac_address::get_mac_address; use once_cell::sync::OnceCell; - use uuid::{timestamp::Timestamp, Context, Uuid}; + use uuid::{Context, Uuid, timestamp::Timestamp}; fn get_node_id() -> [u8; 6] { match get_mac_address() { diff --git a/stdlib/src/zlib.rs b/stdlib/src/zlib.rs index bcfe93aff5..19ed659bbb 100644 --- a/stdlib/src/zlib.rs +++ b/stdlib/src/zlib.rs @@ -5,17 +5,17 @@ pub(crate) use zlib::make_module; #[pymodule] mod zlib { use crate::vm::{ + PyObject, PyPayload, PyResult, VirtualMachine, builtins::{PyBaseExceptionRef, PyBytesRef, PyIntRef, PyTypeRef}, common::lock::PyMutex, convert::TryFromBorrowedObject, function::{ArgBytesLike, ArgPrimitiveIndex, ArgSize, OptionalArg}, types::Constructor, - PyObject, PyPayload, PyResult, VirtualMachine, }; use adler32::RollingAdler32 as Adler32; use flate2::{ - write::ZlibEncoder, Compress, Compression, Decompress, FlushCompress, FlushDecompress, - Status, + Compress, Compression, Decompress, FlushCompress, FlushDecompress, Status, + write::ZlibEncoder, }; use std::io::Write; @@ -420,7 +420,7 @@ mod zlib { fn flush(&self, length: OptionalArg, vm: &VirtualMachine) -> PyResult> { let length = match length { OptionalArg::Present(ArgSize { value }) if value <= 0 => { - return Err(vm.new_value_error("length must be greater than zero".to_owned())) + return Err(vm.new_value_error("length must be greater than zero".to_owned())); } OptionalArg::Present(ArgSize { value }) => value as usize, OptionalArg::Missing => DEF_BUF_SIZE, diff --git a/vm/src/anystr.rs b/vm/src/anystr.rs index f5bfab1c57..0fd1d8f2f6 100644 --- a/vm/src/anystr.rs +++ b/vm/src/anystr.rs @@ -1,9 +1,9 @@ use crate::{ + Py, PyObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine, builtins::{PyIntRef, PyTuple}, cformat::cformat_string, convert::TryFromBorrowedObject, function::OptionalOption, - Py, PyObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine, }; use num_traits::{cast::ToPrimitive, sign::Signed}; @@ -386,11 +386,7 @@ pub trait AnyStr { b'\n' => (keep, 1), b'\r' => { let is_rn = enumerated.next_if(|(_, ch)| **ch == b'\n').is_some(); - if is_rn { - (keep + keep, 2) - } else { - (keep, 1) - } + if is_rn { (keep + keep, 2) } else { (keep, 1) } } _ => continue, }; diff --git a/vm/src/buffer.rs b/vm/src/buffer.rs index bebc7b13a9..3b76002d04 100644 --- a/vm/src/buffer.rs +++ b/vm/src/buffer.rs @@ -1,9 +1,9 @@ use crate::{ + PyObjectRef, PyResult, TryFromObject, VirtualMachine, builtins::{PyBaseExceptionRef, PyBytesRef, PyTuple, PyTupleRef, PyTypeRef}, common::{static_cell, str::wchar_t}, convert::ToPyObject, function::{ArgBytesLike, ArgIntoBool, ArgIntoFloat}, - PyObjectRef, PyResult, TryFromObject, VirtualMachine, }; use half::f16; use itertools::Itertools; @@ -99,8 +99,8 @@ impl fmt::Debug for FormatType { impl FormatType { fn info(self, e: Endianness) -> &'static FormatInfo { - use mem::{align_of, size_of}; use FormatType::*; + use mem::{align_of, size_of}; macro_rules! native_info { ($t:ty) => {{ &FormatInfo { diff --git a/vm/src/builtins/asyncgenerator.rs b/vm/src/builtins/asyncgenerator.rs index f9ed4719d8..3aee327e5b 100644 --- a/vm/src/builtins/asyncgenerator.rs +++ b/vm/src/builtins/asyncgenerator.rs @@ -1,5 +1,6 @@ use super::{PyCode, PyGenericAlias, PyStrRef, PyType, PyTypeRef}; use crate::{ + AsObject, Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, builtins::PyBaseExceptionRef, class::PyClassImpl, coroutine::Coro, @@ -7,7 +8,6 @@ use crate::{ function::OptionalArg, protocol::PyIterReturn, types::{IterNext, Iterable, Representable, SelfIter, Unconstructible}, - AsObject, Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, }; use crossbeam_utils::atomic::AtomicCell; @@ -208,7 +208,7 @@ impl PyAsyncGenASend { AwaitableState::Closed => { return Err(vm.new_runtime_error( "cannot reuse already awaited __anext__()/asend()".to_owned(), - )) + )); } AwaitableState::Iter => val, // already running, all good AwaitableState::Init => { diff --git a/vm/src/builtins/bool.rs b/vm/src/builtins/bool.rs index 63bf6cff2d..93f5f4f1de 100644 --- a/vm/src/builtins/bool.rs +++ b/vm/src/builtins/bool.rs @@ -1,13 +1,13 @@ use super::{PyInt, PyStrRef, PyType, PyTypeRef}; use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyResult, TryFromBorrowedObject, + VirtualMachine, class::PyClassImpl, convert::{IntoPyException, ToPyObject, ToPyResult}, function::OptionalArg, identifier, protocol::PyNumberMethods, types::{AsNumber, Constructor, Representable}, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyResult, TryFromBorrowedObject, - VirtualMachine, }; use malachite_bigint::Sign; use num_traits::Zero; diff --git a/vm/src/builtins/builtin_func.rs b/vm/src/builtins/builtin_func.rs index bc1b082bc9..ff3ef38d3a 100644 --- a/vm/src/builtins/builtin_func.rs +++ b/vm/src/builtins/builtin_func.rs @@ -1,10 +1,10 @@ -use super::{type_, PyStrInterned, PyStrRef, PyType}; +use super::{PyStrInterned, PyStrRef, PyType, type_}; use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, class::PyClassImpl, convert::TryFromObject, function::{FuncArgs, PyComparisonValue, PyMethodDef, PyMethodFlags, PyNativeFn}, types::{Callable, Comparable, PyComparisonOp, Representable, Unconstructible}, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, }; use std::fmt; diff --git a/vm/src/builtins/bytearray.rs b/vm/src/builtins/bytearray.rs index 40e6cf7b5c..36cf8cadcd 100644 --- a/vm/src/builtins/bytearray.rs +++ b/vm/src/builtins/bytearray.rs @@ -4,12 +4,14 @@ use super::{ PyType, PyTypeRef, }; use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, + VirtualMachine, anystr::{self, AnyStr}, atomic_func, byte::{bytes_from_object, value_from_object}, bytesinner::{ - bytes_decode, ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions, - ByteInnerSplitOptions, ByteInnerTranslateOptions, DecodeArgs, PyBytesInner, + ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions, ByteInnerSplitOptions, + ByteInnerTranslateOptions, DecodeArgs, PyBytesInner, bytes_decode, }, class::PyClassImpl, common::{ @@ -33,8 +35,6 @@ use crate::{ DefaultConstructor, Initializer, IterNext, Iterable, PyComparisonOp, Representable, SelfIter, Unconstructible, }, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, - VirtualMachine, }; use bstr::ByteSlice; use std::mem::size_of; diff --git a/vm/src/builtins/bytes.rs b/vm/src/builtins/bytes.rs index 1db6417836..e9f5adc8bb 100644 --- a/vm/src/builtins/bytes.rs +++ b/vm/src/builtins/bytes.rs @@ -2,11 +2,13 @@ use super::{ PositionIterInternal, PyDictRef, PyIntRef, PyStrRef, PyTuple, PyTupleRef, PyType, PyTypeRef, }; use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, + TryFromBorrowedObject, TryFromObject, VirtualMachine, anystr::{self, AnyStr}, atomic_func, bytesinner::{ - bytes_decode, ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions, - ByteInnerSplitOptions, ByteInnerTranslateOptions, DecodeArgs, PyBytesInner, + ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions, ByteInnerSplitOptions, + ByteInnerTranslateOptions, DecodeArgs, PyBytesInner, bytes_decode, }, class::PyClassImpl, common::{hash::PyHash, lock::PyMutex}, @@ -23,8 +25,6 @@ use crate::{ AsBuffer, AsMapping, AsNumber, AsSequence, Callable, Comparable, Constructor, Hashable, IterNext, Iterable, PyComparisonOp, Representable, SelfIter, Unconstructible, }, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, - TryFromBorrowedObject, TryFromObject, VirtualMachine, }; use bstr::ByteSlice; use once_cell::sync::Lazy; diff --git a/vm/src/builtins/classmethod.rs b/vm/src/builtins/classmethod.rs index 02f836199e..94b6e0ca9f 100644 --- a/vm/src/builtins/classmethod.rs +++ b/vm/src/builtins/classmethod.rs @@ -1,9 +1,9 @@ use super::{PyBoundMethod, PyStr, PyType, PyTypeRef}; use crate::{ + AsObject, Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, class::PyClassImpl, common::lock::PyMutex, types::{Constructor, GetDescriptor, Initializer, Representable}, - AsObject, Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, }; /// classmethod(function) -> method diff --git a/vm/src/builtins/code.rs b/vm/src/builtins/code.rs index bedd71f241..d9d8895b19 100644 --- a/vm/src/builtins/code.rs +++ b/vm/src/builtins/code.rs @@ -4,6 +4,7 @@ use super::{PyStrRef, PyTupleRef, PyType, PyTypeRef}; use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyResult, VirtualMachine, builtins::PyStrInterned, bytecode::{self, AsBag, BorrowedConstant, CodeFlags, Constant, ConstantBag}, class::{PyClassImpl, StaticType}, @@ -12,7 +13,6 @@ use crate::{ function::{FuncArgs, OptionalArg}, source_code::OneIndexed, types::Representable, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyResult, VirtualMachine, }; use malachite_bigint::BigInt; use num_traits::Zero; diff --git a/vm/src/builtins/complex.rs b/vm/src/builtins/complex.rs index fb5bc5066c..e665d1e27a 100644 --- a/vm/src/builtins/complex.rs +++ b/vm/src/builtins/complex.rs @@ -1,5 +1,6 @@ -use super::{float, PyStr, PyType, PyTypeRef}; +use super::{PyStr, PyType, PyTypeRef, float}; use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, class::PyClassImpl, convert::{ToPyObject, ToPyResult}, function::{ @@ -11,7 +12,6 @@ use crate::{ protocol::PyNumberMethods, stdlib::warnings, types::{AsNumber, Comparable, Constructor, Hashable, PyComparisonOp, Representable}, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, }; use num_complex::Complex64; use num_traits::Zero; diff --git a/vm/src/builtins/coroutine.rs b/vm/src/builtins/coroutine.rs index db9592bd47..cca2db3293 100644 --- a/vm/src/builtins/coroutine.rs +++ b/vm/src/builtins/coroutine.rs @@ -1,12 +1,12 @@ use super::{PyCode, PyStrRef, PyType}; use crate::{ + AsObject, Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, class::PyClassImpl, coroutine::Coro, frame::FrameRef, function::OptionalArg, protocol::PyIterReturn, types::{IterNext, Iterable, Representable, SelfIter, Unconstructible}, - AsObject, Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, }; #[pyclass(module = false, name = "coroutine")] diff --git a/vm/src/builtins/descriptor.rs b/vm/src/builtins/descriptor.rs index 4b10f63f21..9da4e1d87a 100644 --- a/vm/src/builtins/descriptor.rs +++ b/vm/src/builtins/descriptor.rs @@ -1,10 +1,10 @@ use super::{PyStr, PyStrInterned, PyType}; use crate::{ - builtins::{builtin_func::PyNativeMethod, type_, PyTypeRef}, + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + builtins::{PyTypeRef, builtin_func::PyNativeMethod, type_}, class::PyClassImpl, function::{FuncArgs, PyMethodDef, PyMethodFlags, PySetterValue}, types::{Callable, GetDescriptor, Representable, Unconstructible}, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, }; use rustpython_common::lock::PyRwLock; diff --git a/vm/src/builtins/dict.rs b/vm/src/builtins/dict.rs index 68567e493f..e54d2a1931 100644 --- a/vm/src/builtins/dict.rs +++ b/vm/src/builtins/dict.rs @@ -1,13 +1,14 @@ use super::{ - set::PySetInner, IterStatus, PositionIterInternal, PyBaseExceptionRef, PyGenericAlias, - PyMappingProxy, PySet, PyStr, PyStrRef, PyTupleRef, PyType, PyTypeRef, + IterStatus, PositionIterInternal, PyBaseExceptionRef, PyGenericAlias, PyMappingProxy, PySet, + PyStr, PyStrRef, PyTupleRef, PyType, PyTypeRef, set::PySetInner, }; use crate::{ - atomic_func, + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyRefExact, PyResult, + TryFromObject, atomic_func, builtins::{ + PyTuple, iter::{builtins_iter, builtins_reversed}, type_::PyAttributes, - PyTuple, }, class::{PyClassDef, PyClassImpl}, common::ascii, @@ -21,8 +22,6 @@ use crate::{ Initializer, IterNext, Iterable, PyComparisonOp, Representable, SelfIter, Unconstructible, }, vm::VirtualMachine, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyRefExact, PyResult, - TryFromObject, }; use once_cell::sync::Lazy; use rustpython_common::lock::PyMutex; diff --git a/vm/src/builtins/enumerate.rs b/vm/src/builtins/enumerate.rs index 64d7c1ed36..aa84115074 100644 --- a/vm/src/builtins/enumerate.rs +++ b/vm/src/builtins/enumerate.rs @@ -3,12 +3,12 @@ use super::{ }; use crate::common::lock::{PyMutex, PyRwLock}; use crate::{ + AsObject, Context, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine, class::PyClassImpl, convert::ToPyObject, function::OptionalArg, protocol::{PyIter, PyIterReturn}, types::{Constructor, IterNext, Iterable, SelfIter}, - AsObject, Context, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine, }; use malachite_bigint::BigInt; use num_traits::Zero; diff --git a/vm/src/builtins/filter.rs b/vm/src/builtins/filter.rs index 3b33ff766f..009a1b3eab 100644 --- a/vm/src/builtins/filter.rs +++ b/vm/src/builtins/filter.rs @@ -1,9 +1,9 @@ use super::{PyType, PyTypeRef}; use crate::{ + Context, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine, class::PyClassImpl, protocol::{PyIter, PyIterReturn}, types::{Constructor, IterNext, Iterable, SelfIter}, - Context, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine, }; #[pyclass(module = false, name = "filter", traverse)] diff --git a/vm/src/builtins/float.rs b/vm/src/builtins/float.rs index 27fdff12ee..e33c25cb56 100644 --- a/vm/src/builtins/float.rs +++ b/vm/src/builtins/float.rs @@ -1,9 +1,11 @@ // spell-checker:ignore numer denom use super::{ - try_bigint_to_f64, PyByteArray, PyBytes, PyInt, PyIntRef, PyStr, PyStrRef, PyType, PyTypeRef, + PyByteArray, PyBytes, PyInt, PyIntRef, PyStr, PyStrRef, PyType, PyTypeRef, try_bigint_to_f64, }; use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, + TryFromBorrowedObject, TryFromObject, VirtualMachine, class::PyClassImpl, common::{float_ops, hash}, convert::{IntoPyException, ToPyObject, ToPyResult}, @@ -14,8 +16,6 @@ use crate::{ }, protocol::PyNumberMethods, types::{AsNumber, Callable, Comparable, Constructor, Hashable, PyComparisonOp, Representable}, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, - TryFromBorrowedObject, TryFromObject, VirtualMachine, }; use malachite_bigint::{BigInt, ToBigInt}; use num_complex::Complex64; diff --git a/vm/src/builtins/frame.rs b/vm/src/builtins/frame.rs index 3cc7d788fb..1fd031984a 100644 --- a/vm/src/builtins/frame.rs +++ b/vm/src/builtins/frame.rs @@ -4,11 +4,11 @@ use super::{PyCode, PyDictRef, PyIntRef, PyStrRef}; use crate::{ + AsObject, Context, Py, PyObjectRef, PyRef, PyResult, VirtualMachine, class::PyClassImpl, frame::{Frame, FrameRef}, function::PySetterValue, types::{Representable, Unconstructible}, - AsObject, Context, Py, PyObjectRef, PyRef, PyResult, VirtualMachine, }; use num_traits::Zero; diff --git a/vm/src/builtins/function.rs b/vm/src/builtins/function.rs index eb5a142f0c..63cf8ac5c3 100644 --- a/vm/src/builtins/function.rs +++ b/vm/src/builtins/function.rs @@ -2,8 +2,8 @@ mod jitfunc; use super::{ - tuple::PyTupleTyped, PyAsyncGen, PyCode, PyCoroutine, PyDictRef, PyGenerator, PyStr, PyStrRef, - PyTupleRef, PyType, PyTypeRef, + PyAsyncGen, PyCode, PyCoroutine, PyDictRef, PyGenerator, PyStr, PyStrRef, PyTupleRef, PyType, + PyTypeRef, tuple::PyTupleTyped, }; #[cfg(feature = "jit")] use crate::common::lock::OnceCell; @@ -12,6 +12,7 @@ use crate::convert::ToPyObject; use crate::function::ArgMapping; use crate::object::{Traverse, TraverseFn}; use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, bytecode, class::PyClassImpl, frame::Frame, @@ -20,7 +21,6 @@ use crate::{ types::{ Callable, Comparable, Constructor, GetAttr, GetDescriptor, PyComparisonOp, Representable, }, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, }; use itertools::Itertools; #[cfg(feature = "jit")] diff --git a/vm/src/builtins/function/jitfunc.rs b/vm/src/builtins/function/jitfunc.rs index d46458fc65..a46d9aa0f3 100644 --- a/vm/src/builtins/function/jitfunc.rs +++ b/vm/src/builtins/function/jitfunc.rs @@ -1,9 +1,9 @@ use crate::{ - builtins::{bool_, float, int, PyBaseExceptionRef, PyDictRef, PyFunction, PyStrInterned}, + AsObject, Py, PyObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine, + builtins::{PyBaseExceptionRef, PyDictRef, PyFunction, PyStrInterned, bool_, float, int}, bytecode::CodeFlags, convert::ToPyObject, function::FuncArgs, - AsObject, Py, PyObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine, }; use num_traits::ToPrimitive; use rustpython_jit::{AbiValue, Args, CompiledCode, JitArgumentError, JitType}; diff --git a/vm/src/builtins/generator.rs b/vm/src/builtins/generator.rs index e0ec77006d..db9c263cb2 100644 --- a/vm/src/builtins/generator.rs +++ b/vm/src/builtins/generator.rs @@ -4,13 +4,13 @@ use super::{PyCode, PyStrRef, PyType}; use crate::{ + AsObject, Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, class::PyClassImpl, coroutine::Coro, frame::FrameRef, function::OptionalArg, protocol::PyIterReturn, types::{IterNext, Iterable, Representable, SelfIter, Unconstructible}, - AsObject, Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, }; #[pyclass(module = false, name = "generator")] diff --git a/vm/src/builtins/genericalias.rs b/vm/src/builtins/genericalias.rs index 2746b03128..57c97ba62b 100644 --- a/vm/src/builtins/genericalias.rs +++ b/vm/src/builtins/genericalias.rs @@ -2,7 +2,8 @@ use once_cell::sync::Lazy; use super::type_; use crate::{ - atomic_func, + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, + VirtualMachine, atomic_func, builtins::{PyList, PyStr, PyTuple, PyTupleRef, PyType, PyTypeRef}, class::PyClassImpl, common::hash, @@ -13,8 +14,6 @@ use crate::{ AsMapping, AsNumber, Callable, Comparable, Constructor, GetAttr, Hashable, PyComparisonOp, Representable, }, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, - VirtualMachine, }; use std::fmt; diff --git a/vm/src/builtins/getset.rs b/vm/src/builtins/getset.rs index 603ec021e2..c2e11b770a 100644 --- a/vm/src/builtins/getset.rs +++ b/vm/src/builtins/getset.rs @@ -3,10 +3,10 @@ */ use super::{PyType, PyTypeRef}; use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyResult, VirtualMachine, class::PyClassImpl, function::{IntoPyGetterFunc, IntoPySetterFunc, PyGetterFunc, PySetterFunc, PySetterValue}, types::{GetDescriptor, Unconstructible}, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyResult, VirtualMachine, }; #[pyclass(module = false, name = "getset_descriptor")] diff --git a/vm/src/builtins/int.rs b/vm/src/builtins/int.rs index eb05b4394a..db9c5ea4ed 100644 --- a/vm/src/builtins/int.rs +++ b/vm/src/builtins/int.rs @@ -1,5 +1,7 @@ -use super::{float, PyByteArray, PyBytes, PyStr, PyType, PyTypeRef}; +use super::{PyByteArray, PyBytes, PyStr, PyType, PyTypeRef, float}; use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyRefExact, PyResult, + TryFromBorrowedObject, VirtualMachine, builtins::PyStrRef, bytesinner::PyBytesInner, class::PyClassImpl, @@ -14,8 +16,6 @@ use crate::{ }, protocol::PyNumberMethods, types::{AsNumber, Comparable, Constructor, Hashable, PyComparisonOp, Representable}, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyRefExact, PyResult, - TryFromBorrowedObject, VirtualMachine, }; use malachite_bigint::{BigInt, Sign}; use num_integer::Integer; @@ -109,11 +109,7 @@ fn inner_pow(int1: &BigInt, int2: &BigInt, vm: &VirtualMachine) -> PyResult { } else if int1.is_zero() { 0 } else if int1 == &BigInt::from(-1) { - if int2.is_odd() { - -1 - } else { - 1 - } + if int2.is_odd() { -1 } else { 1 } } else { // missing feature: BigInt exp // practically, exp over u64 is not possible to calculate anyway @@ -426,11 +422,7 @@ impl PyInt { // based on rust-num/num-integer#10, should hopefully be published soon fn normalize(a: BigInt, n: &BigInt) -> BigInt { let a = a % n; - if a.is_negative() { - a + n - } else { - a - } + if a.is_negative() { a + n } else { a } } fn inverse(a: BigInt, n: &BigInt) -> Option { use num_integer::*; @@ -642,7 +634,7 @@ impl PyInt { Sign::Minus if !signed => { return Err( vm.new_overflow_error("can't convert negative int to unsigned".to_owned()) - ) + ); } Sign::NoSign => return Ok(vec![0u8; byte_len].into()), _ => {} diff --git a/vm/src/builtins/iter.rs b/vm/src/builtins/iter.rs index dbe4fc6597..5a47abfac7 100644 --- a/vm/src/builtins/iter.rs +++ b/vm/src/builtins/iter.rs @@ -4,12 +4,12 @@ use super::{PyInt, PyTupleRef, PyType}; use crate::{ + Context, Py, PyObject, PyObjectRef, PyPayload, PyResult, VirtualMachine, class::PyClassImpl, function::ArgCallable, object::{Traverse, TraverseFn}, protocol::{PyIterReturn, PySequence, PySequenceMethods}, types::{IterNext, Iterable, SelfIter}, - Context, Py, PyObject, PyObjectRef, PyPayload, PyResult, VirtualMachine, }; use rustpython_common::{ lock::{PyMutex, PyRwLock, PyRwLockUpgradableReadGuard}, diff --git a/vm/src/builtins/list.rs b/vm/src/builtins/list.rs index 33d6072cd5..3b9624694e 100644 --- a/vm/src/builtins/list.rs +++ b/vm/src/builtins/list.rs @@ -4,6 +4,7 @@ use crate::common::lock::{ PyMappedRwLockReadGuard, PyMutex, PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard, }; use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, class::PyClassImpl, convert::ToPyObject, function::{ArgSize, FuncArgs, OptionalArg, PyComparisonValue}, @@ -18,7 +19,6 @@ use crate::{ }, utils::collection_repr, vm::VirtualMachine, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, }; use std::{fmt, ops::DerefMut}; diff --git a/vm/src/builtins/map.rs b/vm/src/builtins/map.rs index 44bacf587e..555e38c8b9 100644 --- a/vm/src/builtins/map.rs +++ b/vm/src/builtins/map.rs @@ -1,11 +1,11 @@ use super::{PyType, PyTypeRef}; use crate::{ + Context, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine, builtins::PyTupleRef, class::PyClassImpl, function::PosArgs, protocol::{PyIter, PyIterReturn}, types::{Constructor, IterNext, Iterable, SelfIter}, - Context, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine, }; #[pyclass(module = false, name = "map", traverse)] diff --git a/vm/src/builtins/mappingproxy.rs b/vm/src/builtins/mappingproxy.rs index 297ddd0a77..659562cec1 100644 --- a/vm/src/builtins/mappingproxy.rs +++ b/vm/src/builtins/mappingproxy.rs @@ -1,5 +1,6 @@ use super::{PyDict, PyDictRef, PyGenericAlias, PyList, PyTuple, PyType, PyTypeRef}; use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, atomic_func, class::PyClassImpl, convert::ToPyObject, @@ -10,7 +11,6 @@ use crate::{ AsMapping, AsNumber, AsSequence, Comparable, Constructor, Iterable, PyComparisonOp, Representable, }, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, }; use once_cell::sync::Lazy; diff --git a/vm/src/builtins/memory.rs b/vm/src/builtins/memory.rs index dfbbdc5aaf..9426730f40 100644 --- a/vm/src/builtins/memory.rs +++ b/vm/src/builtins/memory.rs @@ -3,7 +3,8 @@ use super::{ PyTupleRef, PyType, PyTypeRef, }; use crate::{ - atomic_func, + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, + TryFromBorrowedObject, TryFromObject, VirtualMachine, atomic_func, buffer::FormatSpec, bytesinner::bytes_to_hex, class::PyClassImpl, @@ -24,8 +25,6 @@ use crate::{ AsBuffer, AsMapping, AsSequence, Comparable, Constructor, Hashable, IterNext, Iterable, PyComparisonOp, Representable, SelfIter, Unconstructible, }, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, - TryFromBorrowedObject, TryFromObject, VirtualMachine, }; use crossbeam_utils::atomic::AtomicCell; use itertools::Itertools; @@ -223,14 +222,16 @@ impl PyMemoryView { fn pos_from_multi_index(&self, indexes: &[isize], vm: &VirtualMachine) -> PyResult { match indexes.len().cmp(&self.desc.ndim()) { Ordering::Less => { - return Err(vm.new_not_implemented_error("sub-views are not implemented".to_owned())) + return Err( + vm.new_not_implemented_error("sub-views are not implemented".to_owned()) + ); } Ordering::Greater => { return Err(vm.new_type_error(format!( "cannot index {}-dimension view with {}-element tuple", self.desc.ndim(), indexes.len() - ))) + ))); } Ordering::Equal => (), } @@ -380,11 +381,7 @@ impl PyMemoryView { } }; ret = vm.bool_eq(&a_val, &b_val); - if let Ok(b) = ret { - !b - } else { - true - } + if let Ok(b) = ret { !b } else { true } }); ret } diff --git a/vm/src/builtins/module.rs b/vm/src/builtins/module.rs index de55b9deae..8c8f22cf58 100644 --- a/vm/src/builtins/module.rs +++ b/vm/src/builtins/module.rs @@ -1,11 +1,11 @@ use super::{PyDictRef, PyStr, PyStrRef, PyType, PyTypeRef}; use crate::{ - builtins::{pystr::AsPyStr, PyStrInterned}, + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + builtins::{PyStrInterned, pystr::AsPyStr}, class::PyClassImpl, convert::ToPyObject, function::{FuncArgs, PyMethodDef}, types::{GetAttr, Initializer, Representable}, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, }; #[pyclass(module = false, name = "module")] @@ -183,11 +183,12 @@ impl Initializer for PyModule { type Args = ModuleInitArgs; fn init(zelf: PyRef, args: Self::Args, vm: &VirtualMachine) -> PyResult<()> { - debug_assert!(zelf - .class() - .slots - .flags - .has_feature(crate::types::PyTypeFlags::HAS_DICT)); + debug_assert!( + zelf.class() + .slots + .flags + .has_feature(crate::types::PyTypeFlags::HAS_DICT) + ); zelf.init_dict(vm.ctx.intern_str(args.name.as_str()), args.doc, vm); Ok(()) } diff --git a/vm/src/builtins/namespace.rs b/vm/src/builtins/namespace.rs index 441fd014f0..38146baa72 100644 --- a/vm/src/builtins/namespace.rs +++ b/vm/src/builtins/namespace.rs @@ -1,5 +1,6 @@ -use super::{tuple::IntoPyTuple, PyTupleRef, PyType}; +use super::{PyTupleRef, PyType, tuple::IntoPyTuple}; use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, builtins::PyDict, class::PyClassImpl, function::{FuncArgs, PyComparisonValue}, @@ -7,7 +8,6 @@ use crate::{ types::{ Comparable, Constructor, DefaultConstructor, Initializer, PyComparisonOp, Representable, }, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, }; /// A simple attribute-based namespace. diff --git a/vm/src/builtins/object.rs b/vm/src/builtins/object.rs index aeb9e93cc6..cce1422d56 100644 --- a/vm/src/builtins/object.rs +++ b/vm/src/builtins/object.rs @@ -2,11 +2,11 @@ use super::{PyDictRef, PyList, PyStr, PyStrRef, PyType, PyTypeRef}; use crate::common::hash::PyHash; use crate::types::PyTypeFlags; use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyResult, VirtualMachine, class::PyClassImpl, convert::ToPyResult, function::{Either, FuncArgs, PyArithmeticValue, PyComparisonValue, PySetterValue}, types::{Constructor, PyComparisonOp}, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyResult, VirtualMachine, }; use itertools::Itertools; diff --git a/vm/src/builtins/property.rs b/vm/src/builtins/property.rs index 61e1ff1692..5bfae5a081 100644 --- a/vm/src/builtins/property.rs +++ b/vm/src/builtins/property.rs @@ -5,10 +5,10 @@ use super::{PyStrRef, PyType, PyTypeRef}; use crate::common::lock::PyRwLock; use crate::function::{IntoFuncArgs, PosArgs}; use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, class::PyClassImpl, function::{FuncArgs, PySetterValue}, types::{Constructor, GetDescriptor, Initializer}, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, }; #[pyclass(module = false, name = "property", traverse)] diff --git a/vm/src/builtins/range.rs b/vm/src/builtins/range.rs index 602ea2d37f..b542a5f191 100644 --- a/vm/src/builtins/range.rs +++ b/vm/src/builtins/range.rs @@ -1,8 +1,9 @@ use super::{ - builtins_iter, tuple::tuple_hash, PyInt, PyIntRef, PySlice, PyTupleRef, PyType, PyTypeRef, + PyInt, PyIntRef, PySlice, PyTupleRef, PyType, PyTypeRef, builtins_iter, tuple::tuple_hash, }; use crate::{ - atomic_func, + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, + VirtualMachine, atomic_func, class::PyClassImpl, common::hash::PyHash, function::{ArgIndex, FuncArgs, OptionalArg, PyComparisonValue}, @@ -11,8 +12,6 @@ use crate::{ AsMapping, AsSequence, Comparable, Hashable, IterNext, Iterable, PyComparisonOp, Representable, SelfIter, Unconstructible, }, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, - VirtualMachine, }; use crossbeam_utils::atomic::AtomicCell; use malachite_bigint::{BigInt, Sign}; diff --git a/vm/src/builtins/set.rs b/vm/src/builtins/set.rs index 1ab1fe21cf..62a66c89b4 100644 --- a/vm/src/builtins/set.rs +++ b/vm/src/builtins/set.rs @@ -2,10 +2,11 @@ * Builtin set type with a sequence of unique items. */ use super::{ - builtins_iter, IterStatus, PositionIterInternal, PyDict, PyDictRef, PyGenericAlias, PyTupleRef, - PyType, PyTypeRef, + IterStatus, PositionIterInternal, PyDict, PyDictRef, PyGenericAlias, PyTupleRef, PyType, + PyTypeRef, builtins_iter, }; use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, atomic_func, class::PyClassImpl, common::{ascii, hash::PyHash, lock::PyMutex, rc::PyRc}, @@ -21,7 +22,6 @@ use crate::{ }, utils::collection_repr, vm::VirtualMachine, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, }; use once_cell::sync::Lazy; use rustpython_common::{ diff --git a/vm/src/builtins/singletons.rs b/vm/src/builtins/singletons.rs index 65b171a262..da0c718c46 100644 --- a/vm/src/builtins/singletons.rs +++ b/vm/src/builtins/singletons.rs @@ -1,10 +1,10 @@ use super::{PyStrRef, PyType, PyTypeRef}; use crate::{ + Context, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine, class::PyClassImpl, convert::ToPyObject, protocol::PyNumberMethods, types::{AsNumber, Constructor, Representable}, - Context, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine, }; #[pyclass(module = false, name = "NoneType")] diff --git a/vm/src/builtins/slice.rs b/vm/src/builtins/slice.rs index 5da3649115..4194360f4a 100644 --- a/vm/src/builtins/slice.rs +++ b/vm/src/builtins/slice.rs @@ -2,13 +2,13 @@ // spell-checker:ignore sliceobject use super::{PyStrRef, PyTupleRef, PyType, PyTypeRef}; use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, class::PyClassImpl, common::hash::{PyHash, PyUHash}, convert::ToPyObject, function::{ArgIndex, FuncArgs, OptionalArg, PyComparisonValue}, sliceable::SaturatedSlice, types::{Comparable, Constructor, Hashable, PyComparisonOp, Representable}, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, }; use malachite_bigint::{BigInt, ToBigInt}; use num_traits::{One, Signed, Zero}; @@ -264,11 +264,7 @@ impl Comparable for PySlice { let eq = vm.identical_or_equal(zelf.start_ref(vm), other.start_ref(vm))? && vm.identical_or_equal(&zelf.stop, &other.stop)? && vm.identical_or_equal(zelf.step_ref(vm), other.step_ref(vm))?; - if op == PyComparisonOp::Ne { - !eq - } else { - eq - } + if op == PyComparisonOp::Ne { !eq } else { eq } } PyComparisonOp::Gt | PyComparisonOp::Ge => None .or_else(|| { diff --git a/vm/src/builtins/staticmethod.rs b/vm/src/builtins/staticmethod.rs index 59a5b18b5d..8e2333da7f 100644 --- a/vm/src/builtins/staticmethod.rs +++ b/vm/src/builtins/staticmethod.rs @@ -1,10 +1,10 @@ use super::{PyStr, PyType, PyTypeRef}; use crate::{ + Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, class::PyClassImpl, common::lock::PyMutex, function::FuncArgs, types::{Callable, Constructor, GetDescriptor, Initializer, Representable}, - Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, }; #[pyclass(module = false, name = "staticmethod", traverse)] diff --git a/vm/src/builtins/str.rs b/vm/src/builtins/str.rs index f37a004297..bf9a9f679c 100644 --- a/vm/src/builtins/str.rs +++ b/vm/src/builtins/str.rs @@ -1,10 +1,12 @@ use super::{ + PositionIterInternal, PyBytesRef, PyDict, PyTupleRef, PyType, PyTypeRef, int::{PyInt, PyIntRef}, iter::IterStatus::{self, Exhausted}, - PositionIterInternal, PyBytesRef, PyDict, PyTupleRef, PyType, PyTypeRef, }; use crate::{ - anystr::{self, adjust_indices, AnyStr, AnyStrContainer, AnyStrWrapper}, + AsObject, Context, Py, PyExact, PyObject, PyObjectRef, PyPayload, PyRef, PyRefExact, PyResult, + TryFromBorrowedObject, VirtualMachine, + anystr::{self, AnyStr, AnyStrContainer, AnyStrWrapper, adjust_indices}, atomic_func, class::PyClassImpl, common::str::{BorrowedStr, PyStrKind, PyStrKindData}, @@ -20,8 +22,6 @@ use crate::{ AsMapping, AsNumber, AsSequence, Comparable, Constructor, Hashable, IterNext, Iterable, PyComparisonOp, Representable, SelfIter, Unconstructible, }, - AsObject, Context, Py, PyExact, PyObject, PyObjectRef, PyPayload, PyRef, PyRefExact, PyResult, - TryFromBorrowedObject, VirtualMachine, }; use ascii::{AsciiStr, AsciiString}; use bstr::ByteSlice; @@ -904,11 +904,7 @@ impl PyStr { '\n' => 1, '\r' => { let is_rn = enumerated.next_if(|(_, ch)| *ch == '\n').is_some(); - if is_rn { - 2 - } else { - 1 - } + if is_rn { 2 } else { 1 } } '\x0b' | '\x0c' | '\x1c' | '\x1d' | '\x1e' | '\u{0085}' | '\u{2028}' | '\u{2029}' => ch.len_utf8(), diff --git a/vm/src/builtins/super.rs b/vm/src/builtins/super.rs index a0192cea5c..5f363ebea5 100644 --- a/vm/src/builtins/super.rs +++ b/vm/src/builtins/super.rs @@ -5,11 +5,11 @@ See also [CPython source code.](https://github.com/python/cpython/blob/50b48572d use super::{PyStr, PyType, PyTypeRef}; use crate::{ + AsObject, Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, class::PyClassImpl, common::lock::PyRwLock, function::{FuncArgs, IntoFuncArgs, OptionalArg}, types::{Callable, Constructor, GetAttr, GetDescriptor, Initializer, Representable}, - AsObject, Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, }; #[pyclass(module = false, name = "super", traverse)] diff --git a/vm/src/builtins/traceback.rs b/vm/src/builtins/traceback.rs index 6506e95363..b0abbd006a 100644 --- a/vm/src/builtins/traceback.rs +++ b/vm/src/builtins/traceback.rs @@ -2,7 +2,7 @@ use rustpython_common::lock::PyMutex; use super::PyType; use crate::{ - class::PyClassImpl, frame::FrameRef, source_code::LineNumber, Context, Py, PyPayload, PyRef, + Context, Py, PyPayload, PyRef, class::PyClassImpl, frame::FrameRef, source_code::LineNumber, }; #[pyclass(module = false, name = "traceback", traverse)] diff --git a/vm/src/builtins/tuple.rs b/vm/src/builtins/tuple.rs index 9d1cc2f5ce..66cb2799a7 100644 --- a/vm/src/builtins/tuple.rs +++ b/vm/src/builtins/tuple.rs @@ -2,6 +2,7 @@ use super::{PositionIterInternal, PyGenericAlias, PyStrRef, PyType, PyTypeRef}; use crate::common::{hash::PyHash, lock::PyMutex}; use crate::object::{Traverse, TraverseFn}; use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, atomic_func, class::PyClassImpl, convert::{ToPyObject, TransmuteFromObject}, @@ -17,7 +18,6 @@ use crate::{ }, utils::collection_repr, vm::VirtualMachine, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, }; use once_cell::sync::Lazy; use std::{fmt, marker::PhantomData}; diff --git a/vm/src/builtins/type.rs b/vm/src/builtins/type.rs index c3fc28cd60..08a15575ed 100644 --- a/vm/src/builtins/type.rs +++ b/vm/src/builtins/type.rs @@ -1,16 +1,18 @@ use super::{ - mappingproxy::PyMappingProxy, object, union_, PyClassMethod, PyDictRef, PyList, PyStr, - PyStrInterned, PyStrRef, PyTuple, PyTupleRef, PyWeak, + PyClassMethod, PyDictRef, PyList, PyStr, PyStrInterned, PyStrRef, PyTuple, PyTupleRef, PyWeak, + mappingproxy::PyMappingProxy, object, union_, }; use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, + VirtualMachine, builtins::{ + PyBaseExceptionRef, descriptor::{ MemberGetter, MemberKind, MemberSetter, PyDescriptorOwned, PyMemberDef, PyMemberDescriptor, }, function::PyCellRef, tuple::{IntoPyTuple, PyTupleTyped}, - PyBaseExceptionRef, }, class::{PyClassImpl, StaticType}, common::{ @@ -26,10 +28,8 @@ use crate::{ types::{ AsNumber, Callable, Constructor, GetAttr, PyTypeFlags, PyTypeSlots, Representable, SetAttr, }, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, - VirtualMachine, }; -use indexmap::{map::Entry, IndexMap}; +use indexmap::{IndexMap, map::Entry}; use itertools::Itertools; use std::{borrow::Borrow, collections::HashSet, fmt, ops::Deref, pin::Pin, ptr::NonNull}; diff --git a/vm/src/builtins/union.rs b/vm/src/builtins/union.rs index 668d87bdce..f9dc8f3131 100644 --- a/vm/src/builtins/union.rs +++ b/vm/src/builtins/union.rs @@ -1,5 +1,6 @@ use super::{genericalias, type_}; use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, atomic_func, builtins::{PyFrozenSet, PyStr, PyTuple, PyTupleRef, PyType}, class::PyClassImpl, @@ -8,7 +9,6 @@ use crate::{ function::PyComparisonValue, protocol::{PyMappingMethods, PyNumberMethods}, types::{AsMapping, AsNumber, Comparable, GetAttr, Hashable, PyComparisonOp, Representable}, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, }; use once_cell::sync::Lazy; use std::fmt; diff --git a/vm/src/builtins/weakproxy.rs b/vm/src/builtins/weakproxy.rs index f271e4cddb..d17bc75118 100644 --- a/vm/src/builtins/weakproxy.rs +++ b/vm/src/builtins/weakproxy.rs @@ -1,6 +1,6 @@ use super::{PyStr, PyStrRef, PyType, PyTypeRef, PyWeak}; use crate::{ - atomic_func, + Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, atomic_func, class::PyClassImpl, common::hash::PyHash, function::{OptionalArg, PyComparisonValue, PySetterValue}, @@ -10,7 +10,6 @@ use crate::{ AsMapping, AsSequence, Comparable, Constructor, GetAttr, Hashable, IterNext, Iterable, PyComparisonOp, Representable, SetAttr, }, - Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, }; use once_cell::sync::Lazy; diff --git a/vm/src/builtins/weakref.rs b/vm/src/builtins/weakref.rs index 1d52225a26..9b2f248aa9 100644 --- a/vm/src/builtins/weakref.rs +++ b/vm/src/builtins/weakref.rs @@ -4,10 +4,10 @@ use crate::common::{ hash::{self, PyHash}, }; use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyResult, VirtualMachine, class::PyClassImpl, function::OptionalArg, types::{Callable, Comparable, Constructor, Hashable, PyComparisonOp, Representable}, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyResult, VirtualMachine, }; pub use crate::object::PyWeak; diff --git a/vm/src/builtins/zip.rs b/vm/src/builtins/zip.rs index 56c88f14c6..abd82b3ccb 100644 --- a/vm/src/builtins/zip.rs +++ b/vm/src/builtins/zip.rs @@ -1,11 +1,11 @@ use super::{PyType, PyTypeRef}; use crate::{ + AsObject, Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, builtins::PyTupleRef, class::PyClassImpl, function::{ArgIntoBool, OptionalArg, PosArgs}, protocol::{PyIter, PyIterReturn}, types::{Constructor, IterNext, Iterable, SelfIter}, - AsObject, Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, }; use rustpython_common::atomic::{self, PyAtomic, Radium}; diff --git a/vm/src/bytesinner.rs b/vm/src/bytesinner.rs index 79d4d96262..3754b75ee1 100644 --- a/vm/src/bytesinner.rs +++ b/vm/src/bytesinner.rs @@ -1,8 +1,9 @@ use crate::{ + AsObject, PyObject, PyObjectRef, PyPayload, PyResult, TryFromBorrowedObject, VirtualMachine, anystr::{self, AnyStr, AnyStrContainer, AnyStrWrapper}, builtins::{ - pystr, PyBaseExceptionRef, PyByteArray, PyBytes, PyBytesRef, PyInt, PyIntRef, PyStr, - PyStrRef, PyTypeRef, + PyBaseExceptionRef, PyByteArray, PyBytes, PyBytesRef, PyInt, PyIntRef, PyStr, PyStrRef, + PyTypeRef, pystr, }, byte::bytes_from_object, cformat::cformat_bytes, @@ -13,7 +14,6 @@ use crate::{ protocol::PyBuffer, sequence::{SequenceExt, SequenceMutExt}, types::PyComparisonOp, - AsObject, PyObject, PyObjectRef, PyPayload, PyResult, TryFromBorrowedObject, VirtualMachine, }; use bstr::ByteSlice; use itertools::Itertools; diff --git a/vm/src/cformat.rs b/vm/src/cformat.rs index 79f4d68833..6e14034d0b 100644 --- a/vm/src/cformat.rs +++ b/vm/src/cformat.rs @@ -2,13 +2,13 @@ //! as per the [Python Docs](https://docs.python.org/3/library/stdtypes.html#printf-style-string-formatting). use crate::{ + AsObject, PyObjectRef, PyResult, TryFromBorrowedObject, TryFromObject, VirtualMachine, builtins::{ - try_f64_to_bigint, tuple, PyBaseExceptionRef, PyByteArray, PyBytes, PyFloat, PyInt, PyStr, + PyBaseExceptionRef, PyByteArray, PyBytes, PyFloat, PyInt, PyStr, try_f64_to_bigint, tuple, }, function::ArgIntoFloat, protocol::PyBuffer, stdlib::builtins, - AsObject, PyObjectRef, PyResult, TryFromBorrowedObject, TryFromObject, VirtualMachine, }; use itertools::Itertools; use num_traits::cast::ToPrimitive; diff --git a/vm/src/class.rs b/vm/src/class.rs index 3b7f1bffcb..bc38d6bd61 100644 --- a/vm/src/class.rs +++ b/vm/src/class.rs @@ -5,7 +5,7 @@ use crate::{ function::PyMethodDef, identifier, object::Py, - types::{hash_not_implemented, PyTypeFlags, PyTypeSlots}, + types::{PyTypeFlags, PyTypeSlots, hash_not_implemented}, vm::Context, }; use rustpython_common::static_cell; diff --git a/vm/src/codecs.rs b/vm/src/codecs.rs index 767bd873b6..e104097413 100644 --- a/vm/src/codecs.rs +++ b/vm/src/codecs.rs @@ -1,9 +1,9 @@ use crate::{ + AsObject, Context, PyObject, PyObjectRef, PyPayload, PyResult, TryFromObject, VirtualMachine, builtins::{PyBaseExceptionRef, PyBytesRef, PyStr, PyStrRef, PyTuple, PyTupleRef}, common::{ascii, lock::PyRwLock}, convert::ToPyObject, function::PyMethodDef, - AsObject, Context, PyObject, PyObjectRef, PyPayload, PyResult, TryFromObject, VirtualMachine, }; use std::{borrow::Cow, collections::HashMap, fmt::Write, ops::Range}; diff --git a/vm/src/convert/to_pyobject.rs b/vm/src/convert/to_pyobject.rs index c61296fe69..840c0c65fb 100644 --- a/vm/src/convert/to_pyobject.rs +++ b/vm/src/convert/to_pyobject.rs @@ -1,4 +1,4 @@ -use crate::{builtins::PyBaseExceptionRef, PyObjectRef, PyResult, VirtualMachine}; +use crate::{PyObjectRef, PyResult, VirtualMachine, builtins::PyBaseExceptionRef}; /// Implemented by any type that can be returned from a built-in Python function. /// diff --git a/vm/src/convert/try_from.rs b/vm/src/convert/try_from.rs index 8abe829803..941e1fef2a 100644 --- a/vm/src/convert/try_from.rs +++ b/vm/src/convert/try_from.rs @@ -1,7 +1,7 @@ use crate::{ + Py, VirtualMachine, builtins::PyFloat, object::{AsObject, PyObject, PyObjectRef, PyPayload, PyRef, PyResult}, - Py, VirtualMachine, }; use num_traits::ToPrimitive; diff --git a/vm/src/coroutine.rs b/vm/src/coroutine.rs index 5dd34a799d..56eb520b2c 100644 --- a/vm/src/coroutine.rs +++ b/vm/src/coroutine.rs @@ -1,9 +1,9 @@ use crate::{ + AsObject, PyObject, PyObjectRef, PyResult, VirtualMachine, builtins::{PyBaseExceptionRef, PyStrRef}, common::lock::PyMutex, frame::{ExecutionResult, FrameRef}, protocol::PyIterReturn, - AsObject, PyObject, PyObjectRef, PyResult, VirtualMachine, }; use crossbeam_utils::atomic::AtomicCell; diff --git a/vm/src/dictdatatype.rs b/vm/src/dictdatatype.rs index 4baeef0bfc..7c8fd23834 100644 --- a/vm/src/dictdatatype.rs +++ b/vm/src/dictdatatype.rs @@ -4,9 +4,9 @@ //! And: http://code.activestate.com/recipes/578375/ use crate::{ + AsObject, Py, PyExact, PyObject, PyObjectRef, PyRefExact, PyResult, VirtualMachine, builtins::{PyInt, PyStr, PyStrInterned, PyStrRef}, convert::ToPyObject, - AsObject, Py, PyExact, PyObject, PyObjectRef, PyRefExact, PyResult, VirtualMachine, }; use crate::{ common::{ @@ -915,7 +915,7 @@ fn str_exact<'a>(obj: &'a PyObject, vm: &VirtualMachine) -> Option<&'a PyStr> { #[cfg(test)] mod tests { use super::*; - use crate::{common::ascii, Interpreter}; + use crate::{Interpreter, common::ascii}; #[test] fn test_insert() { diff --git a/vm/src/eval.rs b/vm/src/eval.rs index 35f27dc9d6..4c48efc700 100644 --- a/vm/src/eval.rs +++ b/vm/src/eval.rs @@ -1,4 +1,4 @@ -use crate::{compiler, scope::Scope, PyResult, VirtualMachine}; +use crate::{PyResult, VirtualMachine, compiler, scope::Scope}; pub fn eval(vm: &VirtualMachine, source: &str, scope: Scope, source_path: &str) -> PyResult { match vm.compile(source, compiler::Mode::Eval, source_path.to_owned()) { diff --git a/vm/src/exceptions.rs b/vm/src/exceptions.rs index 94a16f912e..5882fbe2bc 100644 --- a/vm/src/exceptions.rs +++ b/vm/src/exceptions.rs @@ -2,8 +2,9 @@ use self::types::{PyBaseException, PyBaseExceptionRef}; use crate::common::lock::PyRwLock; use crate::object::{Traverse, TraverseFn}; use crate::{ + AsObject, Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, builtins::{ - traceback::PyTracebackRef, PyNone, PyStr, PyStrRef, PyTuple, PyTupleRef, PyType, PyTypeRef, + PyNone, PyStr, PyStrRef, PyTuple, PyTupleRef, PyType, PyTypeRef, traceback::PyTracebackRef, }, class::{PyClassImpl, StaticType}, convert::{ToPyException, ToPyObject}, @@ -12,7 +13,6 @@ use crate::{ stdlib::sys, suggestion::offer_suggestions, types::{Callable, Constructor, Initializer, Representable}, - AsObject, Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, }; use crossbeam_utils::atomic::AtomicCell; use itertools::Itertools; @@ -1177,13 +1177,13 @@ pub(super) mod types { use crate::common::lock::PyRwLock; #[cfg_attr(target_arch = "wasm32", allow(unused_imports))] use crate::{ + AsObject, PyObjectRef, PyRef, PyResult, VirtualMachine, builtins::{ - traceback::PyTracebackRef, tuple::IntoPyTuple, PyInt, PyStrRef, PyTupleRef, PyTypeRef, + PyInt, PyStrRef, PyTupleRef, PyTypeRef, traceback::PyTracebackRef, tuple::IntoPyTuple, }, convert::ToPyResult, function::FuncArgs, types::{Constructor, Initializer}, - AsObject, PyObjectRef, PyRef, PyResult, VirtualMachine, }; use crossbeam_utils::atomic::AtomicCell; use itertools::Itertools; diff --git a/vm/src/format.rs b/vm/src/format.rs index 8109ea00f4..7e9bb54265 100644 --- a/vm/src/format.rs +++ b/vm/src/format.rs @@ -1,9 +1,9 @@ use crate::{ + PyObject, PyResult, VirtualMachine, builtins::PyBaseExceptionRef, convert::{IntoPyException, ToPyException}, function::FuncArgs, stdlib::builtins, - PyObject, PyResult, VirtualMachine, }; use rustpython_format::*; diff --git a/vm/src/frame.rs b/vm/src/frame.rs index afa5c708e7..7cbe25909b 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -1,11 +1,12 @@ use crate::common::{boxvec::BoxVec, lock::PyMutex}; use crate::{ + AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, builtins::{ + PyBaseExceptionRef, PyCode, PyCoroutine, PyDict, PyDictRef, PyGenerator, PyList, PySet, + PySlice, PyStr, PyStrInterned, PyStrRef, PyTraceback, PyType, asyncgenerator::PyAsyncGenWrappedValue, function::{PyCell, PyCellRef, PyFunction}, tuple::{PyTuple, PyTupleRef, PyTupleTyped}, - PyBaseExceptionRef, PyCode, PyCoroutine, PyDict, PyDictRef, PyGenerator, PyList, PySet, - PySlice, PyStr, PyStrInterned, PyStrRef, PyTraceback, PyType, }, bytecode, convert::{IntoObject, ToPyResult}, @@ -17,7 +18,6 @@ use crate::{ source_code::SourceLocation, stdlib::{builtins, typing::_typing}, vm::{Context, PyMethod}, - AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, }; use indexmap::IndexMap; use itertools::Itertools; @@ -277,15 +277,17 @@ impl Py { } pub fn next_external_frame(&self, vm: &VirtualMachine) -> Option { - self.f_back(vm).map(|mut back| loop { - back = if let Some(back) = back.to_owned().f_back(vm) { - back - } else { - break back; - }; + self.f_back(vm).map(|mut back| { + loop { + back = if let Some(back) = back.to_owned().f_back(vm) { + back + } else { + break back; + }; - if !back.is_internal_frame() { - break back; + if !back.is_internal_frame() { + break back; + } } }) } @@ -638,7 +640,7 @@ impl ExecutingFrame<'_> { match res { Ok(()) => {} Err(e) if e.fast_isinstance(vm.ctx.exceptions.key_error) => { - return Err(name_error(name, vm)) + return Err(name_error(name, vm)); } Err(e) => return Err(e), } @@ -649,7 +651,7 @@ impl ExecutingFrame<'_> { match self.globals.del_item(name, vm) { Ok(()) => {} Err(e) if e.fast_isinstance(vm.ctx.exceptions.key_error) => { - return Err(name_error(name, vm)) + return Err(name_error(name, vm)); } Err(e) => return Err(e), } diff --git a/vm/src/function/argument.rs b/vm/src/function/argument.rs index b0e731ff57..b7fd509ef1 100644 --- a/vm/src/function/argument.rs +++ b/vm/src/function/argument.rs @@ -1,8 +1,8 @@ use crate::{ + AsObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, builtins::{PyBaseExceptionRef, PyTupleRef, PyTypeRef}, convert::ToPyObject, object::{Traverse, TraverseFn}, - AsObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, }; use indexmap::IndexMap; use itertools::Itertools; diff --git a/vm/src/function/arithmetic.rs b/vm/src/function/arithmetic.rs index b40e31e1b6..9f40ca7fec 100644 --- a/vm/src/function/arithmetic.rs +++ b/vm/src/function/arithmetic.rs @@ -1,7 +1,7 @@ use crate::{ + VirtualMachine, convert::{ToPyObject, TryFromObject}, object::{AsObject, PyObjectRef, PyResult}, - VirtualMachine, }; #[derive(result_like::OptionLike)] diff --git a/vm/src/function/buffer.rs b/vm/src/function/buffer.rs index f5d0dd03d6..91379e7a7f 100644 --- a/vm/src/function/buffer.rs +++ b/vm/src/function/buffer.rs @@ -1,9 +1,9 @@ use crate::{ + AsObject, PyObject, PyObjectRef, PyResult, TryFromBorrowedObject, TryFromObject, + VirtualMachine, builtins::{PyStr, PyStrRef}, common::borrow::{BorrowedValue, BorrowedValueMut}, protocol::PyBuffer, - AsObject, PyObject, PyObjectRef, PyResult, TryFromBorrowedObject, TryFromObject, - VirtualMachine, }; // Python/getargs.c diff --git a/vm/src/function/builtin.rs b/vm/src/function/builtin.rs index bf9a5ed345..b8a408453d 100644 --- a/vm/src/function/builtin.rs +++ b/vm/src/function/builtin.rs @@ -1,7 +1,7 @@ use super::{FromArgs, FuncArgs}; use crate::{ - convert::ToPyResult, object::PyThreadingConstraint, Py, PyPayload, PyRef, PyResult, - VirtualMachine, + Py, PyPayload, PyRef, PyResult, VirtualMachine, convert::ToPyResult, + object::PyThreadingConstraint, }; use std::marker::PhantomData; diff --git a/vm/src/function/either.rs b/vm/src/function/either.rs index ceb79d55c9..08b96c7fe3 100644 --- a/vm/src/function/either.rs +++ b/vm/src/function/either.rs @@ -1,5 +1,5 @@ use crate::{ - convert::ToPyObject, AsObject, PyObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine, + AsObject, PyObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine, convert::ToPyObject, }; use std::borrow::Borrow; diff --git a/vm/src/function/fspath.rs b/vm/src/function/fspath.rs index 41e99b0542..69f11eb65d 100644 --- a/vm/src/function/fspath.rs +++ b/vm/src/function/fspath.rs @@ -1,9 +1,9 @@ use crate::{ + PyObjectRef, PyResult, TryFromObject, VirtualMachine, builtins::{PyBytes, PyBytesRef, PyStrRef}, convert::{IntoPyException, ToPyObject}, function::PyStr, protocol::PyBuffer, - PyObjectRef, PyResult, TryFromObject, VirtualMachine, }; use std::{ffi::OsStr, path::PathBuf}; diff --git a/vm/src/function/getset.rs b/vm/src/function/getset.rs index 827158e834..66e668ace6 100644 --- a/vm/src/function/getset.rs +++ b/vm/src/function/getset.rs @@ -2,10 +2,10 @@ */ use crate::{ + Py, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, convert::ToPyResult, function::{BorrowedParam, OwnedParam, RefParam}, object::PyThreadingConstraint, - Py, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, }; #[derive(result_like::OptionLike, is_macro::Is, Debug)] diff --git a/vm/src/function/method.rs b/vm/src/function/method.rs index adf53d7d93..d3d0b85fae 100644 --- a/vm/src/function/method.rs +++ b/vm/src/function/method.rs @@ -1,11 +1,11 @@ use crate::{ + Context, Py, PyObjectRef, PyPayload, PyRef, VirtualMachine, builtins::{ + PyType, builtin_func::{PyNativeFunction, PyNativeMethod}, descriptor::PyMethodDescriptor, - PyType, }, function::{IntoPyNativeFn, PyNativeFn}, - Context, Py, PyObjectRef, PyPayload, PyRef, VirtualMachine, }; bitflags::bitflags! { diff --git a/vm/src/function/mod.rs b/vm/src/function/mod.rs index 3bd6d0f74c..8e517f6ed5 100644 --- a/vm/src/function/mod.rs +++ b/vm/src/function/mod.rs @@ -15,7 +15,7 @@ pub use argument::{ }; pub use arithmetic::{PyArithmeticValue, PyComparisonValue}; pub use buffer::{ArgAsciiBuffer, ArgBytesLike, ArgMemoryBuffer, ArgStrOrBytesLike}; -pub use builtin::{static_func, static_raw_func, IntoPyNativeFn, PyNativeFn}; +pub use builtin::{IntoPyNativeFn, PyNativeFn, static_func, static_raw_func}; pub use either::Either; pub use fspath::FsPath; pub use getset::PySetterValue; @@ -24,7 +24,7 @@ pub use method::{HeapMethodDef, PyMethodDef, PyMethodFlags}; pub use number::{ArgIndex, ArgIntoBool, ArgIntoComplex, ArgIntoFloat, ArgPrimitiveIndex, ArgSize}; pub use protocol::{ArgCallable, ArgIterable, ArgMapping, ArgSequence}; -use crate::{builtins::PyStr, convert::TryFromBorrowedObject, PyObject, PyResult, VirtualMachine}; +use crate::{PyObject, PyResult, VirtualMachine, builtins::PyStr, convert::TryFromBorrowedObject}; use builtin::{BorrowedParam, OwnedParam, RefParam}; #[derive(Clone, Copy, PartialEq, Eq)] diff --git a/vm/src/function/number.rs b/vm/src/function/number.rs index 5f23543395..0e36f57ad1 100644 --- a/vm/src/function/number.rs +++ b/vm/src/function/number.rs @@ -1,5 +1,5 @@ use super::argument::OptionalArg; -use crate::{builtins::PyIntRef, AsObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine}; +use crate::{AsObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine, builtins::PyIntRef}; use malachite_bigint::BigInt; use num_complex::Complex64; use num_traits::PrimInt; diff --git a/vm/src/function/protocol.rs b/vm/src/function/protocol.rs index 295332e480..4b7e4c4cef 100644 --- a/vm/src/function/protocol.rs +++ b/vm/src/function/protocol.rs @@ -1,12 +1,12 @@ use super::IntoFuncArgs; use crate::{ - builtins::{iter::PySequenceIterator, PyDict, PyDictRef}, + AsObject, PyObject, PyObjectRef, PyPayload, PyResult, TryFromObject, VirtualMachine, + builtins::{PyDict, PyDictRef, iter::PySequenceIterator}, convert::ToPyObject, identifier, object::{Traverse, TraverseFn}, protocol::{PyIter, PyIterIter, PyMapping, PyMappingMethods}, types::{AsMapping, GenericMethod}, - AsObject, PyObject, PyObjectRef, PyPayload, PyResult, TryFromObject, VirtualMachine, }; use std::{borrow::Borrow, marker::PhantomData, ops::Deref}; diff --git a/vm/src/import.rs b/vm/src/import.rs index fbd563dabb..860f0b8a16 100644 --- a/vm/src/import.rs +++ b/vm/src/import.rs @@ -2,11 +2,11 @@ * Import mechanics */ use crate::{ - builtins::{list, traceback::PyTraceback, PyBaseExceptionRef, PyCode}, + AsObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, + builtins::{PyBaseExceptionRef, PyCode, list, traceback::PyTraceback}, scope::Scope, version::get_git_revision, - vm::{thread, VirtualMachine}, - AsObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, + vm::{VirtualMachine, thread}, }; pub(crate) fn init_importlib_base(vm: &mut VirtualMachine) -> PyResult { diff --git a/vm/src/intern.rs b/vm/src/intern.rs index 5bb657f222..10aaa53454 100644 --- a/vm/src/intern.rs +++ b/vm/src/intern.rs @@ -1,8 +1,8 @@ use crate::{ + AsObject, Py, PyExact, PyObject, PyObjectRef, PyPayload, PyRef, PyRefExact, VirtualMachine, builtins::{PyStr, PyStrInterned, PyTypeRef}, common::lock::PyRwLock, convert::ToPyObject, - AsObject, Py, PyExact, PyObject, PyObjectRef, PyPayload, PyRef, PyRefExact, VirtualMachine, }; use std::{ borrow::{Borrow, ToOwned}, diff --git a/vm/src/iter.rs b/vm/src/iter.rs index 497dc20adc..1e13243792 100644 --- a/vm/src/iter.rs +++ b/vm/src/iter.rs @@ -1,4 +1,4 @@ -use crate::{types::PyComparisonOp, vm::VirtualMachine, PyObjectRef, PyResult}; +use crate::{PyObjectRef, PyResult, types::PyComparisonOp, vm::VirtualMachine}; use itertools::Itertools; pub trait PyExactSizeIterator<'a>: ExactSizeIterator + Sized { diff --git a/vm/src/object/core.rs b/vm/src/object/core.rs index 61c31a02e3..481111532a 100644 --- a/vm/src/object/core.rs +++ b/vm/src/object/core.rs @@ -11,9 +11,9 @@ //! PyRef may looking like to be called as PyObjectWeak by the rule, //! but not to do to remember it is a PyRef object. use super::{ + PyAtomicRef, ext::{AsObject, PyRefExact, PyResult}, payload::PyObjectPayload, - PyAtomicRef, }; use crate::object::traverse::{Traverse, TraverseFn}; use crate::object::traverse_object::PyObjVTable; diff --git a/vm/src/object/ext.rs b/vm/src/object/ext.rs index 129a6a79bb..8c6d367583 100644 --- a/vm/src/object/ext.rs +++ b/vm/src/object/ext.rs @@ -7,17 +7,17 @@ use crate::common::{ lock::PyRwLockReadGuard, }; use crate::{ + VirtualMachine, builtins::{PyBaseExceptionRef, PyStrInterned, PyType}, convert::{IntoPyException, ToPyObject, ToPyResult, TryFromObject}, vm::Context, - VirtualMachine, }; use std::{ borrow::Borrow, fmt, marker::PhantomData, ops::Deref, - ptr::{null_mut, NonNull}, + ptr::{NonNull, null_mut}, }; /* Python objects and references. diff --git a/vm/src/object/payload.rs b/vm/src/object/payload.rs index d5f3d6330b..6413d6ae06 100644 --- a/vm/src/object/payload.rs +++ b/vm/src/object/payload.rs @@ -1,9 +1,9 @@ use crate::object::{MaybeTraverse, Py, PyObjectRef, PyRef, PyResult}; use crate::{ + PyRefExact, builtins::{PyBaseExceptionRef, PyType, PyTypeRef}, types::PyTypeFlags, vm::{Context, VirtualMachine}, - PyRefExact, }; cfg_if::cfg_if! { diff --git a/vm/src/object/traverse.rs b/vm/src/object/traverse.rs index 43a039d331..5f93dc5c8b 100644 --- a/vm/src/object/traverse.rs +++ b/vm/src/object/traverse.rs @@ -2,7 +2,7 @@ use std::ptr::NonNull; use rustpython_common::lock::{PyMutex, PyRwLock}; -use crate::{function::Either, object::PyObjectPayload, AsObject, PyObject, PyObjectRef, PyRef}; +use crate::{AsObject, PyObject, PyObjectRef, PyRef, function::Either, object::PyObjectPayload}; pub type TraverseFn<'a> = dyn FnMut(&PyObject) + 'a; diff --git a/vm/src/object/traverse_object.rs b/vm/src/object/traverse_object.rs index b690b80f75..682ddf4876 100644 --- a/vm/src/object/traverse_object.rs +++ b/vm/src/object/traverse_object.rs @@ -1,10 +1,10 @@ use std::fmt; use crate::{ + PyObject, object::{ - debug_obj, drop_dealloc_obj, try_trace_obj, Erased, InstanceDict, PyInner, PyObjectPayload, + Erased, InstanceDict, PyInner, PyObjectPayload, debug_obj, drop_dealloc_obj, try_trace_obj, }, - PyObject, }; use super::{Traverse, TraverseFn}; diff --git a/vm/src/ospath.rs b/vm/src/ospath.rs index fe26f227d1..9dda60d621 100644 --- a/vm/src/ospath.rs +++ b/vm/src/ospath.rs @@ -1,9 +1,9 @@ use crate::{ + PyObjectRef, PyResult, VirtualMachine, builtins::PyBaseExceptionRef, convert::{ToPyException, TryFromObject}, function::FsPath, object::AsObject, - PyObjectRef, PyResult, VirtualMachine, }; use std::path::{Path, PathBuf}; diff --git a/vm/src/protocol/buffer.rs b/vm/src/protocol/buffer.rs index 0dfe3f0027..8692e4f78e 100644 --- a/vm/src/protocol/buffer.rs +++ b/vm/src/protocol/buffer.rs @@ -2,6 +2,7 @@ //! https://docs.python.org/3/c-api/buffer.html use crate::{ + Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromBorrowedObject, VirtualMachine, common::{ borrow::{BorrowedValue, BorrowedValueMut}, lock::{MapImmutable, PyMutex, PyMutexGuard}, @@ -9,7 +10,6 @@ use crate::{ object::PyObjectPayload, sliceable::SequenceIndexOp, types::Unconstructible, - Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromBorrowedObject, VirtualMachine, }; use itertools::Itertools; use std::{borrow::Cow, fmt::Debug, ops::Range}; diff --git a/vm/src/protocol/iter.rs b/vm/src/protocol/iter.rs index d9c49ed15f..345914e411 100644 --- a/vm/src/protocol/iter.rs +++ b/vm/src/protocol/iter.rs @@ -1,8 +1,8 @@ use crate::{ + AsObject, PyObject, PyObjectRef, PyPayload, PyResult, TryFromObject, VirtualMachine, builtins::iter::PySequenceIterator, convert::{ToPyObject, ToPyResult}, object::{Traverse, TraverseFn}, - AsObject, PyObject, PyObjectRef, PyPayload, PyResult, TryFromObject, VirtualMachine, }; use std::borrow::Borrow; use std::ops::Deref; diff --git a/vm/src/protocol/mapping.rs b/vm/src/protocol/mapping.rs index 986b806f6c..cbecc8762e 100644 --- a/vm/src/protocol/mapping.rs +++ b/vm/src/protocol/mapping.rs @@ -1,12 +1,12 @@ use crate::{ + AsObject, PyObject, PyObjectRef, PyResult, VirtualMachine, builtins::{ + PyDict, PyStrInterned, dict::{PyDictItems, PyDictKeys, PyDictValues}, type_::PointerSlot, - PyDict, PyStrInterned, }, convert::ToPyResult, object::{Traverse, TraverseFn}, - AsObject, PyObject, PyObjectRef, PyResult, VirtualMachine, }; use crossbeam_utils::atomic::AtomicCell; diff --git a/vm/src/protocol/number.rs b/vm/src/protocol/number.rs index 01829bb553..65c4eaad79 100644 --- a/vm/src/protocol/number.rs +++ b/vm/src/protocol/number.rs @@ -3,13 +3,13 @@ use std::ops::Deref; use crossbeam_utils::atomic::AtomicCell; use crate::{ - builtins::{int, PyByteArray, PyBytes, PyComplex, PyFloat, PyInt, PyIntRef, PyStr}, + AsObject, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromBorrowedObject, + VirtualMachine, + builtins::{PyByteArray, PyBytes, PyComplex, PyFloat, PyInt, PyIntRef, PyStr, int}, common::int::bytes_to_int, function::ArgBytesLike, object::{Traverse, TraverseFn}, stdlib::warnings, - AsObject, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromBorrowedObject, - VirtualMachine, }; pub type PyNumberUnaryFunc = fn(PyNumber, &VirtualMachine) -> PyResult; diff --git a/vm/src/protocol/object.rs b/vm/src/protocol/object.rs index 8b9c0e446c..4e69cf38a2 100644 --- a/vm/src/protocol/object.rs +++ b/vm/src/protocol/object.rs @@ -2,9 +2,10 @@ //! https://docs.python.org/3/c-api/object.html use crate::{ + AsObject, Py, PyObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine, builtins::{ - pystr::AsPyStr, PyAsyncGen, PyBytes, PyDict, PyDictRef, PyGenericAlias, PyInt, PyList, - PyStr, PyStrRef, PyTuple, PyTupleRef, PyType, PyTypeRef, + PyAsyncGen, PyBytes, PyDict, PyDictRef, PyGenericAlias, PyInt, PyList, PyStr, PyStrRef, + PyTuple, PyTupleRef, PyType, PyTypeRef, pystr::AsPyStr, }, bytesinner::ByteInnerNewOptions, common::{hash::PyHash, str::to_ascii}, @@ -14,7 +15,6 @@ use crate::{ object::PyPayload, protocol::{PyIter, PyMapping, PySequence}, types::{Constructor, PyComparisonOp}, - AsObject, Py, PyObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine, }; // RustPython doesn't need these items diff --git a/vm/src/protocol/sequence.rs b/vm/src/protocol/sequence.rs index 62cf828e40..46ac0c8be7 100644 --- a/vm/src/protocol/sequence.rs +++ b/vm/src/protocol/sequence.rs @@ -1,10 +1,10 @@ use crate::{ - builtins::{type_::PointerSlot, PyList, PyListRef, PySlice, PyTuple, PyTupleRef}, + PyObject, PyObjectRef, PyPayload, PyResult, VirtualMachine, + builtins::{PyList, PyListRef, PySlice, PyTuple, PyTupleRef, type_::PointerSlot}, convert::ToPyObject, function::PyArithmeticValue, object::{Traverse, TraverseFn}, protocol::{PyMapping, PyNumberBinaryOp}, - PyObject, PyObjectRef, PyPayload, PyResult, VirtualMachine, }; use crossbeam_utils::atomic::AtomicCell; use itertools::Itertools; diff --git a/vm/src/py_io.rs b/vm/src/py_io.rs index 8091b75afc..0d7c319bc0 100644 --- a/vm/src/py_io.rs +++ b/vm/src/py_io.rs @@ -1,7 +1,7 @@ use crate::{ + PyObject, PyObjectRef, PyResult, VirtualMachine, builtins::{PyBaseExceptionRef, PyBytes, PyStr}, common::ascii, - PyObject, PyObjectRef, PyResult, VirtualMachine, }; use std::{fmt, io, ops}; diff --git a/vm/src/py_serde.rs b/vm/src/py_serde.rs index b4b12e430d..ea72879e42 100644 --- a/vm/src/py_serde.rs +++ b/vm/src/py_serde.rs @@ -3,7 +3,7 @@ use num_traits::sign::Signed; use serde::de::{DeserializeSeed, Visitor}; use serde::ser::{Serialize, SerializeMap, SerializeSeq}; -use crate::builtins::{bool_, dict::PyDictRef, float, int, list::PyList, tuple::PyTuple, PyStr}; +use crate::builtins::{PyStr, bool_, dict::PyDictRef, float, int, list::PyList, tuple::PyTuple}; use crate::{AsObject, PyObject, PyObjectRef, VirtualMachine}; #[inline] diff --git a/vm/src/scope.rs b/vm/src/scope.rs index 47a1e5e3ff..12b878f847 100644 --- a/vm/src/scope.rs +++ b/vm/src/scope.rs @@ -1,4 +1,4 @@ -use crate::{builtins::PyDictRef, function::ArgMapping, VirtualMachine}; +use crate::{VirtualMachine, builtins::PyDictRef, function::ArgMapping}; use std::fmt; #[derive(Clone)] diff --git a/vm/src/sequence.rs b/vm/src/sequence.rs index 3f2d1e94d2..fc6e216809 100644 --- a/vm/src/sequence.rs +++ b/vm/src/sequence.rs @@ -1,10 +1,10 @@ use crate::{ + AsObject, PyObject, PyObjectRef, PyResult, builtins::PyIntRef, function::OptionalArg, sliceable::SequenceIndexOp, types::PyComparisonOp, - vm::{VirtualMachine, MAX_MEMORY_SIZE}, - AsObject, PyObject, PyObjectRef, PyResult, + vm::{MAX_MEMORY_SIZE, VirtualMachine}, }; use optional::Optioned; use std::ops::{Deref, Range}; diff --git a/vm/src/signal.rs b/vm/src/signal.rs index 7489282de2..346664fa24 100644 --- a/vm/src/signal.rs +++ b/vm/src/signal.rs @@ -69,7 +69,7 @@ pub fn assert_in_range(signum: i32, vm: &VirtualMachine) -> PyResult<()> { #[allow(dead_code)] #[cfg(not(target_arch = "wasm32"))] pub fn set_interrupt_ex(signum: i32, vm: &VirtualMachine) -> PyResult<()> { - use crate::stdlib::signal::_signal::{run_signal, SIG_DFL, SIG_IGN}; + use crate::stdlib::signal::_signal::{SIG_DFL, SIG_IGN, run_signal}; assert_in_range(signum, vm)?; match signum as usize { diff --git a/vm/src/sliceable.rs b/vm/src/sliceable.rs index 257d325651..cbc25e4e18 100644 --- a/vm/src/sliceable.rs +++ b/vm/src/sliceable.rs @@ -1,7 +1,7 @@ // export through sliceable module, not slice. use crate::{ - builtins::{int::PyInt, slice::PySlice}, PyObject, PyResult, VirtualMachine, + builtins::{int::PyInt, slice::PySlice}, }; use malachite_bigint::BigInt; use num_traits::{Signed, ToPrimitive}; @@ -357,13 +357,8 @@ impl SaturatedSlice { if step == 0 { return Err(vm.new_value_error("slice step cannot be zero".to_owned())); } - let start = to_isize_index(vm, slice.start_ref(vm))?.unwrap_or_else(|| { - if step.is_negative() { - isize::MAX - } else { - 0 - } - }); + let start = to_isize_index(vm, slice.start_ref(vm))? + .unwrap_or_else(|| if step.is_negative() { isize::MAX } else { 0 }); let stop = to_isize_index(vm, &slice.stop(vm))?.unwrap_or_else(|| { if step.is_negative() { diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs index 616d84368c..7dd893646e 100644 --- a/vm/src/stdlib/ast.rs +++ b/vm/src/stdlib/ast.rs @@ -6,14 +6,14 @@ mod r#gen; use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, + VirtualMachine, builtins::{self, PyDict, PyModule, PyStrRef, PyType}, class::{PyClassImpl, StaticType}, - compiler::core::bytecode::OpArgType, compiler::CompileError, + compiler::core::bytecode::OpArgType, convert::ToPyException, source_code::{LinearLocator, OneIndexed, SourceLocation, SourceRange}, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, - VirtualMachine, }; use num_complex::Complex64; use num_traits::{ToPrimitive, Zero}; @@ -26,9 +26,9 @@ use rustpython_parser as parser; #[pymodule] mod _ast { use crate::{ + AsObject, Context, PyObjectRef, PyPayload, PyResult, VirtualMachine, builtins::{PyStrRef, PyTupleRef}, function::FuncArgs, - AsObject, Context, PyObjectRef, PyPayload, PyResult, VirtualMachine, }; #[pyattr] #[pyclass(module = "_ast", name = "AST")] diff --git a/vm/src/stdlib/atexit.rs b/vm/src/stdlib/atexit.rs index dbeda76741..b1832b5481 100644 --- a/vm/src/stdlib/atexit.rs +++ b/vm/src/stdlib/atexit.rs @@ -3,7 +3,7 @@ pub(crate) use atexit::make_module; #[pymodule] mod atexit { - use crate::{function::FuncArgs, AsObject, PyObjectRef, PyResult, VirtualMachine}; + use crate::{AsObject, PyObjectRef, PyResult, VirtualMachine, function::FuncArgs}; #[pyfunction] fn register(func: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyObjectRef { diff --git a/vm/src/stdlib/builtins.rs b/vm/src/stdlib/builtins.rs index cc9b0a8be7..22aa11dc43 100644 --- a/vm/src/stdlib/builtins.rs +++ b/vm/src/stdlib/builtins.rs @@ -1,7 +1,7 @@ //! Builtin function definitions. //! //! Implements the list of [builtin Python functions](https://docs.python.org/3/library/builtins.html). -use crate::{builtins::PyModule, class::PyClassImpl, Py, VirtualMachine}; +use crate::{Py, VirtualMachine, builtins::PyModule, class::PyClassImpl}; pub(crate) use builtins::{__module_def, DOC}; pub use builtins::{ascii, print, reversed}; @@ -10,13 +10,14 @@ mod builtins { use std::io::IsTerminal; use crate::{ + AsObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, builtins::{ + PyByteArray, PyBytes, PyDictRef, PyStr, PyStrRef, PyTuple, PyTupleRef, PyType, enumerate::PyReverseSequenceIterator, function::{PyCellRef, PyFunction}, int::PyIntRef, iter::PyCallableIterator, list::{PyList, SortOptions}, - PyByteArray, PyBytes, PyDictRef, PyStr, PyStrRef, PyTuple, PyTupleRef, PyType, }, common::{hash::PyHash, str::to_ascii}, function::{ @@ -29,7 +30,6 @@ mod builtins { readline::{Readline, ReadlineResult}, stdlib::sys, types::PyComparisonOp, - AsObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, }; use num_traits::{Signed, ToPrimitive}; @@ -157,8 +157,7 @@ mod builtins { #[cfg(not(feature = "rustpython-parser"))] { - const PARSER_NOT_SUPPORTED: &str = - "can't compile() source code when the `parser` feature of rustpython is disabled"; + const PARSER_NOT_SUPPORTED: &str = "can't compile() source code when the `parser` feature of rustpython is disabled"; Err(vm.new_type_error(PARSER_NOT_SUPPORTED.to_owned())) } #[cfg(feature = "rustpython-parser")] @@ -535,7 +534,7 @@ mod builtins { None => { return default.ok_or_else(|| { vm.new_value_error(format!("{func_name}() arg is an empty sequence")) - }) + }); } }; diff --git a/vm/src/stdlib/codecs.rs b/vm/src/stdlib/codecs.rs index 71317bf484..976545f64b 100644 --- a/vm/src/stdlib/codecs.rs +++ b/vm/src/stdlib/codecs.rs @@ -4,10 +4,10 @@ pub(crate) use _codecs::make_module; mod _codecs { use crate::common::encodings; use crate::{ + AsObject, PyObject, PyObjectRef, PyResult, TryFromBorrowedObject, VirtualMachine, builtins::{PyBaseExceptionRef, PyBytes, PyBytesRef, PyStr, PyStrRef, PyTuple}, codecs, function::{ArgBytesLike, FuncArgs}, - AsObject, PyObject, PyObjectRef, PyResult, TryFromBorrowedObject, VirtualMachine, }; use std::ops::Range; diff --git a/vm/src/stdlib/collections.rs b/vm/src/stdlib/collections.rs index 5a9a172d53..fc867db2b1 100644 --- a/vm/src/stdlib/collections.rs +++ b/vm/src/stdlib/collections.rs @@ -3,6 +3,7 @@ pub(crate) use _collections::make_module; #[pymodule] mod _collections { use crate::{ + AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, atomic_func, builtins::{ IterStatus::{Active, Exhausted}, @@ -20,7 +21,6 @@ mod _collections { Iterable, PyComparisonOp, Representable, SelfIter, }, utils::collection_repr, - AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, }; use crossbeam_utils::atomic::AtomicCell; use std::cmp::max; diff --git a/vm/src/stdlib/ctypes/function.rs b/vm/src/stdlib/ctypes/function.rs index bf7f0b9adf..a7ee07744b 100644 --- a/vm/src/stdlib/ctypes/function.rs +++ b/vm/src/stdlib/ctypes/function.rs @@ -1,5 +1,5 @@ -use crate::stdlib::ctypes::PyCData; use crate::PyObjectRef; +use crate::stdlib::ctypes::PyCData; use crossbeam_utils::atomic::AtomicCell; use rustpython_common::lock::PyRwLock; use std::ffi::c_void; diff --git a/vm/src/stdlib/errno.rs b/vm/src/stdlib/errno.rs index a142d68a34..247e2a2340 100644 --- a/vm/src/stdlib/errno.rs +++ b/vm/src/stdlib/errno.rs @@ -1,4 +1,4 @@ -use crate::{builtins::PyModule, PyRef, VirtualMachine}; +use crate::{PyRef, VirtualMachine, builtins::PyModule}; #[pymodule] mod errno {} @@ -38,9 +38,9 @@ pub mod errors { WSAEPROVIDERFAILEDINIT, WSAEREFUSED, WSAEREMOTE, WSAESHUTDOWN, WSAESOCKTNOSUPPORT, WSAESTALE, WSAETIMEDOUT, WSAETOOMANYREFS, WSAEUSERS, WSAEWOULDBLOCK, WSAHOST_NOT_FOUND, WSAID_ACCEPTEX, WSAID_CONNECTEX, WSAID_DISCONNECTEX, WSAID_GETACCEPTEXSOCKADDRS, - WSAID_TRANSMITFILE, WSAID_TRANSMITPACKETS, WSAID_WSAPOLL, WSAID_WSARECVMSG, - WSANOTINITIALISED, WSANO_DATA, WSANO_RECOVERY, WSAPROTOCOL_LEN, WSASERVICE_NOT_FOUND, - WSASYSCALLFAILURE, WSASYSNOTREADY, WSASYS_STATUS_LEN, WSATRY_AGAIN, WSATYPE_NOT_FOUND, + WSAID_TRANSMITFILE, WSAID_TRANSMITPACKETS, WSAID_WSAPOLL, WSAID_WSARECVMSG, WSANO_DATA, + WSANO_RECOVERY, WSANOTINITIALISED, WSAPROTOCOL_LEN, WSASERVICE_NOT_FOUND, + WSASYS_STATUS_LEN, WSASYSCALLFAILURE, WSASYSNOTREADY, WSATRY_AGAIN, WSATYPE_NOT_FOUND, WSAVERNOTSUPPORTED, }, }; diff --git a/vm/src/stdlib/functools.rs b/vm/src/stdlib/functools.rs index d13b9b84f6..145d95d6ff 100644 --- a/vm/src/stdlib/functools.rs +++ b/vm/src/stdlib/functools.rs @@ -2,7 +2,7 @@ pub(crate) use _functools::make_module; #[pymodule] mod _functools { - use crate::{function::OptionalArg, protocol::PyIter, PyObjectRef, PyResult, VirtualMachine}; + use crate::{PyObjectRef, PyResult, VirtualMachine, function::OptionalArg, protocol::PyIter}; #[pyfunction] fn reduce( diff --git a/vm/src/stdlib/imp.rs b/vm/src/stdlib/imp.rs index d8727e74ff..5c3f4bf61d 100644 --- a/vm/src/stdlib/imp.rs +++ b/vm/src/stdlib/imp.rs @@ -1,11 +1,11 @@ use crate::frozen::FrozenModule; -use crate::{builtins::PyBaseExceptionRef, VirtualMachine}; +use crate::{VirtualMachine, builtins::PyBaseExceptionRef}; pub(crate) use _imp::make_module; #[cfg(feature = "threading")] #[pymodule(sub)] mod lock { - use crate::{stdlib::thread::RawRMutex, PyResult, VirtualMachine}; + use crate::{PyResult, VirtualMachine, stdlib::thread::RawRMutex}; static IMP_LOCK: RawRMutex = RawRMutex::INIT; @@ -60,7 +60,9 @@ impl FrozenError { use FrozenError::*; let msg = match self { BadName | NotFound => format!("No such frozen object named {mod_name}"), - Disabled => format!("Frozen modules are disabled and the frozen object named {mod_name} is not essential"), + Disabled => format!( + "Frozen modules are disabled and the frozen object named {mod_name} is not essential" + ), Excluded => format!("Excluded frozen object named {mod_name}"), Invalid => format!("Frozen object named {mod_name} is invalid"), }; @@ -80,9 +82,10 @@ fn find_frozen(name: &str, vm: &VirtualMachine) -> Result PyBaseExceptionRef { @@ -118,6 +118,8 @@ impl std::os::fd::AsRawFd for Fildes { mod _io { use super::*; use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, + TryFromBorrowedObject, TryFromObject, builtins::{ PyBaseExceptionRef, PyByteArray, PyBytes, PyBytesRef, PyIntRef, PyMemoryView, PyStr, PyStrRef, PyTuple, PyTupleRef, PyType, PyTypeRef, @@ -140,8 +142,6 @@ mod _io { Callable, Constructor, DefaultConstructor, Destructor, Initializer, IterNext, Iterable, }, vm::VirtualMachine, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, - TryFromBorrowedObject, TryFromObject, }; use bstr::ByteSlice; use crossbeam_utils::atomic::AtomicCell; @@ -149,7 +149,7 @@ mod _io { use num_traits::ToPrimitive; use std::{ borrow::Cow, - io::{self, prelude::*, Cursor, SeekFrom}, + io::{self, Cursor, SeekFrom, prelude::*}, ops::Range, }; @@ -319,11 +319,7 @@ mod _io { // For Cursor, fill_buf returns all of the remaining data unlike other BufReads which have outer reading source. // Unless we add other data by write, there will be no more data. let buf = self.cursor.fill_buf().map_err(|err| os_err(vm, err))?; - if size < buf.len() { - &buf[..size] - } else { - buf - } + if size < buf.len() { &buf[..size] } else { buf } }; let buf = match available.find_byte(byte) { Some(i) => available[..=i].to_vec(), @@ -2496,7 +2492,7 @@ mod _io { _ => { return Err( vm.new_value_error(format!("invalid whence ({how}, should be 0, 1 or 2)")) - ) + ); } }; use crate::types::PyComparisonOp; @@ -3006,7 +3002,7 @@ mod _io { } fn parse_decoder_state(state: PyObjectRef, vm: &VirtualMachine) -> PyResult<(PyBytesRef, i32)> { - use crate::builtins::{int, PyTuple}; + use crate::builtins::{PyTuple, int}; let state_err = || vm.new_type_error("illegal decoder state".to_owned()); let state = state.downcast::().map_err(|_| state_err())?; match state.as_slice() { @@ -4032,8 +4028,9 @@ mod _io { #[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))] #[pymodule] mod fileio { - use super::{Offset, _io::*}; + use super::{_io::*, Offset}; use crate::{ + AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, builtins::{PyBaseExceptionRef, PyStr, PyStrRef}, common::crt_fd::Fd, convert::ToPyException, @@ -4041,7 +4038,6 @@ mod fileio { ospath::{IOErrorBuilder, OsPath, OsPathOrFd}, stdlib::os, types::{Constructor, DefaultConstructor, Initializer, Representable}, - AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, }; use crossbeam_utils::atomic::AtomicCell; use std::io::{Read, Write}; diff --git a/vm/src/stdlib/itertools.rs b/vm/src/stdlib/itertools.rs index eb800a0b1f..65c1482057 100644 --- a/vm/src/stdlib/itertools.rs +++ b/vm/src/stdlib/itertools.rs @@ -4,9 +4,11 @@ pub(crate) use decl::make_module; mod decl { use crate::stdlib::itertools::decl::int::get_value; use crate::{ + AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, PyWeakRef, TryFromObject, + VirtualMachine, builtins::{ - int, tuple::IntoPyTuple, PyGenericAlias, PyInt, PyIntRef, PyList, PyTuple, PyTupleRef, - PyTypeRef, + PyGenericAlias, PyInt, PyIntRef, PyList, PyTuple, PyTupleRef, PyTypeRef, int, + tuple::IntoPyTuple, }, common::{ lock::{PyMutex, PyRwLock, PyRwLockWriteGuard}, @@ -18,8 +20,6 @@ mod decl { protocol::{PyIter, PyIterReturn, PyNumber}, stdlib::sys, types::{Constructor, IterNext, Iterable, Representable, SelfIter}, - AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, PyWeakRef, TryFromObject, - VirtualMachine, }; use crossbeam_utils::atomic::AtomicCell; use malachite_bigint::BigInt; diff --git a/vm/src/stdlib/marshal.rs b/vm/src/stdlib/marshal.rs index 35f9b93d5f..fd7332e7c2 100644 --- a/vm/src/stdlib/marshal.rs +++ b/vm/src/stdlib/marshal.rs @@ -5,6 +5,7 @@ mod decl { use crate::builtins::code::{CodeObject, Literal, PyObjBag}; use crate::class::StaticType; use crate::{ + PyObjectRef, PyResult, TryFromObject, VirtualMachine, builtins::{ PyBool, PyByteArray, PyBytes, PyCode, PyComplex, PyDict, PyEllipsis, PyFloat, PyFrozenSet, PyInt, PyList, PyNone, PySet, PyStopIteration, PyStr, PyTuple, @@ -13,7 +14,6 @@ mod decl { function::{ArgBytesLike, OptionalArg}, object::AsObject, protocol::PyBuffer, - PyObjectRef, PyResult, TryFromObject, VirtualMachine, }; use malachite_bigint::BigInt; use num_complex::Complex64; diff --git a/vm/src/stdlib/mod.rs b/vm/src/stdlib/mod.rs index ca8f8f0bfd..529a40e861 100644 --- a/vm/src/stdlib/mod.rs +++ b/vm/src/stdlib/mod.rs @@ -50,7 +50,7 @@ mod winapi; #[cfg(windows)] mod winreg; -use crate::{builtins::PyModule, PyRef, VirtualMachine}; +use crate::{PyRef, VirtualMachine, builtins::PyModule}; use std::{borrow::Cow, collections::HashMap}; pub type StdlibInitFunc = Box PyRef)>; diff --git a/vm/src/stdlib/msvcrt.rs b/vm/src/stdlib/msvcrt.rs index 2572494671..7b3620ad51 100644 --- a/vm/src/stdlib/msvcrt.rs +++ b/vm/src/stdlib/msvcrt.rs @@ -3,10 +3,10 @@ pub use msvcrt::*; #[pymodule] mod msvcrt { use crate::{ + PyRef, PyResult, VirtualMachine, builtins::{PyBytes, PyStrRef}, common::suppress_iph, stdlib::os::errno_err, - PyRef, PyResult, VirtualMachine, }; use itertools::Itertools; use windows_sys::Win32::{ diff --git a/vm/src/stdlib/nt.rs b/vm/src/stdlib/nt.rs index 09d26ba279..34fa8792d5 100644 --- a/vm/src/stdlib/nt.rs +++ b/vm/src/stdlib/nt.rs @@ -1,4 +1,4 @@ -use crate::{builtins::PyModule, PyRef, VirtualMachine}; +use crate::{PyRef, VirtualMachine, builtins::PyModule}; pub use module::raw_set_handle_inheritable; @@ -11,13 +11,13 @@ pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { #[pymodule(name = "nt", with(super::os::_os))] pub(crate) mod module { use crate::{ + PyResult, TryFromObject, VirtualMachine, builtins::{PyDictRef, PyListRef, PyStrRef, PyTupleRef}, common::{crt_fd::Fd, os::last_os_error, suppress_iph}, convert::ToPyException, function::{Either, OptionalArg}, ospath::OsPath, - stdlib::os::{errno_err, DirFd, FollowSymlinks, SupportFunc, TargetIsDirectory, _os}, - PyResult, TryFromObject, VirtualMachine, + stdlib::os::{_os, DirFd, FollowSymlinks, SupportFunc, TargetIsDirectory, errno_err}, }; use libc::intptr_t; use std::{ diff --git a/vm/src/stdlib/operator.rs b/vm/src/stdlib/operator.rs index 19c1f4b61b..d1a4b376e8 100644 --- a/vm/src/stdlib/operator.rs +++ b/vm/src/stdlib/operator.rs @@ -4,6 +4,7 @@ pub(crate) use _operator::make_module; mod _operator { use crate::common::cmp; use crate::{ + AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, builtins::{PyInt, PyIntRef, PyStr, PyStrRef, PyTupleRef, PyTypeRef}, function::Either, function::{ArgBytesLike, FuncArgs, KwArgs, OptionalArg}, @@ -11,7 +12,6 @@ mod _operator { protocol::PyIter, recursion::ReprGuard, types::{Callable, Constructor, PyComparisonOp, Representable}, - AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, }; #[pyfunction] @@ -336,7 +336,7 @@ mod _operator { _ => { return Err(vm.new_type_error( "unsupported operand types(s) or combination of types".to_owned(), - )) + )); } }; Ok(res) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 6a4233f3be..39701cb3a3 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -1,9 +1,9 @@ use crate::{ + AsObject, Py, PyPayload, PyResult, VirtualMachine, builtins::{PyBaseExceptionRef, PyModule, PySet}, common::crt_fd::Fd, convert::ToPyException, function::{ArgumentError, FromArgs, FuncArgs}, - AsObject, Py, PyPayload, PyResult, VirtualMachine, }; use std::{ffi, fs, io, path::Path}; @@ -125,8 +125,9 @@ fn bytes_as_osstr<'a>(b: &'a [u8], vm: &VirtualMachine) -> PyResult<&'a ffi::OsS #[pymodule(sub)] pub(super) mod _os { - use super::{errno_err, DirFd, FollowSymlinks, SupportFunc}; + use super::{DirFd, FollowSymlinks, SupportFunc, errno_err}; use crate::{ + AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, builtins::{ PyBytesRef, PyGenericAlias, PyIntRef, PyStrRef, PyTuple, PyTupleRef, PyTypeRef, }, @@ -144,7 +145,6 @@ pub(super) mod _os { types::{IterNext, Iterable, PyStructSequence, Representable, SelfIter}, utils::ToCString, vm::VirtualMachine, - AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, }; use crossbeam_utils::atomic::AtomicCell; use itertools::Itertools; @@ -303,11 +303,7 @@ pub(super) mod _os { #[cfg(target_os = "redox")] let [] = dir_fd.0; let res = unsafe { libc::mkdir(path.as_ptr(), mode as _) }; - if res < 0 { - Err(errno_err(vm)) - } else { - Ok(()) - } + if res < 0 { Err(errno_err(vm)) } else { Ok(()) } } #[pyfunction] @@ -1011,11 +1007,7 @@ pub(super) mod _os { std::mem::transmute::<[i32; 2], i64>(distance_to_move) } }; - if res < 0 { - Err(errno_err(vm)) - } else { - Ok(res) - } + if res < 0 { Err(errno_err(vm)) } else { Ok(res) } } #[pyfunction] @@ -1100,7 +1092,7 @@ pub(super) mod _os { (Some(_), Some(_)) => { return Err(vm.new_value_error( "utime: you may specify either 'times' or 'ns' but not both".to_owned(), - )) + )); } }; utime_impl(args.path, acc, modif, args.dir_fd, args.follow_symlinks, vm) @@ -1138,11 +1130,7 @@ pub(super) mod _os { }, ) }; - if ret < 0 { - Err(errno_err(vm)) - } else { - Ok(()) - } + if ret < 0 { Err(errno_err(vm)) } else { Ok(()) } } #[cfg(target_os = "redox")] { diff --git a/vm/src/stdlib/posix.rs b/vm/src/stdlib/posix.rs index 77ebf9e620..05b1d8addd 100644 --- a/vm/src/stdlib/posix.rs +++ b/vm/src/stdlib/posix.rs @@ -1,4 +1,4 @@ -use crate::{builtins::PyModule, PyRef, VirtualMachine}; +use crate::{PyRef, VirtualMachine, builtins::PyModule}; use std::os::unix::io::RawFd; pub fn raw_set_inheritable(fd: RawFd, inheritable: bool) -> nix::Result<()> { @@ -21,16 +21,16 @@ pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { #[pymodule(name = "posix", with(super::os::_os))] pub mod module { use crate::{ + AsObject, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine, builtins::{PyDictRef, PyInt, PyListRef, PyStrRef, PyTupleRef, PyTypeRef}, convert::{IntoPyException, ToPyObject, TryFromObject}, function::{Either, KwArgs, OptionalArg}, ospath::{IOErrorBuilder, OsPath, OsPathOrFd}, stdlib::os::{ - errno_err, DirFd, FollowSymlinks, SupportFunc, TargetIsDirectory, _os, fs_metadata, + _os, DirFd, FollowSymlinks, SupportFunc, TargetIsDirectory, errno_err, fs_metadata, }, types::{Constructor, Representable}, utils::ToCString, - AsObject, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine, }; use bitflags::bitflags; use nix::{ @@ -352,11 +352,7 @@ pub mod module { { let [] = args.dir_fd.0; let res = unsafe { libc::symlink(src.as_ptr(), dst.as_ptr()) }; - if res < 0 { - Err(errno_err(vm)) - } else { - Ok(()) - } + if res < 0 { Err(errno_err(vm)) } else { Ok(()) } } } @@ -604,21 +600,13 @@ pub mod module { ) }, }; - if ret != 0 { - Err(errno_err(vm)) - } else { - Ok(()) - } + if ret != 0 { Err(errno_err(vm)) } else { Ok(()) } } #[cfg(target_vendor = "apple")] fn mknod(self, vm: &VirtualMachine) -> PyResult<()> { let [] = self.dir_fd.0; let ret = self._mknod(vm)?; - if ret != 0 { - Err(errno_err(vm)) - } else { - Ok(()) - } + if ret != 0 { Err(errno_err(vm)) } else { Ok(()) } } } @@ -863,7 +851,7 @@ pub mod module { #[pyfunction] fn set_blocking(fd: RawFd, blocking: bool, vm: &VirtualMachine) -> PyResult<()> { let _set_flag = || { - use nix::fcntl::{fcntl, FcntlArg, OFlag}; + use nix::fcntl::{FcntlArg, OFlag, fcntl}; let flags = OFlag::from_bits_truncate(fcntl(fd, FcntlArg::F_GETFL)?); let mut new_flags = flags; @@ -1618,11 +1606,7 @@ pub mod module { #[pyfunction] fn _fcopyfile(in_fd: i32, out_fd: i32, flags: i32, vm: &VirtualMachine) -> PyResult<()> { let ret = unsafe { fcopyfile(in_fd, out_fd, std::ptr::null_mut(), flags as u32) }; - if ret < 0 { - Err(errno_err(vm)) - } else { - Ok(()) - } + if ret < 0 { Err(errno_err(vm)) } else { Ok(()) } } #[pyfunction] diff --git a/vm/src/stdlib/posix_compat.rs b/vm/src/stdlib/posix_compat.rs index 95ed932fe5..334aa597ce 100644 --- a/vm/src/stdlib/posix_compat.rs +++ b/vm/src/stdlib/posix_compat.rs @@ -1,5 +1,5 @@ //! `posix` compatible module for `not(any(unix, windows))` -use crate::{builtins::PyModule, PyRef, VirtualMachine}; +use crate::{PyRef, VirtualMachine, builtins::PyModule}; pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { let module = module::make_module(vm); @@ -10,10 +10,10 @@ pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { #[pymodule(name = "posix", with(super::os::_os))] pub(crate) mod module { use crate::{ + PyObjectRef, PyResult, VirtualMachine, builtins::PyStrRef, ospath::OsPath, - stdlib::os::{DirFd, SupportFunc, TargetIsDirectory, _os}, - PyObjectRef, PyResult, VirtualMachine, + stdlib::os::{_os, DirFd, SupportFunc, TargetIsDirectory}, }; use std::env; diff --git a/vm/src/stdlib/pwd.rs b/vm/src/stdlib/pwd.rs index 0edca9c0a6..f6c277242c 100644 --- a/vm/src/stdlib/pwd.rs +++ b/vm/src/stdlib/pwd.rs @@ -3,11 +3,11 @@ pub(crate) use pwd::make_module; #[pymodule] mod pwd { use crate::{ + PyObjectRef, PyResult, VirtualMachine, builtins::{PyIntRef, PyStrRef}, convert::{IntoPyException, ToPyObject}, exceptions, types::PyStructSequence, - PyObjectRef, PyResult, VirtualMachine, }; use nix::unistd::{self, User}; use std::ptr::NonNull; diff --git a/vm/src/stdlib/signal.rs b/vm/src/stdlib/signal.rs index da7c934552..0c47ca082e 100644 --- a/vm/src/stdlib/signal.rs +++ b/vm/src/stdlib/signal.rs @@ -1,4 +1,4 @@ -use crate::{builtins::PyModule, PyRef, VirtualMachine}; +use crate::{PyRef, VirtualMachine, builtins::PyModule}; pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { let module = _signal::make_module(vm); @@ -13,10 +13,10 @@ pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { pub(crate) mod _signal { #[cfg(any(unix, windows))] use crate::{ - convert::{IntoPyException, TryFromBorrowedObject}, Py, + convert::{IntoPyException, TryFromBorrowedObject}, }; - use crate::{signal, PyObjectRef, PyResult, VirtualMachine}; + use crate::{PyObjectRef, PyResult, VirtualMachine, signal}; use std::sync::atomic::{self, Ordering}; #[cfg(any(unix, windows))] diff --git a/vm/src/stdlib/sre.rs b/vm/src/stdlib/sre.rs index 8442002fa5..7d620c13d9 100644 --- a/vm/src/stdlib/sre.rs +++ b/vm/src/stdlib/sre.rs @@ -3,7 +3,8 @@ pub(crate) use _sre::make_module; #[pymodule] mod _sre { use crate::{ - atomic_func, + Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromBorrowedObject, + TryFromObject, VirtualMachine, atomic_func, builtins::{ PyCallableIterator, PyDictRef, PyGenericAlias, PyInt, PyList, PyListRef, PyStr, PyStrRef, PyTuple, PyTupleRef, PyTypeRef, @@ -14,16 +15,14 @@ mod _sre { protocol::{PyBuffer, PyCallable, PyMappingMethods}, stdlib::sys, types::{AsMapping, Comparable, Hashable, Representable}, - Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromBorrowedObject, - TryFromObject, VirtualMachine, }; use core::str; use crossbeam_utils::atomic::AtomicCell; use itertools::Itertools; use num_traits::ToPrimitive; use rustpython_sre_engine::{ - string::{lower_ascii, lower_unicode, upper_unicode}, Request, SearchIter, SreFlag, State, StrDrive, + string::{lower_ascii, lower_unicode, upper_unicode}, }; #[pyattr] diff --git a/vm/src/stdlib/string.rs b/vm/src/stdlib/string.rs index cedff92d96..8a8a182732 100644 --- a/vm/src/stdlib/string.rs +++ b/vm/src/stdlib/string.rs @@ -7,10 +7,10 @@ pub(crate) use _string::make_module; mod _string { use crate::common::ascii; use crate::{ + PyObjectRef, PyResult, VirtualMachine, builtins::{PyList, PyStrRef}, convert::ToPyException, convert::ToPyObject, - PyObjectRef, PyResult, VirtualMachine, }; use rustpython_format::{ FieldName, FieldNamePart, FieldType, FormatPart, FormatString, FromTemplate, diff --git a/vm/src/stdlib/symtable.rs b/vm/src/stdlib/symtable.rs index 10d79e9a8b..13a4105111 100644 --- a/vm/src/stdlib/symtable.rs +++ b/vm/src/stdlib/symtable.rs @@ -3,7 +3,7 @@ pub(crate) use symtable::make_module; #[pymodule] mod symtable { use crate::{ - builtins::PyStrRef, compiler, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, builtins::PyStrRef, compiler, }; use rustpython_codegen::symboltable::{ Symbol, SymbolFlags, SymbolScope, SymbolTable, SymbolTableType, diff --git a/vm/src/stdlib/sys.rs b/vm/src/stdlib/sys.rs index 59414b5fdd..dfaab20f2a 100644 --- a/vm/src/stdlib/sys.rs +++ b/vm/src/stdlib/sys.rs @@ -1,10 +1,11 @@ -use crate::{builtins::PyModule, convert::ToPyObject, Py, PyResult, VirtualMachine}; +use crate::{Py, PyResult, VirtualMachine, builtins::PyModule, convert::ToPyObject}; -pub(crate) use sys::{UnraisableHookArgs, __module_def, DOC, MAXSIZE, MULTIARCH}; +pub(crate) use sys::{__module_def, DOC, MAXSIZE, MULTIARCH, UnraisableHookArgs}; #[pymodule] mod sys { use crate::{ + AsObject, PyObject, PyObjectRef, PyRef, PyRefExact, PyResult, builtins::{ PyBaseExceptionRef, PyDictRef, PyNamespace, PyStr, PyStrRef, PyTupleRef, PyTypeRef, }, @@ -19,7 +20,6 @@ mod sys { types::PyStructSequence, version, vm::{Settings, VirtualMachine}, - AsObject, PyObject, PyObjectRef, PyRef, PyRefExact, PyResult, }; use num_traits::ToPrimitive; use std::{ @@ -85,11 +85,7 @@ mod sys { #[pyattr] fn default_prefix(_vm: &VirtualMachine) -> &'static str { // TODO: the windows one doesn't really make sense - if cfg!(windows) { - "C:" - } else { - "/usr/local" - } + if cfg!(windows) { "C:" } else { "/usr/local" } } #[pyattr] fn prefix(vm: &VirtualMachine) -> &'static str { @@ -574,9 +570,11 @@ mod sys { if vm.is_none(unraisable.exc_type.as_object()) { // TODO: early return, but with what error? } - assert!(unraisable - .exc_type - .fast_issubclass(vm.ctx.exceptions.base_exception_type)); + assert!( + unraisable + .exc_type + .fast_issubclass(vm.ctx.exceptions.base_exception_type) + ); // TODO: print module name and qualname diff --git a/vm/src/stdlib/sysconfigdata.rs b/vm/src/stdlib/sysconfigdata.rs index 929227ac11..90e46b83b9 100644 --- a/vm/src/stdlib/sysconfigdata.rs +++ b/vm/src/stdlib/sysconfigdata.rs @@ -2,7 +2,7 @@ pub(crate) use _sysconfigdata::make_module; #[pymodule] pub(crate) mod _sysconfigdata { - use crate::{builtins::PyDictRef, convert::ToPyObject, stdlib::sys::MULTIARCH, VirtualMachine}; + use crate::{VirtualMachine, builtins::PyDictRef, convert::ToPyObject, stdlib::sys::MULTIARCH}; #[pyattr] fn build_time_vars(vm: &VirtualMachine) -> PyDictRef { diff --git a/vm/src/stdlib/thread.rs b/vm/src/stdlib/thread.rs index 63e66474d1..bca7930437 100644 --- a/vm/src/stdlib/thread.rs +++ b/vm/src/stdlib/thread.rs @@ -1,20 +1,20 @@ //! Implementation of the _thread module #[cfg_attr(target_arch = "wasm32", allow(unused_imports))] -pub(crate) use _thread::{make_module, RawRMutex}; +pub(crate) use _thread::{RawRMutex, make_module}; #[pymodule] pub(crate) mod _thread { use crate::{ + AsObject, Py, PyPayload, PyRef, PyResult, VirtualMachine, builtins::{PyDictRef, PyStr, PyTupleRef, PyTypeRef}, convert::ToPyException, function::{ArgCallable, Either, FuncArgs, KwArgs, OptionalArg, PySetterValue}, types::{Constructor, GetAttr, Representable, SetAttr}, - AsObject, Py, PyPayload, PyRef, PyResult, VirtualMachine, }; use crossbeam_utils::atomic::AtomicCell; use parking_lot::{ - lock_api::{RawMutex as RawMutexT, RawMutexTimed, RawReentrantMutex}, RawMutex, RawThreadId, + lock_api::{RawMutex as RawMutexT, RawMutexTimed, RawReentrantMutex}, }; use std::{cell::RefCell, fmt, thread, time::Duration}; use thread_local::ThreadLocal; diff --git a/vm/src/stdlib/time.rs b/vm/src/stdlib/time.rs index 3c87550f93..37a518e504 100644 --- a/vm/src/stdlib/time.rs +++ b/vm/src/stdlib/time.rs @@ -2,7 +2,7 @@ // See also: // https://docs.python.org/3/library/time.html -use crate::{builtins::PyModule, PyRef, VirtualMachine}; +use crate::{PyRef, VirtualMachine, builtins::PyModule}; pub use decl::time; @@ -33,14 +33,14 @@ unsafe extern "C" { #[pymodule(name = "time", with(platform))] mod decl { use crate::{ + PyObjectRef, PyResult, TryFromObject, VirtualMachine, builtins::{PyStrRef, PyTypeRef}, function::{Either, FuncArgs, OptionalArg}, types::PyStructSequence, - PyObjectRef, PyResult, TryFromObject, VirtualMachine, }; use chrono::{ - naive::{NaiveDate, NaiveDateTime, NaiveTime}, DateTime, Datelike, Timelike, + naive::{NaiveDate, NaiveDateTime, NaiveTime}, }; use std::time::Duration; #[cfg(target_env = "msvc")] @@ -509,9 +509,9 @@ mod platform { use super::decl::{SEC_TO_NS, US_TO_NS}; #[cfg_attr(target_os = "macos", allow(unused_imports))] use crate::{ + PyObject, PyRef, PyResult, TryFromBorrowedObject, VirtualMachine, builtins::{PyNamespace, PyStrRef}, convert::IntoPyException, - PyObject, PyRef, PyResult, TryFromBorrowedObject, VirtualMachine, }; use nix::{sys::time::TimeSpec, time::ClockId}; use std::time::Duration; @@ -721,7 +721,7 @@ mod platform { target_os = "openbsd", ))] pub(super) fn get_process_time(vm: &VirtualMachine) -> PyResult { - use nix::sys::resource::{getrusage, UsageWho}; + use nix::sys::resource::{UsageWho, getrusage}; fn from_timeval(tv: libc::timeval, vm: &VirtualMachine) -> PyResult { (|tv: libc::timeval| { let t = tv.tv_sec.checked_mul(SEC_TO_NS)?; @@ -743,11 +743,11 @@ mod platform { #[cfg(windows)] #[pymodule] mod platform { - use super::decl::{time_muldiv, MS_TO_NS, SEC_TO_NS}; + use super::decl::{MS_TO_NS, SEC_TO_NS, time_muldiv}; use crate::{ + PyRef, PyResult, VirtualMachine, builtins::{PyNamespace, PyStrRef}, stdlib::os::errno_err, - PyRef, PyResult, VirtualMachine, }; use std::time::Duration; use windows_sys::Win32::{ diff --git a/vm/src/stdlib/typing.rs b/vm/src/stdlib/typing.rs index 29ce516b14..c266e811ca 100644 --- a/vm/src/stdlib/typing.rs +++ b/vm/src/stdlib/typing.rs @@ -3,9 +3,9 @@ pub(crate) use _typing::make_module; #[pymodule] pub(crate) mod _typing { use crate::{ - builtins::{pystr::AsPyStr, PyGenericAlias, PyTupleRef, PyTypeRef}, - function::IntoFuncArgs, PyObjectRef, PyPayload, PyResult, VirtualMachine, + builtins::{PyGenericAlias, PyTupleRef, PyTypeRef, pystr::AsPyStr}, + function::IntoFuncArgs, }; pub(crate) fn _call_typing_func_object<'a>( diff --git a/vm/src/stdlib/warnings.rs b/vm/src/stdlib/warnings.rs index be5ad8131e..a8ffee4579 100644 --- a/vm/src/stdlib/warnings.rs +++ b/vm/src/stdlib/warnings.rs @@ -1,6 +1,6 @@ pub(crate) use _warnings::make_module; -use crate::{builtins::PyType, Py, PyResult, VirtualMachine}; +use crate::{Py, PyResult, VirtualMachine, builtins::PyType}; pub fn warn( category: &Py, @@ -20,9 +20,9 @@ pub fn warn( #[pymodule] mod _warnings { use crate::{ + PyResult, VirtualMachine, builtins::{PyStrRef, PyTypeRef}, function::OptionalArg, - PyResult, VirtualMachine, }; #[derive(FromArgs)] diff --git a/vm/src/stdlib/weakref.rs b/vm/src/stdlib/weakref.rs index 3ef0de6155..7d8924ff52 100644 --- a/vm/src/stdlib/weakref.rs +++ b/vm/src/stdlib/weakref.rs @@ -9,8 +9,8 @@ pub(crate) use _weakref::make_module; #[pymodule] mod _weakref { use crate::{ - builtins::{PyDictRef, PyTypeRef, PyWeak}, PyObjectRef, PyResult, VirtualMachine, + builtins::{PyDictRef, PyTypeRef, PyWeak}, }; #[pyattr(name = "ref")] diff --git a/vm/src/stdlib/winapi.rs b/vm/src/stdlib/winapi.rs index 6cf8c45bfe..c1edb2739e 100644 --- a/vm/src/stdlib/winapi.rs +++ b/vm/src/stdlib/winapi.rs @@ -4,18 +4,18 @@ pub(crate) use _winapi::make_module; #[pymodule] mod _winapi { use crate::{ + PyObjectRef, PyResult, TryFromObject, VirtualMachine, builtins::PyStrRef, common::windows::ToWideString, convert::{ToPyException, ToPyResult}, function::{ArgMapping, ArgSequence, OptionalArg}, stdlib::os::errno_err, windows::WindowsSysResult, - PyObjectRef, PyResult, TryFromObject, VirtualMachine, }; use std::ptr::{null, null_mut}; use windows::{ - core::PCWSTR, Win32::Foundation::{HANDLE, HINSTANCE, MAX_PATH}, + core::PCWSTR, }; use windows_sys::Win32::Foundation::{BOOL, HANDLE as RAW_HANDLE}; diff --git a/vm/src/stdlib/winreg.rs b/vm/src/stdlib/winreg.rs index f25ca0b960..b0dbbfceec 100644 --- a/vm/src/stdlib/winreg.rs +++ b/vm/src/stdlib/winreg.rs @@ -1,6 +1,6 @@ #![allow(non_snake_case)] -use crate::{builtins::PyModule, PyRef, VirtualMachine}; +use crate::{PyRef, VirtualMachine, builtins::PyModule}; pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { let module = winreg::make_module(vm); @@ -29,10 +29,10 @@ pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { mod winreg { use crate::common::lock::{PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard}; use crate::{ - builtins::PyStrRef, convert::ToPyException, PyObjectRef, PyPayload, PyRef, PyResult, - TryFromObject, VirtualMachine, + PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, builtins::PyStrRef, + convert::ToPyException, }; - use ::winreg::{enums::RegType, RegKey, RegValue}; + use ::winreg::{RegKey, RegValue, enums::RegType}; use std::mem::ManuallyDrop; use std::{ffi::OsStr, io}; use windows_sys::Win32::Foundation; diff --git a/vm/src/suggestion.rs b/vm/src/suggestion.rs index bc0556ef8b..2bc9992d43 100644 --- a/vm/src/suggestion.rs +++ b/vm/src/suggestion.rs @@ -1,10 +1,10 @@ use crate::{ + AsObject, Py, PyObjectRef, VirtualMachine, builtins::{PyStr, PyStrRef}, exceptions::types::PyBaseExceptionRef, sliceable::SliceableSequenceOp, - AsObject, Py, PyObjectRef, VirtualMachine, }; -use rustpython_common::str::levenshtein::{levenshtein_distance, MOVE_COST}; +use rustpython_common::str::levenshtein::{MOVE_COST, levenshtein_distance}; use std::iter::ExactSizeIterator; const MAX_CANDIDATE_ITEMS: usize = 750; diff --git a/vm/src/types/slot.rs b/vm/src/types/slot.rs index 75f113977d..de651580d9 100644 --- a/vm/src/types/slot.rs +++ b/vm/src/types/slot.rs @@ -1,5 +1,6 @@ use crate::{ - builtins::{type_::PointerSlot, PyInt, PyStr, PyStrInterned, PyStrRef, PyType, PyTypeRef}, + AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + builtins::{PyInt, PyStr, PyStrInterned, PyStrRef, PyType, PyTypeRef, type_::PointerSlot}, bytecode::ComparisonOperator, common::hash::PyHash, convert::{ToPyObject, ToPyResult}, @@ -12,7 +13,6 @@ use crate::{ PyNumberSlots, PySequence, PySequenceMethods, }, vm::Context, - AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, }; use crossbeam_utils::atomic::AtomicCell; use malachite_bigint::BigInt; diff --git a/vm/src/types/structseq.rs b/vm/src/types/structseq.rs index 516e2085af..a0b445ce7d 100644 --- a/vm/src/types/structseq.rs +++ b/vm/src/types/structseq.rs @@ -1,8 +1,8 @@ use crate::{ + AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, builtins::{PyTuple, PyTupleRef, PyType}, class::{PyClassImpl, StaticType}, vm::Context, - AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, }; #[pyclass] diff --git a/vm/src/types/zoo.rs b/vm/src/types/zoo.rs index 492b584e29..0d39648514 100644 --- a/vm/src/types/zoo.rs +++ b/vm/src/types/zoo.rs @@ -1,4 +1,5 @@ use crate::{ + Py, builtins::{ asyncgenerator, bool_, builtin_func, bytearray, bytes, classmethod, code, complex, coroutine, descriptor, dict, enumerate, filter, float, frame, function, generator, @@ -10,7 +11,6 @@ use crate::{ }, class::StaticType, vm::Context, - Py, }; /// Holder of references to builtin types. diff --git a/vm/src/utils.rs b/vm/src/utils.rs index c7ab7148d1..e2bc993686 100644 --- a/vm/src/utils.rs +++ b/vm/src/utils.rs @@ -1,8 +1,8 @@ use crate::{ + PyObjectRef, PyResult, VirtualMachine, builtins::PyStr, convert::{ToPyException, ToPyObject}, exceptions::cstring_error, - PyObjectRef, PyResult, VirtualMachine, }; pub fn hash_iter<'a, I: IntoIterator>( diff --git a/vm/src/version.rs b/vm/src/version.rs index 3b6d9aa0ea..8c42866a64 100644 --- a/vm/src/version.rs +++ b/vm/src/version.rs @@ -1,7 +1,7 @@ /* Several function to retrieve version information. */ -use chrono::{prelude::DateTime, Local}; +use chrono::{Local, prelude::DateTime}; use std::time::{Duration, UNIX_EPOCH}; // = 3.13.0alpha diff --git a/vm/src/vm/compile.rs b/vm/src/vm/compile.rs index b7c888ab10..a14e986dac 100644 --- a/vm/src/vm/compile.rs +++ b/vm/src/vm/compile.rs @@ -1,9 +1,9 @@ use crate::{ + AsObject, PyObjectRef, PyRef, PyResult, VirtualMachine, builtins::{PyCode, PyDictRef}, compiler::{self, CompileError, CompileOpts}, convert::TryFromObject, scope::Scope, - AsObject, PyObjectRef, PyRef, PyResult, VirtualMachine, }; impl VirtualMachine { diff --git a/vm/src/vm/context.rs b/vm/src/vm/context.rs index 9f031f01ce..54605704a5 100644 --- a/vm/src/vm/context.rs +++ b/vm/src/vm/context.rs @@ -1,6 +1,9 @@ use crate::{ + PyResult, VirtualMachine, builtins::{ - bytes, + PyBaseException, PyBytes, PyComplex, PyDict, PyDictRef, PyEllipsis, PyFloat, PyFrozenSet, + PyInt, PyIntRef, PyList, PyListRef, PyNone, PyNotImplemented, PyStr, PyStrInterned, + PyTuple, PyTupleRef, PyType, PyTypeRef, bytes, code::{self, PyCode}, descriptor::{ MemberGetter, MemberKind, MemberSetter, MemberSetterFunc, PyDescriptorOwned, @@ -9,9 +12,6 @@ use crate::{ getset::PyGetSet, object, pystr, type_::PyAttributes, - PyBaseException, PyBytes, PyComplex, PyDict, PyDictRef, PyEllipsis, PyFloat, PyFrozenSet, - PyInt, PyIntRef, PyList, PyListRef, PyNone, PyNotImplemented, PyStr, PyStrInterned, - PyTuple, PyTupleRef, PyType, PyTypeRef, }, class::{PyClassImpl, StaticType}, common::rc::PyRc, @@ -23,7 +23,6 @@ use crate::{ intern::{InternableString, MaybeInternedString, StringPool}, object::{Py, PyObjectPayload, PyObjectRef, PyPayload, PyRef}, types::{PyTypeFlags, PyTypeSlots, TypeZoo}, - PyResult, VirtualMachine, }; use malachite_bigint::BigInt; use num_complex::Complex64; diff --git a/vm/src/vm/interpreter.rs b/vm/src/vm/interpreter.rs index af28ad87ab..a375dbedc1 100644 --- a/vm/src/vm/interpreter.rs +++ b/vm/src/vm/interpreter.rs @@ -1,5 +1,5 @@ -use super::{setting::Settings, thread, Context, VirtualMachine}; -use crate::{stdlib::atexit, vm::PyBaseExceptionRef, PyResult}; +use super::{Context, VirtualMachine, setting::Settings, thread}; +use crate::{PyResult, stdlib::atexit, vm::PyBaseExceptionRef}; use std::sync::atomic::Ordering; /// The general interface for the VM @@ -140,8 +140,8 @@ impl Interpreter { mod tests { use super::*; use crate::{ - builtins::{int, PyStr}, PyObjectRef, + builtins::{PyStr, int}, }; use malachite_bigint::ToBigInt; diff --git a/vm/src/vm/mod.rs b/vm/src/vm/mod.rs index 3d3351055f..05ac245a0a 100644 --- a/vm/src/vm/mod.rs +++ b/vm/src/vm/mod.rs @@ -15,12 +15,13 @@ mod vm_object; mod vm_ops; use crate::{ + AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, builtins::{ + PyBaseExceptionRef, PyDictRef, PyInt, PyList, PyModule, PyStr, PyStrInterned, PyStrRef, + PyTypeRef, code::PyCode, pystr::AsPyStr, tuple::{PyTuple, PyTupleTyped}, - PyBaseExceptionRef, PyDictRef, PyInt, PyList, PyModule, PyStr, PyStrInterned, PyStrRef, - PyTypeRef, }, codecs::CodecsRegistry, common::{hash::HashSecret, lock::PyMutex, rc::PyRc}, @@ -33,12 +34,11 @@ use crate::{ scope::Scope, signal, stdlib, warn::WarningsState, - AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, }; use crossbeam_utils::atomic::AtomicCell; #[cfg(unix)] use nix::{ - sys::signal::{kill, sigaction, SaFlags, SigAction, SigSet, Signal::SIGINT}, + sys::signal::{SaFlags, SigAction, SigSet, Signal::SIGINT, kill, sigaction}, unistd::getpid, }; use std::sync::atomic::AtomicBool; @@ -356,7 +356,9 @@ impl VirtualMachine { if self.state.settings.allow_external_library && cfg!(feature = "rustpython-compiler") { if let Err(e) = import::init_importlib_package(self, importlib) { - eprintln!("importlib initialization failed. This is critical for many complicated packages."); + eprintln!( + "importlib initialization failed. This is critical for many complicated packages." + ); self.print_exception(e); } } diff --git a/vm/src/vm/vm_new.rs b/vm/src/vm/vm_new.rs index ed0cd5be3f..bd42396299 100644 --- a/vm/src/vm/vm_new.rs +++ b/vm/src/vm/vm_new.rs @@ -1,15 +1,15 @@ use crate::{ + AsObject, Py, PyObject, PyObjectRef, PyRef, builtins::{ + PyBaseException, PyBaseExceptionRef, PyDictRef, PyModule, PyStrRef, PyType, PyTypeRef, builtin_func::PyNativeFunction, descriptor::PyMethodDescriptor, tuple::{IntoPyTuple, PyTupleRef}, - PyBaseException, PyBaseExceptionRef, PyDictRef, PyModule, PyStrRef, PyType, PyTypeRef, }, convert::ToPyObject, function::{IntoPyNativeFn, PyMethodFlags}, scope::Scope, vm::VirtualMachine, - AsObject, Py, PyObject, PyObjectRef, PyRef, }; /// Collection of object creation helpers diff --git a/vm/src/vm/vm_object.rs b/vm/src/vm/vm_object.rs index b687eea34c..103078272d 100644 --- a/vm/src/vm/vm_object.rs +++ b/vm/src/vm/vm_object.rs @@ -1,6 +1,6 @@ use super::PyMethod; use crate::{ - builtins::{pystr::AsPyStr, PyBaseExceptionRef, PyList, PyStrInterned}, + builtins::{PyBaseExceptionRef, PyList, PyStrInterned, pystr::AsPyStr}, function::IntoFuncArgs, identifier, object::{AsObject, PyObject, PyObjectRef, PyResult}, @@ -85,11 +85,7 @@ impl VirtualMachine { obj.is(&self.ctx.none) } pub fn option_if_none(&self, obj: PyObjectRef) -> Option { - if self.is_none(&obj) { - None - } else { - Some(obj) - } + if self.is_none(&obj) { None } else { Some(obj) } } pub fn unwrap_or_none(&self, obj: Option) -> PyObjectRef { obj.unwrap_or_else(|| self.ctx.none()) diff --git a/vm/src/vm/vm_ops.rs b/vm/src/vm/vm_ops.rs index 3aa4144190..09f849a1a1 100644 --- a/vm/src/vm/vm_ops.rs +++ b/vm/src/vm/vm_ops.rs @@ -114,7 +114,7 @@ impl VirtualMachine { Ok(None) } else { Err(e) - } + }; } }; let hint = result diff --git a/vm/src/warn.rs b/vm/src/warn.rs index ab316c8559..b2055225a0 100644 --- a/vm/src/warn.rs +++ b/vm/src/warn.rs @@ -1,11 +1,11 @@ use crate::{ + AsObject, Context, Py, PyObjectRef, PyResult, VirtualMachine, builtins::{ PyDict, PyDictRef, PyListRef, PyStr, PyStrInterned, PyStrRef, PyTuple, PyTupleRef, PyTypeRef, }, convert::{IntoObject, TryFromObject}, types::PyComparisonOp, - AsObject, Context, Py, PyObjectRef, PyResult, VirtualMachine, }; pub struct WarningsState { @@ -17,15 +17,16 @@ pub struct WarningsState { impl WarningsState { fn create_filter(ctx: &Context) -> PyListRef { - ctx.new_list(vec![ctx - .new_tuple(vec![ + ctx.new_list(vec![ + ctx.new_tuple(vec![ ctx.new_str("__main__").into(), ctx.types.none_type.as_object().to_owned(), ctx.exceptions.warning.as_object().to_owned(), ctx.new_str("ACTION").into(), ctx.new_int(0).into(), ]) - .into()]) + .into(), + ]) } pub fn init_state(ctx: &Context) -> WarningsState { diff --git a/vm/src/windows.rs b/vm/src/windows.rs index f98bb1de63..f4f4dad0b3 100644 --- a/vm/src/windows.rs +++ b/vm/src/windows.rs @@ -1,11 +1,11 @@ use crate::common::fileutils::{ - windows::{get_file_information_by_name, FILE_INFO_BY_NAME_CLASS}, StatStruct, + windows::{FILE_INFO_BY_NAME_CLASS, get_file_information_by_name}, }; use crate::{ + PyObjectRef, PyResult, TryFromObject, VirtualMachine, convert::{ToPyObject, ToPyResult}, stdlib::os::errno_err, - PyObjectRef, PyResult, TryFromObject, VirtualMachine, }; use std::{ffi::OsStr, time::SystemTime}; use windows::Win32::Foundation::HANDLE; @@ -139,7 +139,7 @@ fn file_id(path: &OsStr) -> std::io::Result { use windows_sys::Win32::{ Foundation::HANDLE, Storage::FileSystem::{ - GetFileInformationByHandle, BY_HANDLE_FILE_INFORMATION, FILE_FLAG_BACKUP_SEMANTICS, + BY_HANDLE_FILE_INFORMATION, FILE_FLAG_BACKUP_SEMANTICS, GetFileInformationByHandle, }, }; diff --git a/vm/sre_engine/benches/benches.rs b/vm/sre_engine/benches/benches.rs index 915d568333..ee49b036de 100644 --- a/vm/sre_engine/benches/benches.rs +++ b/vm/sre_engine/benches/benches.rs @@ -1,6 +1,6 @@ use rustpython_sre_engine::{Request, State, StrDrive}; -use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion}; +use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; struct Pattern { pattern: &'static str, diff --git a/vm/sre_engine/src/engine.rs b/vm/sre_engine/src/engine.rs index 77f4ccdf36..3425da3371 100644 --- a/vm/sre_engine/src/engine.rs +++ b/vm/sre_engine/src/engine.rs @@ -5,7 +5,7 @@ use crate::string::{ is_uni_word, is_word, lower_ascii, lower_locate, lower_unicode, upper_locate, upper_unicode, }; -use super::{SreAtCode, SreCatCode, SreInfo, SreOpcode, StrDrive, StringCursor, MAXREPEAT}; +use super::{MAXREPEAT, SreAtCode, SreCatCode, SreInfo, SreOpcode, StrDrive, StringCursor}; use optional::Optioned; use std::{convert::TryFrom, ptr::null}; diff --git a/vm/sre_engine/src/lib.rs b/vm/sre_engine/src/lib.rs index fd9f367dc6..08c21de9df 100644 --- a/vm/sre_engine/src/lib.rs +++ b/vm/sre_engine/src/lib.rs @@ -2,7 +2,7 @@ pub mod constants; pub mod engine; pub mod string; -pub use constants::{SreAtCode, SreCatCode, SreFlag, SreInfo, SreOpcode, SRE_MAGIC}; +pub use constants::{SRE_MAGIC, SreAtCode, SreCatCode, SreFlag, SreInfo, SreOpcode}; pub use engine::{Request, SearchIter, State}; pub use string::{StrDrive, StringCursor}; diff --git a/vm/sre_engine/src/string.rs b/vm/sre_engine/src/string.rs index a32d74603a..77e0f3e772 100644 --- a/vm/sre_engine/src/string.rs +++ b/vm/sre_engine/src/string.rs @@ -157,7 +157,7 @@ impl StrDrive for &str { #[inline] unsafe fn next_code_point(ptr: &mut *const u8) -> u32 { // Decode UTF-8 - let x = unsafe { **ptr}; + let x = unsafe { **ptr }; *ptr = unsafe { ptr.offset(1) }; if x < 128 { @@ -187,7 +187,7 @@ unsafe fn next_code_point(ptr: &mut *const u8) -> u32 { // use only the lower 3 bits of `init` // SAFETY: `bytes` produces an UTF-8-like string, // so the iterator must produce a value here. - let w = unsafe { **ptr}; + let w = unsafe { **ptr }; *ptr = unsafe { ptr.offset(1) }; ch = ((init & 7) << 18) | utf8_acc_cont_byte(y_z, w); } diff --git a/wasm/lib/src/browser_module.rs b/wasm/lib/src/browser_module.rs index 86a3ea3ab3..f8d1b2ebc3 100644 --- a/wasm/lib/src/browser_module.rs +++ b/wasm/lib/src/browser_module.rs @@ -7,14 +7,14 @@ mod _browser { use crate::{convert, js_module::PyPromise, vm_class::weak_vm, wasm_builtins::window}; use js_sys::Promise; use rustpython_vm::{ + PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, builtins::{PyDictRef, PyStrRef}, class::PyClassImpl, convert::ToPyObject, function::{ArgCallable, OptionalArg}, import::import_source, - PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, }; - use wasm_bindgen::{prelude::*, JsCast}; + use wasm_bindgen::{JsCast, prelude::*}; use wasm_bindgen_futures::JsFuture; enum FetchResponseFormat { diff --git a/wasm/lib/src/convert.rs b/wasm/lib/src/convert.rs index 0bf1d21c95..d4fb068e6c 100644 --- a/wasm/lib/src/convert.rs +++ b/wasm/lib/src/convert.rs @@ -1,17 +1,18 @@ #![allow(clippy::empty_docs)] // TODO: remove it later. false positive by wasm-bindgen generated code use crate::js_module; -use crate::vm_class::{stored_vm_from_wasm, WASMVirtualMachine}; +use crate::vm_class::{WASMVirtualMachine, stored_vm_from_wasm}; use js_sys::{Array, ArrayBuffer, Object, Promise, Reflect, SyntaxError, Uint8Array}; -use rustpython_parser::{lexer::LexicalErrorType, ParseErrorType}; +use rustpython_parser::{ParseErrorType, lexer::LexicalErrorType}; use rustpython_vm::{ + AsObject, PyObjectRef, PyPayload, PyResult, TryFromBorrowedObject, VirtualMachine, builtins::PyBaseExceptionRef, compiler::{CompileError, CompileErrorType}, exceptions, function::{ArgBytesLike, FuncArgs}, - py_serde, AsObject, PyObjectRef, PyPayload, PyResult, TryFromBorrowedObject, VirtualMachine, + py_serde, }; -use wasm_bindgen::{closure::Closure, prelude::*, JsCast}; +use wasm_bindgen::{JsCast, closure::Closure, prelude::*}; #[wasm_bindgen(inline_js = r" export class PyError extends Error { diff --git a/wasm/lib/src/js_module.rs b/wasm/lib/src/js_module.rs index f0c5378c35..e25499df4d 100644 --- a/wasm/lib/src/js_module.rs +++ b/wasm/lib/src/js_module.rs @@ -5,21 +5,21 @@ use rustpython_vm::VirtualMachine; mod _js { use crate::{ convert, - vm_class::{stored_vm_from_wasm, WASMVirtualMachine}, + vm_class::{WASMVirtualMachine, stored_vm_from_wasm}, weak_vm, }; use js_sys::{Array, Object, Promise, Reflect}; use rustpython_vm::{ + Py, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, builtins::{PyBaseExceptionRef, PyFloat, PyStrRef, PyType, PyTypeRef}, convert::{IntoObject, ToPyObject}, function::{ArgCallable, OptionalArg, OptionalOption, PosArgs}, protocol::PyIterReturn, types::{IterNext, Representable, SelfIter}, - Py, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, }; use std::{cell, fmt, future}; - use wasm_bindgen::{closure::Closure, prelude::*, JsCast}; - use wasm_bindgen_futures::{future_to_promise, JsFuture}; + use wasm_bindgen::{JsCast, closure::Closure, prelude::*}; + use wasm_bindgen_futures::{JsFuture, future_to_promise}; #[wasm_bindgen(inline_js = " export function has_prop(target, prop) { return prop in Object(target); } diff --git a/wasm/lib/src/vm_class.rs b/wasm/lib/src/vm_class.rs index 4ba5760511..c04877f7e3 100644 --- a/wasm/lib/src/vm_class.rs +++ b/wasm/lib/src/vm_class.rs @@ -5,10 +5,10 @@ use crate::{ }; use js_sys::{Object, TypeError}; use rustpython_vm::{ + Interpreter, PyObjectRef, PyPayload, PyRef, PyResult, Settings, VirtualMachine, builtins::{PyModule, PyWeak}, compiler::Mode, scope::Scope, - Interpreter, PyObjectRef, PyPayload, PyRef, PyResult, Settings, VirtualMachine, }; use std::{ cell::RefCell, diff --git a/wasm/lib/src/wasm_builtins.rs b/wasm/lib/src/wasm_builtins.rs index 59d7880af4..79423fc250 100644 --- a/wasm/lib/src/wasm_builtins.rs +++ b/wasm/lib/src/wasm_builtins.rs @@ -4,7 +4,7 @@ //! desktop. //! Implements functions listed here: https://docs.python.org/3/library/builtins.html. -use rustpython_vm::{builtins::PyStrRef, PyObjectRef, PyRef, PyResult, VirtualMachine}; +use rustpython_vm::{PyObjectRef, PyRef, PyResult, VirtualMachine, builtins::PyStrRef}; use web_sys::{self, console}; pub(crate) fn window() -> web_sys::Window { From cebbca9e63f226aa4ed65d95f9228633310e997d Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sat, 22 Feb 2025 20:13:04 -0800 Subject: [PATCH 050/295] cargo-packager config Signed-off-by: Ashwin Naren --- Cargo.toml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index 55cc120335..b27e3157d3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -91,6 +91,21 @@ rev = "2024.02.14" [package.metadata.vcpkg.target] x86_64-pc-windows-msvc = { triplet = "x64-windows-static-md", dev-dependencies = ["openssl" ] } +[package.metadata.packager] +before-packaging-command = "cargo build --release --features ssl,sqlite,jit" +product-name = "RustPython" +identifier = "com.rustpython.rustpython" +description = "An open source Python 3 interpreter written in Rust" +homepage = "https://rustpython.github.io/" +license_file = "LICENSE" +authors = ["RustPython Team"] +publisher = "RustPython Team" +resources = ["LICENSE", "README.md", "Lib"] +icons = ["32x32.png"] + +[package.metadata.packager.nsis] +installer_mode = "both" + [workspace] resolver = "2" members = [ From b2abb1af8499b86d67961b24cf01f5d50a3f9bb9 Mon Sep 17 00:00:00 2001 From: Noa Date: Wed, 26 Feb 2025 23:19:46 -0600 Subject: [PATCH 051/295] Remove redundant lints now that we're on edition2024 --- Cargo.toml | 7 ------- vm/sre_engine/Cargo.toml | 3 +++ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 55cc120335..33fb96e84c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -190,13 +190,6 @@ wasm-bindgen = "0.2.100" unsafe_code = "allow" unsafe_op_in_unsafe_fn = "deny" -# rust_2024_compatibility -missing_unsafe_on_extern = "deny" -unsafe_attr_outside_unsafe = "deny" -deprecated_safe_2024 = "deny" -rust_2024_incompatible_pat = "deny" -keyword_idents_2024 = "deny" - [workspace.lints.clippy] perf = "warn" style = "warn" diff --git a/vm/sre_engine/Cargo.toml b/vm/sre_engine/Cargo.toml index ac0e7b5f16..504652f3a7 100644 --- a/vm/sre_engine/Cargo.toml +++ b/vm/sre_engine/Cargo.toml @@ -21,3 +21,6 @@ optional = "0.5" [dev-dependencies] criterion = { workspace = true } + +[lints] +workspace = true From 26bc4ba3dd08d8c18d8bdebef12a941391f5dfba Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Wed, 26 Feb 2025 22:39:00 -0800 Subject: [PATCH 052/295] add templates --- Cargo.toml | 6 +- installer-config/installer.nsi | 671 +++++++++++++++++++++++++++++++++ installer-config/installer.wxs | 297 +++++++++++++++ 3 files changed, 973 insertions(+), 1 deletion(-) create mode 100644 installer-config/installer.nsi create mode 100644 installer-config/installer.wxs diff --git a/Cargo.toml b/Cargo.toml index b27e3157d3..b5a8440903 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -92,7 +92,6 @@ rev = "2024.02.14" x86_64-pc-windows-msvc = { triplet = "x64-windows-static-md", dev-dependencies = ["openssl" ] } [package.metadata.packager] -before-packaging-command = "cargo build --release --features ssl,sqlite,jit" product-name = "RustPython" identifier = "com.rustpython.rustpython" description = "An open source Python 3 interpreter written in Rust" @@ -105,6 +104,11 @@ icons = ["32x32.png"] [package.metadata.packager.nsis] installer_mode = "both" +template = "installer-config/installer.nsi" + +[package.metadata.packager.wix] +template = "installer-config/installer.wxs" + [workspace] resolver = "2" diff --git a/installer-config/installer.nsi b/installer-config/installer.nsi new file mode 100644 index 0000000000..b68a9fbb59 --- /dev/null +++ b/installer-config/installer.nsi @@ -0,0 +1,671 @@ +; Set the compression algorithm. +!if "{{compression}}" == "" + SetCompressor /SOLID lzma +!else + SetCompressor /SOLID "{{compression}}" +!endif + +Unicode true + +!include MUI2.nsh +!include FileFunc.nsh +!include x64.nsh +!include WordFunc.nsh +!include "FileAssociation.nsh" +!include "StrFunc.nsh" +!include "StrFunc.nsh" +${StrCase} +${StrLoc} + +!define MANUFACTURER "{{manufacturer}}" +!define PRODUCTNAME "{{product_name}}" +!define VERSION "{{version}}" +!define VERSIONWITHBUILD "{{version_with_build}}" +!define SHORTDESCRIPTION "{{short_description}}" +!define INSTALLMODE "{{install_mode}}" +!define LICENSE "{{license}}" +!define INSTALLERICON "{{installer_icon}}" +!define SIDEBARIMAGE "{{sidebar_image}}" +!define HEADERIMAGE "{{header_image}}" +!define MAINBINARYNAME "{{main_binary_name}}" +!define MAINBINARYSRCPATH "{{main_binary_path}}" +!define IDENTIFIER "{{identifier}}" +!define COPYRIGHT "{{copyright}}" +!define OUTFILE "{{out_file}}" +!define ARCH "{{arch}}" +!define PLUGINSPATH "{{additional_plugins_path}}" +!define ALLOWDOWNGRADES "{{allow_downgrades}}" +!define DISPLAYLANGUAGESELECTOR "{{display_language_selector}}" +!define UNINSTKEY "Software\Microsoft\Windows\CurrentVersion\Uninstall\${PRODUCTNAME}" +!define MANUPRODUCTKEY "Software\${MANUFACTURER}\${PRODUCTNAME}" +!define UNINSTALLERSIGNCOMMAND "{{uninstaller_sign_cmd}}" +!define ESTIMATEDSIZE "{{estimated_size}}" + +Name "${PRODUCTNAME}" +BrandingText "${COPYRIGHT}" +OutFile "${OUTFILE}" + +VIProductVersion "${VERSIONWITHBUILD}" +VIAddVersionKey "ProductName" "${PRODUCTNAME}" +VIAddVersionKey "FileDescription" "${SHORTDESCRIPTION}" +VIAddVersionKey "LegalCopyright" "${COPYRIGHT}" +VIAddVersionKey "FileVersion" "${VERSION}" +VIAddVersionKey "ProductVersion" "${VERSION}" + +; Plugins path, currently exists for linux only +!if "${PLUGINSPATH}" != "" + !addplugindir "${PLUGINSPATH}" +!endif + +!if "${UNINSTALLERSIGNCOMMAND}" != "" + !uninstfinalize '${UNINSTALLERSIGNCOMMAND}' +!endif + +; Handle install mode, `perUser`, `perMachine` or `both` +!if "${INSTALLMODE}" == "perMachine" + RequestExecutionLevel highest +!endif + +!if "${INSTALLMODE}" == "currentUser" + RequestExecutionLevel user +!endif + +!if "${INSTALLMODE}" == "both" + !define MULTIUSER_MUI + !define MULTIUSER_INSTALLMODE_INSTDIR "${PRODUCTNAME}" + !define MULTIUSER_INSTALLMODE_COMMANDLINE + !if "${ARCH}" == "x64" + !define MULTIUSER_USE_PROGRAMFILES64 + !else if "${ARCH}" == "arm64" + !define MULTIUSER_USE_PROGRAMFILES64 + !endif + !define MULTIUSER_INSTALLMODE_DEFAULT_REGISTRY_KEY "${UNINSTKEY}" + !define MULTIUSER_INSTALLMODE_DEFAULT_REGISTRY_VALUENAME "CurrentUser" + !define MULTIUSER_INSTALLMODEPAGE_SHOWUSERNAME + !define MULTIUSER_INSTALLMODE_FUNCTION RestorePreviousInstallLocation + !define MULTIUSER_EXECUTIONLEVEL Highest + !include MultiUser.nsh +!endif + +; installer icon +!if "${INSTALLERICON}" != "" + !define MUI_ICON "${INSTALLERICON}" +!endif + +; installer sidebar image +!if "${SIDEBARIMAGE}" != "" + !define MUI_WELCOMEFINISHPAGE_BITMAP "${SIDEBARIMAGE}" +!endif + +; installer header image +!if "${HEADERIMAGE}" != "" + !define MUI_HEADERIMAGE + !define MUI_HEADERIMAGE_BITMAP "${HEADERIMAGE}" +!endif + +; Define registry key to store installer language +!define MUI_LANGDLL_REGISTRY_ROOT "HKCU" +!define MUI_LANGDLL_REGISTRY_KEY "${MANUPRODUCTKEY}" +!define MUI_LANGDLL_REGISTRY_VALUENAME "Installer Language" + +; Installer pages, must be ordered as they appear +; 1. Welcome Page +!define MUI_PAGE_CUSTOMFUNCTION_PRE SkipIfPassive +!insertmacro MUI_PAGE_WELCOME + +; 2. License Page (if defined) +!if "${LICENSE}" != "" + !define MUI_PAGE_CUSTOMFUNCTION_PRE SkipIfPassive + !insertmacro MUI_PAGE_LICENSE "${LICENSE}" +!endif + +; 3. Install mode (if it is set to `both`) +!if "${INSTALLMODE}" == "both" + !define MUI_PAGE_CUSTOMFUNCTION_PRE SkipIfPassive + !insertmacro MULTIUSER_PAGE_INSTALLMODE +!endif + + +; 4. Custom page to ask user if he wants to reinstall/uninstall +; only if a previous installtion was detected +Var ReinstallPageCheck +Page custom PageReinstall PageLeaveReinstall +Function PageReinstall + ; Uninstall previous WiX installation if exists. + ; + ; A WiX installer stores the isntallation info in registry + ; using a UUID and so we have to loop through all keys under + ; `HKLM\SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall` + ; and check if `DisplayName` and `Publisher` keys match ${PRODUCTNAME} and ${MANUFACTURER} + ; + ; This has a potentional issue that there maybe another installation that matches + ; our ${PRODUCTNAME} and ${MANUFACTURER} but wasn't installed by our WiX installer, + ; however, this should be fine since the user will have to confirm the uninstallation + ; and they can chose to abort it if doesn't make sense. + StrCpy $0 0 + wix_loop: + EnumRegKey $1 HKLM "SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall" $0 + StrCmp $1 "" wix_done ; Exit loop if there is no more keys to loop on + IntOp $0 $0 + 1 + ReadRegStr $R0 HKLM "SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\$1" "DisplayName" + ReadRegStr $R1 HKLM "SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\$1" "Publisher" + StrCmp "$R0$R1" "${PRODUCTNAME}${MANUFACTURER}" 0 wix_loop + ReadRegStr $R0 HKLM "SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\$1" "UninstallString" + ${StrCase} $R1 $R0 "L" + ${StrLoc} $R0 $R1 "msiexec" ">" + StrCmp $R0 0 0 wix_done + StrCpy $R7 "wix" + StrCpy $R6 "SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\$1" + Goto compare_version + wix_done: + + ; Check if there is an existing installation, if not, abort the reinstall page + ReadRegStr $R0 SHCTX "${UNINSTKEY}" "" + ReadRegStr $R1 SHCTX "${UNINSTKEY}" "UninstallString" + ${IfThen} "$R0$R1" == "" ${|} Abort ${|} + + ; Compare this installar version with the existing installation + ; and modify the messages presented to the user accordingly + compare_version: + StrCpy $R4 "$(older)" + ${If} $R7 == "wix" + ReadRegStr $R0 HKLM "$R6" "DisplayVersion" + ${Else} + ReadRegStr $R0 SHCTX "${UNINSTKEY}" "DisplayVersion" + ${EndIf} + ${IfThen} $R0 == "" ${|} StrCpy $R4 "$(unknown)" ${|} + + nsis_tauri_utils::SemverCompare "${VERSION}" $R0 + Pop $R0 + ; Reinstalling the same version + ${If} $R0 == 0 + StrCpy $R1 "$(alreadyInstalledLong)" + StrCpy $R2 "$(addOrReinstall)" + StrCpy $R3 "$(uninstallApp)" + !insertmacro MUI_HEADER_TEXT "$(alreadyInstalled)" "$(chooseMaintenanceOption)" + StrCpy $R5 "2" + ; Upgrading + ${ElseIf} $R0 == 1 + StrCpy $R1 "$(olderOrUnknownVersionInstalled)" + StrCpy $R2 "$(uninstallBeforeInstalling)" + StrCpy $R3 "$(dontUninstall)" + !insertmacro MUI_HEADER_TEXT "$(alreadyInstalled)" "$(choowHowToInstall)" + StrCpy $R5 "1" + ; Downgrading + ${ElseIf} $R0 == -1 + StrCpy $R1 "$(newerVersionInstalled)" + StrCpy $R2 "$(uninstallBeforeInstalling)" + !if "${ALLOWDOWNGRADES}" == "true" + StrCpy $R3 "$(dontUninstall)" + !else + StrCpy $R3 "$(dontUninstallDowngrade)" + !endif + !insertmacro MUI_HEADER_TEXT "$(alreadyInstalled)" "$(choowHowToInstall)" + StrCpy $R5 "1" + ${Else} + Abort + ${EndIf} + + Call SkipIfPassive + + nsDialogs::Create 1018 + Pop $R4 + ${IfThen} $(^RTL) == 1 ${|} nsDialogs::SetRTL $(^RTL) ${|} + + ${NSD_CreateLabel} 0 0 100% 24u $R1 + Pop $R1 + + ${NSD_CreateRadioButton} 30u 50u -30u 8u $R2 + Pop $R2 + ${NSD_OnClick} $R2 PageReinstallUpdateSelection + + ${NSD_CreateRadioButton} 30u 70u -30u 8u $R3 + Pop $R3 + ; disable this radio button if downgrading and downgrades are disabled + !if "${ALLOWDOWNGRADES}" == "false" + ${IfThen} $R0 == -1 ${|} EnableWindow $R3 0 ${|} + !endif + ${NSD_OnClick} $R3 PageReinstallUpdateSelection + + ; Check the first radio button if this the first time + ; we enter this page or if the second button wasn't + ; selected the last time we were on this page + ${If} $ReinstallPageCheck != 2 + SendMessage $R2 ${BM_SETCHECK} ${BST_CHECKED} 0 + ${Else} + SendMessage $R3 ${BM_SETCHECK} ${BST_CHECKED} 0 + ${EndIf} + + ${NSD_SetFocus} $R2 + nsDialogs::Show +FunctionEnd +Function PageReinstallUpdateSelection + ${NSD_GetState} $R2 $R1 + ${If} $R1 == ${BST_CHECKED} + StrCpy $ReinstallPageCheck 1 + ${Else} + StrCpy $ReinstallPageCheck 2 + ${EndIf} +FunctionEnd +Function PageLeaveReinstall + ${NSD_GetState} $R2 $R1 + + ; $R5 holds whether we are reinstalling the same version or not + ; $R5 == "1" -> different versions + ; $R5 == "2" -> same version + ; + ; $R1 holds the radio buttons state. its meaning is dependant on the context + StrCmp $R5 "1" 0 +2 ; Existing install is not the same version? + StrCmp $R1 "1" reinst_uninstall reinst_done ; $R1 == "1", then user chose to uninstall existing version, otherwise skip uninstalling + StrCmp $R1 "1" reinst_done ; Same version? skip uninstalling + + reinst_uninstall: + HideWindow + ClearErrors + + ${If} $R7 == "wix" + ReadRegStr $R1 HKLM "$R6" "UninstallString" + ExecWait '$R1' $0 + ${Else} + ReadRegStr $4 SHCTX "${MANUPRODUCTKEY}" "" + ReadRegStr $R1 SHCTX "${UNINSTKEY}" "UninstallString" + ExecWait '$R1 /P _?=$4' $0 + ${EndIf} + + BringToFront + + ${IfThen} ${Errors} ${|} StrCpy $0 2 ${|} ; ExecWait failed, set fake exit code + + ${If} $0 <> 0 + ${OrIf} ${FileExists} "$INSTDIR\${MAINBINARYNAME}.exe" + ${If} $0 = 1 ; User aborted uninstaller? + StrCmp $R5 "2" 0 +2 ; Is the existing install the same version? + Quit ; ...yes, already installed, we are done + Abort + ${EndIf} + MessageBox MB_ICONEXCLAMATION "$(unableToUninstall)" + Abort + ${Else} + StrCpy $0 $R1 1 + ${IfThen} $0 == '"' ${|} StrCpy $R1 $R1 -1 1 ${|} ; Strip quotes from UninstallString + Delete $R1 + RMDir $INSTDIR + ${EndIf} + reinst_done: +FunctionEnd + +; 5. Choose install directoy page +!define MUI_PAGE_CUSTOMFUNCTION_PRE SkipIfPassive +!insertmacro MUI_PAGE_DIRECTORY + +; 6. Start menu shortcut page +!define MUI_PAGE_CUSTOMFUNCTION_PRE SkipIfPassive +Var AppStartMenuFolder +!insertmacro MUI_PAGE_STARTMENU Application $AppStartMenuFolder + +; 7. Installation page +!insertmacro MUI_PAGE_INSTFILES + +; 8. Finish page +; +; Don't auto jump to finish page after installation page, +; because the installation page has useful info that can be used debug any issues with the installer. +!define MUI_FINISHPAGE_NOAUTOCLOSE +; Use show readme button in the finish page as a button create a desktop shortcut +!define MUI_FINISHPAGE_SHOWREADME +!define MUI_FINISHPAGE_SHOWREADME_TEXT "$(createDesktop)" +!define MUI_FINISHPAGE_SHOWREADME_FUNCTION CreateDesktopShortcut +; Show run app after installation. +!define MUI_FINISHPAGE_RUN "$INSTDIR\${MAINBINARYNAME}.exe" +!define MUI_PAGE_CUSTOMFUNCTION_PRE SkipIfPassive +!insertmacro MUI_PAGE_FINISH + +; Uninstaller Pages +; 1. Confirm uninstall page +{{#if appdata_paths}} +Var DeleteAppDataCheckbox +Var DeleteAppDataCheckboxState +!define /ifndef WS_EX_LAYOUTRTL 0x00400000 +!define MUI_PAGE_CUSTOMFUNCTION_SHOW un.ConfirmShow +Function un.ConfirmShow + FindWindow $1 "#32770" "" $HWNDPARENT ; Find inner dialog + ${If} $(^RTL) == 1 + System::Call 'USER32::CreateWindowEx(i${__NSD_CheckBox_EXSTYLE}|${WS_EX_LAYOUTRTL},t"${__NSD_CheckBox_CLASS}",t "$(deleteAppData)",i${__NSD_CheckBox_STYLE},i 50,i 100,i 400, i 25,i$1,i0,i0,i0)i.s' + ${Else} + System::Call 'USER32::CreateWindowEx(i${__NSD_CheckBox_EXSTYLE},t"${__NSD_CheckBox_CLASS}",t "$(deleteAppData)",i${__NSD_CheckBox_STYLE},i 0,i 100,i 400, i 25,i$1,i0,i0,i0)i.s' + ${EndIf} + Pop $DeleteAppDataCheckbox + SendMessage $HWNDPARENT ${WM_GETFONT} 0 0 $1 + SendMessage $DeleteAppDataCheckbox ${WM_SETFONT} $1 1 +FunctionEnd +!define MUI_PAGE_CUSTOMFUNCTION_LEAVE un.ConfirmLeave +Function un.ConfirmLeave + SendMessage $DeleteAppDataCheckbox ${BM_GETCHECK} 0 0 $DeleteAppDataCheckboxState +FunctionEnd +{{/if}} +!insertmacro MUI_UNPAGE_CONFIRM + +; 2. Uninstalling Page +!insertmacro MUI_UNPAGE_INSTFILES + +;Languages +{{#each languages}} +!insertmacro MUI_LANGUAGE "{{this}}" +{{/each}} +!insertmacro MUI_RESERVEFILE_LANGDLL +{{#each language_files}} + !include "{{this}}" +{{/each}} + +!macro SetContext + !if "${INSTALLMODE}" == "currentUser" + SetShellVarContext current + !else if "${INSTALLMODE}" == "perMachine" + SetShellVarContext all + !endif + + ${If} ${RunningX64} + !if "${ARCH}" == "x64" + SetRegView 64 + !else if "${ARCH}" == "arm64" + SetRegView 64 + !else + SetRegView 32 + !endif + ${EndIf} +!macroend + +Var PassiveMode +Function .onInit + ${GetOptions} $CMDLINE "/P" $PassiveMode + IfErrors +2 0 + StrCpy $PassiveMode 1 + + !if "${DISPLAYLANGUAGESELECTOR}" == "true" + !insertmacro MUI_LANGDLL_DISPLAY + !endif + + !insertmacro SetContext + + ${If} $INSTDIR == "" + ; Set default install location + !if "${INSTALLMODE}" == "perMachine" + ${If} ${RunningX64} + !if "${ARCH}" == "x64" + StrCpy $INSTDIR "$PROGRAMFILES64\${PRODUCTNAME}" + !else if "${ARCH}" == "arm64" + StrCpy $INSTDIR "$PROGRAMFILES64\${PRODUCTNAME}" + !else + StrCpy $INSTDIR "$PROGRAMFILES\${PRODUCTNAME}" + !endif + ${Else} + StrCpy $INSTDIR "$PROGRAMFILES\${PRODUCTNAME}" + ${EndIf} + !else if "${INSTALLMODE}" == "currentUser" + StrCpy $INSTDIR "$LOCALAPPDATA\${PRODUCTNAME}" + !endif + + Call RestorePreviousInstallLocation + ${EndIf} + + + !if "${INSTALLMODE}" == "both" + !insertmacro MULTIUSER_INIT + !endif +FunctionEnd + + +Section EarlyChecks + ; Abort silent installer if downgrades is disabled + !if "${ALLOWDOWNGRADES}" == "false" + IfSilent 0 silent_downgrades_done + ; If downgrading + ${If} $R0 == -1 + System::Call 'kernel32::AttachConsole(i -1)i.r0' + ${If} $0 != 0 + System::Call 'kernel32::GetStdHandle(i -11)i.r0' + System::call 'kernel32::SetConsoleTextAttribute(i r0, i 0x0004)' ; set red color + FileWrite $0 "$(silentDowngrades)" + ${EndIf} + Abort + ${EndIf} + silent_downgrades_done: + !endif + +SectionEnd + +{{#if preinstall_section}} +{{unescape_newlines preinstall_section}} +{{/if}} + +!macro CheckIfAppIsRunning + nsis_tauri_utils::FindProcess "${MAINBINARYNAME}.exe" + Pop $R0 + ${If} $R0 = 0 + IfSilent kill 0 + ${IfThen} $PassiveMode != 1 ${|} MessageBox MB_OKCANCEL "$(appRunningOkKill)" IDOK kill IDCANCEL cancel ${|} + kill: + nsis_tauri_utils::KillProcess "${MAINBINARYNAME}.exe" + Pop $R0 + Sleep 500 + ${If} $R0 = 0 + Goto app_check_done + ${Else} + IfSilent silent ui + silent: + System::Call 'kernel32::AttachConsole(i -1)i.r0' + ${If} $0 != 0 + System::Call 'kernel32::GetStdHandle(i -11)i.r0' + System::call 'kernel32::SetConsoleTextAttribute(i r0, i 0x0004)' ; set red color + FileWrite $0 "$(appRunning)$\n" + ${EndIf} + Abort + ui: + Abort "$(failedToKillApp)" + ${EndIf} + cancel: + Abort "$(appRunning)" + ${EndIf} + app_check_done: +!macroend + +Section Install + SetOutPath $INSTDIR + + !insertmacro CheckIfAppIsRunning + + ; Copy main executable + File "${MAINBINARYSRCPATH}" + + ; Create resources directory structure + {{#each resources_dirs}} + CreateDirectory "$INSTDIR\\{{this}}" + {{/each}} + + ; Copy resources + {{#each resources}} + File /a "/oname={{this}}" "{{@key}}" + {{/each}} + + ; Copy external binaries + {{#each binaries}} + File /a "/oname={{this}}" "{{@key}}" + {{/each}} + + ; Create file associations + {{#each file_associations as |association| ~}} + {{#each association.ext as |ext| ~}} + !insertmacro APP_ASSOCIATE "{{ext}}" "{{or association.name ext}}" "{{association-description association.description ext}}" "$INSTDIR\${MAINBINARYNAME}.exe,0" "Open with ${PRODUCTNAME}" "$INSTDIR\${MAINBINARYNAME}.exe $\"%1$\"" + {{/each}} + {{/each}} + + ; Register deep links + {{#each deep_link_protocols as |protocol| ~}} + WriteRegStr SHCTX "Software\Classes\\{{protocol}}" "URL Protocol" "" + WriteRegStr SHCTX "Software\Classes\\{{protocol}}" "" "URL:${BUNDLEID} protocol" + WriteRegStr SHCTX "Software\Classes\\{{protocol}}\DefaultIcon" "" "$\"$INSTDIR\${MAINBINARYNAME}.exe$\",0" + WriteRegStr SHCTX "Software\Classes\\{{protocol}}\shell\open\command" "" "$\"$INSTDIR\${MAINBINARYNAME}.exe$\" $\"%1$\"" + {{/each}} + + ; Create uninstaller + WriteUninstaller "$INSTDIR\uninstall.exe" + + ; Save $INSTDIR in registry for future installations + WriteRegStr SHCTX "${MANUPRODUCTKEY}" "" $INSTDIR + + !if "${INSTALLMODE}" == "both" + ; Save install mode to be selected by default for the next installation such as updating + ; or when uninstalling + WriteRegStr SHCTX "${UNINSTKEY}" $MultiUser.InstallMode 1 + !endif + + ; Registry information for add/remove programs + WriteRegStr SHCTX "${UNINSTKEY}" "DisplayName" "${PRODUCTNAME}" + WriteRegStr SHCTX "${UNINSTKEY}" "DisplayIcon" "$\"$INSTDIR\${MAINBINARYNAME}.exe$\"" + WriteRegStr SHCTX "${UNINSTKEY}" "DisplayVersion" "${VERSION}" + WriteRegStr SHCTX "${UNINSTKEY}" "Publisher" "${MANUFACTURER}" + WriteRegStr SHCTX "${UNINSTKEY}" "InstallLocation" "$\"$INSTDIR$\"" + WriteRegStr SHCTX "${UNINSTKEY}" "UninstallString" "$\"$INSTDIR\uninstall.exe$\"" + WriteRegDWORD SHCTX "${UNINSTKEY}" "NoModify" "1" + WriteRegDWORD SHCTX "${UNINSTKEY}" "NoRepair" "1" + WriteRegDWORD SHCTX "${UNINSTKEY}" "EstimatedSize" "${ESTIMATEDSIZE}" + + ; Create start menu shortcut (GUI) + !insertmacro MUI_STARTMENU_WRITE_BEGIN Application + Call CreateStartMenuShortcut + !insertmacro MUI_STARTMENU_WRITE_END + + ; Create shortcuts for silent and passive installers, which + ; can be disabled by passing `/NS` flag + ; GUI installer has buttons for users to control creating them + IfSilent check_ns_flag 0 + ${IfThen} $PassiveMode == 1 ${|} Goto check_ns_flag ${|} + Goto shortcuts_done + check_ns_flag: + ${GetOptions} $CMDLINE "/NS" $R0 + IfErrors 0 shortcuts_done + Call CreateDesktopShortcut + Call CreateStartMenuShortcut + shortcuts_done: + + ; Auto close this page for passive mode + ${IfThen} $PassiveMode == 1 ${|} SetAutoClose true ${|} +SectionEnd + +Function .onInstSuccess + ; Check for `/R` flag only in silent and passive installers because + ; GUI installer has a toggle for the user to (re)start the app + IfSilent check_r_flag 0 + ${IfThen} $PassiveMode == 1 ${|} Goto check_r_flag ${|} + Goto run_done + check_r_flag: + ${GetOptions} $CMDLINE "/R" $R0 + IfErrors run_done 0 + Exec '"$INSTDIR\${MAINBINARYNAME}.exe"' + run_done: +FunctionEnd + +Function un.onInit + !insertmacro SetContext + + !if "${INSTALLMODE}" == "both" + !insertmacro MULTIUSER_UNINIT + !endif + + !insertmacro MUI_UNGETLANGUAGE +FunctionEnd + +Section Uninstall + !insertmacro CheckIfAppIsRunning + + ; Delete the app directory and its content from disk + ; Copy main executable + Delete "$INSTDIR\${MAINBINARYNAME}.exe" + + ; Delete resources + {{#each resources}} + Delete "$INSTDIR\\{{this}}" + {{/each}} + + ; Delete external binaries + {{#each binaries}} + Delete "$INSTDIR\\{{this}}" + {{/each}} + + ; Delete app associations + {{#each file_associations as |association| ~}} + {{#each association.ext as |ext| ~}} + !insertmacro APP_UNASSOCIATE "{{ext}}" "{{or association.name ext}}" + {{/each}} + {{/each}} + + ; Delete deep links + {{#each deep_link_protocols as |protocol| ~}} + ReadRegStr $R7 SHCTX "Software\Classes\\{{protocol}}\shell\open\command" "" + !if $R7 == "$\"$INSTDIR\${MAINBINARYNAME}.exe$\" $\"%1$\"" + DeleteRegKey SHCTX "Software\Classes\\{{protocol}}" + !endif + {{/each}} + + ; Delete uninstaller + Delete "$INSTDIR\uninstall.exe" + + {{#each resources_dirs}} + RMDir /REBOOTOK "$INSTDIR\\{{this}}" + {{/each}} + RMDir "$INSTDIR" + + ; Remove start menu shortcut + !insertmacro MUI_STARTMENU_GETFOLDER Application $AppStartMenuFolder + Delete "$SMPROGRAMS\$AppStartMenuFolder\${PRODUCTNAME}.lnk" + RMDir "$SMPROGRAMS\$AppStartMenuFolder" + + ; Remove desktop shortcuts + Delete "$DESKTOP\${PRODUCTNAME}.lnk" + + ; Remove registry information for add/remove programs + !if "${INSTALLMODE}" == "both" + DeleteRegKey SHCTX "${UNINSTKEY}" + !else if "${INSTALLMODE}" == "perMachine" + DeleteRegKey HKLM "${UNINSTKEY}" + !else + DeleteRegKey HKCU "${UNINSTKEY}" + !endif + + DeleteRegValue HKCU "${MANUPRODUCTKEY}" "Installer Language" + + ; Delete app data + {{#if appdata_paths}} + ${If} $DeleteAppDataCheckboxState == 1 + SetShellVarContext current + {{#each appdata_paths}} + RmDir /r "{{unescape_dollar_sign this}}" + {{/each}} + ${EndIf} + {{/if}} + + ${GetOptions} $CMDLINE "/P" $R0 + IfErrors +2 0 + SetAutoClose true +SectionEnd + +Function RestorePreviousInstallLocation + ReadRegStr $4 SHCTX "${MANUPRODUCTKEY}" "" + StrCmp $4 "" +2 0 + StrCpy $INSTDIR $4 +FunctionEnd + +Function SkipIfPassive + ${IfThen} $PassiveMode == 1 ${|} Abort ${|} +FunctionEnd + +Function CreateDesktopShortcut + CreateShortcut "$DESKTOP\${PRODUCTNAME}.lnk" "$INSTDIR\${MAINBINARYNAME}.exe" + ApplicationID::Set "$DESKTOP\${PRODUCTNAME}.lnk" "${IDENTIFIER}" +FunctionEnd + +Function CreateStartMenuShortcut + CreateDirectory "$SMPROGRAMS\$AppStartMenuFolder" + CreateShortcut "$SMPROGRAMS\$AppStartMenuFolder\${PRODUCTNAME}.lnk" "$INSTDIR\${MAINBINARYNAME}.exe" + ApplicationID::Set "$SMPROGRAMS\$AppStartMenuFolder\${PRODUCTNAME}.lnk" "${IDENTIFIER}" +FunctionEnd \ No newline at end of file diff --git a/installer-config/installer.wxs b/installer-config/installer.wxs new file mode 100644 index 0000000000..46b23e5fc6 --- /dev/null +++ b/installer-config/installer.wxs @@ -0,0 +1,297 @@ + + + + + + + + + + + + + + + + + + + + {{#if allow_downgrades}} + + {{else}} + + {{/if}} + + + Installed AND NOT UPGRADINGPRODUCTCODE + + + + + {{#if banner_path}} + + {{/if}} + {{#if dialog_image_path}} + + {{/if}} + {{#if license}} + + {{/if}} + + {{#if icon_path}} + + + {{/if}} + + + + + + + + + + + + + + + + + WIXUI_EXITDIALOGOPTIONALCHECKBOX = 1 and NOT Installed + + + + {{#unless license}} + + 1 + 1 + {{/unless}} + + + + + + + + + + + + + + + + + + + + + + + + + + + {{#each deep_link_protocols as |protocol| ~}} + + + + + + + + + + + {{/each~}} + + + + {{#each file_associations as |association| ~}} + {{#each association.ext as |ext| ~}} + + + + + + {{/each~}} + {{/each~}} + + {{#each binaries as |bin| ~}} + + + + {{/each~}} + {{resources}} + + + + + + + + + + + + + + + + + + + + + {{#each merge_modules as |msm| ~}} + + + + + + + + {{/each~}} + + + + + + {{#each resource_file_ids as |resource_file_id| ~}} + + {{/each~}} + + + + + + + + + + + {{#each binaries as |bin| ~}} + + {{/each~}} + + + + + {{#each component_group_refs as |id| ~}} + + {{/each~}} + {{#each component_refs as |id| ~}} + + {{/each~}} + {{#each feature_group_refs as |id| ~}} + + {{/each~}} + {{#each feature_refs as |id| ~}} + + {{/each~}} + {{#each merge_refs as |id| ~}} + + {{/each~}} + + + {{#each custom_action_refs as |id| ~}} + + {{/each~}} + + {{#if install_webview}} + + + + + + + {{#if download_bootstrapper}} + + + + + + + {{/if}} + + + {{#if webview2_bootstrapper_path}} + + + + + + + + {{/if}} + + + {{#if webview2_installer_path}} + + + + + + + + {{/if}} + + {{/if}} + + + + \ No newline at end of file From aa4774fe3235ac75277885e0c407929f09622452 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Thu, 27 Feb 2025 08:44:42 -0800 Subject: [PATCH 053/295] update release.yml --- .github/workflows/release.yml | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 99035d04fc..de0cb25973 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -50,6 +50,7 @@ jobs: steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable + - uses: cargo-bins/cargo-binstall@main - name: Set up Environment shell: bash @@ -78,6 +79,31 @@ jobs: run: cp target/${{ matrix.platform.target }}/release/rustpython.exe target/rustpython-release-${{ runner.os }}-${{ matrix.platform.target }}.exe if: runner.os == 'Windows' + - name: Install cargo-packager + run: cargo binstall cargo-packager + + - name: Generate MSI + if: runner.os == 'Windows' + run: cargo packager -f wix --release -o installer + + - name: Upload MSI + if: runner.os == 'Windows' + uses: actions/upload-artifact@v4 + with: + name: rustpython-installer-msi-${{ runner.os }}-${{ matrix.platform.target }} + path: installer/*.msi + + - name: Generate DMG + if: runner.os == 'macOS' + run: cargo packager -f dmg --release -o installer + + - name: Upload DMG + if: runner.os == 'macOS' + uses: actions/upload-artifact@v4 + with: + name: rustpython-installer-dmg-${{ runner.os }}-${{ matrix.platform.target }} + path: installer/*.dmg + - name: Upload Binary Artifacts uses: actions/upload-artifact@v4 with: @@ -114,7 +140,7 @@ jobs: uses: actions/download-artifact@v4 with: path: bin - pattern: rustpython-release-* + pattern: rustpython-* merge-multiple: true - name: List Binaries @@ -142,4 +168,4 @@ jobs: --target="$tag" \ --generate-notes \ $PRERELEASE_ARG \ - bin/rustpython-release-* \ No newline at end of file + bin/rustpython-release-* From 12c3fa0b8778b5fd27f854f58a49d66aadcfc24a Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Thu, 27 Feb 2025 08:48:22 -0800 Subject: [PATCH 054/295] update wix installer config --- installer-config/installer.wxs | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/installer-config/installer.wxs b/installer-config/installer.wxs index 46b23e5fc6..1fdc77e01a 100644 --- a/installer-config/installer.wxs +++ b/installer-config/installer.wxs @@ -106,6 +106,18 @@ + + + ALLUSERS=1 OR (ALLUSERS=2 AND Privileged) + + + + + + ALLUSERS="" OR (ALLUSERS=2 AND (NOT Privileged)) + + + @@ -207,7 +219,6 @@ {{#each resource_file_ids as |resource_file_id| ~}} {{/each~}} - @@ -230,6 +241,15 @@ + + + + + {{#each component_group_refs as |id| ~}} From 544182ebfcaed73e265f3cc33759f68bdca4e587 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Thu, 27 Feb 2025 08:56:13 -0800 Subject: [PATCH 055/295] update license dates --- LICENSE | 2 +- vm/sre_engine/LICENSE | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LICENSE b/LICENSE index 7213274e0f..e2aa2ed952 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2020 RustPython Team +Copyright (c) 2025 RustPython Team Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/vm/sre_engine/LICENSE b/vm/sre_engine/LICENSE index 7213274e0f..e2aa2ed952 100644 --- a/vm/sre_engine/LICENSE +++ b/vm/sre_engine/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2020 RustPython Team +Copyright (c) 2025 RustPython Team Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal From 6731c4b1ab112da6d5051aaf80ca80cc94f9b399 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Thu, 27 Feb 2025 08:53:41 -0800 Subject: [PATCH 056/295] fix devcontainer.json --- .devcontainer/devcontainer.json | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index d8641749b5..d60eee2130 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,6 +1,4 @@ { - "image": "mcr.microsoft.com/devcontainers/universal:2", - "features": { - "ghcr.io/devcontainers/features/rust:1": {} - } -} + "image": "mcr.microsoft.com/devcontainers/base:jammy", + "onCreateCommand": "curl https://sh.rustup.rs -sSf | sh -s -- -y" +} \ No newline at end of file From 8ff856d7ce310356f6fb95f222cbd86b83470737 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Fri, 28 Feb 2025 20:48:28 -0800 Subject: [PATCH 057/295] _ctypes addressof and Structure (#5573) * _ctypes.addressof Signed-off-by: Ashwin Naren * ctypes.Structure implementation Signed-off-by: Ashwin Naren * clippy fix Signed-off-by: Ashwin Naren * formatting Signed-off-by: Ashwin Naren --------- Signed-off-by: Ashwin Naren --- vm/src/stdlib/ctypes.rs | 10 ++++++ vm/src/stdlib/ctypes/structure.rs | 58 ++++++++++++++++++++++++++++++- 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/vm/src/stdlib/ctypes.rs b/vm/src/stdlib/ctypes.rs index 2580939b62..701094a375 100644 --- a/vm/src/stdlib/ctypes.rs +++ b/vm/src/stdlib/ctypes.rs @@ -214,6 +214,16 @@ pub(crate) mod _ctypes { } } + #[pyfunction] + fn addressof(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if obj.is_instance(PyCSimple::static_type().as_ref(), vm)? { + let simple = obj.downcast_ref::().unwrap(); + Ok(simple.value.as_ptr() as usize) + } else { + Err(vm.new_type_error("expected a ctypes instance".to_string())) + } + } + #[pyfunction] fn get_errno() -> i32 { errno::errno().0 diff --git a/vm/src/stdlib/ctypes/structure.rs b/vm/src/stdlib/ctypes/structure.rs index 13cca6c260..c6cf158953 100644 --- a/vm/src/stdlib/ctypes/structure.rs +++ b/vm/src/stdlib/ctypes/structure.rs @@ -1,5 +1,61 @@ +use crate::builtins::{PyList, PyStr, PyTuple, PyTypeRef}; +use crate::function::FuncArgs; +use crate::types::GetAttr; +use crate::{AsObject, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine}; +use rustpython_common::lock::PyRwLock; +use rustpython_vm::types::Constructor; +use std::collections::HashMap; +use std::fmt::Debug; + #[pyclass(name = "Structure", module = "_ctypes")] -pub struct PyCStructure {} +#[derive(PyPayload, Debug)] +pub struct PyCStructure { + #[allow(dead_code)] + field_data: PyRwLock>, + data: PyRwLock>, +} + +impl Constructor for PyCStructure { + type Args = FuncArgs; + + fn py_new(cls: PyTypeRef, _args: Self::Args, vm: &VirtualMachine) -> PyResult { + let fields_attr = cls + .get_class_attr(vm.ctx.interned_str("_fields_").unwrap()) + .ok_or_else(|| { + vm.new_attribute_error("Structure must have a _fields_ attribute".to_string()) + })?; + // downcast into list + let fields = fields_attr.downcast_ref::().ok_or_else(|| { + vm.new_type_error("Structure _fields_ attribute must be a list".to_string()) + })?; + let fields = fields.borrow_vec(); + let mut field_data = HashMap::new(); + for field in fields.iter() { + let field = field + .downcast_ref::() + .ok_or_else(|| vm.new_type_error("Field must be a tuple".to_string()))?; + let name = field + .first() + .unwrap() + .downcast_ref::() + .ok_or_else(|| vm.new_type_error("Field name must be a string".to_string()))?; + let typ = field.get(1).unwrap().clone(); + field_data.insert(name.as_str().to_string(), typ); + } + todo!("Implement PyCStructure::py_new") + } +} + +impl GetAttr for PyCStructure { + fn getattro(zelf: &Py, name: &Py, vm: &VirtualMachine) -> PyResult { + let name = name.to_string(); + let data = zelf.data.read(); + match data.get(&name) { + Some(value) => Ok(value.clone()), + None => Err(vm.new_attribute_error(format!("No attribute named {}", name))), + } + } +} #[pyclass(flags(BASETYPE, IMMUTABLETYPE))] impl PyCStructure {} From 6daee1b00e11f4ed0568189528a2d1618e9b8298 Mon Sep 17 00:00:00 2001 From: Noa Date: Thu, 27 Feb 2025 18:55:42 -0600 Subject: [PATCH 058/295] Warn on elided_lifetimes_in_paths --- Cargo.toml | 1 + common/src/borrow.rs | 2 +- common/src/boxvec.rs | 12 ++++----- common/src/lock/thread_mutex.rs | 12 ++++----- compiler/codegen/src/compile.rs | 2 +- compiler/codegen/src/error.rs | 2 +- compiler/codegen/src/symboltable.rs | 4 +-- compiler/core/src/bytecode.rs | 36 +++++++++++++------------- compiler/core/src/marshal.rs | 2 +- compiler/src/lib.rs | 2 +- derive-impl/src/compile_bytecode.rs | 2 +- derive-impl/src/pyclass.rs | 2 +- derive-impl/src/pymodule.rs | 2 +- jit/src/instructions.rs | 2 +- stdlib/src/contextvars.rs | 2 +- stdlib/src/csv.rs | 4 +-- stdlib/src/hashlib.rs | 4 +-- stdlib/src/mmap.rs | 6 ++--- stdlib/src/overlapped.rs | 6 ++--- stdlib/src/posixsubprocess.rs | 4 +-- stdlib/src/scproxy.rs | 5 +--- stdlib/src/socket.rs | 2 +- stdlib/src/sqlite.rs | 20 +++++++++------ vm/src/builtins/code.rs | 8 +++--- vm/src/builtins/dict.rs | 2 +- vm/src/builtins/function.rs | 2 +- vm/src/builtins/genericalias.rs | 2 +- vm/src/builtins/iter.rs | 4 +-- vm/src/builtins/list.rs | 2 +- vm/src/builtins/mappingproxy.rs | 2 +- vm/src/builtins/memory.rs | 8 +++--- vm/src/builtins/set.rs | 8 +++--- vm/src/builtins/str.rs | 6 ++--- vm/src/builtins/tuple.rs | 4 +-- vm/src/builtins/type.rs | 10 ++++---- vm/src/builtins/union.rs | 2 +- vm/src/dictdatatype.rs | 6 ++--- vm/src/exceptions.rs | 4 +-- vm/src/frame.rs | 10 ++++---- vm/src/function/argument.rs | 10 ++++---- vm/src/function/protocol.rs | 6 ++--- vm/src/object/core.rs | 19 ++++++++------ vm/src/object/ext.rs | 6 ++--- vm/src/object/traverse.rs | 28 ++++++++++---------- vm/src/object/traverse_object.rs | 10 ++++---- vm/src/protocol/buffer.rs | 16 ++++++------ vm/src/protocol/callable.rs | 2 +- vm/src/protocol/iter.rs | 8 +++--- vm/src/protocol/mapping.rs | 10 ++++---- vm/src/protocol/number.rs | 6 ++--- vm/src/protocol/sequence.rs | 20 ++++++++------- vm/src/py_io.rs | 8 +++--- vm/src/py_serde.rs | 4 +-- vm/src/scope.rs | 2 +- vm/src/signal.rs | 4 +-- vm/src/stdlib/ctypes/library.rs | 4 +-- vm/src/stdlib/io.rs | 10 ++++---- vm/src/stdlib/sre.rs | 2 +- vm/src/stdlib/symtable.rs | 4 +-- vm/src/stdlib/thread.rs | 4 +-- vm/src/stdlib/time.rs | 2 +- vm/src/types/slot.rs | 8 +++--- vm/src/vm/mod.rs | 4 +-- vm/sre_engine/src/engine.rs | 40 ++++++++++++++--------------- wasm/lib/src/js_module.rs | 2 +- wasm/lib/src/lib.rs | 2 +- 66 files changed, 232 insertions(+), 225 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 33fb96e84c..807e821ca7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -189,6 +189,7 @@ wasm-bindgen = "0.2.100" [workspace.lints.rust] unsafe_code = "allow" unsafe_op_in_unsafe_fn = "deny" +elided_lifetimes_in_paths = "warn" [workspace.lints.clippy] perf = "warn" diff --git a/common/src/borrow.rs b/common/src/borrow.rs index 0e4b9d6be3..ce86d71e27 100644 --- a/common/src/borrow.rs +++ b/common/src/borrow.rs @@ -68,7 +68,7 @@ impl Deref for BorrowedValue<'_, T> { } impl fmt::Display for BorrowedValue<'_, str> { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fmt::Display::fmt(self.deref(), f) } } diff --git a/common/src/boxvec.rs b/common/src/boxvec.rs index 3b1f7e90a0..25afbcebb3 100644 --- a/common/src/boxvec.rs +++ b/common/src/boxvec.rs @@ -262,7 +262,7 @@ impl BoxVec { /// /// **Panics** if the starting point is greater than the end point or if /// the end point is greater than the length of the vector. - pub fn drain(&mut self, range: R) -> Drain + pub fn drain(&mut self, range: R) -> Drain<'_, T> where R: RangeBounds, { @@ -290,7 +290,7 @@ impl BoxVec { self.drain_range(start, end) } - fn drain_range(&mut self, start: usize, end: usize) -> Drain { + fn drain_range(&mut self, start: usize, end: usize) -> Drain<'_, T> { let len = self.len(); // bounds check happens here (before length is changed!) @@ -438,7 +438,7 @@ impl fmt::Debug for IntoIter where T: fmt::Debug, { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_list().entries(&self.v[self.index..]).finish() } } @@ -658,7 +658,7 @@ impl fmt::Debug for BoxVec where T: fmt::Debug, { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { (**self).fmt(f) } } @@ -691,13 +691,13 @@ const CAPERROR: &str = "insufficient capacity"; impl std::error::Error for CapacityError {} impl fmt::Display for CapacityError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{CAPERROR}") } } impl fmt::Debug for CapacityError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "capacity error: {CAPERROR}") } } diff --git a/common/src/lock/thread_mutex.rs b/common/src/lock/thread_mutex.rs index 35b0b9ac6d..d730818d8f 100644 --- a/common/src/lock/thread_mutex.rs +++ b/common/src/lock/thread_mutex.rs @@ -100,7 +100,7 @@ impl From for ThreadMutex { } } impl ThreadMutex { - pub fn lock(&self) -> Option> { + pub fn lock(&self) -> Option> { if self.raw.lock() { Some(ThreadMutexGuard { mu: self, @@ -110,7 +110,7 @@ impl ThreadMutex { None } } - pub fn try_lock(&self) -> Result, TryLockThreadError> { + pub fn try_lock(&self) -> Result, TryLockThreadError> { match self.raw.try_lock() { Some(true) => Ok(ThreadMutexGuard { mu: self, @@ -218,14 +218,14 @@ impl Drop for ThreadMutexGuard<'_, R, G, impl fmt::Display for ThreadMutexGuard<'_, R, G, T> { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fmt::Display::fmt(&**self, f) } } impl fmt::Debug for ThreadMutexGuard<'_, R, G, T> { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fmt::Debug::fmt(&**self, f) } } @@ -285,14 +285,14 @@ impl Drop for MappedThreadMutexGuard<'_, impl fmt::Display for MappedThreadMutexGuard<'_, R, G, T> { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fmt::Display::fmt(&**self, f) } } impl fmt::Debug for MappedThreadMutexGuard<'_, R, G, T> { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fmt::Debug::fmt(&**self, f) } } diff --git a/compiler/codegen/src/compile.rs b/compiler/codegen/src/compile.rs index 6b1d15c720..ce2e5627f0 100644 --- a/compiler/codegen/src/compile.rs +++ b/compiler/codegen/src/compile.rs @@ -3484,7 +3484,7 @@ mod tests { use rustpython_parser_core::source_code::LinearLocator; fn compile_exec(source: &str) -> CodeObject { - let mut locator: LinearLocator = LinearLocator::new(source); + let mut locator: LinearLocator<'_> = LinearLocator::new(source); use rustpython_parser::ast::fold::Fold; let mut compiler: Compiler = Compiler::new( CompileOpts::default(), diff --git a/compiler/codegen/src/error.rs b/compiler/codegen/src/error.rs index 27333992df..581f229712 100644 --- a/compiler/codegen/src/error.rs +++ b/compiler/codegen/src/error.rs @@ -38,7 +38,7 @@ pub enum CodegenErrorType { impl std::error::Error for CodegenErrorType {} impl fmt::Display for CodegenErrorType { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { use CodegenErrorType::*; match self { Assign(target) => write!(f, "cannot assign to {target}"), diff --git a/compiler/codegen/src/symboltable.rs b/compiler/codegen/src/symboltable.rs index 7db5c4edbe..53d4ff8eb4 100644 --- a/compiler/codegen/src/symboltable.rs +++ b/compiler/codegen/src/symboltable.rs @@ -74,7 +74,7 @@ pub enum SymbolTableType { } impl fmt::Display for SymbolTableType { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { SymbolTableType::Module => write!(f, "module"), SymbolTableType::Class => write!(f, "class"), @@ -195,7 +195,7 @@ impl SymbolTable { } impl std::fmt::Debug for SymbolTable { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!( f, "SymbolTable({:?} symbols, {:?} sub scopes)", diff --git a/compiler/core/src/bytecode.rs b/compiler/core/src/bytecode.rs index 11e49a47db..66284efdc9 100644 --- a/compiler/core/src/bytecode.rs +++ b/compiler/core/src/bytecode.rs @@ -15,12 +15,12 @@ pub trait Constant: Sized { type Name: AsRef; /// Transforms the given Constant to a BorrowedConstant - fn borrow_constant(&self) -> BorrowedConstant; + fn borrow_constant(&self) -> BorrowedConstant<'_, Self>; } impl Constant for ConstantData { type Name = String; - fn borrow_constant(&self) -> BorrowedConstant { + fn borrow_constant(&self) -> BorrowedConstant<'_, Self> { use BorrowedConstant::*; match self { ConstantData::Integer { value } => Integer { value }, @@ -40,7 +40,7 @@ impl Constant for ConstantData { /// A Constant Bag pub trait ConstantBag: Sized + Copy { type Constant: Constant; - fn make_constant(&self, constant: BorrowedConstant) -> Self::Constant; + fn make_constant(&self, constant: BorrowedConstant<'_, C>) -> Self::Constant; fn make_int(&self, value: BigInt) -> Self::Constant; fn make_tuple(&self, elements: impl Iterator) -> Self::Constant; fn make_code(&self, code: CodeObject) -> Self::Constant; @@ -65,7 +65,7 @@ pub struct BasicBag; impl ConstantBag for BasicBag { type Constant = ConstantData; - fn make_constant(&self, constant: BorrowedConstant) -> Self::Constant { + fn make_constant(&self, constant: BorrowedConstant<'_, C>) -> Self::Constant { constant.to_owned() } fn make_int(&self, value: BigInt) -> Self::Constant { @@ -306,7 +306,7 @@ impl PartialEq for Arg { impl Eq for Arg {} impl fmt::Debug for Arg { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "Arg<{}>", std::any::type_name::()) } } @@ -329,7 +329,7 @@ impl OpArgType for Label { } impl fmt::Display for Label { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { self.0.fmt(f) } } @@ -752,7 +752,7 @@ impl Clone for BorrowedConstant<'_, C> { } impl BorrowedConstant<'_, C> { - pub fn fmt_display(&self, f: &mut fmt::Formatter) -> fmt::Result { + pub fn fmt_display(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { BorrowedConstant::Integer { value } => write!(f, "{value}"), BorrowedConstant::Float { value } => write!(f, "{value}"), @@ -939,7 +939,7 @@ impl> fmt::Debug for Arguments<'_, N> { impl CodeObject { /// Get all arguments of the code object /// like inspect.getargs - pub fn arg_names(&self) -> Arguments { + pub fn arg_names(&self) -> Arguments<'_, C::Name> { let nargs = self.arg_count as usize; let nkwargs = self.kwonlyarg_count as usize; let mut varargs_pos = nargs + nkwargs; @@ -984,7 +984,7 @@ impl CodeObject { fn display_inner( &self, - f: &mut fmt::Formatter, + f: &mut fmt::Formatter<'_>, expand_code_objects: bool, level: usize, ) -> fmt::Result { @@ -1034,7 +1034,7 @@ impl CodeObject { pub fn display_expand_code_objects(&self) -> impl fmt::Display + '_ { struct Display<'a, C: Constant>(&'a CodeObject); impl fmt::Display for Display<'_, C> { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { self.0.display_inner(f, true, 1) } } @@ -1107,7 +1107,7 @@ impl CodeObject { } impl fmt::Display for CodeObject { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { self.display_inner(f, false, 1)?; for constant in &*self.constants { if let BorrowedConstant::Code { code } = constant.borrow_constant() { @@ -1302,19 +1302,19 @@ impl Instruction { ctx: &'a impl InstrDisplayContext, ) -> impl fmt::Display + 'a { struct FmtFn(F); - impl fmt::Result> fmt::Display for FmtFn { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + impl) -> fmt::Result> fmt::Display for FmtFn { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { (self.0)(f) } } - FmtFn(move |f: &mut fmt::Formatter| self.fmt_dis(arg, f, ctx, false, 0, 0)) + FmtFn(move |f: &mut fmt::Formatter<'_>| self.fmt_dis(arg, f, ctx, false, 0, 0)) } #[allow(clippy::too_many_arguments)] fn fmt_dis( &self, arg: OpArg, - f: &mut fmt::Formatter, + f: &mut fmt::Formatter<'_>, ctx: &impl InstrDisplayContext, expand_code_objects: bool, pad: usize, @@ -1346,7 +1346,7 @@ impl Instruction { let cell_name = |i: u32| ctx.get_cell_name(i as usize); let fmt_const = - |op: &str, arg: OpArg, f: &mut fmt::Formatter, idx: &Arg| -> fmt::Result { + |op: &str, arg: OpArg, f: &mut fmt::Formatter<'_>, idx: &Arg| -> fmt::Result { let value = ctx.get_constant(idx.get(arg) as usize); match value.borrow_constant() { BorrowedConstant::Code { code } if expand_code_objects => { @@ -1496,13 +1496,13 @@ impl InstrDisplayContext for CodeObject { } impl fmt::Display for ConstantData { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { self.borrow_constant().fmt_display(f) } } impl fmt::Debug for CodeObject { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!( f, "", diff --git a/compiler/core/src/marshal.rs b/compiler/core/src/marshal.rs index 9f3bb8e22e..cdc8b8e4ef 100644 --- a/compiler/core/src/marshal.rs +++ b/compiler/core/src/marshal.rs @@ -21,7 +21,7 @@ pub enum MarshalError { } impl std::fmt::Display for MarshalError { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Self::Eof => f.write_str("unexpected end of data"), Self::InvalidBytecode => f.write_str("invalid bytecode"), diff --git a/compiler/src/lib.rs b/compiler/src/lib.rs index 7d226cd1ce..659dc9974d 100644 --- a/compiler/src/lib.rs +++ b/compiler/src/lib.rs @@ -25,7 +25,7 @@ impl std::error::Error for CompileErrorType { } } impl std::fmt::Display for CompileErrorType { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { CompileErrorType::Codegen(e) => e.fmt(f), CompileErrorType::Parse(e) => e.fmt(f), diff --git a/derive-impl/src/compile_bytecode.rs b/derive-impl/src/compile_bytecode.rs index 2a57134197..2c702b35fe 100644 --- a/derive-impl/src/compile_bytecode.rs +++ b/derive-impl/src/compile_bytecode.rs @@ -316,7 +316,7 @@ impl PyCompileArgs { } } -fn parse_str(input: ParseStream) -> ParseResult { +fn parse_str(input: ParseStream<'_>) -> ParseResult { let span = input.span(); if input.peek(LitStr) { input.parse() diff --git a/derive-impl/src/pyclass.rs b/derive-impl/src/pyclass.rs index 077bd36bb8..b704e083c6 100644 --- a/derive-impl/src/pyclass.rs +++ b/derive-impl/src/pyclass.rs @@ -25,7 +25,7 @@ enum AttrName { } impl std::fmt::Display for AttrName { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let s = match self { Self::Method => "pymethod", Self::ClassMethod => "pyclassmethod", diff --git a/derive-impl/src/pymodule.rs b/derive-impl/src/pymodule.rs index c59e40678e..eb1e4bba9e 100644 --- a/derive-impl/src/pymodule.rs +++ b/derive-impl/src/pymodule.rs @@ -19,7 +19,7 @@ enum AttrName { } impl std::fmt::Display for AttrName { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let s = match self { Self::Function => "pyfunction", Self::Attr => "pyattr", diff --git a/jit/src/instructions.rs b/jit/src/instructions.rs index 2d5e990dd0..b495c6e1f6 100644 --- a/jit/src/instructions.rs +++ b/jit/src/instructions.rs @@ -188,7 +188,7 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> { fn prepare_const( &mut self, - constant: BorrowedConstant, + constant: BorrowedConstant<'_, C>, ) -> Result { let value = match constant { BorrowedConstant::Integer { value } => { diff --git a/stdlib/src/contextvars.rs b/stdlib/src/contextvars.rs index 1e27b8b9e5..bcc372ed55 100644 --- a/stdlib/src/contextvars.rs +++ b/stdlib/src/contextvars.rs @@ -301,7 +301,7 @@ mod _contextvars { } impl std::fmt::Debug for ContextVar { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("ContextVar").finish() } } diff --git a/stdlib/src/csv.rs b/stdlib/src/csv.rs index 03a5429ba4..f07b40b3a2 100644 --- a/stdlib/src/csv.rs +++ b/stdlib/src/csv.rs @@ -910,7 +910,7 @@ mod _csv { } impl fmt::Debug for Reader { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "_csv.reader") } } @@ -1064,7 +1064,7 @@ mod _csv { } impl fmt::Debug for Writer { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "_csv.writer") } } diff --git a/stdlib/src/hashlib.rs b/stdlib/src/hashlib.rs index 6124b6d242..f140515c13 100644 --- a/stdlib/src/hashlib.rs +++ b/stdlib/src/hashlib.rs @@ -91,7 +91,7 @@ pub mod _hashlib { } impl std::fmt::Debug for PyHasher { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "HASH {}", self.name) } } @@ -155,7 +155,7 @@ pub mod _hashlib { } impl std::fmt::Debug for PyHasherXof { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "HASHXOF {}", self.name) } } diff --git a/stdlib/src/mmap.rs b/stdlib/src/mmap.rs index e96339c370..cbc86bb2c9 100644 --- a/stdlib/src/mmap.rs +++ b/stdlib/src/mmap.rs @@ -483,7 +483,7 @@ mod mmap { flags(BASETYPE) )] impl PyMmap { - fn as_bytes_mut(&self) -> BorrowedValueMut<[u8]> { + fn as_bytes_mut(&self) -> BorrowedValueMut<'_, [u8]> { PyMutexGuard::map(self.mmap.lock(), |m| { match m.as_mut().expect("mmap closed or invalid") { MmapObj::Read(_) => panic!("mmap can't modify a readonly memory map."), @@ -493,7 +493,7 @@ mod mmap { .into() } - fn as_bytes(&self) -> BorrowedValue<[u8]> { + fn as_bytes(&self) -> BorrowedValue<'_, [u8]> { PyMutexGuard::map_immutable(self.mmap.lock(), |m| { match m.as_ref().expect("mmap closed or invalid") { MmapObj::Read(mmap) => &mmap[..], @@ -536,7 +536,7 @@ mod mmap { } } - fn check_valid(&self, vm: &VirtualMachine) -> PyResult>> { + fn check_valid(&self, vm: &VirtualMachine) -> PyResult>> { let m = self.mmap.lock(); if m.is_none() { diff --git a/stdlib/src/overlapped.rs b/stdlib/src/overlapped.rs index 45eac5f51b..f40494a432 100644 --- a/stdlib/src/overlapped.rs +++ b/stdlib/src/overlapped.rs @@ -52,7 +52,7 @@ mod _overlapped { unsafe impl Send for OverlappedInner {} impl std::fmt::Debug for Overlapped { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let zelf = self.inner.lock(); f.debug_struct("Overlapped") // .field("overlapped", &(self.overlapped as *const _ as usize)) @@ -93,7 +93,7 @@ mod _overlapped { } impl std::fmt::Debug for OverlappedReadFrom { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("OverlappedReadFrom") .field("result", &self.result) .field("allocated_buffer", &self.allocated_buffer) @@ -114,7 +114,7 @@ mod _overlapped { } impl std::fmt::Debug for OverlappedReadFromInto { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("OverlappedReadFromInto") .field("result", &self.result) .field("user_buffer", &self.user_buffer) diff --git a/stdlib/src/posixsubprocess.rs b/stdlib/src/posixsubprocess.rs index cff00a70aa..595caf524b 100644 --- a/stdlib/src/posixsubprocess.rs +++ b/stdlib/src/posixsubprocess.rs @@ -152,7 +152,7 @@ struct ProcArgs<'a> { extra_groups: Option<&'a [Gid]>, } -fn exec(args: &ForkExecArgs, procargs: ProcArgs) -> ! { +fn exec(args: &ForkExecArgs, procargs: ProcArgs<'_>) -> ! { let mut ctx = ExecErrorContext::NoExec; match exec_inner(args, procargs, &mut ctx) { Ok(x) => match x {}, @@ -183,7 +183,7 @@ impl ExecErrorContext { fn exec_inner( args: &ForkExecArgs, - procargs: ProcArgs, + procargs: ProcArgs<'_>, ctx: &mut ExecErrorContext, ) -> nix::Result { for &fd in args.fds_to_keep.as_slice() { diff --git a/stdlib/src/scproxy.rs b/stdlib/src/scproxy.rs index 9bf29626ab..c63acf5343 100644 --- a/stdlib/src/scproxy.rs +++ b/stdlib/src/scproxy.rs @@ -56,10 +56,7 @@ mod _scproxy { .map(|s| { unsafe { CFType::from_void(*s) } .downcast::() - .map(|s| { - let a_string: std::borrow::Cow = (&s).into(); - PyStr::from(a_string.into_owned()) - }) + .map(|s| PyStr::from(s.to_string())) .to_pyobject(vm) }) .collect(); diff --git a/stdlib/src/socket.rs b/stdlib/src/socket.rs index a38b4f123c..533a0a0b24 100644 --- a/stdlib/src/socket.rs +++ b/stdlib/src/socket.rs @@ -1920,7 +1920,7 @@ mod _socket { }; let host = opts.host.as_ref().map(|s| s.as_str()); - let port = opts.port.as_ref().map(|p| -> std::borrow::Cow { + let port = opts.port.as_ref().map(|p| -> std::borrow::Cow<'_, str> { match p { Either::A(s) => s.as_str().into(), Either::B(i) => i.to_string().into(), diff --git a/stdlib/src/sqlite.rs b/stdlib/src/sqlite.rs index 5487511e20..85ce8d80fd 100644 --- a/stdlib/src/sqlite.rs +++ b/stdlib/src/sqlite.rs @@ -316,7 +316,7 @@ mod _sqlite { } unsafe impl Traverse for ConnectArgs { - fn traverse(&self, tracer_fn: &mut TraverseFn) { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { self.isolation_level.traverse(tracer_fn); self.factory.traverse(tracer_fn); } @@ -337,7 +337,7 @@ mod _sqlite { } unsafe impl Traverse for BackupArgs { - fn traverse(&self, tracer_fn: &mut TraverseFn) { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { self.progress.traverse(tracer_fn); self.name.traverse(tracer_fn); } @@ -867,12 +867,12 @@ mod _sqlite { }) } - fn db_lock(&self, vm: &VirtualMachine) -> PyResult> { + fn db_lock(&self, vm: &VirtualMachine) -> PyResult> { self.check_thread(vm)?; self._db_lock(vm) } - fn _db_lock(&self, vm: &VirtualMachine) -> PyResult> { + fn _db_lock(&self, vm: &VirtualMachine) -> PyResult> { let guard = self.db.lock(); if guard.is_some() { Ok(PyMutexGuard::map(guard, |x| unsafe { @@ -1437,7 +1437,7 @@ mod _sqlite { } } - fn inner(&self, vm: &VirtualMachine) -> PyResult> { + fn inner(&self, vm: &VirtualMachine) -> PyResult> { let guard = self.inner.lock(); if guard.is_some() { Ok(PyMutexGuard::map(guard, |x| unsafe { @@ -2103,7 +2103,7 @@ mod _sqlite { self.close() } - fn inner(&self, vm: &VirtualMachine) -> PyResult> { + fn inner(&self, vm: &VirtualMachine) -> PyResult> { let guard = self.inner.lock(); if guard.is_some() { Ok(PyMutexGuard::map(guard, |x| unsafe { @@ -2314,7 +2314,7 @@ mod _sqlite { })) } - fn lock(&self) -> PyMutexGuard { + fn lock(&self) -> PyMutexGuard<'_, SqliteStatement> { self.st.lock() } } @@ -2675,7 +2675,11 @@ mod _sqlite { Ok(()) } - fn bind_parameters_sequence(self, seq: PySequence, vm: &VirtualMachine) -> PyResult<()> { + fn bind_parameters_sequence( + self, + seq: PySequence<'_>, + vm: &VirtualMachine, + ) -> PyResult<()> { let num_needed = unsafe { sqlite3_bind_parameter_count(self.st) }; if seq.length(vm)? != num_needed as usize { return Err(new_programming_error( diff --git a/vm/src/builtins/code.rs b/vm/src/builtins/code.rs index d9d8895b19..3fa332b2f3 100644 --- a/vm/src/builtins/code.rs +++ b/vm/src/builtins/code.rs @@ -58,7 +58,7 @@ impl From for PyObjectRef { } } -fn borrow_obj_constant(obj: &PyObject) -> BorrowedConstant { +fn borrow_obj_constant(obj: &PyObject) -> BorrowedConstant<'_, Literal> { match_class!(match obj { ref i @ super::int::PyInt => { let value = i.as_bigint(); @@ -96,7 +96,7 @@ fn borrow_obj_constant(obj: &PyObject) -> BorrowedConstant { impl Constant for Literal { type Name = &'static PyStrInterned; - fn borrow_constant(&self) -> BorrowedConstant { + fn borrow_constant(&self) -> BorrowedConstant<'_, Self> { borrow_obj_constant(&self.0) } } @@ -120,7 +120,7 @@ pub struct PyObjBag<'a>(pub &'a Context); impl ConstantBag for PyObjBag<'_> { type Constant = Literal; - fn make_constant(&self, constant: BorrowedConstant) -> Self::Constant { + fn make_constant(&self, constant: BorrowedConstant<'_, C>) -> Self::Constant { let ctx = self.0; let obj = match constant { bytecode::BorrowedConstant::Integer { value } => ctx.new_bigint(value).into(), @@ -208,7 +208,7 @@ impl PyCode { } impl fmt::Debug for PyCode { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "code: {:?}", self.code) } } diff --git a/vm/src/builtins/dict.rs b/vm/src/builtins/dict.rs index e54d2a1931..c8da40dc01 100644 --- a/vm/src/builtins/dict.rs +++ b/vm/src/builtins/dict.rs @@ -37,7 +37,7 @@ pub struct PyDict { pub type PyDictRef = PyRef; impl fmt::Debug for PyDict { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { // TODO: implement more detailed, non-recursive Debug formatter f.write_str("dict") } diff --git a/vm/src/builtins/function.rs b/vm/src/builtins/function.rs index 63cf8ac5c3..f7b5d39993 100644 --- a/vm/src/builtins/function.rs +++ b/vm/src/builtins/function.rs @@ -44,7 +44,7 @@ pub struct PyFunction { } unsafe impl Traverse for PyFunction { - fn traverse(&self, tracer_fn: &mut TraverseFn) { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { self.globals.traverse(tracer_fn); self.closure.traverse(tracer_fn); self.defaults_and_kwdefaults.traverse(tracer_fn); diff --git a/vm/src/builtins/genericalias.rs b/vm/src/builtins/genericalias.rs index 57c97ba62b..c03e3145b2 100644 --- a/vm/src/builtins/genericalias.rs +++ b/vm/src/builtins/genericalias.rs @@ -36,7 +36,7 @@ pub struct PyGenericAlias { } impl fmt::Debug for PyGenericAlias { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str("GenericAlias") } } diff --git a/vm/src/builtins/iter.rs b/vm/src/builtins/iter.rs index 5a47abfac7..0bd1994801 100644 --- a/vm/src/builtins/iter.rs +++ b/vm/src/builtins/iter.rs @@ -26,7 +26,7 @@ pub enum IterStatus { } unsafe impl Traverse for IterStatus { - fn traverse(&self, tracer_fn: &mut TraverseFn) { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { match self { IterStatus::Active(r) => r.traverse(tracer_fn), IterStatus::Exhausted => (), @@ -41,7 +41,7 @@ pub struct PositionIterInternal { } unsafe impl Traverse for PositionIterInternal { - fn traverse(&self, tracer_fn: &mut TraverseFn) { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { self.status.traverse(tracer_fn) } } diff --git a/vm/src/builtins/list.rs b/vm/src/builtins/list.rs index 3b9624694e..1d8b4a30e0 100644 --- a/vm/src/builtins/list.rs +++ b/vm/src/builtins/list.rs @@ -29,7 +29,7 @@ pub struct PyList { } impl fmt::Debug for PyList { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { // TODO: implement more detailed, non-recursive Debug formatter f.write_str("list") } diff --git a/vm/src/builtins/mappingproxy.rs b/vm/src/builtins/mappingproxy.rs index 659562cec1..385d18df0b 100644 --- a/vm/src/builtins/mappingproxy.rs +++ b/vm/src/builtins/mappingproxy.rs @@ -27,7 +27,7 @@ enum MappingProxyInner { } unsafe impl Traverse for MappingProxyInner { - fn traverse(&self, tracer_fn: &mut TraverseFn) { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { match self { MappingProxyInner::Class(r) => r.traverse(tracer_fn), MappingProxyInner::Mapping(arg) => arg.traverse(tracer_fn), diff --git a/vm/src/builtins/memory.rs b/vm/src/builtins/memory.rs index 9426730f40..e411d312db 100644 --- a/vm/src/builtins/memory.rs +++ b/vm/src/builtins/memory.rs @@ -386,7 +386,7 @@ impl PyMemoryView { ret } - fn obj_bytes(&self) -> BorrowedValue<[u8]> { + fn obj_bytes(&self) -> BorrowedValue<'_, [u8]> { if self.desc.is_contiguous() { BorrowedValue::map(self.buffer.obj_bytes(), |x| { &x[self.start..self.start + self.desc.len] @@ -396,7 +396,7 @@ impl PyMemoryView { } } - fn obj_bytes_mut(&self) -> BorrowedValueMut<[u8]> { + fn obj_bytes_mut(&self) -> BorrowedValueMut<'_, [u8]> { if self.desc.is_contiguous() { BorrowedValueMut::map(self.buffer.obj_bytes_mut(), |x| { &mut x[self.start..self.start + self.desc.len] @@ -406,7 +406,7 @@ impl PyMemoryView { } } - fn as_contiguous(&self) -> Option> { + fn as_contiguous(&self) -> Option> { self.desc.is_contiguous().then(|| { BorrowedValue::map(self.buffer.obj_bytes(), |x| { &x[self.start..self.start + self.desc.len] @@ -414,7 +414,7 @@ impl PyMemoryView { }) } - fn _as_contiguous_mut(&self) -> Option> { + fn _as_contiguous_mut(&self) -> Option> { self.desc.is_contiguous().then(|| { BorrowedValueMut::map(self.buffer.obj_bytes_mut(), |x| { &mut x[self.start..self.start + self.desc.len] diff --git a/vm/src/builtins/set.rs b/vm/src/builtins/set.rs index 62a66c89b4..a135af1bde 100644 --- a/vm/src/builtins/set.rs +++ b/vm/src/builtins/set.rs @@ -138,14 +138,14 @@ impl PyFrozenSet { } impl fmt::Debug for PySet { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { // TODO: implement more detailed, non-recursive Debug formatter f.write_str("set") } } impl fmt::Debug for PyFrozenSet { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { // TODO: implement more detailed, non-recursive Debug formatter f.write_str("PyFrozenSet ")?; f.debug_set().entries(self.elements().iter()).finish() @@ -170,7 +170,7 @@ pub(super) struct PySetInner { } unsafe impl crate::object::Traverse for PySetInner { - fn traverse(&self, tracer_fn: &mut crate::object::TraverseFn) { + fn traverse(&self, tracer_fn: &mut crate::object::TraverseFn<'_>) { // FIXME(discord9): Rc means shared ref, so should it be traced? self.content.traverse(tracer_fn) } @@ -1266,7 +1266,7 @@ pub(crate) struct PySetIterator { } impl fmt::Debug for PySetIterator { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { // TODO: implement more detailed, non-recursive Debug formatter f.write_str("set_iterator") } diff --git a/vm/src/builtins/str.rs b/vm/src/builtins/str.rs index bf9a9f679c..3446a3d54c 100644 --- a/vm/src/builtins/str.rs +++ b/vm/src/builtins/str.rs @@ -62,7 +62,7 @@ pub struct PyStr { } impl fmt::Debug for PyStr { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("PyStr") .field("value", &self.as_str()) .field("kind", &self.kind) @@ -192,7 +192,7 @@ pub struct PyStrIterator { } unsafe impl Traverse for PyStrIterator { - fn traverse(&self, tracer: &mut TraverseFn) { + fn traverse(&self, tracer: &mut TraverseFn<'_>) { // No need to worry about deadlock, for inner is a PyStr and can't make ref cycle self.internal.lock().0.traverse(tracer); } @@ -351,7 +351,7 @@ impl PyStr { } } - fn borrow(&self) -> &BorrowedStr { + fn borrow(&self) -> &BorrowedStr<'_> { unsafe { std::mem::transmute(self) } } diff --git a/vm/src/builtins/tuple.rs b/vm/src/builtins/tuple.rs index 66cb2799a7..c9af04d300 100644 --- a/vm/src/builtins/tuple.rs +++ b/vm/src/builtins/tuple.rs @@ -28,7 +28,7 @@ pub struct PyTuple { } impl fmt::Debug for PyTuple { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { // TODO: implement more informational, non-recursive Debug formatter f.write_str("tuple") } @@ -513,7 +513,7 @@ unsafe impl Traverse for PyTupleTyped where T: TransmuteFromObject + Traverse, { - fn traverse(&self, tracer_fn: &mut TraverseFn) { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { self.tuple.traverse(tracer_fn); } } diff --git a/vm/src/builtins/type.rs b/vm/src/builtins/type.rs index 08a15575ed..e8ad67a666 100644 --- a/vm/src/builtins/type.rs +++ b/vm/src/builtins/type.rs @@ -45,7 +45,7 @@ pub struct PyType { } unsafe impl crate::object::Traverse for PyType { - fn traverse(&self, tracer_fn: &mut crate::object::TraverseFn) { + fn traverse(&self, tracer_fn: &mut crate::object::TraverseFn<'_>) { self.base.traverse(tracer_fn); self.bases.traverse(tracer_fn); self.mro.traverse(tracer_fn); @@ -119,7 +119,7 @@ cfg_if::cfg_if! { pub type PyAttributes = IndexMap<&'static PyStrInterned, PyObjectRef, ahash::RandomState>; unsafe impl Traverse for PyAttributes { - fn traverse(&self, tracer_fn: &mut TraverseFn) { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { self.values().for_each(|v| v.traverse(tracer_fn)); } } @@ -131,7 +131,7 @@ impl fmt::Display for PyType { } impl fmt::Debug for PyType { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "[PyType {}]", &self.name()) } } @@ -406,14 +406,14 @@ impl PyType { } } - pub fn slot_name(&self) -> BorrowedValue { + pub fn slot_name(&self) -> BorrowedValue<'_, str> { self.name_inner( |name| name.into(), |ext| PyRwLockReadGuard::map(ext.name.read(), |name| name.as_str()).into(), ) } - pub fn name(&self) -> BorrowedValue { + pub fn name(&self) -> BorrowedValue<'_, str> { self.name_inner( |name| name.rsplit_once('.').map_or(name, |(_, name)| name).into(), |ext| PyRwLockReadGuard::map(ext.name.read(), |name| name.as_str()).into(), diff --git a/vm/src/builtins/union.rs b/vm/src/builtins/union.rs index f9dc8f3131..2798724ae5 100644 --- a/vm/src/builtins/union.rs +++ b/vm/src/builtins/union.rs @@ -22,7 +22,7 @@ pub struct PyUnion { } impl fmt::Debug for PyUnion { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str("UnionObject") } } diff --git a/vm/src/dictdatatype.rs b/vm/src/dictdatatype.rs index 7c8fd23834..912aa8e776 100644 --- a/vm/src/dictdatatype.rs +++ b/vm/src/dictdatatype.rs @@ -35,7 +35,7 @@ pub struct Dict { } unsafe impl Traverse for Dict { - fn traverse(&self, tracer_fn: &mut TraverseFn) { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { self.inner.traverse(tracer_fn); } } @@ -79,7 +79,7 @@ struct DictInner { } unsafe impl Traverse for DictInner { - fn traverse(&self, tracer_fn: &mut TraverseFn) { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { self.entries .iter() .map(|v| { @@ -548,7 +548,7 @@ impl Dict { vm: &VirtualMachine, key: &K, hash_value: HashValue, - mut lock: Option>>, + mut lock: Option>>, ) -> PyResult { let mut idxs = None; let mut free_slot = None; diff --git a/vm/src/exceptions.rs b/vm/src/exceptions.rs index 5882fbe2bc..d2f410b307 100644 --- a/vm/src/exceptions.rs +++ b/vm/src/exceptions.rs @@ -22,7 +22,7 @@ use std::{ }; unsafe impl Traverse for PyBaseException { - fn traverse(&self, tracer_fn: &mut TraverseFn) { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { self.traceback.traverse(tracer_fn); self.cause.traverse(tracer_fn); self.context.traverse(tracer_fn); @@ -31,7 +31,7 @@ unsafe impl Traverse for PyBaseException { } impl std::fmt::Debug for PyBaseException { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { // TODO: implement more detailed, non-recursive Debug formatter f.write_str("PyBaseException") } diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 7cbe25909b..ec5dad10a2 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -221,7 +221,7 @@ impl Frame { impl Py { #[inline(always)] - fn with_exec(&self, f: impl FnOnce(ExecutingFrame) -> R) -> R { + fn with_exec(&self, f: impl FnOnce(ExecutingFrame<'_>) -> R) -> R { let mut state = self.state.lock(); let exec = ExecutingFrame { code: &self.code, @@ -308,7 +308,7 @@ struct ExecutingFrame<'a> { } impl fmt::Debug for ExecutingFrame<'_> { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("ExecutingFrame") .field("code", self.code) // .field("scope", self.scope) @@ -371,7 +371,7 @@ impl ExecutingFrame<'_> { Err(exception) => { #[cold] fn handle_exception( - frame: &mut ExecutingFrame, + frame: &mut ExecutingFrame<'_>, exception: PyBaseExceptionRef, idx: usize, vm: &VirtualMachine, @@ -2081,7 +2081,7 @@ impl ExecutingFrame<'_> { } } - fn pop_multiple(&mut self, count: usize) -> crate::common::boxvec::Drain { + fn pop_multiple(&mut self, count: usize) -> crate::common::boxvec::Drain<'_, PyObjectRef> { let stack_len = self.state.stack.len(); self.state.stack.drain(stack_len - count..) } @@ -2119,7 +2119,7 @@ impl ExecutingFrame<'_> { } impl fmt::Debug for Frame { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let state = self.state.lock(); let stack_str = state.stack.iter().fold(String::new(), |mut s, elem| { if elem.payload_is::() { diff --git a/vm/src/function/argument.rs b/vm/src/function/argument.rs index b7fd509ef1..197cfe7b96 100644 --- a/vm/src/function/argument.rs +++ b/vm/src/function/argument.rs @@ -67,7 +67,7 @@ pub struct FuncArgs { } unsafe impl Traverse for IndexMap { - fn traverse(&self, tracer_fn: &mut TraverseFn) { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { self.values().for_each(|v| v.traverse(tracer_fn)); } } @@ -336,7 +336,7 @@ unsafe impl Traverse for KwArgs where T: Traverse, { - fn traverse(&self, tracer_fn: &mut TraverseFn) { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { self.0.iter().map(|(_, v)| v.traverse(tracer_fn)).count(); } } @@ -402,7 +402,7 @@ unsafe impl Traverse for PosArgs where T: Traverse, { - fn traverse(&self, tracer_fn: &mut TraverseFn) { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { self.0.traverse(tracer_fn) } } @@ -416,7 +416,7 @@ impl PosArgs { self.0 } - pub fn iter(&self) -> std::slice::Iter { + pub fn iter(&self) -> std::slice::Iter<'_, T> { self.0.iter() } } @@ -495,7 +495,7 @@ unsafe impl Traverse for OptionalArg where T: Traverse, { - fn traverse(&self, tracer_fn: &mut TraverseFn) { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { match self { OptionalArg::Present(o) => o.traverse(tracer_fn), OptionalArg::Missing => (), diff --git a/vm/src/function/protocol.rs b/vm/src/function/protocol.rs index 4b7e4c4cef..2f4b4d160a 100644 --- a/vm/src/function/protocol.rs +++ b/vm/src/function/protocol.rs @@ -81,7 +81,7 @@ pub struct ArgIterable { } unsafe impl Traverse for ArgIterable { - fn traverse(&self, tracer_fn: &mut TraverseFn) { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { self.iterable.traverse(tracer_fn) } } @@ -143,7 +143,7 @@ impl ArgMapping { } #[inline(always)] - pub fn mapping(&self) -> PyMapping { + pub fn mapping(&self) -> PyMapping<'_> { PyMapping { obj: &self.obj, methods: self.methods, @@ -200,7 +200,7 @@ impl TryFromObject for ArgMapping { pub struct ArgSequence(Vec); unsafe impl Traverse for ArgSequence { - fn traverse(&self, tracer_fn: &mut TraverseFn) { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { self.0.traverse(tracer_fn); } } diff --git a/vm/src/object/core.rs b/vm/src/object/core.rs index 481111532a..56ab419c01 100644 --- a/vm/src/object/core.rs +++ b/vm/src/object/core.rs @@ -81,14 +81,17 @@ pub(super) unsafe fn drop_dealloc_obj(x: *mut PyObject) { } pub(super) unsafe fn debug_obj( x: &PyObject, - f: &mut fmt::Formatter, + f: &mut fmt::Formatter<'_>, ) -> fmt::Result { let x = unsafe { &*(x as *const PyObject as *const PyInner) }; fmt::Debug::fmt(x, f) } /// Call `try_trace` on payload -pub(super) unsafe fn try_trace_obj(x: &PyObject, tracer_fn: &mut TraverseFn) { +pub(super) unsafe fn try_trace_obj( + x: &PyObject, + tracer_fn: &mut TraverseFn<'_>, +) { let x = unsafe { &*(x as *const PyObject as *const PyInner) }; let payload = &x.payload; payload.try_traverse(tracer_fn) @@ -113,7 +116,7 @@ pub(super) struct PyInner { } impl fmt::Debug for PyInner { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "[PyObject {:?}]", &self.payload) } } @@ -121,7 +124,7 @@ impl fmt::Debug for PyInner { unsafe impl Traverse for Py { /// DO notice that call `trace` on `Py` means apply `tracer_fn` on `Py`'s children, /// not like call `trace` on `PyRef` which apply `tracer_fn` on `PyRef` itself - fn traverse(&self, tracer_fn: &mut TraverseFn) { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { self.0.traverse(tracer_fn) } } @@ -129,7 +132,7 @@ unsafe impl Traverse for Py { unsafe impl Traverse for PyObject { /// DO notice that call `trace` on `PyObject` means apply `tracer_fn` on `PyObject`'s children, /// not like call `trace` on `PyObjectRef` which apply `tracer_fn` on `PyObjectRef` itself - fn traverse(&self, tracer_fn: &mut TraverseFn) { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { self.0.traverse(tracer_fn) } } @@ -139,7 +142,7 @@ pub(super) struct WeakRefList { } impl fmt::Debug for WeakRefList { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("WeakRefList").finish_non_exhaustive() } } @@ -972,7 +975,7 @@ where } impl fmt::Debug for Py { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { (**self).fmt(f) } } @@ -999,7 +1002,7 @@ cfg_if::cfg_if! { } impl fmt::Debug for PyRef { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { (**self).fmt(f) } } diff --git a/vm/src/object/ext.rs b/vm/src/object/ext.rs index 8c6d367583..b2bc6eec46 100644 --- a/vm/src/object/ext.rs +++ b/vm/src/object/ext.rs @@ -43,7 +43,7 @@ impl fmt::Display for PyRef where T: PyObjectPayload + fmt::Display, { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fmt::Display::fmt(&**self, f) } } @@ -51,7 +51,7 @@ impl fmt::Display for Py where T: PyObjectPayload + fmt::Display, { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fmt::Display::fmt(&**self, f) } } @@ -534,7 +534,7 @@ impl fmt::Display for PyLease<'_, T> where T: PyPayload + fmt::Display, { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fmt::Display::fmt(&**self, f) } } diff --git a/vm/src/object/traverse.rs b/vm/src/object/traverse.rs index 5f93dc5c8b..9ff0f88343 100644 --- a/vm/src/object/traverse.rs +++ b/vm/src/object/traverse.rs @@ -14,7 +14,7 @@ pub trait MaybeTraverse { /// if is traceable, will be used by vtable to determine const IS_TRACE: bool = false; // if this type is traceable, then call with tracer_fn, default to do nothing - fn try_traverse(&self, traverse_fn: &mut TraverseFn); + fn try_traverse(&self, traverse_fn: &mut TraverseFn<'_>); } /// Type that need traverse it's children should impl `Traverse`(Not `MaybeTraverse`) @@ -27,28 +27,28 @@ pub unsafe trait Traverse { /// but if some field is called repeatedly, panic and deadlock can happen. /// /// - _**DO NOT**_ clone a `PyObjectRef` or `Pyef` in `traverse()` - fn traverse(&self, traverse_fn: &mut TraverseFn); + fn traverse(&self, traverse_fn: &mut TraverseFn<'_>); } unsafe impl Traverse for PyObjectRef { - fn traverse(&self, traverse_fn: &mut TraverseFn) { + fn traverse(&self, traverse_fn: &mut TraverseFn<'_>) { traverse_fn(self) } } unsafe impl Traverse for PyRef { - fn traverse(&self, traverse_fn: &mut TraverseFn) { + fn traverse(&self, traverse_fn: &mut TraverseFn<'_>) { traverse_fn(self.as_object()) } } unsafe impl Traverse for () { - fn traverse(&self, _traverse_fn: &mut TraverseFn) {} + fn traverse(&self, _traverse_fn: &mut TraverseFn<'_>) {} } unsafe impl Traverse for Option { #[inline] - fn traverse(&self, traverse_fn: &mut TraverseFn) { + fn traverse(&self, traverse_fn: &mut TraverseFn<'_>) { if let Some(v) = self { v.traverse(traverse_fn); } @@ -60,7 +60,7 @@ where T: Traverse, { #[inline] - fn traverse(&self, traverse_fn: &mut TraverseFn) { + fn traverse(&self, traverse_fn: &mut TraverseFn<'_>) { for elem in self { elem.traverse(traverse_fn); } @@ -72,7 +72,7 @@ where T: Traverse, { #[inline] - fn traverse(&self, traverse_fn: &mut TraverseFn) { + fn traverse(&self, traverse_fn: &mut TraverseFn<'_>) { for elem in &**self { elem.traverse(traverse_fn); } @@ -84,7 +84,7 @@ where T: Traverse, { #[inline] - fn traverse(&self, traverse_fn: &mut TraverseFn) { + fn traverse(&self, traverse_fn: &mut TraverseFn<'_>) { for elem in self { elem.traverse(traverse_fn); } @@ -93,7 +93,7 @@ where unsafe impl Traverse for PyRwLock { #[inline] - fn traverse(&self, traverse_fn: &mut TraverseFn) { + fn traverse(&self, traverse_fn: &mut TraverseFn<'_>) { // if can't get a lock, this means something else is holding the lock, // but since gc stopped the world, during gc the lock is always held // so it is safe to ignore those in gc @@ -109,7 +109,7 @@ unsafe impl Traverse for PyRwLock { /// and refcnt is atomic, so it should be fine?) unsafe impl Traverse for PyMutex { #[inline] - fn traverse(&self, traverse_fn: &mut TraverseFn) { + fn traverse(&self, traverse_fn: &mut TraverseFn<'_>) { let mut chs: Vec> = Vec::new(); if let Some(obj) = self.try_lock() { obj.traverse(&mut |ch| { @@ -130,7 +130,7 @@ macro_rules! trace_tuple { ($(($NAME: ident, $NUM: tt)),*) => { unsafe impl<$($NAME: Traverse),*> Traverse for ($($NAME),*) { #[inline] - fn traverse(&self, traverse_fn: &mut TraverseFn) { + fn traverse(&self, traverse_fn: &mut TraverseFn<'_>) { $( self.$NUM.traverse(traverse_fn); )* @@ -142,7 +142,7 @@ macro_rules! trace_tuple { unsafe impl Traverse for Either { #[inline] - fn traverse(&self, tracer_fn: &mut TraverseFn) { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { match self { Either::A(a) => a.traverse(tracer_fn), Either::B(b) => b.traverse(tracer_fn), @@ -154,7 +154,7 @@ unsafe impl Traverse for Either { // because long tuple is extremely rare in almost every case unsafe impl Traverse for (A,) { #[inline] - fn traverse(&self, tracer_fn: &mut TraverseFn) { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { self.0.traverse(tracer_fn); } } diff --git a/vm/src/object/traverse_object.rs b/vm/src/object/traverse_object.rs index 682ddf4876..2cf4fba2d3 100644 --- a/vm/src/object/traverse_object.rs +++ b/vm/src/object/traverse_object.rs @@ -11,8 +11,8 @@ use super::{Traverse, TraverseFn}; pub(in crate::object) struct PyObjVTable { pub(in crate::object) drop_dealloc: unsafe fn(*mut PyObject), - pub(in crate::object) debug: unsafe fn(&PyObject, &mut fmt::Formatter) -> fmt::Result, - pub(in crate::object) trace: Option, + pub(in crate::object) debug: unsafe fn(&PyObject, &mut fmt::Formatter<'_>) -> fmt::Result, + pub(in crate::object) trace: Option)>, } impl PyObjVTable { @@ -32,14 +32,14 @@ impl PyObjVTable { } unsafe impl Traverse for InstanceDict { - fn traverse(&self, tracer_fn: &mut TraverseFn) { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { self.d.traverse(tracer_fn) } } unsafe impl Traverse for PyInner { /// Because PyObject hold a `PyInner`, so we need to trace it - fn traverse(&self, tracer_fn: &mut TraverseFn) { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { // 1. trace `dict` and `slots` field(`typ` can't trace for it's a AtomicRef while is leaked by design) // 2. call vtable's trace function to trace payload // self.typ.trace(tracer_fn); @@ -58,7 +58,7 @@ unsafe impl Traverse for PyInner { unsafe impl Traverse for PyInner { /// Type is known, so we can call `try_trace` directly instead of using erased type vtable - fn traverse(&self, tracer_fn: &mut TraverseFn) { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { // 1. trace `dict` and `slots` field(`typ` can't trace for it's a AtomicRef while is leaked by design) // 2. call corresponding `try_trace` function to trace payload // (No need to call vtable's trace function because we already know the type) diff --git a/vm/src/protocol/buffer.rs b/vm/src/protocol/buffer.rs index 8692e4f78e..e3b03b4f80 100644 --- a/vm/src/protocol/buffer.rs +++ b/vm/src/protocol/buffer.rs @@ -15,8 +15,8 @@ use itertools::Itertools; use std::{borrow::Cow, fmt::Debug, ops::Range}; pub struct BufferMethods { - pub obj_bytes: fn(&PyBuffer) -> BorrowedValue<[u8]>, - pub obj_bytes_mut: fn(&PyBuffer) -> BorrowedValueMut<[u8]>, + pub obj_bytes: fn(&PyBuffer) -> BorrowedValue<'_, [u8]>, + pub obj_bytes_mut: fn(&PyBuffer) -> BorrowedValueMut<'_, [u8]>, pub release: fn(&PyBuffer), pub retain: fn(&PyBuffer), } @@ -52,13 +52,13 @@ impl PyBuffer { zelf } - pub fn as_contiguous(&self) -> Option> { + pub fn as_contiguous(&self) -> Option> { self.desc .is_contiguous() .then(|| unsafe { self.contiguous_unchecked() }) } - pub fn as_contiguous_mut(&self) -> Option> { + pub fn as_contiguous_mut(&self) -> Option> { (!self.desc.readonly && self.desc.is_contiguous()) .then(|| unsafe { self.contiguous_mut_unchecked() }) } @@ -74,13 +74,13 @@ impl PyBuffer { /// # Safety /// assume the buffer is contiguous - pub unsafe fn contiguous_unchecked(&self) -> BorrowedValue<[u8]> { + pub unsafe fn contiguous_unchecked(&self) -> BorrowedValue<'_, [u8]> { self.obj_bytes() } /// # Safety /// assume the buffer is contiguous and writable - pub unsafe fn contiguous_mut_unchecked(&self) -> BorrowedValueMut<[u8]> { + pub unsafe fn contiguous_mut_unchecked(&self) -> BorrowedValueMut<'_, [u8]> { self.obj_bytes_mut() } @@ -113,11 +113,11 @@ impl PyBuffer { unsafe { self.obj.downcast_unchecked_ref() } } - pub fn obj_bytes(&self) -> BorrowedValue<[u8]> { + pub fn obj_bytes(&self) -> BorrowedValue<'_, [u8]> { (self.methods.obj_bytes)(self) } - pub fn obj_bytes_mut(&self) -> BorrowedValueMut<[u8]> { + pub fn obj_bytes_mut(&self) -> BorrowedValueMut<'_, [u8]> { (self.methods.obj_bytes_mut)(self) } diff --git a/vm/src/protocol/callable.rs b/vm/src/protocol/callable.rs index ca10555720..1444b6bf73 100644 --- a/vm/src/protocol/callable.rs +++ b/vm/src/protocol/callable.rs @@ -62,7 +62,7 @@ enum TraceEvent { } impl std::fmt::Display for TraceEvent { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { use TraceEvent::*; match self { Call => write!(f, "call"), diff --git a/vm/src/protocol/iter.rs b/vm/src/protocol/iter.rs index 345914e411..a7491a3897 100644 --- a/vm/src/protocol/iter.rs +++ b/vm/src/protocol/iter.rs @@ -16,7 +16,7 @@ where O: Borrow; unsafe impl> Traverse for PyIter { - fn traverse(&self, tracer_fn: &mut TraverseFn) { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { self.0.borrow().traverse(tracer_fn); } } @@ -70,7 +70,7 @@ where impl PyIter { /// Returns an iterator over this sequence of objects. - pub fn into_iter(self, vm: &VirtualMachine) -> PyResult> { + pub fn into_iter(self, vm: &VirtualMachine) -> PyResult> { let length_hint = vm.length_hint_opt(self.as_object().to_owned())?; Ok(PyIterIter::new(vm, self.0, length_hint)) } @@ -157,7 +157,7 @@ pub enum PyIterReturn { } unsafe impl Traverse for PyIterReturn { - fn traverse(&self, tracer_fn: &mut TraverseFn) { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { match self { PyIterReturn::Return(r) => r.traverse(tracer_fn), PyIterReturn::StopIteration(Some(obj)) => obj.traverse(tracer_fn), @@ -233,7 +233,7 @@ unsafe impl Traverse for PyIterIter<'_, T, O> where O: Traverse + Borrow, { - fn traverse(&self, tracer_fn: &mut TraverseFn) { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { self.obj.traverse(tracer_fn) } } diff --git a/vm/src/protocol/mapping.rs b/vm/src/protocol/mapping.rs index cbecc8762e..6b60ae6e05 100644 --- a/vm/src/protocol/mapping.rs +++ b/vm/src/protocol/mapping.rs @@ -22,15 +22,15 @@ impl PyObject { #[allow(clippy::type_complexity)] #[derive(Default)] pub struct PyMappingMethods { - pub length: AtomicCell PyResult>>, - pub subscript: AtomicCell PyResult>>, + pub length: AtomicCell, &VirtualMachine) -> PyResult>>, + pub subscript: AtomicCell, &PyObject, &VirtualMachine) -> PyResult>>, pub ass_subscript: AtomicCell< - Option, &VirtualMachine) -> PyResult<()>>, + Option, &PyObject, Option, &VirtualMachine) -> PyResult<()>>, >, } impl std::fmt::Debug for PyMappingMethods { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "mapping methods") } } @@ -64,7 +64,7 @@ pub struct PyMapping<'a> { } unsafe impl Traverse for PyMapping<'_> { - fn traverse(&self, tracer_fn: &mut TraverseFn) { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { self.obj.traverse(tracer_fn) } } diff --git a/vm/src/protocol/number.rs b/vm/src/protocol/number.rs index 65c4eaad79..2b6720e843 100644 --- a/vm/src/protocol/number.rs +++ b/vm/src/protocol/number.rs @@ -12,13 +12,13 @@ use crate::{ stdlib::warnings, }; -pub type PyNumberUnaryFunc = fn(PyNumber, &VirtualMachine) -> PyResult; +pub type PyNumberUnaryFunc = fn(PyNumber<'_>, &VirtualMachine) -> PyResult; pub type PyNumberBinaryFunc = fn(&PyObject, &PyObject, &VirtualMachine) -> PyResult; pub type PyNumberTernaryFunc = fn(&PyObject, &PyObject, &PyObject, &VirtualMachine) -> PyResult; impl PyObject { #[inline] - pub fn to_number(&self) -> PyNumber { + pub fn to_number(&self) -> PyNumber<'_> { PyNumber(self) } @@ -427,7 +427,7 @@ impl PyNumberSlots { pub struct PyNumber<'a>(&'a PyObject); unsafe impl Traverse for PyNumber<'_> { - fn traverse(&self, tracer_fn: &mut TraverseFn) { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { self.0.traverse(tracer_fn) } } diff --git a/vm/src/protocol/sequence.rs b/vm/src/protocol/sequence.rs index 46ac0c8be7..5d5622c156 100644 --- a/vm/src/protocol/sequence.rs +++ b/vm/src/protocol/sequence.rs @@ -28,16 +28,18 @@ impl PyObject { #[allow(clippy::type_complexity)] #[derive(Default)] pub struct PySequenceMethods { - pub length: AtomicCell PyResult>>, - pub concat: AtomicCell PyResult>>, - pub repeat: AtomicCell PyResult>>, - pub item: AtomicCell PyResult>>, + pub length: AtomicCell, &VirtualMachine) -> PyResult>>, + pub concat: AtomicCell, &PyObject, &VirtualMachine) -> PyResult>>, + pub repeat: AtomicCell, isize, &VirtualMachine) -> PyResult>>, + pub item: AtomicCell, isize, &VirtualMachine) -> PyResult>>, pub ass_item: AtomicCell< - Option, &VirtualMachine) -> PyResult<()>>, + Option, isize, Option, &VirtualMachine) -> PyResult<()>>, >, - pub contains: AtomicCell PyResult>>, - pub inplace_concat: AtomicCell PyResult>>, - pub inplace_repeat: AtomicCell PyResult>>, + pub contains: + AtomicCell, &PyObject, &VirtualMachine) -> PyResult>>, + pub inplace_concat: + AtomicCell, &PyObject, &VirtualMachine) -> PyResult>>, + pub inplace_repeat: AtomicCell, isize, &VirtualMachine) -> PyResult>>, } impl Debug for PySequenceMethods { @@ -67,7 +69,7 @@ pub struct PySequence<'a> { } unsafe impl Traverse for PySequence<'_> { - fn traverse(&self, tracer_fn: &mut TraverseFn) { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { self.obj.traverse(tracer_fn) } } diff --git a/vm/src/py_io.rs b/vm/src/py_io.rs index 0d7c319bc0..c50f09e2bf 100644 --- a/vm/src/py_io.rs +++ b/vm/src/py_io.rs @@ -7,7 +7,7 @@ use std::{fmt, io, ops}; pub trait Write { type Error; - fn write_fmt(&mut self, args: fmt::Arguments) -> Result<(), Self::Error>; + fn write_fmt(&mut self, args: fmt::Arguments<'_>) -> Result<(), Self::Error>; } #[repr(transparent)] @@ -37,14 +37,14 @@ where W: io::Write, { type Error = io::Error; - fn write_fmt(&mut self, args: fmt::Arguments) -> io::Result<()> { + fn write_fmt(&mut self, args: fmt::Arguments<'_>) -> io::Result<()> { ::write_fmt(&mut self.0, args) } } impl Write for String { type Error = fmt::Error; - fn write_fmt(&mut self, args: fmt::Arguments) -> fmt::Result { + fn write_fmt(&mut self, args: fmt::Arguments<'_>) -> fmt::Result { ::write_fmt(self, args) } } @@ -53,7 +53,7 @@ pub struct PyWriter<'vm>(pub PyObjectRef, pub &'vm VirtualMachine); impl Write for PyWriter<'_> { type Error = PyBaseExceptionRef; - fn write_fmt(&mut self, args: fmt::Arguments) -> Result<(), Self::Error> { + fn write_fmt(&mut self, args: fmt::Arguments<'_>) -> Result<(), Self::Error> { let PyWriter(obj, vm) = self; vm.call_method(obj, "write", (args.to_string(),)).map(drop) } diff --git a/vm/src/py_serde.rs b/vm/src/py_serde.rs index ea72879e42..6c23f924e1 100644 --- a/vm/src/py_serde.rs +++ b/vm/src/py_serde.rs @@ -41,7 +41,7 @@ impl<'s> PyObjectSerializer<'s> { PyObjectSerializer { pyobject, vm } } - fn clone_with_object(&self, pyobject: &'s PyObjectRef) -> PyObjectSerializer { + fn clone_with_object(&self, pyobject: &'s PyObjectRef) -> PyObjectSerializer<'_> { PyObjectSerializer { pyobject, vm: self.vm, @@ -130,7 +130,7 @@ impl<'de> DeserializeSeed<'de> for PyObjectDeserializer<'de> { impl<'de> Visitor<'de> for PyObjectDeserializer<'de> { type Value = PyObjectRef; - fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { formatter.write_str("a type that can deserialize in Python") } diff --git a/vm/src/scope.rs b/vm/src/scope.rs index 12b878f847..e01209857c 100644 --- a/vm/src/scope.rs +++ b/vm/src/scope.rs @@ -8,7 +8,7 @@ pub struct Scope { } impl fmt::Debug for Scope { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { // TODO: have a more informative Debug impl that DOESN'T recurse and cause a stack overflow f.write_str("Scope") } diff --git a/vm/src/signal.rs b/vm/src/signal.rs index 346664fa24..846114794b 100644 --- a/vm/src/signal.rs +++ b/vm/src/signal.rs @@ -107,14 +107,14 @@ impl UserSignalSender { pub struct UserSignalSendError(pub UserSignal); impl fmt::Debug for UserSignalSendError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("UserSignalSendError") .finish_non_exhaustive() } } impl fmt::Display for UserSignalSendError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str("sending a signal to a exited vm") } } diff --git a/vm/src/stdlib/ctypes/library.rs b/vm/src/stdlib/ctypes/library.rs index 94b6327440..f777d26bc0 100644 --- a/vm/src/stdlib/ctypes/library.rs +++ b/vm/src/stdlib/ctypes/library.rs @@ -12,7 +12,7 @@ pub struct SharedLibrary { } impl fmt::Debug for SharedLibrary { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "SharedLibrary") } } @@ -30,7 +30,7 @@ impl SharedLibrary { unsafe { inner .get(name.as_bytes()) - .map(|f: libloading::Symbol<*mut c_void>| *f) + .map(|f: libloading::Symbol<'_, *mut c_void>| *f) .map_err(|err| err.to_string()) } } else { diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index 1b5eb6d60c..0f472fd940 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -1392,7 +1392,7 @@ mod _io { const WRITABLE: bool; const SEEKABLE: bool = false; fn data(&self) -> &PyThreadMutex; - fn lock(&self, vm: &VirtualMachine) -> PyResult> { + fn lock(&self, vm: &VirtualMachine) -> PyResult> { self.data() .lock() .ok_or_else(|| vm.new_runtime_error("reentrant call inside buffered io".to_owned())) @@ -2273,13 +2273,13 @@ mod _io { fn lock_opt( &self, vm: &VirtualMachine, - ) -> PyResult>> { + ) -> PyResult>> { self.data .lock() .ok_or_else(|| vm.new_runtime_error("reentrant call inside textio".to_owned())) } - fn lock(&self, vm: &VirtualMachine) -> PyResult> { + fn lock(&self, vm: &VirtualMachine) -> PyResult> { let lock = self.lock_opt(vm)?; PyThreadMutexGuard::try_map(lock, |x| x.as_mut()) .map_err(|_| vm.new_value_error("I/O operation on uninitialized object".to_owned())) @@ -3210,7 +3210,7 @@ mod _io { fn lock_opt( &self, vm: &VirtualMachine, - ) -> PyResult>> { + ) -> PyResult>> { self.data .lock() .ok_or_else(|| vm.new_runtime_error("reentrant call inside nldecoder".to_owned())) @@ -3219,7 +3219,7 @@ mod _io { fn lock( &self, vm: &VirtualMachine, - ) -> PyResult> { + ) -> PyResult> { let lock = self.lock_opt(vm)?; PyThreadMutexGuard::try_map(lock, |x| x.as_mut()).map_err(|_| { vm.new_value_error("I/O operation on uninitialized nldecoder".to_owned()) diff --git a/vm/src/stdlib/sre.rs b/vm/src/stdlib/sre.rs index 7d620c13d9..193976a62d 100644 --- a/vm/src/stdlib/sre.rs +++ b/vm/src/stdlib/sre.rs @@ -53,7 +53,7 @@ mod _sre { trait SreStr: StrDrive { fn slice(&self, start: usize, end: usize, vm: &VirtualMachine) -> PyObjectRef; - fn create_request(self, pattern: &Pattern, start: usize, end: usize) -> Request { + fn create_request(self, pattern: &Pattern, start: usize, end: usize) -> Request<'_, Self> { Request::new(self, start, end, &pattern.code, false) } } diff --git a/vm/src/stdlib/symtable.rs b/vm/src/stdlib/symtable.rs index 13a4105111..7a575cb089 100644 --- a/vm/src/stdlib/symtable.rs +++ b/vm/src/stdlib/symtable.rs @@ -41,7 +41,7 @@ mod symtable { } impl fmt::Debug for PySymbolTable { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "SymbolTable()") } } @@ -157,7 +157,7 @@ mod symtable { } impl fmt::Debug for PySymbol { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "Symbol()") } } diff --git a/vm/src/stdlib/thread.rs b/vm/src/stdlib/thread.rs index bca7930437..a664d6cdc5 100644 --- a/vm/src/stdlib/thread.rs +++ b/vm/src/stdlib/thread.rs @@ -115,7 +115,7 @@ pub(crate) mod _thread { } impl fmt::Debug for Lock { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.pad("Lock") } } @@ -192,7 +192,7 @@ pub(crate) mod _thread { } impl fmt::Debug for RLock { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.pad("RLock") } } diff --git a/vm/src/stdlib/time.rs b/vm/src/stdlib/time.rs index 37a518e504..c464dc3abf 100644 --- a/vm/src/stdlib/time.rs +++ b/vm/src/stdlib/time.rs @@ -450,7 +450,7 @@ mod decl { } impl std::fmt::Debug for PyStructTime { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "struct_time()") } } diff --git a/vm/src/types/slot.rs b/vm/src/types/slot.rs index de651580d9..2d8c825817 100644 --- a/vm/src/types/slot.rs +++ b/vm/src/types/slot.rs @@ -1232,7 +1232,7 @@ pub trait AsMapping: PyPayload { fn as_mapping() -> &'static PyMappingMethods; #[inline] - fn mapping_downcast(mapping: PyMapping) -> &Py { + fn mapping_downcast(mapping: PyMapping<'_>) -> &Py { unsafe { mapping.obj.downcast_unchecked_ref() } } } @@ -1243,7 +1243,7 @@ pub trait AsSequence: PyPayload { fn as_sequence() -> &'static PySequenceMethods; #[inline] - fn sequence_downcast(seq: PySequence) -> &Py { + fn sequence_downcast(seq: PySequence<'_>) -> &Py { unsafe { seq.obj.downcast_unchecked_ref() } } } @@ -1259,12 +1259,12 @@ pub trait AsNumber: PyPayload { } #[inline] - fn number_downcast(num: PyNumber) -> &Py { + fn number_downcast(num: PyNumber<'_>) -> &Py { unsafe { num.obj().downcast_unchecked_ref() } } #[inline] - fn number_downcast_exact(num: PyNumber, vm: &VirtualMachine) -> PyRef { + fn number_downcast_exact(num: PyNumber<'_>, vm: &VirtualMachine) -> PyRef { if let Some(zelf) = num.downcast_ref_if_exact::(vm) { zelf.to_owned() } else { diff --git a/vm/src/vm/mod.rs b/vm/src/vm/mod.rs index 05ac245a0a..dd647e36f8 100644 --- a/vm/src/vm/mod.rs +++ b/vm/src/vm/mod.rs @@ -497,7 +497,7 @@ impl VirtualMachine { } } - pub fn current_frame(&self) -> Option> { + pub fn current_frame(&self) -> Option> { let frames = self.frames.borrow(); if frames.is_empty() { None @@ -514,7 +514,7 @@ impl VirtualMachine { .locals(self) } - pub fn current_globals(&self) -> Ref { + pub fn current_globals(&self) -> Ref<'_, PyDictRef> { let frame = self .current_frame() .expect("called current_globals but no frames on the stack"); diff --git a/vm/sre_engine/src/engine.rs b/vm/sre_engine/src/engine.rs index 3425da3371..bf0a6046fa 100644 --- a/vm/sre_engine/src/engine.rs +++ b/vm/sre_engine/src/engine.rs @@ -122,14 +122,14 @@ pub struct State { } impl State { - pub fn reset(&mut self, req: &Request, start: usize) { + pub fn reset(&mut self, req: &Request<'_, S>, start: usize) { self.marks.clear(); self.repeat_stack.clear(); self.start = start; req.string.adjust_cursor(&mut self.cursor, start); } - pub fn pymatch(&mut self, req: &Request) -> bool { + pub fn pymatch(&mut self, req: &Request<'_, S>) -> bool { self.start = req.start; req.string.adjust_cursor(&mut self.cursor, self.start); @@ -144,7 +144,7 @@ impl State { _match(req, self, ctx) } - pub fn search(&mut self, mut req: Request) -> bool { + pub fn search(&mut self, mut req: Request<'_, S>) -> bool { self.start = req.start; req.string.adjust_cursor(&mut self.cursor, self.start); @@ -279,7 +279,7 @@ enum Jump { PossessiveRepeat4, } -fn _match(req: &Request, state: &mut State, mut ctx: MatchContext) -> bool { +fn _match(req: &Request<'_, S>, state: &mut State, mut ctx: MatchContext) -> bool { let mut context_stack = vec![]; let mut popped_result = false; @@ -882,7 +882,7 @@ fn _match(req: &Request, state: &mut State, mut ctx: MatchContex } fn search_info_literal( - req: &mut Request, + req: &mut Request<'_, S>, state: &mut State, mut ctx: MatchContext, ) -> bool { @@ -998,7 +998,7 @@ fn search_info_literal( } fn search_info_charset( - req: &mut Request, + req: &mut Request<'_, S>, state: &mut State, mut ctx: MatchContext, ) -> bool { @@ -1054,11 +1054,11 @@ impl MatchContext { &req.pattern_codes[self.code_position..] } - fn remaining_codes(&self, req: &Request) -> usize { + fn remaining_codes(&self, req: &Request<'_, S>) -> usize { req.pattern_codes.len() - self.code_position } - fn remaining_chars(&self, req: &Request) -> usize { + fn remaining_chars(&self, req: &Request<'_, S>) -> usize { req.end - self.cursor.position } @@ -1086,11 +1086,11 @@ impl MatchContext { S::back_advance(&mut self.cursor) } - fn peek_code(&self, req: &Request, peek: usize) -> u32 { + fn peek_code(&self, req: &Request<'_, S>, peek: usize) -> u32 { req.pattern_codes[self.code_position + peek] } - fn try_peek_code_as(&self, req: &Request, peek: usize) -> Result + fn try_peek_code_as(&self, req: &Request<'_, S>, peek: usize) -> Result where T: TryFrom, { @@ -1101,7 +1101,7 @@ impl MatchContext { self.code_position += skip; } - fn skip_code_from(&mut self, req: &Request, peek: usize) { + fn skip_code_from(&mut self, req: &Request<'_, S>, peek: usize) { self.skip_code(self.peek_code(req, peek) as usize + 1); } @@ -1110,17 +1110,17 @@ impl MatchContext { self.cursor.position == 0 } - fn at_end(&self, req: &Request) -> bool { + fn at_end(&self, req: &Request<'_, S>) -> bool { self.cursor.position == req.end } - fn at_linebreak(&self, req: &Request) -> bool { + fn at_linebreak(&self, req: &Request<'_, S>) -> bool { !self.at_end(req) && is_linebreak(self.peek_char::()) } fn at_boundary bool>( &self, - req: &Request, + req: &Request<'_, S>, mut word_checker: F, ) -> bool { if self.at_beginning() && self.at_end(req) { @@ -1133,7 +1133,7 @@ impl MatchContext { fn at_non_boundary bool>( &self, - req: &Request, + req: &Request<'_, S>, mut word_checker: F, ) -> bool { if self.at_beginning() && self.at_end(req) { @@ -1144,7 +1144,7 @@ impl MatchContext { this == that } - fn can_success(&self, req: &Request) -> bool { + fn can_success(&self, req: &Request<'_, S>) -> bool { if !self.toplevel { return true; } @@ -1158,7 +1158,7 @@ impl MatchContext { } #[must_use] - fn next_peek_from(&mut self, peek: usize, req: &Request, jump: Jump) -> Self { + fn next_peek_from(&mut self, peek: usize, req: &Request<'_, S>, jump: Jump) -> Self { self.next_offset(self.peek_code(req, peek) as usize + 1, jump) } @@ -1179,7 +1179,7 @@ impl MatchContext { } } -fn at(req: &Request, ctx: &MatchContext, atcode: SreAtCode) -> bool { +fn at(req: &Request<'_, S>, ctx: &MatchContext, atcode: SreAtCode) -> bool { match atcode { SreAtCode::BEGINNING | SreAtCode::BEGINNING_STRING => ctx.at_beginning(), SreAtCode::BEGINNING_LINE => ctx.at_beginning() || is_linebreak(ctx.back_peek_char::()), @@ -1327,7 +1327,7 @@ fn charset(set: &[u32], ch: u32) -> bool { } fn _count( - req: &Request, + req: &Request<'_, S>, state: &mut State, ctx: &mut MatchContext, max_count: usize, @@ -1399,7 +1399,7 @@ fn _count( } fn general_count_literal bool>( - req: &Request, + req: &Request<'_, S>, ctx: &mut MatchContext, end: usize, mut f: F, diff --git a/wasm/lib/src/js_module.rs b/wasm/lib/src/js_module.rs index e25499df4d..a5b7281481 100644 --- a/wasm/lib/src/js_module.rs +++ b/wasm/lib/src/js_module.rs @@ -297,7 +297,7 @@ mod _js { } impl fmt::Debug for JsClosure { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.pad("JsClosure") } } diff --git a/wasm/lib/src/lib.rs b/wasm/lib/src/lib.rs index a5f1a5895a..8d1d19ddae 100644 --- a/wasm/lib/src/lib.rs +++ b/wasm/lib/src/lib.rs @@ -14,7 +14,7 @@ pub(crate) use vm_class::weak_vm; use wasm_bindgen::prelude::*; /// Sets error info on the window object, and prints the backtrace to console -pub fn panic_hook(info: &panic::PanicHookInfo) { +pub fn panic_hook(info: &panic::PanicHookInfo<'_>) { // If something errors, just ignore it; we don't want to panic in the panic hook let try_set_info = || { let msg = &info.to_string(); From 56196890f56cf3341e10c4a0ae21e9e8c88ffde9 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sun, 2 Mar 2025 01:18:17 -0800 Subject: [PATCH 059/295] Actions caching for nodejs (#5575) * caching for nodejs and various CI dependency updates * commit the package-lock.json --- .github/workflows/ci.yaml | 10 +- wasm/demo/package-lock.json | 5623 +++++++++++++++++++++++++++++++++++ 2 files changed, 5630 insertions(+), 3 deletions(-) create mode 100644 wasm/demo/package-lock.json diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 23f47b574c..73b5cc1cd6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -4,6 +4,7 @@ on: pull_request: types: [unlabeled, opened, synchronize, reopened] merge_group: + workflow_dispatch: name: CI @@ -360,15 +361,18 @@ jobs: run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh - name: install geckodriver run: | - wget https://github.com/mozilla/geckodriver/releases/download/v0.34.0/geckodriver-v0.34.0-linux64.tar.gz + wget https://github.com/mozilla/geckodriver/releases/download/v0.36.0/geckodriver-v0.36.0-linux64.tar.gz mkdir geckodriver - tar -xzf geckodriver-v0.34.0-linux64.tar.gz -C geckodriver + tar -xzf geckodriver-v0.36.0-linux64.tar.gz -C geckodriver - uses: actions/setup-python@v5 with: python-version: ${{ env.PYTHON_VERSION }} - run: python -m pip install -r requirements.txt working-directory: ./wasm/tests - uses: actions/setup-node@v4 + with: + cache: "npm" + cache-dependency-path: "wasm/demo/package-lock.json" - name: run test run: | export PATH=$PATH:`pwd`/../../geckodriver @@ -378,7 +382,7 @@ jobs: NODE_OPTIONS: "--openssl-legacy-provider" working-directory: ./wasm/demo - uses: mwilliamson/setup-wabt-action@v3 - with: { wabt-version: "1.0.30" } + with: { wabt-version: "1.0.36" } - name: check wasm32-unknown without js run: | cd wasm/wasm-unknown-test diff --git a/wasm/demo/package-lock.json b/wasm/demo/package-lock.json new file mode 100644 index 0000000000..01753cf48f --- /dev/null +++ b/wasm/demo/package-lock.json @@ -0,0 +1,5623 @@ +{ + "name": "app", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "app", + "version": "1.0.0", + "license": "MIT", + "dependencies": { + "@codemirror/lang-python": "^6.1.6", + "@xterm/addon-fit": "^0.10.0", + "@xterm/xterm": "^5.3.0", + "codemirror": "^6.0.1", + "upgrade": "^1.1.0", + "xterm-readline": "^1.1.2" + }, + "devDependencies": { + "@wasm-tool/wasm-pack-plugin": "^1.7.0", + "css-loader": "^7.1.2", + "html-webpack-plugin": "^5.6.3", + "mini-css-extract-plugin": "^2.9.2", + "serve": "^14.2.4", + "webpack": "^5.97.1", + "webpack-cli": "^6.0.1", + "webpack-dev-server": "^5.2.0" + } + }, + "node_modules/@codemirror/autocomplete": { + "version": "6.18.6", + "resolved": "https://registry.npmjs.org/@codemirror/autocomplete/-/autocomplete-6.18.6.tgz", + "integrity": "sha512-PHHBXFomUs5DF+9tCOM/UoW6XQ4R44lLNNhRaW9PKPTU0D7lIjRg3ElxaJnTwsl/oHiR93WSXDBrekhoUGCPtg==", + "license": "MIT", + "dependencies": { + "@codemirror/language": "^6.0.0", + "@codemirror/state": "^6.0.0", + "@codemirror/view": "^6.17.0", + "@lezer/common": "^1.0.0" + } + }, + "node_modules/@codemirror/commands": { + "version": "6.8.0", + "resolved": "https://registry.npmjs.org/@codemirror/commands/-/commands-6.8.0.tgz", + "integrity": "sha512-q8VPEFaEP4ikSlt6ZxjB3zW72+7osfAYW9i8Zu943uqbKuz6utc1+F170hyLUCUltXORjQXRyYQNfkckzA/bPQ==", + "license": "MIT", + "dependencies": { + "@codemirror/language": "^6.0.0", + "@codemirror/state": "^6.4.0", + "@codemirror/view": "^6.27.0", + "@lezer/common": "^1.1.0" + } + }, + "node_modules/@codemirror/lang-python": { + "version": "6.1.7", + "resolved": "https://registry.npmjs.org/@codemirror/lang-python/-/lang-python-6.1.7.tgz", + "integrity": "sha512-mZnFTsL4lW5p9ch8uKNKeRU3xGGxr1QpESLilfON2E3fQzOa/OygEMkaDvERvXDJWJA9U9oN/D4w0ZuUzNO4+g==", + "license": "MIT", + "dependencies": { + "@codemirror/autocomplete": "^6.3.2", + "@codemirror/language": "^6.8.0", + "@codemirror/state": "^6.0.0", + "@lezer/common": "^1.2.1", + "@lezer/python": "^1.1.4" + } + }, + "node_modules/@codemirror/language": { + "version": "6.10.8", + "resolved": "https://registry.npmjs.org/@codemirror/language/-/language-6.10.8.tgz", + "integrity": "sha512-wcP8XPPhDH2vTqf181U8MbZnW+tDyPYy0UzVOa+oHORjyT+mhhom9vBd7dApJwoDz9Nb/a8kHjJIsuA/t8vNFw==", + "license": "MIT", + "dependencies": { + "@codemirror/state": "^6.0.0", + "@codemirror/view": "^6.23.0", + "@lezer/common": "^1.1.0", + "@lezer/highlight": "^1.0.0", + "@lezer/lr": "^1.0.0", + "style-mod": "^4.0.0" + } + }, + "node_modules/@codemirror/lint": { + "version": "6.8.4", + "resolved": "https://registry.npmjs.org/@codemirror/lint/-/lint-6.8.4.tgz", + "integrity": "sha512-u4q7PnZlJUojeRe8FJa/njJcMctISGgPQ4PnWsd9268R4ZTtU+tfFYmwkBvgcrK2+QQ8tYFVALVb5fVJykKc5A==", + "license": "MIT", + "dependencies": { + "@codemirror/state": "^6.0.0", + "@codemirror/view": "^6.35.0", + "crelt": "^1.0.5" + } + }, + "node_modules/@codemirror/search": { + "version": "6.5.10", + "resolved": "https://registry.npmjs.org/@codemirror/search/-/search-6.5.10.tgz", + "integrity": "sha512-RMdPdmsrUf53pb2VwflKGHEe1XVM07hI7vV2ntgw1dmqhimpatSJKva4VA9h4TLUDOD4EIF02201oZurpnEFsg==", + "license": "MIT", + "dependencies": { + "@codemirror/state": "^6.0.0", + "@codemirror/view": "^6.0.0", + "crelt": "^1.0.5" + } + }, + "node_modules/@codemirror/state": { + "version": "6.5.2", + "resolved": "https://registry.npmjs.org/@codemirror/state/-/state-6.5.2.tgz", + "integrity": "sha512-FVqsPqtPWKVVL3dPSxy8wEF/ymIEuVzF1PK3VbUgrxXpJUSHQWWZz4JMToquRxnkw+36LTamCZG2iua2Ptq0fA==", + "license": "MIT", + "dependencies": { + "@marijn/find-cluster-break": "^1.0.0" + } + }, + "node_modules/@codemirror/view": { + "version": "6.36.3", + "resolved": "https://registry.npmjs.org/@codemirror/view/-/view-6.36.3.tgz", + "integrity": "sha512-N2bilM47QWC8Hnx0rMdDxO2x2ImJ1FvZWXubwKgjeoOrWwEiFrtpA7SFHcuZ+o2Ze2VzbkgbzWVj4+V18LVkeg==", + "license": "MIT", + "dependencies": { + "@codemirror/state": "^6.5.0", + "style-mod": "^4.1.0", + "w3c-keyname": "^2.2.4" + } + }, + "node_modules/@discoveryjs/json-ext": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/@discoveryjs/json-ext/-/json-ext-0.6.3.tgz", + "integrity": "sha512-4B4OijXeVNOPZlYA2oEwWOTkzyltLao+xbotHQeqN++Rv27Y6s818+n2Qkp8q+Fxhn0t/5lA5X1Mxktud8eayQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.17.0" + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.8", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.8.tgz", + "integrity": "sha512-imAbBGkb+ebQyxKgzv5Hu2nmROxoDOXHh80evxdoXNOrvAnVx7zimzc1Oo5h9RlfV4vPXaE2iM5pOFbvOCClWA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/set-array": "^1.2.1", + "@jridgewell/sourcemap-codec": "^1.4.10", + "@jridgewell/trace-mapping": "^0.3.24" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/set-array": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.2.1.tgz", + "integrity": "sha512-R8gLRTZeyp03ymzP/6Lil/28tGeGEzhx1q2k703KGWRAI1VdvPIXdG70VJc2pAMw3NA6JKL5hhFu1sJX0Mnn/A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/source-map": { + "version": "0.3.6", + "resolved": "https://registry.npmjs.org/@jridgewell/source-map/-/source-map-0.3.6.tgz", + "integrity": "sha512-1ZJTZebgqllO79ue2bm3rIGud/bOe0pP5BjSRCRxxYkEZS8STV7zN84UBbiYu7jy+eCKSnVIUgoWWE/tt+shMQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.25" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.0.tgz", + "integrity": "sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.25", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.25.tgz", + "integrity": "sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@jsonjoy.com/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-q6XAnWQDIMA3+FTiOYajoYqySkO+JSat0ytXGSuRdq9uXE7o92gzuQwQM14xaCRlBLGq3v5miDGC4vkVTn54xA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=10.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + }, + "peerDependencies": { + "tslib": "2" + } + }, + "node_modules/@jsonjoy.com/json-pack": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/json-pack/-/json-pack-1.1.1.tgz", + "integrity": "sha512-osjeBqMJ2lb/j/M8NCPjs1ylqWIcTRTycIhVB5pt6LgzgeRSb0YRZ7j9RfA8wIUrsr/medIuhVyonXRZWLyfdw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@jsonjoy.com/base64": "^1.1.1", + "@jsonjoy.com/util": "^1.1.2", + "hyperdyperid": "^1.2.0", + "thingies": "^1.20.0" + }, + "engines": { + "node": ">=10.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + }, + "peerDependencies": { + "tslib": "2" + } + }, + "node_modules/@jsonjoy.com/util": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/util/-/util-1.5.0.tgz", + "integrity": "sha512-ojoNsrIuPI9g6o8UxhraZQSyF2ByJanAY4cTFbc8Mf2AXEF4aQRGY1dJxyJpuyav8r9FGflEt/Ff3u5Nt6YMPA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=10.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + }, + "peerDependencies": { + "tslib": "2" + } + }, + "node_modules/@leichtgewicht/ip-codec": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@leichtgewicht/ip-codec/-/ip-codec-2.0.5.tgz", + "integrity": "sha512-Vo+PSpZG2/fmgmiNzYK9qWRh8h/CHrwD0mo1h1DzL4yzHNSfWYujGTYsWGreD000gcgmZ7K4Ys6Tx9TxtsKdDw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@lezer/common": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@lezer/common/-/common-1.2.3.tgz", + "integrity": "sha512-w7ojc8ejBqr2REPsWxJjrMFsA/ysDCFICn8zEOR9mrqzOu2amhITYuLD8ag6XZf0CFXDrhKqw7+tW8cX66NaDA==", + "license": "MIT" + }, + "node_modules/@lezer/highlight": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@lezer/highlight/-/highlight-1.2.1.tgz", + "integrity": "sha512-Z5duk4RN/3zuVO7Jq0pGLJ3qynpxUVsh7IbUbGj88+uV2ApSAn6kWg2au3iJb+0Zi7kKtqffIESgNcRXWZWmSA==", + "license": "MIT", + "dependencies": { + "@lezer/common": "^1.0.0" + } + }, + "node_modules/@lezer/lr": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/@lezer/lr/-/lr-1.4.2.tgz", + "integrity": "sha512-pu0K1jCIdnQ12aWNaAVU5bzi7Bd1w54J3ECgANPmYLtQKP0HBj2cE/5coBD66MT10xbtIuUr7tg0Shbsvk0mDA==", + "license": "MIT", + "dependencies": { + "@lezer/common": "^1.0.0" + } + }, + "node_modules/@lezer/python": { + "version": "1.1.15", + "resolved": "https://registry.npmjs.org/@lezer/python/-/python-1.1.15.tgz", + "integrity": "sha512-aVQ43m2zk4FZYedCqL0KHPEUsqZOrmAvRhkhHlVPnDD1HODDyyQv5BRIuod4DadkgBEZd53vQOtXTonNbEgjrQ==", + "license": "MIT", + "dependencies": { + "@lezer/common": "^1.2.0", + "@lezer/highlight": "^1.0.0", + "@lezer/lr": "^1.0.0" + } + }, + "node_modules/@marijn/find-cluster-break": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@marijn/find-cluster-break/-/find-cluster-break-1.0.2.tgz", + "integrity": "sha512-l0h88YhZFyKdXIFNfSWpyjStDjGHwZ/U7iobcK1cQQD8sejsONdQtTVU+1wVN1PBw40PiiHB1vA5S7VTfQiP9g==", + "license": "MIT" + }, + "node_modules/@types/body-parser": { + "version": "1.19.5", + "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.5.tgz", + "integrity": "sha512-fB3Zu92ucau0iQ0JMCFQE7b/dv8Ot07NI3KaZIkIUNXq82k4eBAqUaneXfleGY9JWskeS9y+u0nXMyspcuQrCg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/connect": "*", + "@types/node": "*" + } + }, + "node_modules/@types/bonjour": { + "version": "3.5.13", + "resolved": "https://registry.npmjs.org/@types/bonjour/-/bonjour-3.5.13.tgz", + "integrity": "sha512-z9fJ5Im06zvUL548KvYNecEVlA7cVDkGUi6kZusb04mpyEFKCIZJvloCcmpmLaIahDpOQGHaHmG6imtPMmPXGQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/connect": { + "version": "3.4.38", + "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz", + "integrity": "sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/connect-history-api-fallback": { + "version": "1.5.4", + "resolved": "https://registry.npmjs.org/@types/connect-history-api-fallback/-/connect-history-api-fallback-1.5.4.tgz", + "integrity": "sha512-n6Cr2xS1h4uAulPRdlw6Jl6s1oG8KrVilPN2yUITEs+K48EzMJJ3W1xy8K5eWuFvjp3R74AOIGSmp2UfBJ8HFw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/express-serve-static-core": "*", + "@types/node": "*" + } + }, + "node_modules/@types/eslint": { + "version": "9.6.1", + "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-9.6.1.tgz", + "integrity": "sha512-FXx2pKgId/WyYo2jXw63kk7/+TY7u7AziEJxJAnSFzHlqTAS3Ync6SvgYAN/k4/PQpnnVuzoMuVnByKK2qp0ag==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "*", + "@types/json-schema": "*" + } + }, + "node_modules/@types/eslint-scope": { + "version": "3.7.7", + "resolved": "https://registry.npmjs.org/@types/eslint-scope/-/eslint-scope-3.7.7.tgz", + "integrity": "sha512-MzMFlSLBqNF2gcHWO0G1vP/YQyfvrxZ0bF+u7mzUdZ1/xK4A4sru+nraZz5i3iEIk1l1uyicaDVTB4QbbEkAYg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/eslint": "*", + "@types/estree": "*" + } + }, + "node_modules/@types/estree": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.6.tgz", + "integrity": "sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/express": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.21.tgz", + "integrity": "sha512-ejlPM315qwLpaQlQDTjPdsUFSc6ZsP4AN6AlWnogPjQ7CVi7PYF3YVz+CY3jE2pwYf7E/7HlDAN0rV2GxTG0HQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/body-parser": "*", + "@types/express-serve-static-core": "^4.17.33", + "@types/qs": "*", + "@types/serve-static": "*" + } + }, + "node_modules/@types/express-serve-static-core": { + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-5.0.6.tgz", + "integrity": "sha512-3xhRnjJPkULekpSzgtoNYYcTWgEZkp4myc+Saevii5JPnHNvHMRlBSHDbs7Bh1iPPoVTERHEZXyhyLbMEsExsA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "@types/qs": "*", + "@types/range-parser": "*", + "@types/send": "*" + } + }, + "node_modules/@types/express/node_modules/@types/express-serve-static-core": { + "version": "4.19.6", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.19.6.tgz", + "integrity": "sha512-N4LZ2xG7DatVqhCZzOGb1Yi5lMbXSZcmdLDe9EzSndPV2HpWYWzRbaerl2n27irrm94EPpprqa8KpskPT085+A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "@types/qs": "*", + "@types/range-parser": "*", + "@types/send": "*" + } + }, + "node_modules/@types/html-minifier-terser": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/@types/html-minifier-terser/-/html-minifier-terser-6.1.0.tgz", + "integrity": "sha512-oh/6byDPnL1zeNXFrDXFLyZjkr1MsBG667IM792caf1L2UPOOMf65NFzjUH/ltyfwjAGfs1rsX1eftK0jC/KIg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/http-errors": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@types/http-errors/-/http-errors-2.0.4.tgz", + "integrity": "sha512-D0CFMMtydbJAegzOyHjtiKPLlvnm3iTZyZRSZoLq2mRhDdmLfIWOCYPfQJ4cu2erKghU++QvjcUjp/5h7hESpA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/http-proxy": { + "version": "1.17.16", + "resolved": "https://registry.npmjs.org/@types/http-proxy/-/http-proxy-1.17.16.tgz", + "integrity": "sha512-sdWoUajOB1cd0A8cRRQ1cfyWNbmFKLAqBB89Y8x5iYyG/mkJHc0YUH8pdWBy2omi9qtCpiIgGjuwO0dQST2l5w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/json-schema": { + "version": "7.0.15", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", + "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/mime": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz", + "integrity": "sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "22.13.8", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.13.8.tgz", + "integrity": "sha512-G3EfaZS+iOGYWLLRCEAXdWK9my08oHNZ+FHluRiggIYJPOXzhOiDgpVCUHaUvyIC5/fj7C/p637jdzC666AOKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.20.0" + } + }, + "node_modules/@types/node-forge": { + "version": "1.3.11", + "resolved": "https://registry.npmjs.org/@types/node-forge/-/node-forge-1.3.11.tgz", + "integrity": "sha512-FQx220y22OKNTqaByeBGqHWYz4cl94tpcxeFdvBo3wjG6XPBuZ0BNgNZRV5J5TFmmcsJ4IzsLkmGRiQbnYsBEQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/qs": { + "version": "6.9.18", + "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.18.tgz", + "integrity": "sha512-kK7dgTYDyGqS+e2Q4aK9X3D7q234CIZ1Bv0q/7Z5IwRDoADNU81xXJK/YVyLbLTZCoIwUoDoffFeF+p/eIklAA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/range-parser": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.7.tgz", + "integrity": "sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/retry": { + "version": "0.12.2", + "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.2.tgz", + "integrity": "sha512-XISRgDJ2Tc5q4TRqvgJtzsRkFYNJzZrhTdtMoGVBttwzzQJkPnS3WWTFc7kuDRoPtPakl+T+OfdEUjYJj7Jbow==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/send": { + "version": "0.17.4", + "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.4.tgz", + "integrity": "sha512-x2EM6TJOybec7c52BX0ZspPodMsQUd5L6PRwOunVyVUhXiBSKf3AezDL8Dgvgt5o0UfKNfuA0eMLr2wLT4AiBA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/mime": "^1", + "@types/node": "*" + } + }, + "node_modules/@types/serve-index": { + "version": "1.9.4", + "resolved": "https://registry.npmjs.org/@types/serve-index/-/serve-index-1.9.4.tgz", + "integrity": "sha512-qLpGZ/c2fhSs5gnYsQxtDEq3Oy8SXPClIXkW5ghvAvsNuVSA8k+gCONcUCS/UjLEYvYps+e8uBtfgXgvhwfNug==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/express": "*" + } + }, + "node_modules/@types/serve-static": { + "version": "1.15.7", + "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.7.tgz", + "integrity": "sha512-W8Ym+h8nhuRwaKPaDw34QUkwsGi6Rc4yYqvKFo5rm2FUEhCFbzVWrxXUxuKK8TASjWsysJY0nsmNCGhCOIsrOw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/http-errors": "*", + "@types/node": "*", + "@types/send": "*" + } + }, + "node_modules/@types/sockjs": { + "version": "0.3.36", + "resolved": "https://registry.npmjs.org/@types/sockjs/-/sockjs-0.3.36.tgz", + "integrity": "sha512-MK9V6NzAS1+Ud7JV9lJLFqW85VbC9dq3LmwZCuBe4wBDgKC0Kj/jd8Xl+nSviU+Qc3+m7umHHyHg//2KSa0a0Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/ws": { + "version": "8.5.14", + "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.14.tgz", + "integrity": "sha512-bd/YFLW+URhBzMXurx7lWByOu+xzU9+kb3RboOteXYDfW+tr+JZa99OyNmPINEGB/ahzKrEuc8rcv4gnpJmxTw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@wasm-tool/wasm-pack-plugin": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/@wasm-tool/wasm-pack-plugin/-/wasm-pack-plugin-1.7.0.tgz", + "integrity": "sha512-WikzYsw7nTd5CZxH75h7NxM/FLJAgqfWt+/gk3EL3wYKxiIlpMIYPja+sHQl3ARiicIYy4BDfxkbAVjRYlouTA==", + "dev": true, + "license": "MIT", + "dependencies": { + "chalk": "^2.4.1", + "command-exists": "^1.2.7", + "watchpack": "^2.1.1", + "which": "^2.0.2" + } + }, + "node_modules/@webassemblyjs/ast": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.14.1.tgz", + "integrity": "sha512-nuBEDgQfm1ccRp/8bCQrx1frohyufl4JlbMMZ4P1wpeOfDhF6FQkxZJ1b/e+PLwr6X1Nhw6OLme5usuBWYBvuQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@webassemblyjs/helper-numbers": "1.13.2", + "@webassemblyjs/helper-wasm-bytecode": "1.13.2" + } + }, + "node_modules/@webassemblyjs/floating-point-hex-parser": { + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.13.2.tgz", + "integrity": "sha512-6oXyTOzbKxGH4steLbLNOu71Oj+C8Lg34n6CqRvqfS2O71BxY6ByfMDRhBytzknj9yGUPVJ1qIKhRlAwO1AovA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@webassemblyjs/helper-api-error": { + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-api-error/-/helper-api-error-1.13.2.tgz", + "integrity": "sha512-U56GMYxy4ZQCbDZd6JuvvNV/WFildOjsaWD3Tzzvmw/mas3cXzRJPMjP83JqEsgSbyrmaGjBfDtV7KDXV9UzFQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@webassemblyjs/helper-buffer": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-buffer/-/helper-buffer-1.14.1.tgz", + "integrity": "sha512-jyH7wtcHiKssDtFPRB+iQdxlDf96m0E39yb0k5uJVhFGleZFoNw1c4aeIcVUPPbXUVJ94wwnMOAqUHyzoEPVMA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@webassemblyjs/helper-numbers": { + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-numbers/-/helper-numbers-1.13.2.tgz", + "integrity": "sha512-FE8aCmS5Q6eQYcV3gI35O4J789wlQA+7JrqTTpJqn5emA4U2hvwJmvFRC0HODS+3Ye6WioDklgd6scJ3+PLnEA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@webassemblyjs/floating-point-hex-parser": "1.13.2", + "@webassemblyjs/helper-api-error": "1.13.2", + "@xtuc/long": "4.2.2" + } + }, + "node_modules/@webassemblyjs/helper-wasm-bytecode": { + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.13.2.tgz", + "integrity": "sha512-3QbLKy93F0EAIXLh0ogEVR6rOubA9AoZ+WRYhNbFyuB70j3dRdwH9g+qXhLAO0kiYGlg3TxDV+I4rQTr/YNXkA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@webassemblyjs/helper-wasm-section": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.14.1.tgz", + "integrity": "sha512-ds5mXEqTJ6oxRoqjhWDU83OgzAYjwsCV8Lo/N+oRsNDmx/ZDpqalmrtgOMkHwxsG0iI//3BwWAErYRHtgn0dZw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@webassemblyjs/ast": "1.14.1", + "@webassemblyjs/helper-buffer": "1.14.1", + "@webassemblyjs/helper-wasm-bytecode": "1.13.2", + "@webassemblyjs/wasm-gen": "1.14.1" + } + }, + "node_modules/@webassemblyjs/ieee754": { + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/@webassemblyjs/ieee754/-/ieee754-1.13.2.tgz", + "integrity": "sha512-4LtOzh58S/5lX4ITKxnAK2USuNEvpdVV9AlgGQb8rJDHaLeHciwG4zlGr0j/SNWlr7x3vO1lDEsuePvtcDNCkw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@xtuc/ieee754": "^1.2.0" + } + }, + "node_modules/@webassemblyjs/leb128": { + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/@webassemblyjs/leb128/-/leb128-1.13.2.tgz", + "integrity": "sha512-Lde1oNoIdzVzdkNEAWZ1dZ5orIbff80YPdHx20mrHwHrVNNTjNr8E3xz9BdpcGqRQbAEa+fkrCb+fRFTl/6sQw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@xtuc/long": "4.2.2" + } + }, + "node_modules/@webassemblyjs/utf8": { + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/@webassemblyjs/utf8/-/utf8-1.13.2.tgz", + "integrity": "sha512-3NQWGjKTASY1xV5m7Hr0iPeXD9+RDobLll3T9d2AO+g3my8xy5peVyjSag4I50mR1bBSN/Ct12lo+R9tJk0NZQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@webassemblyjs/wasm-edit": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-edit/-/wasm-edit-1.14.1.tgz", + "integrity": "sha512-RNJUIQH/J8iA/1NzlE4N7KtyZNHi3w7at7hDjvRNm5rcUXa00z1vRz3glZoULfJ5mpvYhLybmVcwcjGrC1pRrQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@webassemblyjs/ast": "1.14.1", + "@webassemblyjs/helper-buffer": "1.14.1", + "@webassemblyjs/helper-wasm-bytecode": "1.13.2", + "@webassemblyjs/helper-wasm-section": "1.14.1", + "@webassemblyjs/wasm-gen": "1.14.1", + "@webassemblyjs/wasm-opt": "1.14.1", + "@webassemblyjs/wasm-parser": "1.14.1", + "@webassemblyjs/wast-printer": "1.14.1" + } + }, + "node_modules/@webassemblyjs/wasm-gen": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-gen/-/wasm-gen-1.14.1.tgz", + "integrity": "sha512-AmomSIjP8ZbfGQhumkNvgC33AY7qtMCXnN6bL2u2Js4gVCg8fp735aEiMSBbDR7UQIj90n4wKAFUSEd0QN2Ukg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@webassemblyjs/ast": "1.14.1", + "@webassemblyjs/helper-wasm-bytecode": "1.13.2", + "@webassemblyjs/ieee754": "1.13.2", + "@webassemblyjs/leb128": "1.13.2", + "@webassemblyjs/utf8": "1.13.2" + } + }, + "node_modules/@webassemblyjs/wasm-opt": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-opt/-/wasm-opt-1.14.1.tgz", + "integrity": "sha512-PTcKLUNvBqnY2U6E5bdOQcSM+oVP/PmrDY9NzowJjislEjwP/C4an2303MCVS2Mg9d3AJpIGdUFIQQWbPds0Sw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@webassemblyjs/ast": "1.14.1", + "@webassemblyjs/helper-buffer": "1.14.1", + "@webassemblyjs/wasm-gen": "1.14.1", + "@webassemblyjs/wasm-parser": "1.14.1" + } + }, + "node_modules/@webassemblyjs/wasm-parser": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-parser/-/wasm-parser-1.14.1.tgz", + "integrity": "sha512-JLBl+KZ0R5qB7mCnud/yyX08jWFw5MsoalJ1pQ4EdFlgj9VdXKGuENGsiCIjegI1W7p91rUlcB/LB5yRJKNTcQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@webassemblyjs/ast": "1.14.1", + "@webassemblyjs/helper-api-error": "1.13.2", + "@webassemblyjs/helper-wasm-bytecode": "1.13.2", + "@webassemblyjs/ieee754": "1.13.2", + "@webassemblyjs/leb128": "1.13.2", + "@webassemblyjs/utf8": "1.13.2" + } + }, + "node_modules/@webassemblyjs/wast-printer": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wast-printer/-/wast-printer-1.14.1.tgz", + "integrity": "sha512-kPSSXE6De1XOR820C90RIo2ogvZG+c3KiHzqUoO/F34Y2shGzesfqv7o57xrxovZJH/MetF5UjroJ/R/3isoiw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@webassemblyjs/ast": "1.14.1", + "@xtuc/long": "4.2.2" + } + }, + "node_modules/@webpack-cli/configtest": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@webpack-cli/configtest/-/configtest-3.0.1.tgz", + "integrity": "sha512-u8d0pJ5YFgneF/GuvEiDA61Tf1VDomHHYMjv/wc9XzYj7nopltpG96nXN5dJRstxZhcNpV1g+nT6CydO7pHbjA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18.12.0" + }, + "peerDependencies": { + "webpack": "^5.82.0", + "webpack-cli": "6.x.x" + } + }, + "node_modules/@webpack-cli/info": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@webpack-cli/info/-/info-3.0.1.tgz", + "integrity": "sha512-coEmDzc2u/ffMvuW9aCjoRzNSPDl/XLuhPdlFRpT9tZHmJ/039az33CE7uH+8s0uL1j5ZNtfdv0HkfaKRBGJsQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18.12.0" + }, + "peerDependencies": { + "webpack": "^5.82.0", + "webpack-cli": "6.x.x" + } + }, + "node_modules/@webpack-cli/serve": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@webpack-cli/serve/-/serve-3.0.1.tgz", + "integrity": "sha512-sbgw03xQaCLiT6gcY/6u3qBDn01CWw/nbaXl3gTdTFuJJ75Gffv3E3DBpgvY2fkkrdS1fpjaXNOmJlnbtKauKg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18.12.0" + }, + "peerDependencies": { + "webpack": "^5.82.0", + "webpack-cli": "6.x.x" + }, + "peerDependenciesMeta": { + "webpack-dev-server": { + "optional": true + } + } + }, + "node_modules/@xterm/addon-fit": { + "version": "0.10.0", + "resolved": "https://registry.npmjs.org/@xterm/addon-fit/-/addon-fit-0.10.0.tgz", + "integrity": "sha512-UFYkDm4HUahf2lnEyHvio51TNGiLK66mqP2JoATy7hRZeXaGMRDr00JiSF7m63vR5WKATF605yEggJKsw0JpMQ==", + "license": "MIT", + "peerDependencies": { + "@xterm/xterm": "^5.0.0" + } + }, + "node_modules/@xterm/xterm": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/@xterm/xterm/-/xterm-5.5.0.tgz", + "integrity": "sha512-hqJHYaQb5OptNunnyAnkHyM8aCjZ1MEIDTQu1iIbbTD/xops91NB5yq1ZK/dC2JDbVWtF23zUtl9JE2NqwT87A==", + "license": "MIT" + }, + "node_modules/@xtuc/ieee754": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@xtuc/ieee754/-/ieee754-1.2.0.tgz", + "integrity": "sha512-DX8nKgqcGwsc0eJSqYt5lwP4DH5FlHnmuWWBRy7X0NcaGR0ZtuyeESgMwTYVEtxmsNGY+qit4QYT/MIYTOTPeA==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@xtuc/long": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/@xtuc/long/-/long-4.2.2.tgz", + "integrity": "sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==", + "dev": true, + "license": "Apache-2.0" + }, + "node_modules/@zeit/schemas": { + "version": "2.36.0", + "resolved": "https://registry.npmjs.org/@zeit/schemas/-/schemas-2.36.0.tgz", + "integrity": "sha512-7kjMwcChYEzMKjeex9ZFXkt1AyNov9R5HZtjBKVsmVpw7pa7ZtlCGvCBC2vnnXctaYN+aRI61HjIqeetZW5ROg==", + "dev": true, + "license": "MIT" + }, + "node_modules/accepts": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "dev": true, + "license": "MIT", + "dependencies": { + "mime-types": "~2.1.34", + "negotiator": "0.6.3" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/acorn": { + "version": "8.14.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.14.0.tgz", + "integrity": "sha512-cl669nCJTZBsL97OF4kUQm5g5hC2uihk0NxY3WENAC0TYdILVkAyHymAntgxGkl7K+t0cXIrH5siy5S4XkFycA==", + "dev": true, + "license": "MIT", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/ajv": { + "version": "8.17.1", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", + "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-2.1.1.tgz", + "integrity": "sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/ajv-keywords": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-5.1.0.tgz", + "integrity": "sha512-YCS/JNFAUyr5vAuhk1DWm1CBxRHW9LbJ2ozWeemrIqpbsqKjHVxYPyi5GC0rjZIT5JxJ3virVTS8wk4i/Z+krw==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3" + }, + "peerDependencies": { + "ajv": "^8.8.2" + } + }, + "node_modules/ansi-align": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ansi-align/-/ansi-align-3.0.1.tgz", + "integrity": "sha512-IOfwwBF5iczOjp/WeY4YxyjqAFMQoZufdQWDd19SEExbVLNXqvpzSJ/M7Za4/sCPmQ0+GRquoA7bGcINcxew6w==", + "dev": true, + "license": "ISC", + "dependencies": { + "string-width": "^4.1.0" + } + }, + "node_modules/ansi-align/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, + "node_modules/ansi-align/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-html-community": { + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/ansi-html-community/-/ansi-html-community-0.0.8.tgz", + "integrity": "sha512-1APHAyr3+PCamwNw3bXCPp4HFLONZt/yIH0sZp0/469KWNTEy+qN5jQ3GVX6DMZ1UXAi34yVwtTeaG/HpBuuzw==", + "dev": true, + "engines": [ + "node >= 0.8.0" + ], + "license": "Apache-2.0", + "bin": { + "ansi-html": "bin/ansi-html" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/anymatch": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", + "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", + "dev": true, + "license": "ISC", + "dependencies": { + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/arch": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/arch/-/arch-2.2.0.tgz", + "integrity": "sha512-Of/R0wqp83cgHozfIYLbBMnej79U/SVGOOyuB3VVFv1NRM/PSFMK12x9KVtiYzJqmnU5WR2qp0Z5rHb7sWGnFQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/arg": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/arg/-/arg-5.0.2.tgz", + "integrity": "sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==", + "dev": true, + "license": "MIT" + }, + "node_modules/array-flatten": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==", + "dev": true, + "license": "MIT" + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true, + "license": "MIT" + }, + "node_modules/batch": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/batch/-/batch-0.6.1.tgz", + "integrity": "sha512-x+VAiMRL6UPkx+kudNvxTl6hB2XNNCG2r+7wixVfIYwu/2HKRXimwQyaumLjMveWvT2Hkd/cAJw+QBMfJ/EKVw==", + "dev": true, + "license": "MIT" + }, + "node_modules/binary-extensions": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", + "integrity": "sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/body-parser": { + "version": "1.20.3", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz", + "integrity": "sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==", + "dev": true, + "license": "MIT", + "dependencies": { + "bytes": "3.1.2", + "content-type": "~1.0.5", + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "on-finished": "2.4.1", + "qs": "6.13.0", + "raw-body": "2.5.2", + "type-is": "~1.6.18", + "unpipe": "1.0.0" + }, + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/body-parser/node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/bonjour-service": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/bonjour-service/-/bonjour-service-1.3.0.tgz", + "integrity": "sha512-3YuAUiSkWykd+2Azjgyxei8OWf8thdn8AITIog2M4UICzoqfjlqr64WIjEXZllf/W6vK1goqleSR6brGomxQqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "multicast-dns": "^7.2.5" + } + }, + "node_modules/boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==", + "dev": true, + "license": "ISC" + }, + "node_modules/boxen": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/boxen/-/boxen-7.0.0.tgz", + "integrity": "sha512-j//dBVuyacJbvW+tvZ9HuH03fZ46QcaKvvhZickZqtB271DxJ7SNRSNxrV/dZX0085m7hISRZWbzWlJvx/rHSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-align": "^3.0.1", + "camelcase": "^7.0.0", + "chalk": "^5.0.1", + "cli-boxes": "^3.0.0", + "string-width": "^5.1.2", + "type-fest": "^2.13.0", + "widest-line": "^4.0.1", + "wrap-ansi": "^8.0.1" + }, + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/boxen/node_modules/chalk": { + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.4.1.tgz", + "integrity": "sha512-zgVZuo2WcZgfUEmsn6eO3kINexW8RAE4maiQ8QNs8CtpPCSyMiYsULR3HQYkm3w8FIA3SberyMJMSldGsW+U3w==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^12.17.0 || ^14.13 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/braces": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fill-range": "^7.1.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/browserslist": { + "version": "4.24.4", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.24.4.tgz", + "integrity": "sha512-KDi1Ny1gSePi1vm0q4oxSF8b4DR44GF4BbmS2YdhPLOEqd8pDviZOGH/GsmRwoWJ2+5Lr085X7naowMwKHDG1A==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "caniuse-lite": "^1.0.30001688", + "electron-to-chromium": "^1.5.73", + "node-releases": "^2.0.19", + "update-browserslist-db": "^1.1.1" + }, + "bin": { + "browserslist": "cli.js" + }, + "engines": { + "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" + } + }, + "node_modules/buffer-from": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/bundle-name": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bundle-name/-/bundle-name-4.1.0.tgz", + "integrity": "sha512-tjwM5exMg6BGRI+kNmTntNsvdZS1X8BFYS6tnJ2hdH0kVxM6/eVZ2xy+FqStSWvYmtfFMDLIxurorHwDKfDz5Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "run-applescript": "^7.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/bytes": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", + "integrity": "sha512-pMhOfFDPiv9t5jjIXkHosWmkSyQbvsgEVNkz0ERHbuLh2T/7j4Mqqpz523Fe8MVY89KC6Sh/QfS2sM+SjgFDcw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.3.tgz", + "integrity": "sha512-YTd+6wGlNlPxSuri7Y6X8tY2dmm12UMH66RpKMhiX6rsk5wXXnYgbUcOt8kiS31/AjfoTOvCsE+w8nZQLQnzHA==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "get-intrinsic": "^1.2.6" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/camel-case": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/camel-case/-/camel-case-4.1.2.tgz", + "integrity": "sha512-gxGWBrTT1JuMx6R+o5PTXMmUnhnVzLQ9SNutD4YqKtI6ap897t3tKECYla6gCWEkplXnlNybEkZg9GEGxKFCgw==", + "dev": true, + "license": "MIT", + "dependencies": { + "pascal-case": "^3.1.2", + "tslib": "^2.0.3" + } + }, + "node_modules/camelcase": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-7.0.1.tgz", + "integrity": "sha512-xlx1yCK2Oc1APsPXDL2LdlNP6+uu8OCDdhOBSVT279M/S+y75O30C2VuD8T2ogdePBBl7PfPF4504tnLgX3zfw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/caniuse-lite": { + "version": "1.0.30001701", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001701.tgz", + "integrity": "sha512-faRs/AW3jA9nTwmJBSO1PQ6L/EOgsB5HMQQq4iCu5zhPgVVgO/pZRHlmatwijZKetFw8/Pr4q6dEN8sJuq8qTw==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "CC-BY-4.0" + }, + "node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/chalk-template": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/chalk-template/-/chalk-template-0.4.0.tgz", + "integrity": "sha512-/ghrgmhfY8RaSdeo43hNXxpoHAtxdbskUHjPpfqUWGttFgycUhYPGx3YZBCnUCvOa7Doivn1IZec3DEGFoMgLg==", + "dev": true, + "license": "MIT", + "dependencies": { + "chalk": "^4.1.2" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/chalk-template?sponsor=1" + } + }, + "node_modules/chalk-template/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/chalk-template/node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/chalk-template/node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/chalk-template/node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true, + "license": "MIT" + }, + "node_modules/chalk-template/node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/chalk-template/node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/chokidar": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz", + "integrity": "sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==", + "dev": true, + "license": "MIT", + "dependencies": { + "anymatch": "~3.1.2", + "braces": "~3.0.2", + "glob-parent": "~5.1.2", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.6.0" + }, + "engines": { + "node": ">= 8.10.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + }, + "optionalDependencies": { + "fsevents": "~2.3.2" + } + }, + "node_modules/chrome-trace-event": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/chrome-trace-event/-/chrome-trace-event-1.0.4.tgz", + "integrity": "sha512-rNjApaLzuwaOTjCiT8lSDdGN1APCiqkChLMJxJPWLunPAt5fy8xgU9/jNOchV84wfIxrA0lRQB7oCT8jrn/wrQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0" + } + }, + "node_modules/clean-css": { + "version": "5.3.3", + "resolved": "https://registry.npmjs.org/clean-css/-/clean-css-5.3.3.tgz", + "integrity": "sha512-D5J+kHaVb/wKSFcyyV75uCn8fiY4sV38XJoe4CUyGQ+mOU/fMVYUdH1hJC+CJQ5uY3EnW27SbJYS4X8BiLrAFg==", + "dev": true, + "license": "MIT", + "dependencies": { + "source-map": "~0.6.0" + }, + "engines": { + "node": ">= 10.0" + } + }, + "node_modules/cli-boxes": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/cli-boxes/-/cli-boxes-3.0.0.tgz", + "integrity": "sha512-/lzGpEWL/8PfI0BmBOPRwp0c/wFNX1RdUML3jK/RcSBA9T8mZDdQpqYBKtCFTOfQbwPqWEOpjqW+Fnayc0969g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/clipboardy": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/clipboardy/-/clipboardy-3.0.0.tgz", + "integrity": "sha512-Su+uU5sr1jkUy1sGRpLKjKrvEOVXgSgiSInwa/qeID6aJ07yh+5NWc3h2QfjHjBnfX4LhtFcuAWKUsJ3r+fjbg==", + "dev": true, + "license": "MIT", + "dependencies": { + "arch": "^2.2.0", + "execa": "^5.1.1", + "is-wsl": "^2.2.0" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/clone-deep": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/clone-deep/-/clone-deep-4.0.1.tgz", + "integrity": "sha512-neHB9xuzh/wk0dIHweyAXv2aPGZIVk3pLMe+/RNzINf17fe0OG96QroktYAUm7SM1PBnzTabaLboqqxDyMU+SQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-plain-object": "^2.0.4", + "kind-of": "^6.0.2", + "shallow-clone": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/codemirror": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/codemirror/-/codemirror-6.0.1.tgz", + "integrity": "sha512-J8j+nZ+CdWmIeFIGXEFbFPtpiYacFMDR8GlHK3IyHQJMCaVRfGx9NT+Hxivv1ckLWPvNdZqndbr/7lVhrf/Svg==", + "license": "MIT", + "dependencies": { + "@codemirror/autocomplete": "^6.0.0", + "@codemirror/commands": "^6.0.0", + "@codemirror/language": "^6.0.0", + "@codemirror/lint": "^6.0.0", + "@codemirror/search": "^6.0.0", + "@codemirror/state": "^6.0.0", + "@codemirror/view": "^6.0.0" + } + }, + "node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", + "dev": true, + "license": "MIT" + }, + "node_modules/colorette": { + "version": "2.0.20", + "resolved": "https://registry.npmjs.org/colorette/-/colorette-2.0.20.tgz", + "integrity": "sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==", + "dev": true, + "license": "MIT" + }, + "node_modules/command-exists": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/command-exists/-/command-exists-1.2.9.tgz", + "integrity": "sha512-LTQ/SGc+s0Xc0Fu5WaKnR0YiygZkm9eKFvyS+fRsU7/ZWFF8ykFM6Pc9aCVf1+xasOOZpO3BAVgVrKvsqKHV7w==", + "dev": true, + "license": "MIT" + }, + "node_modules/commander": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-8.3.0.tgz", + "integrity": "sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, + "node_modules/compressible": { + "version": "2.0.18", + "resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz", + "integrity": "sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==", + "dev": true, + "license": "MIT", + "dependencies": { + "mime-db": ">= 1.43.0 < 2" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/compression": { + "version": "1.7.4", + "resolved": "https://registry.npmjs.org/compression/-/compression-1.7.4.tgz", + "integrity": "sha512-jaSIDzP9pZVS4ZfQ+TzvtiWhdpFhE2RDHz8QJkpX9SIpLq88VueF5jJw6t+6CUQcAoA6t+x89MLrWAqpfDE8iQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "accepts": "~1.3.5", + "bytes": "3.0.0", + "compressible": "~2.0.16", + "debug": "2.6.9", + "on-headers": "~1.0.2", + "safe-buffer": "5.1.2", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "dev": true, + "license": "MIT" + }, + "node_modules/connect-history-api-fallback": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/connect-history-api-fallback/-/connect-history-api-fallback-2.0.0.tgz", + "integrity": "sha512-U73+6lQFmfiNPrYbXqr6kZ1i1wiRqXnp2nhMsINseWXO8lDau0LGEffJ8kQi4EjLZympVgRdvqjAgiZ1tgzDDA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.8" + } + }, + "node_modules/content-disposition": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.2.tgz", + "integrity": "sha512-kRGRZw3bLlFISDBgwTSA1TMBFN6J6GWDeubmDE3AF+3+yXL8hTWv8r5rkLbqYXY4RjPk/EzHnClI3zQf1cFmHA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie": { + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.1.tgz", + "integrity": "sha512-6DnInpx7SJ2AK3+CTUE/ZM0vWTUboZCegxhC2xiIydHR9jNuTAASBrfEpHhiGOZw/nX51bHt6YQl8jsGo4y/0w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", + "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/crelt": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/crelt/-/crelt-1.0.6.tgz", + "integrity": "sha512-VQ2MBenTq1fWZUH9DJNGti7kKv6EeAuYr3cLwxUWhIu1baTaXh4Ib5W2CqHVqib4/MqbYGJqiL3Zb8GJZr3l4g==", + "license": "MIT" + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/css-loader": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/css-loader/-/css-loader-7.1.2.tgz", + "integrity": "sha512-6WvYYn7l/XEGN8Xu2vWFt9nVzrCn39vKyTEFf/ExEyoksJjjSZV/0/35XPlMbpnr6VGhZIUg5yJrL8tGfes/FA==", + "dev": true, + "license": "MIT", + "dependencies": { + "icss-utils": "^5.1.0", + "postcss": "^8.4.33", + "postcss-modules-extract-imports": "^3.1.0", + "postcss-modules-local-by-default": "^4.0.5", + "postcss-modules-scope": "^3.2.0", + "postcss-modules-values": "^4.0.0", + "postcss-value-parser": "^4.2.0", + "semver": "^7.5.4" + }, + "engines": { + "node": ">= 18.12.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "@rspack/core": "0.x || 1.x", + "webpack": "^5.27.0" + }, + "peerDependenciesMeta": { + "@rspack/core": { + "optional": true + }, + "webpack": { + "optional": true + } + } + }, + "node_modules/css-select": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-4.3.0.tgz", + "integrity": "sha512-wPpOYtnsVontu2mODhA19JrqWxNsfdatRKd64kmpRbQgh1KtItko5sTnEpPdpSaJszTOhEMlF/RPz28qj4HqhQ==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0", + "css-what": "^6.0.1", + "domhandler": "^4.3.1", + "domutils": "^2.8.0", + "nth-check": "^2.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/css-what": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.1.0.tgz", + "integrity": "sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">= 6" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/cssesc": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/cssesc/-/cssesc-3.0.0.tgz", + "integrity": "sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==", + "dev": true, + "license": "MIT", + "bin": { + "cssesc": "bin/cssesc" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/default-browser": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/default-browser/-/default-browser-5.2.1.tgz", + "integrity": "sha512-WY/3TUME0x3KPYdRRxEJJvXRHV4PyPoUsxtZa78lwItwRQRHhd2U9xOscaT/YTf8uCXIAjeJOFBVEh/7FtD8Xg==", + "dev": true, + "license": "MIT", + "dependencies": { + "bundle-name": "^4.1.0", + "default-browser-id": "^5.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/default-browser-id": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/default-browser-id/-/default-browser-id-5.0.0.tgz", + "integrity": "sha512-A6p/pu/6fyBcA1TRz/GqWYPViplrftcW2gZC9q79ngNCKAeR/X3gcEdXQHl4KNXV+3wgIJ1CPkJQ3IHM6lcsyA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/define-lazy-prop": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/define-lazy-prop/-/define-lazy-prop-3.0.0.tgz", + "integrity": "sha512-N+MeXYoqr3pOgn8xfyRPREN7gHakLYjhsHhWGT3fWAiL4IkAt0iDw14QiiEm2bE30c5XX5q0FtAA3CK5f9/BUg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/destroy": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", + "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/detect-node": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/detect-node/-/detect-node-2.1.0.tgz", + "integrity": "sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==", + "dev": true, + "license": "MIT" + }, + "node_modules/dns-packet": { + "version": "5.6.1", + "resolved": "https://registry.npmjs.org/dns-packet/-/dns-packet-5.6.1.tgz", + "integrity": "sha512-l4gcSouhcgIKRvyy99RNVOgxXiicE+2jZoNmaNmZ6JXiGajBOJAesk1OBlJuM5k2c+eudGdLxDqXuPCKIj6kpw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@leichtgewicht/ip-codec": "^2.0.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/dom-converter": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/dom-converter/-/dom-converter-0.2.0.tgz", + "integrity": "sha512-gd3ypIPfOMr9h5jIKq8E3sHOTCjeirnl0WK5ZdS1AW0Odt0b1PaWaHdJ4Qk4klv+YB9aJBS7mESXjFoDQPu6DA==", + "dev": true, + "license": "MIT", + "dependencies": { + "utila": "~0.4" + } + }, + "node_modules/dom-serializer": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-1.4.1.tgz", + "integrity": "sha512-VHwB3KfrcOOkelEG2ZOfxqLZdfkil8PtJi4P8N2MMXucZq2yLp75ClViUlOVwyoHEDjYU433Aq+5zWP61+RGag==", + "dev": true, + "license": "MIT", + "dependencies": { + "domelementtype": "^2.0.1", + "domhandler": "^4.2.0", + "entities": "^2.0.0" + }, + "funding": { + "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" + } + }, + "node_modules/domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "BSD-2-Clause" + }, + "node_modules/domhandler": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-4.3.1.tgz", + "integrity": "sha512-GrwoxYN+uWlzO8uhUXRl0P+kHE4GtVPfYzVLcUxPL7KNdHKj66vvlhiweIHqYYXWlw+T8iLMp42Lm67ghw4WMQ==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "domelementtype": "^2.2.0" + }, + "engines": { + "node": ">= 4" + }, + "funding": { + "url": "https://github.com/fb55/domhandler?sponsor=1" + } + }, + "node_modules/domutils": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-2.8.0.tgz", + "integrity": "sha512-w96Cjofp72M5IIhpjgobBimYEfoPjx1Vx0BSX9P30WBdZW2WIKU0T1Bd0kz2eNZ9ikjKgHbEyKx8BB6H1L3h3A==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "dom-serializer": "^1.0.1", + "domelementtype": "^2.2.0", + "domhandler": "^4.2.0" + }, + "funding": { + "url": "https://github.com/fb55/domutils?sponsor=1" + } + }, + "node_modules/dot-case": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/dot-case/-/dot-case-3.0.4.tgz", + "integrity": "sha512-Kv5nKlh6yRrdrGvxeJ2e5y2eRUpkUosIW4A2AS38zwSz27zu7ufDwQPi5Jhs3XAlGNetl3bmnGhQsMtkKJnj3w==", + "dev": true, + "license": "MIT", + "dependencies": { + "no-case": "^3.0.4", + "tslib": "^2.0.3" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/eastasianwidth": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", + "dev": true, + "license": "MIT" + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "dev": true, + "license": "MIT" + }, + "node_modules/electron-to-chromium": { + "version": "1.5.109", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.109.tgz", + "integrity": "sha512-AidaH9JETVRr9DIPGfp1kAarm/W6hRJTPuCnkF+2MqhF4KaAgRIcBc8nvjk+YMXZhwfISof/7WG29eS4iGxQLQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", + "dev": true, + "license": "MIT" + }, + "node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/enhanced-resolve": { + "version": "5.18.1", + "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.18.1.tgz", + "integrity": "sha512-ZSW3ma5GkcQBIpwZTSRAI8N71Uuwgs93IezB7mf7R60tC8ZbJideoDNKjHn2O9KIlx6rkGTTEk1xUCK2E1Y2Yg==", + "dev": true, + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.4", + "tapable": "^2.2.0" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/entities": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-2.2.0.tgz", + "integrity": "sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A==", + "dev": true, + "license": "BSD-2-Clause", + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/envinfo": { + "version": "7.14.0", + "resolved": "https://registry.npmjs.org/envinfo/-/envinfo-7.14.0.tgz", + "integrity": "sha512-CO40UI41xDQzhLB1hWyqUKgFhs250pNcGbyGKe1l/e4FSaI/+YE4IMG76GDt0In67WLPACIITC+sOi08x4wIvg==", + "dev": true, + "license": "MIT", + "bin": { + "envinfo": "dist/cli.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-module-lexer": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.6.0.tgz", + "integrity": "sha512-qqnD1yMU6tk/jnaMosogGySTZP8YtUgAffA9nMN+E/rjxcfRQ6IEk7IiozUjgxKoFHBGjTLnrHB/YC45r/59EQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "dev": true, + "license": "MIT" + }, + "node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/eslint-scope": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.1.tgz", + "integrity": "sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "esrecurse": "^4.3.0", + "estraverse": "^4.1.1" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/esrecurse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "estraverse": "^5.2.0" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/esrecurse/node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estraverse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz", + "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/eventemitter3": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", + "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", + "dev": true, + "license": "MIT" + }, + "node_modules/events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.8.x" + } + }, + "node_modules/execa": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", + "integrity": "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==", + "dev": true, + "license": "MIT", + "dependencies": { + "cross-spawn": "^7.0.3", + "get-stream": "^6.0.0", + "human-signals": "^2.1.0", + "is-stream": "^2.0.0", + "merge-stream": "^2.0.0", + "npm-run-path": "^4.0.1", + "onetime": "^5.1.2", + "signal-exit": "^3.0.3", + "strip-final-newline": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sindresorhus/execa?sponsor=1" + } + }, + "node_modules/express": { + "version": "4.21.2", + "resolved": "https://registry.npmjs.org/express/-/express-4.21.2.tgz", + "integrity": "sha512-28HqgMZAmih1Czt9ny7qr6ek2qddF4FclbMzwhCREB6OFfH+rXAnuNCwo1/wFvrtbgsQDb4kSbX9de9lFbrXnA==", + "dev": true, + "license": "MIT", + "dependencies": { + "accepts": "~1.3.8", + "array-flatten": "1.1.1", + "body-parser": "1.20.3", + "content-disposition": "0.5.4", + "content-type": "~1.0.4", + "cookie": "0.7.1", + "cookie-signature": "1.0.6", + "debug": "2.6.9", + "depd": "2.0.0", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "1.3.1", + "fresh": "0.5.2", + "http-errors": "2.0.0", + "merge-descriptors": "1.0.3", + "methods": "~1.1.2", + "on-finished": "2.4.1", + "parseurl": "~1.3.3", + "path-to-regexp": "0.1.12", + "proxy-addr": "~2.0.7", + "qs": "6.13.0", + "range-parser": "~1.2.1", + "safe-buffer": "5.2.1", + "send": "0.19.0", + "serve-static": "1.16.2", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/express/node_modules/content-disposition": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "safe-buffer": "5.2.1" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/express/node_modules/path-to-regexp": { + "version": "0.1.12", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.12.tgz", + "integrity": "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/express/node_modules/range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/express/node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-uri": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.0.6.tgz", + "integrity": "sha512-Atfo14OibSv5wAp4VWNsFYE1AchQRTv9cBGWET4pZWHzYshFSS9NQI6I57rdKn9croWVMbYFbLhJ+yJvmZIIHw==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/fastest-levenshtein": { + "version": "1.0.16", + "resolved": "https://registry.npmjs.org/fastest-levenshtein/-/fastest-levenshtein-1.0.16.tgz", + "integrity": "sha512-eRnCtTTtGZFpQCwhJiUOuxPQWRXVKYDn0b2PeHfXL6/Zi53SLAzAHfVhVWK2AryC/WH05kGfxhFIPvTF0SXQzg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4.9.1" + } + }, + "node_modules/faye-websocket": { + "version": "0.11.4", + "resolved": "https://registry.npmjs.org/faye-websocket/-/faye-websocket-0.11.4.tgz", + "integrity": "sha512-CzbClwlXAuiRQAlUyfqPgvPoNKTckTPGfwZV4ZdAhVcP2lh9KUxJg2b5GkE7XbjKQ3YJnQ9z6D9ntLAlB+tP8g==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "websocket-driver": ">=0.5.1" + }, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/fill-range": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", + "dev": true, + "license": "MIT", + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/finalhandler": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.1.tgz", + "integrity": "sha512-6BN9trH7bp3qvnrRyzsBz+g3lZxTNZTbVO2EV1CS0WIcDbawYVdYvGflME/9QP0h0pYlCDBCTjYa9nZzMDpyxQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "debug": "2.6.9", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "on-finished": "2.4.1", + "parseurl": "~1.3.3", + "statuses": "2.0.1", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dev": true, + "license": "MIT", + "dependencies": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/flat": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/flat/-/flat-5.0.2.tgz", + "integrity": "sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ==", + "dev": true, + "license": "BSD-3-Clause", + "bin": { + "flat": "cli.js" + } + }, + "node_modules/follow-redirects": { + "version": "1.15.9", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.9.tgz", + "integrity": "sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "license": "MIT", + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "dev": true, + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/get-stream": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", + "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "dev": true, + "license": "ISC", + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/glob-to-regexp": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz", + "integrity": "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==", + "dev": true, + "license": "BSD-2-Clause" + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/handle-thing": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/handle-thing/-/handle-thing-2.0.1.tgz", + "integrity": "sha512-9Qn4yBxelxoh2Ow62nP+Ka/kMnOXRi8BXnRaUwezLNhqelnN49xKz4F/dPP8OYLxLxq6JDtZb2i9XznUQbNPTg==", + "dev": true, + "license": "MIT" + }, + "node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/he": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", + "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==", + "dev": true, + "license": "MIT", + "bin": { + "he": "bin/he" + } + }, + "node_modules/hpack.js": { + "version": "2.1.6", + "resolved": "https://registry.npmjs.org/hpack.js/-/hpack.js-2.1.6.tgz", + "integrity": "sha512-zJxVehUdMGIKsRaNt7apO2Gqp0BdqW5yaiGHXXmbpvxgBYVZnAql+BJb4RO5ad2MgpbZKn5G6nMnegrH1FcNYQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "inherits": "^2.0.1", + "obuf": "^1.0.0", + "readable-stream": "^2.0.1", + "wbuf": "^1.1.0" + } + }, + "node_modules/hpack.js/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "dev": true, + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/hpack.js/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dev": true, + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/html-minifier-terser": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/html-minifier-terser/-/html-minifier-terser-6.1.0.tgz", + "integrity": "sha512-YXxSlJBZTP7RS3tWnQw74ooKa6L9b9i9QYXY21eUEvhZ3u9XLfv6OnFsQq6RxkhHygsaUMvYsZRV5rU/OVNZxw==", + "dev": true, + "license": "MIT", + "dependencies": { + "camel-case": "^4.1.2", + "clean-css": "^5.2.2", + "commander": "^8.3.0", + "he": "^1.2.0", + "param-case": "^3.0.4", + "relateurl": "^0.2.7", + "terser": "^5.10.0" + }, + "bin": { + "html-minifier-terser": "cli.js" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/html-webpack-plugin": { + "version": "5.6.3", + "resolved": "https://registry.npmjs.org/html-webpack-plugin/-/html-webpack-plugin-5.6.3.tgz", + "integrity": "sha512-QSf1yjtSAsmf7rYBV7XX86uua4W/vkhIt0xNXKbsi2foEeW7vjJQz4bhnpL3xH+l1ryl1680uNv968Z+X6jSYg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/html-minifier-terser": "^6.0.0", + "html-minifier-terser": "^6.0.2", + "lodash": "^4.17.21", + "pretty-error": "^4.0.0", + "tapable": "^2.0.0" + }, + "engines": { + "node": ">=10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/html-webpack-plugin" + }, + "peerDependencies": { + "@rspack/core": "0.x || 1.x", + "webpack": "^5.20.0" + }, + "peerDependenciesMeta": { + "@rspack/core": { + "optional": true + }, + "webpack": { + "optional": true + } + } + }, + "node_modules/htmlparser2": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-6.1.0.tgz", + "integrity": "sha512-gyyPk6rgonLFEDGoeRgQNaEUvdJ4ktTmmUh/h2t7s+M8oPpIPxgNACWa+6ESR57kXstwqPiCut0V8NRpcwgU7A==", + "dev": true, + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "MIT", + "dependencies": { + "domelementtype": "^2.0.1", + "domhandler": "^4.0.0", + "domutils": "^2.5.2", + "entities": "^2.0.0" + } + }, + "node_modules/http-deceiver": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/http-deceiver/-/http-deceiver-1.2.7.tgz", + "integrity": "sha512-LmpOGxTfbpgtGVxJrj5k7asXHCgNZp5nLfp+hWc8QQRqtb7fUy6kRY3BO1h9ddF6yIPYUARgxGOwB42DnxIaNw==", + "dev": true, + "license": "MIT" + }, + "node_modules/http-errors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", + "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "depd": "2.0.0", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "toidentifier": "1.0.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/http-parser-js": { + "version": "0.5.9", + "resolved": "https://registry.npmjs.org/http-parser-js/-/http-parser-js-0.5.9.tgz", + "integrity": "sha512-n1XsPy3rXVxlqxVioEWdC+0+M+SQw0DpJynwtOPo1X+ZlvdzTLtDBIJJlDQTnwZIFJrZSzSGmIOUdP8tu+SgLw==", + "dev": true, + "license": "MIT" + }, + "node_modules/http-proxy": { + "version": "1.18.1", + "resolved": "https://registry.npmjs.org/http-proxy/-/http-proxy-1.18.1.tgz", + "integrity": "sha512-7mz/721AbnJwIVbnaSv1Cz3Am0ZLT/UBwkC92VlxhXv/k/BBQfM2fXElQNC27BVGr0uwUpplYPQM9LnaBMR5NQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "eventemitter3": "^4.0.0", + "follow-redirects": "^1.0.0", + "requires-port": "^1.0.0" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/http-proxy-middleware": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-2.0.7.tgz", + "integrity": "sha512-fgVY8AV7qU7z/MmXJ/rxwbrtQH4jBQ9m7kp3llF0liB7glmFeVZFBepQb32T3y8n8k2+AEYuMPCpinYW+/CuRA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/http-proxy": "^1.17.8", + "http-proxy": "^1.18.1", + "is-glob": "^4.0.1", + "is-plain-obj": "^3.0.0", + "micromatch": "^4.0.2" + }, + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "@types/express": "^4.17.13" + }, + "peerDependenciesMeta": { + "@types/express": { + "optional": true + } + } + }, + "node_modules/human-signals": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz", + "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=10.17.0" + } + }, + "node_modules/hyperdyperid": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/hyperdyperid/-/hyperdyperid-1.2.0.tgz", + "integrity": "sha512-Y93lCzHYgGWdrJ66yIktxiaGULYc6oGiABxhcO5AufBeOyoIdZF7bIfLaOrbM0iGIOXQQgxxRrFEnb+Y6w1n4A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10.18" + } + }, + "node_modules/iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "dev": true, + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/icss-utils": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/icss-utils/-/icss-utils-5.1.0.tgz", + "integrity": "sha512-soFhflCVWLfRNOPU3iv5Z9VUdT44xFRbzjLsEzSr5AQmgqPMTHdU3PMT1Cf1ssx8fLNJDA1juftYl+PUcv3MqA==", + "dev": true, + "license": "ISC", + "engines": { + "node": "^10 || ^12 || >= 14" + }, + "peerDependencies": { + "postcss": "^8.1.0" + } + }, + "node_modules/import-local": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/import-local/-/import-local-3.2.0.tgz", + "integrity": "sha512-2SPlun1JUPWoM6t3F0dw0FkCF/jWY8kttcY4f599GLTSjh2OCuuhdTkJQsEcZzBqbXZGKMK2OqW1oZsjtf/gQA==", + "dev": true, + "license": "MIT", + "dependencies": { + "pkg-dir": "^4.2.0", + "resolve-cwd": "^3.0.0" + }, + "bin": { + "import-local-fixture": "fixtures/cli.js" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", + "dev": true, + "license": "ISC" + }, + "node_modules/interpret": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/interpret/-/interpret-3.1.1.tgz", + "integrity": "sha512-6xwYfHbajpoF0xLW+iwLkhwgvLoZDfjYfoFNu8ftMoXINzwuymNLd9u/KmwtdT2GbR+/Cz66otEGEVVUHX9QLQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/ipaddr.js": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.2.0.tgz", + "integrity": "sha512-Ag3wB2o37wslZS19hZqorUnrnzSkpOVy+IiiDEiTqNubEYpYuHWIf6K4psgN2ZWKExS4xhVCrRVfb/wfW8fWJA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 10" + } + }, + "node_modules/is-binary-path": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", + "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", + "dev": true, + "license": "MIT", + "dependencies": { + "binary-extensions": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/is-core-module": { + "version": "2.16.1", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.1.tgz", + "integrity": "sha512-UfoeMA6fIJ8wTYFEUjelnaGI67v6+N7qXJEvQuIGa99l4xsCruSYOVSQ0uPANn4dAzm8lkYPaKLrrijLq7x23w==", + "dev": true, + "license": "MIT", + "dependencies": { + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-docker": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-2.2.1.tgz", + "integrity": "sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ==", + "dev": true, + "license": "MIT", + "bin": { + "is-docker": "cli.js" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-inside-container": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-inside-container/-/is-inside-container-1.0.0.tgz", + "integrity": "sha512-KIYLCCJghfHZxqjYBE7rEy0OBuTd5xCHS7tHVgvCLkx7StIoaxwNW3hCALgEUjFfeRk+MG/Qxmp/vtETEF3tRA==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-docker": "^3.0.0" + }, + "bin": { + "is-inside-container": "cli.js" + }, + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-inside-container/node_modules/is-docker": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-3.0.0.tgz", + "integrity": "sha512-eljcgEDlEns/7AXFosB5K/2nCM4P7FQPkGc/DWLy5rmFEWvZayGrik1d9/QIY5nJ4f9YsVvBkA6kJpHn9rISdQ==", + "dev": true, + "license": "MIT", + "bin": { + "is-docker": "cli.js" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-network-error": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-network-error/-/is-network-error-1.1.0.tgz", + "integrity": "sha512-tUdRRAnhT+OtCZR/LxZelH/C7QtjtFrTu5tXCA8pl55eTUElUHT+GPYV8MBMBvea/j+NxQqVt3LbWMRir7Gx9g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/is-plain-obj": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-3.0.0.tgz", + "integrity": "sha512-gwsOE28k+23GP1B6vFl1oVh/WOzmawBrKwo5Ev6wMKzPkaXaCDIQKzLnvsA42DRlbVTWorkgTKIviAKCWkfUwA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-plain-object": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", + "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", + "dev": true, + "license": "MIT", + "dependencies": { + "isobject": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-port-reachable": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-port-reachable/-/is-port-reachable-4.0.0.tgz", + "integrity": "sha512-9UoipoxYmSk6Xy7QFgRv2HDyaysmgSG75TFQs6S+3pDM7ZhKTF/bskZV+0UlABHzKjNVhPjYCLfeZUEg1wXxig==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-wsl": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz", + "integrity": "sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-docker": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true, + "license": "ISC" + }, + "node_modules/isobject": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", + "integrity": "sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/jest-worker": { + "version": "27.5.1", + "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-27.5.1.tgz", + "integrity": "sha512-7vuh85V5cdDofPyxn58nrPjBktZo0u9x1g8WtjQol+jZDaE+fhN+cIvTj11GndBnMnyfrUOG1sZQxCdjKh+DKg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "merge-stream": "^2.0.0", + "supports-color": "^8.0.0" + }, + "engines": { + "node": ">= 10.13.0" + } + }, + "node_modules/jest-worker/node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/jest-worker/node_modules/supports-color": { + "version": "8.1.1", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", + "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/supports-color?sponsor=1" + } + }, + "node_modules/json-parse-even-better-errors": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", + "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", + "dev": true, + "license": "MIT" + }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "dev": true, + "license": "MIT" + }, + "node_modules/kind-of": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/launch-editor": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/launch-editor/-/launch-editor-2.10.0.tgz", + "integrity": "sha512-D7dBRJo/qcGX9xlvt/6wUYzQxjh5G1RvZPgPv8vi4KRU99DVQL/oW7tnVOCCTm2HGeo3C5HvGE5Yrh6UBoZ0vA==", + "dev": true, + "license": "MIT", + "dependencies": { + "picocolors": "^1.0.0", + "shell-quote": "^1.8.1" + } + }, + "node_modules/loader-runner": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-4.3.0.tgz", + "integrity": "sha512-3R/1M+yS3j5ou80Me59j7F9IMs4PXs3VqRrm0TU3AbKPxlmpoY1TNscJV/oGJXo8qCatFGTfDbY6W6ipGOYXfg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.11.5" + } + }, + "node_modules/locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-locate": "^4.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", + "dev": true, + "license": "MIT" + }, + "node_modules/lower-case": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/lower-case/-/lower-case-2.0.2.tgz", + "integrity": "sha512-7fm3l3NAF9WfN6W3JOmf5drwpVqX78JtoGJ3A6W0a6ZnldM41w2fV5D490psKFTpMds8TJse/eHLFFsNHHjHgg==", + "dev": true, + "license": "MIT", + "dependencies": { + "tslib": "^2.0.3" + } + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/memfs": { + "version": "4.17.0", + "resolved": "https://registry.npmjs.org/memfs/-/memfs-4.17.0.tgz", + "integrity": "sha512-4eirfZ7thblFmqFjywlTmuWVSvccHAJbn1r8qQLzmTO11qcqpohOjmY2mFce6x7x7WtskzRqApPD0hv+Oa74jg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@jsonjoy.com/json-pack": "^1.0.3", + "@jsonjoy.com/util": "^1.3.0", + "tree-dump": "^1.0.1", + "tslib": "^2.0.0" + }, + "engines": { + "node": ">= 4.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + } + }, + "node_modules/merge-descriptors": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", + "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/merge-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", + "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", + "dev": true, + "license": "MIT" + }, + "node_modules/methods": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/micromatch": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", + "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", + "dev": true, + "license": "MIT", + "dependencies": { + "braces": "^3.0.3", + "picomatch": "^2.3.1" + }, + "engines": { + "node": ">=8.6" + } + }, + "node_modules/mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "dev": true, + "license": "MIT", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/mime-db": { + "version": "1.53.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.53.0.tgz", + "integrity": "sha512-oHlN/w+3MQ3rba9rqFr6V/ypF10LSkdwUysQL7GkXoTgIWeV+tcXGA852TBxH+gsh8UWoyhR1hKcoMJTuWflpg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "dev": true, + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types/node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mimic-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", + "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/mini-css-extract-plugin": { + "version": "2.9.2", + "resolved": "https://registry.npmjs.org/mini-css-extract-plugin/-/mini-css-extract-plugin-2.9.2.tgz", + "integrity": "sha512-GJuACcS//jtq4kCtd5ii/M0SZf7OZRH+BxdqXZHaJfb8TJiVl+NgQRPwiYt2EuqeSkNydn/7vP+bcE27C5mb9w==", + "dev": true, + "license": "MIT", + "dependencies": { + "schema-utils": "^4.0.0", + "tapable": "^2.2.1" + }, + "engines": { + "node": ">= 12.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "webpack": "^5.0.0" + } + }, + "node_modules/minimalistic-assert": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz", + "integrity": "sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A==", + "dev": true, + "license": "ISC" + }, + "node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "dev": true, + "license": "MIT" + }, + "node_modules/multicast-dns": { + "version": "7.2.5", + "resolved": "https://registry.npmjs.org/multicast-dns/-/multicast-dns-7.2.5.tgz", + "integrity": "sha512-2eznPJP8z2BFLX50tf0LuODrpINqP1RVIm/CObbTcBRITQgmC/TjcREF1NeTBzIcR5XO/ukWo+YHOjBbFwIupg==", + "dev": true, + "license": "MIT", + "dependencies": { + "dns-packet": "^5.2.2", + "thunky": "^1.0.2" + }, + "bin": { + "multicast-dns": "cli.js" + } + }, + "node_modules/nanoid": { + "version": "3.3.8", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.8.tgz", + "integrity": "sha512-WNLf5Sd8oZxOm+TzppcYk8gVOgP+l58xNy58D0nbUnOxOWRWvlcCV4kUF7ltmI6PsrLl/BgKEyS4mqsGChFN0w==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/negotiator": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/neo-async": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz", + "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==", + "dev": true, + "license": "MIT" + }, + "node_modules/no-case": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/no-case/-/no-case-3.0.4.tgz", + "integrity": "sha512-fgAN3jGAh+RoxUGZHTSOLJIqUc2wmoBwGR4tbpNAKmmovFoWq0OdRkb0VkldReO2a2iBT/OEulG9XSUc10r3zg==", + "dev": true, + "license": "MIT", + "dependencies": { + "lower-case": "^2.0.2", + "tslib": "^2.0.3" + } + }, + "node_modules/node-forge": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.3.1.tgz", + "integrity": "sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA==", + "dev": true, + "license": "(BSD-3-Clause OR GPL-2.0)", + "engines": { + "node": ">= 6.13.0" + } + }, + "node_modules/node-releases": { + "version": "2.0.19", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.19.tgz", + "integrity": "sha512-xxOWJsBKtzAq7DY0J+DTzuz58K8e7sJbdgwkbMWQe8UYB6ekmsQ45q0M/tJDsGaZmbC+l7n57UV8Hl5tHxO9uw==", + "dev": true, + "license": "MIT" + }, + "node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/npm-run-path": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", + "integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/nth-check": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz", + "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0" + }, + "funding": { + "url": "https://github.com/fb55/nth-check?sponsor=1" + } + }, + "node_modules/object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/obuf": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/obuf/-/obuf-1.1.2.tgz", + "integrity": "sha512-PX1wu0AmAdPqOL1mWhqmlOd8kOIZQwGZw6rh7uby9fTc5lhaOWFLX3I6R1hrF9k3zUY40e6igsLGkDXK92LJNg==", + "dev": true, + "license": "MIT" + }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "dev": true, + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/on-headers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.0.2.tgz", + "integrity": "sha512-pZAE+FJLoyITytdqK0U5s+FIpjN0JP3OzFi/u8Rx+EV5/W+JTWGXG8xFzevE7AjBfDqHv/8vL8qQsIhHnqRkrA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/onetime": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", + "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "mimic-fn": "^2.1.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/open": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/open/-/open-10.1.0.tgz", + "integrity": "sha512-mnkeQ1qP5Ue2wd+aivTD3NHd/lZ96Lu0jgf0pwktLPtx6cTZiH7tyeGRRHs0zX0rbrahXPnXlUnbeXyaBBuIaw==", + "dev": true, + "license": "MIT", + "dependencies": { + "default-browser": "^5.2.1", + "define-lazy-prop": "^3.0.0", + "is-inside-container": "^1.0.0", + "is-wsl": "^3.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/open/node_modules/is-wsl": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-3.1.0.tgz", + "integrity": "sha512-UcVfVfaK4Sc4m7X3dUSoHoozQGBEFeDC+zVo06t98xe8CzHSZZBekNXH+tu0NalHolcJ/QAGqS46Hef7QXBIMw==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-inside-container": "^1.0.0" + }, + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-try": "^2.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-limit": "^2.2.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/p-retry": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-6.2.1.tgz", + "integrity": "sha512-hEt02O4hUct5wtwg4H4KcWgDdm+l1bOaEy/hWzd8xtXB9BqxTWBBhb+2ImAtH4Cv4rPjV76xN3Zumqk3k3AhhQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/retry": "0.12.2", + "is-network-error": "^1.0.0", + "retry": "^0.13.1" + }, + "engines": { + "node": ">=16.17" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-try": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/param-case": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/param-case/-/param-case-3.0.4.tgz", + "integrity": "sha512-RXlj7zCYokReqWpOPH9oYivUzLYZ5vAPIfEmCTNViosC78F8F0H9y7T7gG2M39ymgutxF5gcFEsyZQSph9Bp3A==", + "dev": true, + "license": "MIT", + "dependencies": { + "dot-case": "^3.0.4", + "tslib": "^2.0.3" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/pascal-case": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/pascal-case/-/pascal-case-3.1.2.tgz", + "integrity": "sha512-uWlGT3YSnK9x3BQJaOdcZwrnV6hPpd8jFH1/ucpiLRPh/2zCVJKS19E4GvYHvaCcACn3foXZ0cLB9Wrx1KGe5g==", + "dev": true, + "license": "MIT", + "dependencies": { + "no-case": "^3.0.4", + "tslib": "^2.0.3" + } + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-is-inside": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/path-is-inside/-/path-is-inside-1.0.2.tgz", + "integrity": "sha512-DUWJr3+ULp4zXmol/SZkFf3JGsS9/SIv+Y3Rt93/UjPpDpklB5f1er4O3POIbUuUJ3FXgqte2Q7SrU6zAqwk8w==", + "dev": true, + "license": "(WTFPL OR MIT)" + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-parse": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", + "dev": true, + "license": "MIT" + }, + "node_modules/path-to-regexp": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-3.3.0.tgz", + "integrity": "sha512-qyCH421YQPS2WFDxDjftfc1ZR5WKQzVzqsp4n9M2kQhVOo/ByahFoUNJfl58kOcEGfQ//7weFTDhm+ss8Ecxgw==", + "dev": true, + "license": "MIT" + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/pkg-dir": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", + "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "find-up": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/postcss": { + "version": "8.5.3", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.3.tgz", + "integrity": "sha512-dle9A3yYxlBSrt8Fu+IpjGT8SY8hN0mlaA6GY8t0P5PjIOZemULz/E2Bnm/2dcUOena75OTNkHI76uZBNUUq3A==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.8", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/postcss-modules-extract-imports": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/postcss-modules-extract-imports/-/postcss-modules-extract-imports-3.1.0.tgz", + "integrity": "sha512-k3kNe0aNFQDAZGbin48pL2VNidTF0w4/eASDsxlyspobzU3wZQLOGj7L9gfRe0Jo9/4uud09DsjFNH7winGv8Q==", + "dev": true, + "license": "ISC", + "engines": { + "node": "^10 || ^12 || >= 14" + }, + "peerDependencies": { + "postcss": "^8.1.0" + } + }, + "node_modules/postcss-modules-local-by-default": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/postcss-modules-local-by-default/-/postcss-modules-local-by-default-4.2.0.tgz", + "integrity": "sha512-5kcJm/zk+GJDSfw+V/42fJ5fhjL5YbFDl8nVdXkJPLLW+Vf9mTD5Xe0wqIaDnLuL2U6cDNpTr+UQ+v2HWIBhzw==", + "dev": true, + "license": "MIT", + "dependencies": { + "icss-utils": "^5.0.0", + "postcss-selector-parser": "^7.0.0", + "postcss-value-parser": "^4.1.0" + }, + "engines": { + "node": "^10 || ^12 || >= 14" + }, + "peerDependencies": { + "postcss": "^8.1.0" + } + }, + "node_modules/postcss-modules-scope": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/postcss-modules-scope/-/postcss-modules-scope-3.2.1.tgz", + "integrity": "sha512-m9jZstCVaqGjTAuny8MdgE88scJnCiQSlSrOWcTQgM2t32UBe+MUmFSO5t7VMSfAf/FJKImAxBav8ooCHJXCJA==", + "dev": true, + "license": "ISC", + "dependencies": { + "postcss-selector-parser": "^7.0.0" + }, + "engines": { + "node": "^10 || ^12 || >= 14" + }, + "peerDependencies": { + "postcss": "^8.1.0" + } + }, + "node_modules/postcss-modules-values": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/postcss-modules-values/-/postcss-modules-values-4.0.0.tgz", + "integrity": "sha512-RDxHkAiEGI78gS2ofyvCsu7iycRv7oqw5xMWn9iMoR0N/7mf9D50ecQqUo5BZ9Zh2vH4bCUR/ktCqbB9m8vJjQ==", + "dev": true, + "license": "ISC", + "dependencies": { + "icss-utils": "^5.0.0" + }, + "engines": { + "node": "^10 || ^12 || >= 14" + }, + "peerDependencies": { + "postcss": "^8.1.0" + } + }, + "node_modules/postcss-selector-parser": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-7.1.0.tgz", + "integrity": "sha512-8sLjZwK0R+JlxlYcTuVnyT2v+htpdrjDOKuMcOVdYjt52Lh8hWRYpxBPoKx/Zg+bcjc3wx6fmQevMmUztS/ccA==", + "dev": true, + "license": "MIT", + "dependencies": { + "cssesc": "^3.0.0", + "util-deprecate": "^1.0.2" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/postcss-value-parser": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-4.2.0.tgz", + "integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/pretty-error": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/pretty-error/-/pretty-error-4.0.0.tgz", + "integrity": "sha512-AoJ5YMAcXKYxKhuJGdcvse+Voc6v1RgnsR3nWcYU7q4t6z0Q6T86sv5Zq8VIRbOWWFpvdGE83LtdSMNd+6Y0xw==", + "dev": true, + "license": "MIT", + "dependencies": { + "lodash": "^4.17.20", + "renderkid": "^3.0.0" + } + }, + "node_modules/process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", + "dev": true, + "license": "MIT" + }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "dev": true, + "license": "MIT", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/proxy-addr/node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/qs": { + "version": "6.13.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", + "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "side-channel": "^1.0.6" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/randombytes": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", + "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "safe-buffer": "^5.1.0" + } + }, + "node_modules/range-parser": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.0.tgz", + "integrity": "sha512-kA5WQoNVo4t9lNx2kQNFCxKeBl5IbbSNBl1M/tLkw9WCn+hxNBAW5Qh8gdhs63CJnhjJ2zQWFoqPJP2sK1AV5A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/raw-body": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz", + "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==", + "dev": true, + "license": "MIT", + "dependencies": { + "bytes": "3.1.2", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "unpipe": "1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/raw-body/node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "dev": true, + "license": "(BSD-2-Clause OR MIT OR Apache-2.0)", + "dependencies": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "bin": { + "rc": "cli.js" + } + }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/readdirp": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", + "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", + "dev": true, + "license": "MIT", + "dependencies": { + "picomatch": "^2.2.1" + }, + "engines": { + "node": ">=8.10.0" + } + }, + "node_modules/rechoir": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.8.0.tgz", + "integrity": "sha512-/vxpCXddiX8NGfGO/mTafwjq4aFa/71pvamip0++IQk3zG8cbCj0fifNPrjjF1XMXUne91jL9OoxmdykoEtifQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "resolve": "^1.20.0" + }, + "engines": { + "node": ">= 10.13.0" + } + }, + "node_modules/registry-auth-token": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/registry-auth-token/-/registry-auth-token-3.3.2.tgz", + "integrity": "sha512-JL39c60XlzCVgNrO+qq68FoNb56w/m7JYvGR2jT5iR1xBrUA3Mfx5Twk5rqTThPmQKMWydGmq8oFtDlxfrmxnQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "rc": "^1.1.6", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/registry-url": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/registry-url/-/registry-url-3.1.0.tgz", + "integrity": "sha512-ZbgR5aZEdf4UKZVBPYIgaglBmSF2Hi94s2PcIHhRGFjKYu+chjJdYfHn4rt3hB6eCKLJ8giVIIfgMa1ehDfZKA==", + "dev": true, + "license": "MIT", + "dependencies": { + "rc": "^1.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/relateurl": { + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/relateurl/-/relateurl-0.2.7.tgz", + "integrity": "sha512-G08Dxvm4iDN3MLM0EsP62EDV9IuhXPR6blNz6Utcp7zyV3tr4HVNINt6MpaRWbxoOHT3Q7YN2P+jaHX8vUbgog==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/renderkid": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/renderkid/-/renderkid-3.0.0.tgz", + "integrity": "sha512-q/7VIQA8lmM1hF+jn+sFSPWGlMkSAeNYcPLmDQx2zzuiDfaLrOmumR8iaUKlenFgh0XRPIUeSPlH3A+AW3Z5pg==", + "dev": true, + "license": "MIT", + "dependencies": { + "css-select": "^4.1.3", + "dom-converter": "^0.2.0", + "htmlparser2": "^6.1.0", + "lodash": "^4.17.21", + "strip-ansi": "^6.0.1" + } + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/requires-port": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", + "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/resolve": { + "version": "1.22.10", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.10.tgz", + "integrity": "sha512-NPRy+/ncIMeDlTAsuqwKIiferiawhefFJtkNSW0qZJEqMEb+qBt/77B/jGeeek+F0uOeN05CDa6HXbbIgtVX4w==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-core-module": "^2.16.0", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + }, + "bin": { + "resolve": "bin/resolve" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/resolve-cwd": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/resolve-cwd/-/resolve-cwd-3.0.0.tgz", + "integrity": "sha512-OrZaX2Mb+rJCpH/6CpSqt9xFVpN++x01XnN2ie9g6P5/3xelLAkXWVADpdz1IHD/KFfEXyE6V0U01OQ3UO2rEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "resolve-from": "^5.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/resolve-from": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", + "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/retry": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", + "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/run-applescript": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/run-applescript/-/run-applescript-7.0.0.tgz", + "integrity": "sha512-9by4Ij99JUr/MCFBUkDKLWK3G9HVXmabKz9U5MlIAIuvuzkiOicRYs8XJLxX+xahD+mLiiCYDqF9dKAgtzKP1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "dev": true, + "license": "MIT" + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "dev": true, + "license": "MIT" + }, + "node_modules/schema-utils": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-4.3.0.tgz", + "integrity": "sha512-Gf9qqc58SpCA/xdziiHz35F4GNIWYWZrEshUc/G/r5BnLph6xpKuLeoJoQuj5WfBIx/eQLf+hmVPYHaxJu7V2g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/json-schema": "^7.0.9", + "ajv": "^8.9.0", + "ajv-formats": "^2.1.1", + "ajv-keywords": "^5.1.0" + }, + "engines": { + "node": ">= 10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + } + }, + "node_modules/select-hose": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/select-hose/-/select-hose-2.0.0.tgz", + "integrity": "sha512-mEugaLK+YfkijB4fx0e6kImuJdCIt2LxCRcbEYPqRGCs4F2ogyfZU5IAZRdjCP8JPq2AtdNoC/Dux63d9Kiryg==", + "dev": true, + "license": "MIT" + }, + "node_modules/selfsigned": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/selfsigned/-/selfsigned-2.4.1.tgz", + "integrity": "sha512-th5B4L2U+eGLq1TVh7zNRGBapioSORUeymIydxgFpwww9d2qyKvtuPU2jJuHvYAwwqi2Y596QBL3eEqcPEYL8Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node-forge": "^1.3.0", + "node-forge": "^1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/semver": { + "version": "7.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.1.tgz", + "integrity": "sha512-hlq8tAfn0m/61p4BVRcPzIGr6LKiMwo4VM6dGi6pt4qcRkmNzTcWq6eCEjEh+qXjkMDvPlOFFSGwQjoEa6gyMA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/send": { + "version": "0.19.0", + "resolved": "https://registry.npmjs.org/send/-/send-0.19.0.tgz", + "integrity": "sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==", + "dev": true, + "license": "MIT", + "dependencies": { + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "0.5.2", + "http-errors": "2.0.0", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "2.4.1", + "range-parser": "~1.2.1", + "statuses": "2.0.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/send/node_modules/encodeurl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", + "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/send/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/send/node_modules/range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/serialize-javascript": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.2.tgz", + "integrity": "sha512-Saa1xPByTTq2gdeFZYLLo+RFE35NHZkAbqZeWNd3BpzppeVisAqpDjcp8dyf6uIvEqJRd46jemmyA4iFIeVk8g==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "randombytes": "^2.1.0" + } + }, + "node_modules/serve": { + "version": "14.2.4", + "resolved": "https://registry.npmjs.org/serve/-/serve-14.2.4.tgz", + "integrity": "sha512-qy1S34PJ/fcY8gjVGszDB3EXiPSk5FKhUa7tQe0UPRddxRidc2V6cNHPNewbE1D7MAkgLuWEt3Vw56vYy73tzQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@zeit/schemas": "2.36.0", + "ajv": "8.12.0", + "arg": "5.0.2", + "boxen": "7.0.0", + "chalk": "5.0.1", + "chalk-template": "0.4.0", + "clipboardy": "3.0.0", + "compression": "1.7.4", + "is-port-reachable": "4.0.0", + "serve-handler": "6.1.6", + "update-check": "1.5.4" + }, + "bin": { + "serve": "build/main.js" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/serve-handler": { + "version": "6.1.6", + "resolved": "https://registry.npmjs.org/serve-handler/-/serve-handler-6.1.6.tgz", + "integrity": "sha512-x5RL9Y2p5+Sh3D38Fh9i/iQ5ZK+e4xuXRd/pGbM4D13tgo/MGwbttUk8emytcr1YYzBYs+apnUngBDFYfpjPuQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "bytes": "3.0.0", + "content-disposition": "0.5.2", + "mime-types": "2.1.18", + "minimatch": "3.1.2", + "path-is-inside": "1.0.2", + "path-to-regexp": "3.3.0", + "range-parser": "1.2.0" + } + }, + "node_modules/serve-handler/node_modules/mime-db": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.33.0.tgz", + "integrity": "sha512-BHJ/EKruNIqJf/QahvxwQZXKygOQ256myeN/Ew+THcAa5q+PjyTTMMeNQC4DZw5AwfvelsUrA6B67NKMqXDbzQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/serve-handler/node_modules/mime-types": { + "version": "2.1.18", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.18.tgz", + "integrity": "sha512-lc/aahn+t4/SWV/qcmumYjymLsWfN3ELhpmVuUFjgsORruuZPVSwAQryq+HHGvO/SI2KVX26bx+En+zhM8g8hQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "mime-db": "~1.33.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/serve-index": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/serve-index/-/serve-index-1.9.1.tgz", + "integrity": "sha512-pXHfKNP4qujrtteMrSBb0rc8HJ9Ms/GrXwcUtUtD5s4ewDJI8bT3Cz2zTVRMKtri49pLx2e0Ya8ziP5Ya2pZZw==", + "dev": true, + "license": "MIT", + "dependencies": { + "accepts": "~1.3.4", + "batch": "0.6.1", + "debug": "2.6.9", + "escape-html": "~1.0.3", + "http-errors": "~1.6.2", + "mime-types": "~2.1.17", + "parseurl": "~1.3.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/serve-index/node_modules/depd": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", + "integrity": "sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/serve-index/node_modules/http-errors": { + "version": "1.6.3", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz", + "integrity": "sha512-lks+lVC8dgGyh97jxvxeYTWQFvh4uw4yC12gVl63Cg30sjPX4wuGcdkICVXDAESr6OJGjqGA8Iz5mkeN6zlD7A==", + "dev": true, + "license": "MIT", + "dependencies": { + "depd": "~1.1.2", + "inherits": "2.0.3", + "setprototypeof": "1.1.0", + "statuses": ">= 1.4.0 < 2" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/serve-index/node_modules/inherits": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "integrity": "sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw==", + "dev": true, + "license": "ISC" + }, + "node_modules/serve-index/node_modules/setprototypeof": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.0.tgz", + "integrity": "sha512-BvE/TwpZX4FXExxOxZyRGQQv651MSwmWKZGqvmPcRIjDqWub67kTKuIMx43cZZrS/cBBzwBcNDWoFxt2XEFIpQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/serve-index/node_modules/statuses": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", + "integrity": "sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/serve-static": { + "version": "1.16.2", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.2.tgz", + "integrity": "sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw==", + "dev": true, + "license": "MIT", + "dependencies": { + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "0.19.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/serve/node_modules/ajv": { + "version": "8.12.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.12.0.tgz", + "integrity": "sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/serve/node_modules/chalk": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.0.1.tgz", + "integrity": "sha512-Fo07WOYGqMfCWHOzSXOt2CxDbC6skS/jO9ynEcmpANMoPrD+W1r1K6Vx7iNm+AQmETU1Xr2t+n8nzkV9t6xh3w==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^12.17.0 || ^14.13 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "dev": true, + "license": "ISC" + }, + "node_modules/shallow-clone": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/shallow-clone/-/shallow-clone-3.0.1.tgz", + "integrity": "sha512-/6KqX+GVUdqPuPPd2LxDDxzX6CAbjJehAAOKlNpqqUpAqPM6HeL8f+o3a+JsyGjn2lv0WY8UsTgUJjU9Ok55NA==", + "dev": true, + "license": "MIT", + "dependencies": { + "kind-of": "^6.0.2" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/shell-quote": { + "version": "1.8.2", + "resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.8.2.tgz", + "integrity": "sha512-AzqKpGKjrj7EM6rKVQEPpB288oCfnrEIuyoT9cyF4nmGa7V8Zk6f7RRqYisX8X9m+Q7bd632aZW4ky7EhbQztA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz", + "integrity": "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.3", + "side-channel-list": "^1.0.0", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.0.tgz", + "integrity": "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/sockjs": { + "version": "0.3.24", + "resolved": "https://registry.npmjs.org/sockjs/-/sockjs-0.3.24.tgz", + "integrity": "sha512-GJgLTZ7vYb/JtPSSZ10hsOYIvEYsjbNU+zPdIHcUaWVNUEPivzxku31865sSSud0Da0W4lEeOPlmw93zLQchuQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "faye-websocket": "^0.11.3", + "uuid": "^8.3.2", + "websocket-driver": "^0.7.4" + } + }, + "node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/source-map-support": { + "version": "0.5.21", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz", + "integrity": "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==", + "dev": true, + "license": "MIT", + "dependencies": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, + "node_modules/spdy": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/spdy/-/spdy-4.0.2.tgz", + "integrity": "sha512-r46gZQZQV+Kl9oItvl1JZZqJKGr+oEkB08A6BzkiR7593/7IbtuncXHd2YoYeTsG4157ZssMu9KYvUHLcjcDoA==", + "dev": true, + "license": "MIT", + "dependencies": { + "debug": "^4.1.0", + "handle-thing": "^2.0.0", + "http-deceiver": "^1.2.7", + "select-hose": "^2.0.0", + "spdy-transport": "^3.0.0" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/spdy-transport": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/spdy-transport/-/spdy-transport-3.0.0.tgz", + "integrity": "sha512-hsLVFE5SjA6TCisWeJXFKniGGOpBgMLmerfO2aCyCU5s7nJ/rpAepqmFifv/GCbSbueEeAJJnmSQ2rKC/g8Fcw==", + "dev": true, + "license": "MIT", + "dependencies": { + "debug": "^4.1.0", + "detect-node": "^2.0.4", + "hpack.js": "^2.1.6", + "obuf": "^1.1.2", + "readable-stream": "^3.0.6", + "wbuf": "^1.7.3" + } + }, + "node_modules/spdy-transport/node_modules/debug": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", + "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/spdy-transport/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/spdy/node_modules/debug": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", + "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/spdy/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/statuses": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", + "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dev": true, + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/string_decoder/node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/string-width/node_modules/ansi-regex": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.1.0.tgz", + "integrity": "sha512-7HSX4QQb4CspciLpVFwyRe79O3xsIZDDLER21kERQ71oaPodF8jL725AgJMFAYbooIqolJoRLuM81SpeUkpkvA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/string-width/node_modules/strip-ansi": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", + "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^6.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-final-newline": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz", + "integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/style-mod": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/style-mod/-/style-mod-4.1.2.tgz", + "integrity": "sha512-wnD1HyVqpJUI2+eKZ+eo1UwghftP6yuFheBqqe+bWCotBjC2K1YnteJILRMs3SM4V/0dLEW1SC27MWP5y+mwmw==", + "license": "MIT" + }, + "node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/supports-preserve-symlinks-flag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", + "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/tapable": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.2.1.tgz", + "integrity": "sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/terser": { + "version": "5.39.0", + "resolved": "https://registry.npmjs.org/terser/-/terser-5.39.0.tgz", + "integrity": "sha512-LBAhFyLho16harJoWMg/nZsQYgTrg5jXOn2nCYjRUcZZEdE3qa2zb8QEDRUGVZBW4rlazf2fxkg8tztybTaqWw==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "@jridgewell/source-map": "^0.3.3", + "acorn": "^8.8.2", + "commander": "^2.20.0", + "source-map-support": "~0.5.20" + }, + "bin": { + "terser": "bin/terser" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/terser-webpack-plugin": { + "version": "5.3.12", + "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.3.12.tgz", + "integrity": "sha512-jDLYqo7oF8tJIttjXO6jBY5Hk8p3A8W4ttih7cCEq64fQFWmgJ4VqAQjKr7WwIDlmXKEc6QeoRb5ecjZ+2afcg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.25", + "jest-worker": "^27.4.5", + "schema-utils": "^4.3.0", + "serialize-javascript": "^6.0.2", + "terser": "^5.31.1" + }, + "engines": { + "node": ">= 10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "webpack": "^5.1.0" + }, + "peerDependenciesMeta": { + "@swc/core": { + "optional": true + }, + "esbuild": { + "optional": true + }, + "uglify-js": { + "optional": true + } + } + }, + "node_modules/terser/node_modules/commander": { + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/thingies": { + "version": "1.21.0", + "resolved": "https://registry.npmjs.org/thingies/-/thingies-1.21.0.tgz", + "integrity": "sha512-hsqsJsFMsV+aD4s3CWKk85ep/3I9XzYV/IXaSouJMYIoDlgyi11cBhsqYe9/geRfB0YIikBQg6raRaM+nIMP9g==", + "dev": true, + "license": "Unlicense", + "engines": { + "node": ">=10.18" + }, + "peerDependencies": { + "tslib": "^2" + } + }, + "node_modules/thunky": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/thunky/-/thunky-1.1.0.tgz", + "integrity": "sha512-eHY7nBftgThBqOyHGVN+l8gF0BucP09fMo0oO/Lb0w1OF80dJv+lDVpXG60WMQvkcxAkNybKsrEIE3ZtKGmPrA==", + "dev": true, + "license": "MIT" + }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/tree-dump": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/tree-dump/-/tree-dump-1.0.2.tgz", + "integrity": "sha512-dpev9ABuLWdEubk+cIaI9cHwRNNDjkBBLXTwI4UCUFdQ5xXKqNXoK4FEciw/vxf+NQ7Cb7sGUyeUtORvHIdRXQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=10.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + }, + "peerDependencies": { + "tslib": "2" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "dev": true, + "license": "0BSD" + }, + "node_modules/type-fest": { + "version": "2.19.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-2.19.0.tgz", + "integrity": "sha512-RAH822pAdBgcNMAfWnCBU3CFZcfZ/i1eZjwFU/dsLKumyuuP3niueg2UAukXYF0E2AAoc82ZSSf9J0WQBinzHA==", + "dev": true, + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=12.20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "dev": true, + "license": "MIT", + "dependencies": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/undici-types": { + "version": "6.20.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.20.0.tgz", + "integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==", + "dev": true, + "license": "MIT" + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/update-browserslist-db": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.1.3.tgz", + "integrity": "sha512-UxhIZQ+QInVdunkDAaiazvvT/+fXL5Osr0JZlJulepYu6Jd7qJtDZjlur0emRlT71EN3ScPoE7gvsuIKKNavKw==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "escalade": "^3.2.0", + "picocolors": "^1.1.1" + }, + "bin": { + "update-browserslist-db": "cli.js" + }, + "peerDependencies": { + "browserslist": ">= 4.21.0" + } + }, + "node_modules/update-check": { + "version": "1.5.4", + "resolved": "https://registry.npmjs.org/update-check/-/update-check-1.5.4.tgz", + "integrity": "sha512-5YHsflzHP4t1G+8WGPlvKbJEbAJGCgw+Em+dGR1KmBUbr1J36SJBqlHLjR7oob7sco5hWHGQVcr9B2poIVDDTQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "registry-auth-token": "3.3.2", + "registry-url": "3.1.0" + } + }, + "node_modules/upgrade": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/upgrade/-/upgrade-1.1.0.tgz", + "integrity": "sha512-NtkVvqVCqsJo5U3mYRum2Tw6uCltOxfIJ/AfTZeTmw6U39IB5X23xF+kRZ9aiPaORqeiQQ7Q209/ibhOvxzwHA==", + "license": "MIT" + }, + "node_modules/uri-js": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "punycode": "^2.1.0" + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "dev": true, + "license": "MIT" + }, + "node_modules/utila": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/utila/-/utila-0.4.0.tgz", + "integrity": "sha512-Z0DbgELS9/L/75wZbro8xAnT50pBVFQZ+hUEueGDU5FN51YSCYM+jdxsfCiHjwNP/4LCDD0i/graKpeBnOXKRA==", + "dev": true, + "license": "MIT" + }, + "node_modules/utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "dev": true, + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/w3c-keyname": { + "version": "2.2.8", + "resolved": "https://registry.npmjs.org/w3c-keyname/-/w3c-keyname-2.2.8.tgz", + "integrity": "sha512-dpojBhNsCNN7T82Tm7k26A6G9ML3NkhDsnw9n/eoxSRlVBB4CEtIQ/KTCLI2Fwf3ataSXRhYFkQi3SlnFwPvPQ==", + "license": "MIT" + }, + "node_modules/watchpack": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.4.2.tgz", + "integrity": "sha512-TnbFSbcOCcDgjZ4piURLCbJ3nJhznVh9kw6F6iokjiFPl8ONxe9A6nMDVXDiNbrSfLILs6vB07F7wLBrwPYzJw==", + "dev": true, + "license": "MIT", + "dependencies": { + "glob-to-regexp": "^0.4.1", + "graceful-fs": "^4.1.2" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/wbuf": { + "version": "1.7.3", + "resolved": "https://registry.npmjs.org/wbuf/-/wbuf-1.7.3.tgz", + "integrity": "sha512-O84QOnr0icsbFGLS0O3bI5FswxzRr8/gHwWkDlQFskhSPryQXvrTMxjxGP4+iWYoauLoBvfDpkrOauZ+0iZpDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "minimalistic-assert": "^1.0.0" + } + }, + "node_modules/webpack": { + "version": "5.98.0", + "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.98.0.tgz", + "integrity": "sha512-UFynvx+gM44Gv9qFgj0acCQK2VE1CtdfwFdimkapco3hlPCJ/zeq73n2yVKimVbtm+TnApIugGhLJnkU6gjYXA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/eslint-scope": "^3.7.7", + "@types/estree": "^1.0.6", + "@webassemblyjs/ast": "^1.14.1", + "@webassemblyjs/wasm-edit": "^1.14.1", + "@webassemblyjs/wasm-parser": "^1.14.1", + "acorn": "^8.14.0", + "browserslist": "^4.24.0", + "chrome-trace-event": "^1.0.2", + "enhanced-resolve": "^5.17.1", + "es-module-lexer": "^1.2.1", + "eslint-scope": "5.1.1", + "events": "^3.2.0", + "glob-to-regexp": "^0.4.1", + "graceful-fs": "^4.2.11", + "json-parse-even-better-errors": "^2.3.1", + "loader-runner": "^4.2.0", + "mime-types": "^2.1.27", + "neo-async": "^2.6.2", + "schema-utils": "^4.3.0", + "tapable": "^2.1.1", + "terser-webpack-plugin": "^5.3.11", + "watchpack": "^2.4.1", + "webpack-sources": "^3.2.3" + }, + "bin": { + "webpack": "bin/webpack.js" + }, + "engines": { + "node": ">=10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependenciesMeta": { + "webpack-cli": { + "optional": true + } + } + }, + "node_modules/webpack-cli": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/webpack-cli/-/webpack-cli-6.0.1.tgz", + "integrity": "sha512-MfwFQ6SfwinsUVi0rNJm7rHZ31GyTcpVE5pgVA3hwFRb7COD4TzjUUwhGWKfO50+xdc2MQPuEBBJoqIMGt3JDw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@discoveryjs/json-ext": "^0.6.1", + "@webpack-cli/configtest": "^3.0.1", + "@webpack-cli/info": "^3.0.1", + "@webpack-cli/serve": "^3.0.1", + "colorette": "^2.0.14", + "commander": "^12.1.0", + "cross-spawn": "^7.0.3", + "envinfo": "^7.14.0", + "fastest-levenshtein": "^1.0.12", + "import-local": "^3.0.2", + "interpret": "^3.1.1", + "rechoir": "^0.8.0", + "webpack-merge": "^6.0.1" + }, + "bin": { + "webpack-cli": "bin/cli.js" + }, + "engines": { + "node": ">=18.12.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "webpack": "^5.82.0" + }, + "peerDependenciesMeta": { + "webpack-bundle-analyzer": { + "optional": true + }, + "webpack-dev-server": { + "optional": true + } + } + }, + "node_modules/webpack-cli/node_modules/commander": { + "version": "12.1.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-12.1.0.tgz", + "integrity": "sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/webpack-dev-middleware": { + "version": "7.4.2", + "resolved": "https://registry.npmjs.org/webpack-dev-middleware/-/webpack-dev-middleware-7.4.2.tgz", + "integrity": "sha512-xOO8n6eggxnwYpy1NlzUKpvrjfJTvae5/D6WOK0S2LSo7vjmo5gCM1DbLUmFqrMTJP+W/0YZNctm7jasWvLuBA==", + "dev": true, + "license": "MIT", + "dependencies": { + "colorette": "^2.0.10", + "memfs": "^4.6.0", + "mime-types": "^2.1.31", + "on-finished": "^2.4.1", + "range-parser": "^1.2.1", + "schema-utils": "^4.0.0" + }, + "engines": { + "node": ">= 18.12.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "webpack": "^5.0.0" + }, + "peerDependenciesMeta": { + "webpack": { + "optional": true + } + } + }, + "node_modules/webpack-dev-middleware/node_modules/range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/webpack-dev-server": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/webpack-dev-server/-/webpack-dev-server-5.2.0.tgz", + "integrity": "sha512-90SqqYXA2SK36KcT6o1bvwvZfJFcmoamqeJY7+boioffX9g9C0wjjJRGUrQIuh43pb0ttX7+ssavmj/WN2RHtA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/bonjour": "^3.5.13", + "@types/connect-history-api-fallback": "^1.5.4", + "@types/express": "^4.17.21", + "@types/serve-index": "^1.9.4", + "@types/serve-static": "^1.15.5", + "@types/sockjs": "^0.3.36", + "@types/ws": "^8.5.10", + "ansi-html-community": "^0.0.8", + "bonjour-service": "^1.2.1", + "chokidar": "^3.6.0", + "colorette": "^2.0.10", + "compression": "^1.7.4", + "connect-history-api-fallback": "^2.0.0", + "express": "^4.21.2", + "graceful-fs": "^4.2.6", + "http-proxy-middleware": "^2.0.7", + "ipaddr.js": "^2.1.0", + "launch-editor": "^2.6.1", + "open": "^10.0.3", + "p-retry": "^6.2.0", + "schema-utils": "^4.2.0", + "selfsigned": "^2.4.1", + "serve-index": "^1.9.1", + "sockjs": "^0.3.24", + "spdy": "^4.0.2", + "webpack-dev-middleware": "^7.4.2", + "ws": "^8.18.0" + }, + "bin": { + "webpack-dev-server": "bin/webpack-dev-server.js" + }, + "engines": { + "node": ">= 18.12.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "webpack": "^5.0.0" + }, + "peerDependenciesMeta": { + "webpack": { + "optional": true + }, + "webpack-cli": { + "optional": true + } + } + }, + "node_modules/webpack-merge": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/webpack-merge/-/webpack-merge-6.0.1.tgz", + "integrity": "sha512-hXXvrjtx2PLYx4qruKl+kyRSLc52V+cCvMxRjmKwoA+CBbbF5GfIBtR6kCvl0fYGqTUPKB+1ktVmTHqMOzgCBg==", + "dev": true, + "license": "MIT", + "dependencies": { + "clone-deep": "^4.0.1", + "flat": "^5.0.2", + "wildcard": "^2.0.1" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/webpack-sources": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-3.2.3.tgz", + "integrity": "sha512-/DyMEOrDgLKKIG0fmvtz+4dUX/3Ghozwgm6iPp8KRhvn+eQf9+Q7GWxVNMk3+uCPWfdXYC4ExGBckIXdFEfH1w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/websocket-driver": { + "version": "0.7.4", + "resolved": "https://registry.npmjs.org/websocket-driver/-/websocket-driver-0.7.4.tgz", + "integrity": "sha512-b17KeDIQVjvb0ssuSDF2cYXSg2iztliJ4B9WdsuB6J952qCPKmnVq4DyW5motImXHDC1cBT/1UezrJVsKw5zjg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "http-parser-js": ">=0.5.1", + "safe-buffer": ">=5.1.0", + "websocket-extensions": ">=0.1.1" + }, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/websocket-extensions": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/websocket-extensions/-/websocket-extensions-0.1.4.tgz", + "integrity": "sha512-OqedPIGOfsDlo31UNwYbCFMSaO9m9G/0faIHj5/dZFDMFqPTcx6UwqyOy3COEaEOg/9VsGIpdqn62W5KhoKSpg==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/widest-line": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/widest-line/-/widest-line-4.0.1.tgz", + "integrity": "sha512-o0cyEG0e8GPzT4iGHphIOh0cJOV8fivsXxddQasHPHfoZf1ZexrfeA21w2NaEN1RHE+fXlfISmOE8R9N3u3Qig==", + "dev": true, + "license": "MIT", + "dependencies": { + "string-width": "^5.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/wildcard": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/wildcard/-/wildcard-2.0.1.tgz", + "integrity": "sha512-CC1bOL87PIWSBhDcTrdeLo6eGT7mCFtrg0uIJtqJUFyK+eJnzl8A1niH56uu7KMa5XFrtiV+AQuHO3n7DsHnLQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi/node_modules/ansi-regex": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.1.0.tgz", + "integrity": "sha512-7HSX4QQb4CspciLpVFwyRe79O3xsIZDDLER21kERQ71oaPodF8jL725AgJMFAYbooIqolJoRLuM81SpeUkpkvA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/wrap-ansi/node_modules/ansi-styles": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", + "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/wrap-ansi/node_modules/strip-ansi": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", + "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^6.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/ws": { + "version": "8.18.1", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.1.tgz", + "integrity": "sha512-RKW2aJZMXeMxVpnZ6bck+RswznaxmzdULiBr6KY7XkTnW8uvt0iT9H5DkHUChXrc+uurzwa0rVI16n/Xzjdz1w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/xterm-readline": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/xterm-readline/-/xterm-readline-1.1.2.tgz", + "integrity": "sha512-1+W2nVuQvCYz9OUYwFBiolrSQUui51aDDyacKXt4PuxeBHqzvabQEJ2kwdBDzsmOjz5BwlDTAjJmYpH2OGqLFA==", + "license": "MIT", + "dependencies": { + "string-width": "^4" + }, + "peerDependencies": { + "@xterm/xterm": "^5.5.0" + } + }, + "node_modules/xterm-readline/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/xterm-readline/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + } + } +} From 40e3f49ab71d40e172f6dc94c78bca857da7bf4c Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sun, 2 Mar 2025 22:53:04 -0800 Subject: [PATCH 060/295] _ctypes pt. 3 (#5530) * Initial CFuncPtr implementation * function calling via libffi * working no-arg function call Signed-off-by: Ashwin Naren --- Cargo.lock | 35 ++--- extra_tests/snippets/builtins_ctypes.py | 134 ++++++++++++++++++ vm/Cargo.toml | 5 +- vm/src/stdlib/ctypes.rs | 27 +++- vm/src/stdlib/ctypes/base.rs | 25 ++++ vm/src/stdlib/ctypes/function.rs | 176 ++++++++++++++++++++++-- vm/src/stdlib/ctypes/library.rs | 35 ++--- vm/src/stdlib/mod.rs | 10 +- 8 files changed, 386 insertions(+), 61 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 67e0813b7b..ec6e5f0295 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -62,9 +62,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.95" +version = "1.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" +checksum = "6b964d184e89d9b6b67dd2715bc8e74cf3107fb2b529990c90cf517326150bf4" [[package]] name = "approx" @@ -1757,7 +1757,7 @@ checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.1", - "zerocopy 0.8.18", + "zerocopy 0.8.20", ] [[package]] @@ -1796,7 +1796,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a88e0da7a2c97baa202165137c158d0a2e824ac465d13d81046727b34cb247d3" dependencies = [ "getrandom 0.3.1", - "zerocopy 0.8.18", + "zerocopy 0.8.20", ] [[package]] @@ -2282,6 +2282,7 @@ dependencies = [ "itertools 0.14.0", "junction", "libc", + "libffi", "libloading", "log", "malachite-bigint", @@ -2421,9 +2422,9 @@ checksum = "f79dfe2d285b0488816f30e700a7438c5a73d816b5b7d3ac72fbc48b0d185e03" [[package]] name = "serde" -version = "1.0.217" +version = "1.0.218" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" +checksum = "e8dfc9d19bdbf6d17e22319da49161d5d0108e4188e8b680aef6299eed22df60" dependencies = [ "serde_derive", ] @@ -2452,9 +2453,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.217" +version = "1.0.218" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" +checksum = "f09503e191f4e797cb8aac08e9a4a4695c5edf6a2e70e376d961ddd5c969f82b" dependencies = [ "proc-macro2", "quote", @@ -2463,9 +2464,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.138" +version = "1.0.139" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d434192e7da787e94a6ea7e9670b26a036d0ca41e0b7efb2676dd32bae872949" +checksum = "44f86c3acccc9c65b153fe1b85a3be07fe5515274ec9f0653b4a0875731c72a6" dependencies = [ "itoa", "memchr", @@ -2937,9 +2938,9 @@ checksum = "623f59e6af2a98bdafeb93fa277ac8e1e40440973001ca15cf4ae1541cd16d56" [[package]] name = "unicode-ident" -version = "1.0.16" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a210d160f08b701c8721ba1c726c11662f877ea6b7094007e1ca9a1041945034" +checksum = "00e2473a93778eb0bad35909dff6a10d28e63f792f16ed15e404fca9d5eeedbe" [[package]] name = "unicode-normalization" @@ -3449,11 +3450,11 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.18" +version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79386d31a42a4996e3336b0919ddb90f81112af416270cff95b5f5af22b839c2" +checksum = "dde3bb8c68a8f3f1ed4ac9221aad6b10cece3e60a8e2ea54a6a2dec806d0084c" dependencies = [ - "zerocopy-derive 0.8.18", + "zerocopy-derive 0.8.20", ] [[package]] @@ -3469,9 +3470,9 @@ dependencies = [ [[package]] name = "zerocopy-derive" -version = "0.8.18" +version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76331675d372f91bf8d17e13afbd5fe639200b73d01f0fc748bb059f9cca2db7" +checksum = "eea57037071898bf96a6da35fd626f4f27e9cee3ead2a6c703cf09d472b2e700" dependencies = [ "proc-macro2", "quote", diff --git a/extra_tests/snippets/builtins_ctypes.py b/extra_tests/snippets/builtins_ctypes.py index 5bd6e5ef25..c5c563a48e 100644 --- a/extra_tests/snippets/builtins_ctypes.py +++ b/extra_tests/snippets/builtins_ctypes.py @@ -1,9 +1,29 @@ import os as _os, sys as _sys +import types as _types +from _ctypes import RTLD_LOCAL, RTLD_GLOBAL from _ctypes import sizeof from _ctypes import _SimpleCData +from _ctypes import CFuncPtr as _CFuncPtr + from struct import calcsize as _calcsize + +DEFAULT_MODE = RTLD_LOCAL +if _os.name == "posix" and _sys.platform == "darwin": + # On OS X 10.3, we use RTLD_GLOBAL as default mode + # because RTLD_LOCAL does not work at least on some + # libraries. OS X 10.3 is Darwin 7, so we check for + # that. + + if int(_os.uname().release.split('.')[0]) < 8: + DEFAULT_MODE = RTLD_GLOBAL + +from _ctypes import FUNCFLAG_CDECL as _FUNCFLAG_CDECL, \ + FUNCFLAG_PYTHONAPI as _FUNCFLAG_PYTHONAPI, \ + FUNCFLAG_USE_ERRNO as _FUNCFLAG_USE_ERRNO, \ + FUNCFLAG_USE_LASTERROR as _FUNCFLAG_USE_LASTERROR + def create_string_buffer(init, size=None): """create_string_buffer(aBytes) -> character array create_string_buffer(anInteger) -> character array @@ -131,3 +151,117 @@ class c_bool(_SimpleCData): # s = create_string_buffer(b'\000' * 32) assert i.value == 42 assert abs(f.value - 3.14) < 1e-06 + +if _os.name == "nt": + from _ctypes import LoadLibrary as _dlopen + from _ctypes import FUNCFLAG_STDCALL as _FUNCFLAG_STDCALL +elif _os.name == "posix": + from _ctypes import dlopen as _dlopen + +class CDLL(object): + """An instance of this class represents a loaded dll/shared + library, exporting functions using the standard C calling + convention (named 'cdecl' on Windows). + + The exported functions can be accessed as attributes, or by + indexing with the function name. Examples: + + .qsort -> callable object + ['qsort'] -> callable object + + Calling the functions releases the Python GIL during the call and + reacquires it afterwards. + """ + _func_flags_ = _FUNCFLAG_CDECL + _func_restype_ = c_int + # default values for repr + _name = '' + _handle = 0 + _FuncPtr = None + + def __init__(self, name, mode=DEFAULT_MODE, handle=None, + use_errno=False, + use_last_error=False, + winmode=None): + self._name = name + flags = self._func_flags_ + if use_errno: + flags |= _FUNCFLAG_USE_ERRNO + if use_last_error: + flags |= _FUNCFLAG_USE_LASTERROR + if _sys.platform.startswith("aix"): + """When the name contains ".a(" and ends with ")", + e.g., "libFOO.a(libFOO.so)" - this is taken to be an + archive(member) syntax for dlopen(), and the mode is adjusted. + Otherwise, name is presented to dlopen() as a file argument. + """ + if name and name.endswith(")") and ".a(" in name: + mode |= ( _os.RTLD_MEMBER | _os.RTLD_NOW ) + if _os.name == "nt": + if winmode is not None: + mode = winmode + else: + import nt + mode = 4096 + if '/' in name or '\\' in name: + self._name = nt._getfullpathname(self._name) + mode |= nt._LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR + + class _FuncPtr(_CFuncPtr): + _flags_ = flags + _restype_ = self._func_restype_ + self._FuncPtr = _FuncPtr + + if handle is None: + self._handle = _dlopen(self._name, mode) + else: + self._handle = handle + + def __repr__(self): + return "<%s '%s', handle %x at %#x>" % \ + (self.__class__.__name__, self._name, + (self._handle & (_sys.maxsize*2 + 1)), + id(self) & (_sys.maxsize*2 + 1)) + + def __getattr__(self, name): + if name.startswith('__') and name.endswith('__'): + raise AttributeError(name) + func = self.__getitem__(name) + setattr(self, name, func) + return func + + def __getitem__(self, name_or_ordinal): + func = self._FuncPtr((name_or_ordinal, self)) + if not isinstance(name_or_ordinal, int): + func.__name__ = name_or_ordinal + return func + +class LibraryLoader(object): + def __init__(self, dlltype): + self._dlltype = dlltype + + def __getattr__(self, name): + if name[0] == '_': + raise AttributeError(name) + try: + dll = self._dlltype(name) + except OSError: + raise AttributeError(name) + setattr(self, name, dll) + return dll + + def __getitem__(self, name): + return getattr(self, name) + + def LoadLibrary(self, name): + return self._dlltype(name) + + __class_getitem__ = classmethod(_types.GenericAlias) + +cdll = LibraryLoader(CDLL) + +if _os.name == "posix" or _sys.platform == "darwin": + pass +else: + libc = cdll.msvcrt + print("rand", libc.rand()) diff --git a/vm/Cargo.toml b/vm/Cargo.toml index acc645bb74..330a2beab5 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -100,9 +100,12 @@ uname = "0.1.1" rustyline = { workspace = true } which = "6" errno = "0.3" -libloading = "0.8" widestring = { workspace = true } +[target.'cfg(all(any(target_os = "linux", target_os = "macos", target_os = "windows"), not(any(target_env = "musl", target_env = "sgx"))))'.dependencies] +libffi = "3.2" +libloading = "0.8" + [target.'cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))'.dependencies] num_cpus = "1.13.1" diff --git a/vm/src/stdlib/ctypes.rs b/vm/src/stdlib/ctypes.rs index 701094a375..99866bae70 100644 --- a/vm/src/stdlib/ctypes.rs +++ b/vm/src/stdlib/ctypes.rs @@ -37,7 +37,7 @@ pub(crate) mod _ctypes { use super::base::PyCSimple; use crate::builtins::PyTypeRef; use crate::class::StaticType; - use crate::function::Either; + use crate::function::{Either, OptionalArg}; use crate::stdlib::ctypes::library; use crate::{AsObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine}; use crossbeam_utils::atomic::AtomicCell; @@ -180,12 +180,31 @@ pub(crate) mod _ctypes { } #[pyfunction(name = "LoadLibrary")] - fn load_library(name: String, vm: &VirtualMachine) -> PyResult { + fn load_library_windows( + name: String, + _load_flags: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + // TODO: audit functions first + // TODO: load_flags + let cache = library::libcache(); + let mut cache_write = cache.write(); + let (id, _) = cache_write.get_or_insert_lib(&name, vm).unwrap(); + Ok(id) + } + + #[pyfunction(name = "dlopen")] + fn load_library_unix( + name: String, + _load_flags: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { // TODO: audit functions first + // TODO: load_flags let cache = library::libcache(); let mut cache_write = cache.write(); - let lib_ref = cache_write.get_or_insert_lib(&name, vm).unwrap(); - Ok(lib_ref.get_pointer()) + let (id, _) = cache_write.get_or_insert_lib(&name, vm).unwrap(); + Ok(id) } #[pyfunction(name = "FreeLibrary")] diff --git a/vm/src/stdlib/ctypes/base.rs b/vm/src/stdlib/ctypes/base.rs index a4147c62b2..5c5396be29 100644 --- a/vm/src/stdlib/ctypes/base.rs +++ b/vm/src/stdlib/ctypes/base.rs @@ -10,6 +10,31 @@ use num_traits::ToPrimitive; use rustpython_common::lock::PyRwLock; use std::fmt::Debug; +pub fn ffi_type_from_str(_type_: &str) -> Option { + match _type_ { + "c" => Some(libffi::middle::Type::u8()), + "u" => Some(libffi::middle::Type::u32()), + "b" => Some(libffi::middle::Type::i8()), + "B" => Some(libffi::middle::Type::u8()), + "h" => Some(libffi::middle::Type::i16()), + "H" => Some(libffi::middle::Type::u16()), + "i" => Some(libffi::middle::Type::i32()), + "I" => Some(libffi::middle::Type::u32()), + "l" => Some(libffi::middle::Type::i32()), + "L" => Some(libffi::middle::Type::u32()), + "q" => Some(libffi::middle::Type::i64()), + "Q" => Some(libffi::middle::Type::u64()), + "f" => Some(libffi::middle::Type::f32()), + "d" => Some(libffi::middle::Type::f64()), + "g" => Some(libffi::middle::Type::f64()), + "?" => Some(libffi::middle::Type::u8()), + "z" => Some(libffi::middle::Type::u64()), + "Z" => Some(libffi::middle::Type::u64()), + "P" => Some(libffi::middle::Type::u64()), + _ => None, + } +} + #[allow(dead_code)] fn set_primitive(_type_: &str, value: &PyObjectRef, vm: &VirtualMachine) -> PyResult { match _type_ { diff --git a/vm/src/stdlib/ctypes/function.rs b/vm/src/stdlib/ctypes/function.rs index a7ee07744b..7d8dc0386a 100644 --- a/vm/src/stdlib/ctypes/function.rs +++ b/vm/src/stdlib/ctypes/function.rs @@ -1,24 +1,176 @@ -use crate::PyObjectRef; +use crate::builtins::{PyStr, PyTupleRef, PyTypeRef}; +use crate::convert::ToPyObject; +use crate::function::FuncArgs; use crate::stdlib::ctypes::PyCData; +use crate::stdlib::ctypes::base::{PyCSimple, ffi_type_from_str}; +use crate::types::{Callable, Constructor}; +use crate::{Py, PyObjectRef, PyResult, VirtualMachine}; use crossbeam_utils::atomic::AtomicCell; +use libffi::middle::{Arg, Cif, CodePtr, Type}; +use libloading::Symbol; +use num_traits::ToPrimitive; use rustpython_common::lock::PyRwLock; -use std::ffi::c_void; +use std::fmt::Debug; + +// https://github.com/python/cpython/blob/4f8bb3947cfbc20f970ff9d9531e1132a9e95396/Modules/_ctypes/callproc.c#L15 #[derive(Debug)] pub struct Function { - _pointer: *mut c_void, - _arguments: Vec<()>, - _return_type: Box<()>, + // TODO: no protection from use-after-free + pointer: CodePtr, + cif: Cif, +} + +unsafe impl Send for Function {} +unsafe impl Sync for Function {} + +type FP = unsafe extern "C" fn(); + +impl Function { + pub unsafe fn load( + library: &libloading::Library, + function: &str, + args: &[PyObjectRef], + ret_type: &Option, + vm: &VirtualMachine, + ) -> PyResult { + // map each arg to a PyCSimple + let args = args + .iter() + .map(|arg| { + if let Some(data) = arg.downcast_ref::() { + Ok(ffi_type_from_str(&data._type_).unwrap()) + } else { + Err(vm.new_type_error("Expected a ctypes simple type".to_string())) + } + }) + .collect::>>()?; + let terminated = format!("{}\0", function); + let pointer: Symbol<'_, FP> = unsafe { + library + .get(terminated.as_bytes()) + .map_err(|err| err.to_string()) + .map_err(|err| vm.new_value_error(err))? + }; + let code_ptr = CodePtr(*pointer as *mut _); + let return_type = match ret_type { + // TODO: Fix this + Some(_t) => { + return Err(vm.new_not_implemented_error("Return type not implemented".to_string())); + } + None => Type::c_int(), + }; + let cif = Cif::new(args, return_type); + Ok(Function { + cif, + pointer: code_ptr, + }) + } + + pub unsafe fn call( + &self, + args: Vec, + vm: &VirtualMachine, + ) -> PyResult { + let args = args + .into_iter() + .map(|arg| { + if let Some(data) = arg.downcast_ref::() { + dbg!(&data); + todo!("HANDLE ARGUMENTS") + } else { + Err(vm.new_type_error("Expected a ctypes simple type".to_string())) + } + }) + .collect::>>()?; + // TODO: FIX return type + let result: i32 = unsafe { self.cif.call(self.pointer, &args) }; + Ok(vm.ctx.new_int(result).into()) + } } #[pyclass(module = "_ctypes", name = "CFuncPtr", base = "PyCData")] +#[derive(PyPayload)] pub struct PyCFuncPtr { - pub _name_: String, - pub _argtypes_: AtomicCell>, - pub _restype_: AtomicCell, - _handle: PyObjectRef, - _f: PyRwLock, + pub name: PyRwLock, + pub _flags_: AtomicCell, + // FIXME(arihant2math): This shouldn't be an option, setting the default as the none type should work + // This is a workaround for now and I'll fix it later + pub _restype_: PyRwLock>, + pub handler: PyObjectRef, +} + +impl Debug for PyCFuncPtr { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PyCFuncPtr") + .field("name", &self.name) + .finish() + } } -#[pyclass] -impl PyCFuncPtr {} +impl Constructor for PyCFuncPtr { + type Args = (PyTupleRef, FuncArgs); + + fn py_new(_cls: PyTypeRef, (tuple, _args): Self::Args, vm: &VirtualMachine) -> PyResult { + let name = tuple + .first() + .ok_or(vm.new_type_error("Expected a tuple with at least 2 elements".to_string()))? + .downcast_ref::() + .ok_or(vm.new_type_error("Expected a string".to_string()))? + .to_string(); + let handler = tuple + .into_iter() + .nth(1) + .ok_or(vm.new_type_error("Expected a tuple with at least 2 elements".to_string()))? + .clone(); + Ok(Self { + _flags_: AtomicCell::new(0), + name: PyRwLock::new(name), + _restype_: PyRwLock::new(None), + handler, + } + .to_pyobject(vm)) + } +} + +impl Callable for PyCFuncPtr { + type Args = FuncArgs; + fn call(zelf: &Py, args: Self::Args, vm: &VirtualMachine) -> PyResult { + unsafe { + let handle = zelf.handler.get_attr("_handle", vm)?; + let handle = handle.try_int(vm)?.as_bigint().clone(); + let library_cache = crate::stdlib::ctypes::library::libcache().read(); + let library = library_cache + .get_lib( + handle + .to_usize() + .ok_or(vm.new_value_error("Invalid handle".to_string()))?, + ) + .ok_or_else(|| vm.new_value_error("Library not found".to_string()))?; + let inner_lib = library.lib.lock(); + let name = zelf.name.read(); + let res_type = zelf._restype_.read(); + let func = Function::load( + inner_lib.as_ref().unwrap(), + &name, + &args.args, + &res_type, + vm, + )?; + func.call(args.args, vm) + } + } +} + +#[pyclass(flags(BASETYPE), with(Callable, Constructor))] +impl PyCFuncPtr { + #[pygetset(magic)] + fn name(&self) -> String { + self.name.read().clone() + } + + #[pygetset(setter, magic)] + fn set_name(&self, name: String) { + *self.name.write() = name; + } +} diff --git a/vm/src/stdlib/ctypes/library.rs b/vm/src/stdlib/ctypes/library.rs index f777d26bc0..74a601a488 100644 --- a/vm/src/stdlib/ctypes/library.rs +++ b/vm/src/stdlib/ctypes/library.rs @@ -1,14 +1,13 @@ use crate::VirtualMachine; -use crossbeam_utils::atomic::AtomicCell; use libloading::Library; -use rustpython_common::lock::PyRwLock; +use rustpython_common::lock::{PyMutex, PyRwLock}; use std::collections::HashMap; use std::ffi::c_void; use std::fmt; use std::ptr::null; pub struct SharedLibrary { - lib: AtomicCell>, + pub(crate) lib: PyMutex>, } impl fmt::Debug for SharedLibrary { @@ -20,26 +19,13 @@ impl fmt::Debug for SharedLibrary { impl SharedLibrary { pub fn new(name: &str) -> Result { Ok(SharedLibrary { - lib: AtomicCell::new(Some(unsafe { Library::new(name)? })), + lib: PyMutex::new(unsafe { Some(Library::new(name)?) }), }) } - #[allow(dead_code)] - pub fn get_sym(&self, name: &str) -> Result<*mut c_void, String> { - if let Some(inner) = unsafe { &*self.lib.as_ptr() } { - unsafe { - inner - .get(name.as_bytes()) - .map(|f: libloading::Symbol<'_, *mut c_void>| *f) - .map_err(|err| err.to_string()) - } - } else { - Err("The library has been closed".to_string()) - } - } - pub fn get_pointer(&self) -> usize { - if let Some(l) = unsafe { &*self.lib.as_ptr() } { + let lib_lock = self.lib.lock(); + if let Some(l) = &*lib_lock { l as *const Library as usize } else { null::() as usize @@ -47,13 +33,12 @@ impl SharedLibrary { } pub fn is_closed(&self) -> bool { - unsafe { &*self.lib.as_ptr() }.is_none() + let lib_lock = self.lib.lock(); + lib_lock.is_none() } pub fn close(&self) { - let old = self.lib.take(); - self.lib.store(None); - drop(old); + *self.lib.lock() = None; } } @@ -83,7 +68,7 @@ impl ExternalLibs { &mut self, library_path: &str, _vm: &VirtualMachine, - ) -> Result<&SharedLibrary, libloading::Error> { + ) -> Result<(usize, &SharedLibrary), libloading::Error> { let nlib = SharedLibrary::new(library_path)?; let key = nlib.get_pointer(); @@ -98,7 +83,7 @@ impl ExternalLibs { } }; - Ok(self.libraries.get(&key).unwrap()) + Ok((key, self.libraries.get(&key).unwrap())) } pub fn drop_lib(&mut self, key: usize) { diff --git a/vm/src/stdlib/mod.rs b/vm/src/stdlib/mod.rs index 529a40e861..a3180c1c0f 100644 --- a/vm/src/stdlib/mod.rs +++ b/vm/src/stdlib/mod.rs @@ -37,7 +37,10 @@ pub mod posix; #[path = "posix_compat.rs"] pub mod posix; -#[cfg(any(target_family = "unix", target_family = "windows"))] +#[cfg(all( + any(target_os = "linux", target_os = "macos", target_os = "windows"), + not(any(target_env = "musl", target_env = "sgx")) +))] mod ctypes; #[cfg(windows)] pub(crate) mod msvcrt; @@ -126,7 +129,10 @@ pub fn get_module_inits() -> StdlibMap { "_winapi" => winapi::make_module, "winreg" => winreg::make_module, } - #[cfg(any(target_family = "unix", target_family = "windows"))] + #[cfg(all( + any(target_os = "linux", target_os = "macos", target_os = "windows"), + not(any(target_env = "musl", target_env = "sgx")) + ))] { "_ctypes" => ctypes::make_module, } From defcadafbb1c627384936ba193ff995628dae6bf Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sun, 2 Mar 2025 19:21:01 -0800 Subject: [PATCH 061/295] publish demo on weekly release --- .github/workflows/release.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 99035d04fc..0d4b3a270c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -106,6 +106,28 @@ jobs: name: rustpython-release-wasm32-wasip1 path: target/rustpython-release-wasm32-wasip1.wasm + - name: install wasm-pack + run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh + - uses: actions/setup-node@v4 + - uses: mwilliamson/setup-wabt-action@v3 + with: { wabt-version: "1.0.30" } + - name: build notebook demo + if: github.ref == 'refs/heads/release' + run: | + npm install + npm run dist + mv dist ../demo/dist/notebook + env: + NODE_OPTIONS: "--openssl-legacy-provider" + working-directory: ./wasm/notebook + - name: Deploy demo to Github Pages + uses: peaceiris/actions-gh-pages@v4 + env: + ACTIONS_DEPLOY_KEY: ${{ secrets.ACTIONS_DEMO_DEPLOY_KEY }} + PUBLISH_DIR: ./wasm/demo/dist + EXTERNAL_REPOSITORY: RustPython/demo + PUBLISH_BRANCH: master + release: runs-on: ubuntu-latest needs: [build, build-wasm] From 6804dd4363b4368113750b2f29814f231ec24375 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sun, 2 Mar 2025 19:26:29 -0800 Subject: [PATCH 062/295] use rust-toolchain targets options instead of using rustup --- .github/workflows/release.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0d4b3a270c..98b7de4823 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -89,10 +89,8 @@ jobs: steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable - - - name: Set up Environment - shell: bash - run: rustup target add wasm32-wasip1 + with: + targets: wasm32-wasip1 - name: Build RustPython run: cargo build --target wasm32-wasip1 --no-default-features --features freeze-stdlib,stdlib --release @@ -112,7 +110,6 @@ jobs: - uses: mwilliamson/setup-wabt-action@v3 with: { wabt-version: "1.0.30" } - name: build notebook demo - if: github.ref == 'refs/heads/release' run: | npm install npm run dist From 3ae1160868a5aa4b341eb5956935f0e03b4169e7 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Mon, 3 Mar 2025 01:09:07 -0800 Subject: [PATCH 063/295] Remove winapi dependency --- Cargo.lock | 1 - stdlib/Cargo.toml | 6 ---- stdlib/src/socket.rs | 66 ++++++++++++++++++++++++++++++++------------ 3 files changed, 48 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ec6e5f0295..7ff4313548 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2252,7 +2252,6 @@ dependencies = [ "unicode_names2", "uuid", "widestring", - "winapi", "windows-sys 0.52.0", "xml-rs", ] diff --git a/stdlib/Cargo.toml b/stdlib/Cargo.toml index 1086dea090..f2ace7b26a 100644 --- a/stdlib/Cargo.toml +++ b/stdlib/Cargo.toml @@ -117,12 +117,6 @@ paste = { workspace = true } schannel = { workspace = true } widestring = { workspace = true } -[target.'cfg(windows)'.dependencies.winapi] -version = "0.3.9" -features = [ - "winsock2", "ifdef", "netioapi", "ws2tcpip", -] - [target.'cfg(windows)'.dependencies.windows-sys] workspace = true features = [ diff --git a/stdlib/src/socket.rs b/stdlib/src/socket.rs index 533a0a0b24..940fa3299e 100644 --- a/stdlib/src/socket.rs +++ b/stdlib/src/socket.rs @@ -35,17 +35,27 @@ mod _socket { use libc as c; #[cfg(windows)] mod c { - pub use winapi::shared::netioapi::{if_indextoname, if_nametoindex}; - pub use winapi::shared::ws2def::{ - INADDR_ANY, INADDR_BROADCAST, INADDR_LOOPBACK, INADDR_NONE, - }; - pub use winapi::um::winsock2::{ - SO_EXCLUSIVEADDRUSE, getprotobyname, getservbyname, getservbyport, getsockopt, + pub use windows_sys::Win32::NetworkManagement::IpHelper::{if_indextoname, if_nametoindex}; + + pub const INADDR_ANY: u32 = 0x00000000; + pub const INADDR_LOOPBACK: u32 = 0x7f000001; + pub const INADDR_BROADCAST: u32 = 0xffffffff; + pub const INADDR_NONE: u32 = 0xffffffff; + + pub use windows_sys::Win32::Networking::WinSock::{ + SO_REUSEADDR as SO_EXCLUSIVEADDRUSE, getprotobyname, getservbyname, getservbyport, getsockopt, setsockopt, }; - pub use winapi::um::ws2tcpip::{ - EAI_AGAIN, EAI_BADFLAGS, EAI_FAIL, EAI_FAMILY, EAI_MEMORY, EAI_NODATA, EAI_NONAME, - EAI_SERVICE, EAI_SOCKTYPE, + pub use windows_sys::Win32::Networking::WinSock::{ + WSATRY_AGAIN as EAI_AGAIN, + WSAEINVAL as EAI_BADFLAGS, + WSANO_RECOVERY as EAI_FAIL, + WSAEAFNOSUPPORT as EAI_FAMILY, + WSA_NOT_ENOUGH_MEMORY as EAI_MEMORY, + WSAHOST_NOT_FOUND as EAI_NODATA, + WSAHOST_NOT_FOUND as EAI_NONAME, + WSATYPE_NOT_FOUND as EAI_SERVICE, + WSAESOCKTNOSUPPORT as EAI_SOCKTYPE, }; pub use windows_sys::Win32::Networking::WinSock::{ AF_APPLETALK, AF_DECnet, AF_IPX, AF_LINK, AI_ADDRCONFIG, AI_ALL, AI_CANONNAME, @@ -755,7 +765,7 @@ mod _socket { } #[cfg(windows)] - use winapi::shared::netioapi; + use windows_sys::Win32::NetworkManagement::IpHelper; fn get_raw_sock(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { #[cfg(unix)] @@ -1755,6 +1765,9 @@ mod _socket { .map(|s| s.to_cstring(vm)) .transpose()?; let cstr_proto = cstr_opt_as_ptr(&cstr_proto); + #[cfg(windows)] + let serv = unsafe { c::getservbyname(cstr_name.as_ptr() as windows_sys::core::PCSTR, cstr_proto as windows_sys::core::PCSTR) }; + #[cfg(not(windows))] let serv = unsafe { c::getservbyname(cstr_name.as_ptr(), cstr_proto) }; if serv.is_null() { return Err(vm.new_os_error("service/proto not found".to_owned())); @@ -1777,10 +1790,16 @@ mod _socket { .map(|s| s.to_cstring(vm)) .transpose()?; let cstr_proto = cstr_opt_as_ptr(&cstr_proto); + #[cfg(windows)] + let serv = unsafe { c::getservbyport(port.to_be() as _, cstr_proto as windows_sys::core::PCSTR) }; + #[cfg(not(windows))] let serv = unsafe { c::getservbyport(port.to_be() as _, cstr_proto) }; if serv.is_null() { return Err(vm.new_os_error("port/proto not found".to_owned())); } + #[cfg(windows)] + let s = unsafe { ffi::CStr::from_ptr((*serv).s_name as *const i8) }; + #[cfg(not(windows))] let s = unsafe { ffi::CStr::from_ptr((*serv).s_name) }; Ok(s.to_string_lossy().into_owned()) } @@ -2033,6 +2052,9 @@ mod _socket { #[pyfunction] fn getprotobyname(name: PyStrRef, vm: &VirtualMachine) -> PyResult { let cstr = name.to_cstring(vm)?; + #[cfg(windows)] + let proto = unsafe { c::getprotobyname(cstr.as_ptr() as *const u8) }; + #[cfg(not(windows))] let proto = unsafe { c::getprotobyname(cstr.as_ptr()) }; if proto.is_null() { return Err(vm.new_os_error("protocol not found".to_owned())); @@ -2111,13 +2133,16 @@ mod _socket { #[cfg(all(unix, not(target_os = "redox")))] type IfIndex = c::c_uint; #[cfg(windows)] - type IfIndex = winapi::shared::ifdef::NET_IFINDEX; + type IfIndex = u32; #[cfg(not(target_os = "redox"))] #[pyfunction] fn if_nametoindex(name: FsPath, vm: &VirtualMachine) -> PyResult { let name = name.to_cstring(vm)?; + #[cfg(windows)] + let ret = unsafe { c::if_nametoindex(name.as_ptr() as *const u8) }; + #[cfg(not(windows))] let ret = unsafe { c::if_nametoindex(name.as_ptr()) }; if ret == 0 { Err(vm.new_os_error("no interface with this name".to_owned())) @@ -2134,6 +2159,9 @@ mod _socket { if ret.is_null() { Err(crate::vm::stdlib::os::errno_err(vm)) } else { + #[cfg(windows)] + let buf = unsafe { ffi::CStr::from_ptr(buf.as_ptr() as *const i8) }; + #[cfg(not(windows))] let buf = unsafe { ffi::CStr::from_ptr(buf.as_ptr()) }; Ok(buf.to_string_lossy().into_owned()) } @@ -2152,6 +2180,8 @@ mod _socket { ))] #[pyfunction] fn if_nameindex(vm: &VirtualMachine) -> PyResult> { + use windows_sys::Win32::NetworkManagement::Ndis::NET_LUID_LH; + #[cfg(not(windows))] { let list = nix::net::if_::if_nameindex() @@ -2182,11 +2212,11 @@ mod _socket { return Ok(list); fn get_name( - luid: &winapi::shared::ifdef::NET_LUID, + luid: &NET_LUID_LH, ) -> io::Result { let mut buf = [0; c::IF_NAMESIZE + 1]; let ret = unsafe { - netioapi::ConvertInterfaceLuidToNameW(luid, buf.as_mut_ptr(), buf.len()) + IpHelper::ConvertInterfaceLuidToNameW(luid, buf.as_mut_ptr(), buf.len()) }; if ret == 0 { Ok(widestring::WideCString::from_ustr_truncate( @@ -2197,12 +2227,12 @@ mod _socket { } } struct MibTable { - ptr: ptr::NonNull, + ptr: ptr::NonNull, } impl MibTable { fn get_raw() -> io::Result { let mut ptr = ptr::null_mut(); - let ret = unsafe { netioapi::GetIfTable2Ex(netioapi::MibIfTableRaw, &mut ptr) }; + let ret = unsafe { IpHelper::GetIfTable2Ex(IpHelper::MibIfTableRaw, &mut ptr) }; if ret == 0 { let ptr = unsafe { ptr::NonNull::new_unchecked(ptr) }; Ok(Self { ptr }) @@ -2212,17 +2242,17 @@ mod _socket { } } impl MibTable { - fn as_slice(&self) -> &[netioapi::MIB_IF_ROW2] { + fn as_slice(&self) -> &[IpHelper::MIB_IF_ROW2] { unsafe { let p = self.ptr.as_ptr(); - let ptr = &raw const (*p).Table as *const netioapi::MIB_IF_ROW2; + let ptr = &raw const (*p).Table as *const IpHelper::MIB_IF_ROW2; std::slice::from_raw_parts(ptr, (*p).NumEntries as usize) } } } impl Drop for MibTable { fn drop(&mut self) { - unsafe { netioapi::FreeMibTable(self.ptr.as_ptr() as *mut _) } + unsafe { IpHelper::FreeMibTable(self.ptr.as_ptr() as *mut _) }; } } } From 4d9804f188c9604ac4ce63f89a1f7befd7453562 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Mon, 3 Mar 2025 01:37:27 -0800 Subject: [PATCH 064/295] formatting --- stdlib/src/socket.rs | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/stdlib/src/socket.rs b/stdlib/src/socket.rs index 940fa3299e..cfdf4c8a4e 100644 --- a/stdlib/src/socket.rs +++ b/stdlib/src/socket.rs @@ -36,27 +36,12 @@ mod _socket { #[cfg(windows)] mod c { pub use windows_sys::Win32::NetworkManagement::IpHelper::{if_indextoname, if_nametoindex}; - + pub const INADDR_ANY: u32 = 0x00000000; pub const INADDR_LOOPBACK: u32 = 0x7f000001; pub const INADDR_BROADCAST: u32 = 0xffffffff; pub const INADDR_NONE: u32 = 0xffffffff; - - pub use windows_sys::Win32::Networking::WinSock::{ - SO_REUSEADDR as SO_EXCLUSIVEADDRUSE, getprotobyname, getservbyname, getservbyport, getsockopt, - setsockopt, - }; - pub use windows_sys::Win32::Networking::WinSock::{ - WSATRY_AGAIN as EAI_AGAIN, - WSAEINVAL as EAI_BADFLAGS, - WSANO_RECOVERY as EAI_FAIL, - WSAEAFNOSUPPORT as EAI_FAMILY, - WSA_NOT_ENOUGH_MEMORY as EAI_MEMORY, - WSAHOST_NOT_FOUND as EAI_NODATA, - WSAHOST_NOT_FOUND as EAI_NONAME, - WSATYPE_NOT_FOUND as EAI_SERVICE, - WSAESOCKTNOSUPPORT as EAI_SOCKTYPE, - }; + pub use windows_sys::Win32::Networking::WinSock::{ AF_APPLETALK, AF_DECnet, AF_IPX, AF_LINK, AI_ADDRCONFIG, AI_ALL, AI_CANONNAME, AI_NUMERICSERV, AI_V4MAPPED, IP_ADD_MEMBERSHIP, IP_DROP_MEMBERSHIP, IP_HDRINCL, @@ -78,6 +63,17 @@ mod _socket { SOL_SOCKET, SOMAXCONN, TCP_NODELAY, WSAEBADF, WSAECONNRESET, WSAENOTSOCK, WSAEWOULDBLOCK, }; + pub use windows_sys::Win32::Networking::WinSock::{ + SO_REUSEADDR as SO_EXCLUSIVEADDRUSE, getprotobyname, getservbyname, getservbyport, + getsockopt, setsockopt, + }; + pub use windows_sys::Win32::Networking::WinSock::{ + WSA_NOT_ENOUGH_MEMORY as EAI_MEMORY, WSAEAFNOSUPPORT as EAI_FAMILY, + WSAEINVAL as EAI_BADFLAGS, WSAESOCKTNOSUPPORT as EAI_SOCKTYPE, + WSAHOST_NOT_FOUND as EAI_NODATA, WSAHOST_NOT_FOUND as EAI_NONAME, + WSANO_RECOVERY as EAI_FAIL, WSATRY_AGAIN as EAI_AGAIN, + WSATYPE_NOT_FOUND as EAI_SERVICE, + }; pub const IF_NAMESIZE: usize = windows_sys::Win32::NetworkManagement::Ndis::IF_MAX_STRING_SIZE as _; pub const AF_UNSPEC: i32 = windows_sys::Win32::Networking::WinSock::AF_UNSPEC as _; @@ -1766,7 +1762,12 @@ mod _socket { .transpose()?; let cstr_proto = cstr_opt_as_ptr(&cstr_proto); #[cfg(windows)] - let serv = unsafe { c::getservbyname(cstr_name.as_ptr() as windows_sys::core::PCSTR, cstr_proto as windows_sys::core::PCSTR) }; + let serv = unsafe { + c::getservbyname( + cstr_name.as_ptr() as windows_sys::core::PCSTR, + cstr_proto as windows_sys::core::PCSTR, + ) + }; #[cfg(not(windows))] let serv = unsafe { c::getservbyname(cstr_name.as_ptr(), cstr_proto) }; if serv.is_null() { @@ -1791,7 +1792,8 @@ mod _socket { .transpose()?; let cstr_proto = cstr_opt_as_ptr(&cstr_proto); #[cfg(windows)] - let serv = unsafe { c::getservbyport(port.to_be() as _, cstr_proto as windows_sys::core::PCSTR) }; + let serv = + unsafe { c::getservbyport(port.to_be() as _, cstr_proto as windows_sys::core::PCSTR) }; #[cfg(not(windows))] let serv = unsafe { c::getservbyport(port.to_be() as _, cstr_proto) }; if serv.is_null() { @@ -2211,9 +2213,7 @@ mod _socket { let list = list.collect::>()?; return Ok(list); - fn get_name( - luid: &NET_LUID_LH, - ) -> io::Result { + fn get_name(luid: &NET_LUID_LH) -> io::Result { let mut buf = [0; c::IF_NAMESIZE + 1]; let ret = unsafe { IpHelper::ConvertInterfaceLuidToNameW(luid, buf.as_mut_ptr(), buf.len()) From 8d2c6807d28000c8ee3dc4b7d65d86c8f63bcb52 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Mon, 3 Mar 2025 10:50:19 -0800 Subject: [PATCH 065/295] fix non-windows build --- stdlib/src/socket.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/stdlib/src/socket.rs b/stdlib/src/socket.rs index cfdf4c8a4e..b46514eb70 100644 --- a/stdlib/src/socket.rs +++ b/stdlib/src/socket.rs @@ -2182,8 +2182,6 @@ mod _socket { ))] #[pyfunction] fn if_nameindex(vm: &VirtualMachine) -> PyResult> { - use windows_sys::Win32::NetworkManagement::Ndis::NET_LUID_LH; - #[cfg(not(windows))] { let list = nix::net::if_::if_nameindex() @@ -2202,6 +2200,7 @@ mod _socket { #[cfg(windows)] { use std::ptr; + use windows_sys::Win32::NetworkManagement::Ndis::NET_LUID_LH; let table = MibTable::get_raw().map_err(|err| err.into_pyexception(vm))?; let list = table.as_slice().iter().map(|entry| { From 05cb8c0b73918f12ca6e86f432f5b4b11dc523b5 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Tue, 4 Mar 2025 01:28:53 -0800 Subject: [PATCH 066/295] Update socket.rs Co-authored-by: Jeong, YunWon <69878+youknowone@users.noreply.github.com> --- stdlib/src/socket.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/src/socket.rs b/stdlib/src/socket.rs index b46514eb70..149408a6f3 100644 --- a/stdlib/src/socket.rs +++ b/stdlib/src/socket.rs @@ -2135,7 +2135,7 @@ mod _socket { #[cfg(all(unix, not(target_os = "redox")))] type IfIndex = c::c_uint; #[cfg(windows)] - type IfIndex = u32; + type IfIndex = u32; // NET_IFINDEX but windows-sys 0.59 doesn't have it #[cfg(not(target_os = "redox"))] #[pyfunction] From 33940726a8ad2f91cf7b19c8d24989f74298091b Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Wed, 5 Mar 2025 10:36:04 -0800 Subject: [PATCH 067/295] upgrade to windows-sys 0.59.0 --- Cargo.lock | 6 +++--- Cargo.toml | 2 +- common/src/fileutils.rs | 10 +++++----- stdlib/Cargo.toml | 1 + stdlib/src/overlapped.rs | 42 ++++++++++++++++++++++------------------ stdlib/src/socket.rs | 8 ++------ vm/Cargo.toml | 1 + vm/src/stdlib/nt.rs | 4 ++-- vm/src/stdlib/winapi.rs | 34 ++++++++++++++++---------------- vm/src/windows.rs | 2 +- 10 files changed, 56 insertions(+), 54 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7ff4313548..1c428c2d73 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2023,7 +2023,7 @@ dependencies = [ "siphasher 0.3.11", "volatile", "widestring", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -2252,7 +2252,7 @@ dependencies = [ "unicode_names2", "uuid", "widestring", - "windows-sys 0.52.0", + "windows-sys 0.59.0", "xml-rs", ] @@ -2332,7 +2332,7 @@ dependencies = [ "which", "widestring", "windows", - "windows-sys 0.52.0", + "windows-sys 0.59.0", "winreg", ] diff --git a/Cargo.toml b/Cargo.toml index 807e821ca7..14e66b39eb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -181,7 +181,7 @@ thiserror = "2.0" thread_local = "1.1.8" unicode_names2 = "1.3.0" widestring = "1.1.0" -windows-sys = "0.52.0" +windows-sys = "0.59.0" wasm-bindgen = "0.2.100" # Lints diff --git a/common/src/fileutils.rs b/common/src/fileutils.rs index 7d5ff01942..ce4aa8250a 100644 --- a/common/src/fileutils.rs +++ b/common/src/fileutils.rs @@ -116,7 +116,7 @@ pub mod windows { let h = h?; // reset stat? - let file_type = unsafe { GetFileType(h) }; + let file_type = unsafe { GetFileType(h as _) }; if file_type == FILE_TYPE_UNKNOWN { return Err(std::io::Error::last_os_error()); } @@ -138,10 +138,10 @@ pub mod windows { let mut basic_info: FILE_BASIC_INFO = unsafe { std::mem::zeroed() }; let mut id_info: FILE_ID_INFO = unsafe { std::mem::zeroed() }; - if unsafe { GetFileInformationByHandle(h, &mut info) } == 0 + if unsafe { GetFileInformationByHandle(h as _, &mut info) } == 0 || unsafe { GetFileInformationByHandleEx( - h, + h as _, FileBasicInfo, &mut basic_info as *mut _ as *mut _, std::mem::size_of_val(&basic_info) as u32, @@ -153,7 +153,7 @@ pub mod windows { let p_id_info = if unsafe { GetFileInformationByHandleEx( - h, + h as _, FileIdInfo, &mut id_info as *mut _ as *mut _, std::mem::size_of_val(&id_info) as u32, @@ -320,7 +320,7 @@ pub mod windows { .get_or_init(|| { let library_name = OsString::from("api-ms-win-core-file-l2-1-4").to_wide_with_nul(); let module = unsafe { LoadLibraryW(library_name.as_ptr()) }; - if module == 0 { + if module == std::ptr::null_mut() { return None; } let name = CString::new("GetFileInformationByName").unwrap(); diff --git a/stdlib/Cargo.toml b/stdlib/Cargo.toml index f2ace7b26a..d7552844be 100644 --- a/stdlib/Cargo.toml +++ b/stdlib/Cargo.toml @@ -125,6 +125,7 @@ features = [ "Win32_NetworkManagement_Ndis", "Win32_Security_Cryptography", "Win32_System_Environment", + "Win32_System_IO" ] [target.'cfg(target_os = "macos")'.dependencies] diff --git a/stdlib/src/overlapped.rs b/stdlib/src/overlapped.rs index f40494a432..d3c46cad0a 100644 --- a/stdlib/src/overlapped.rs +++ b/stdlib/src/overlapped.rs @@ -24,13 +24,17 @@ mod _overlapped { use windows_sys::Win32::{ Foundation::{ ERROR_IO_PENDING, ERROR_NETNAME_DELETED, ERROR_OPERATION_ABORTED, ERROR_PIPE_BUSY, - ERROR_PORT_UNREACHABLE, ERROR_SEM_TIMEOUT, INVALID_HANDLE_VALUE, + ERROR_PORT_UNREACHABLE, ERROR_SEM_TIMEOUT, }, Networking::WinSock::{ SO_UPDATE_ACCEPT_CONTEXT, SO_UPDATE_CONNECT_CONTEXT, TF_REUSE_SOCKET, }, System::Threading::INFINITE, }; + #[pyattr(once)] + fn INVALID_HANDLE_VALUE(_vm: &VirtualMachine) -> isize { + windows_sys::Win32::Foundation::INVALID_HANDLE_VALUE as isize + } #[pyattr] const NULL: isize = 0; @@ -126,7 +130,7 @@ mod _overlapped { fn mark_as_completed(ov: &mut OVERLAPPED) { ov.Internal = 0; - if ov.hEvent != 0 { + if ov.hEvent != std::ptr::null_mut() { unsafe { windows_sys::Win32::System::Threading::SetEvent(ov.hEvent) }; } } @@ -164,7 +168,7 @@ mod _overlapped { fn WSARecv_inner( inner: &mut OverlappedInner, - handle: HANDLE, + handle: isize, buf: &[u8], mut flags: u32, vm: &VirtualMachine, @@ -209,7 +213,7 @@ mod _overlapped { #[pymethod] fn WSARecv( zelf: &Py, - handle: HANDLE, + handle: isize, size: u32, flags: u32, vm: &VirtualMachine, @@ -224,9 +228,9 @@ mod _overlapped { let buf = vec![0u8; std::cmp::max(size, 1) as usize]; let buf = vm.ctx.new_bytes(buf); - inner.handle = handle; + inner.handle = handle as _; - let r = Self::WSARecv_inner(&mut inner, handle, buf.as_bytes(), flags, vm); + let r = Self::WSARecv_inner(&mut inner, handle as _, buf.as_bytes(), flags, vm); inner.data = OverlappedData::Read(buf); r } @@ -256,30 +260,30 @@ mod _overlapped { } impl Constructor for Overlapped { - type Args = (HANDLE,); + type Args = (isize,); fn py_new(cls: PyTypeRef, (mut event,): Self::Args, vm: &VirtualMachine) -> PyResult { - if event == INVALID_HANDLE_VALUE { + if event as isize == INVALID_HANDLE_VALUE as isize { event = unsafe { windows_sys::Win32::System::Threading::CreateEventA( std::ptr::null(), Foundation::TRUE, Foundation::FALSE, std::ptr::null(), - ) + ) as isize }; - if event == NULL { + if event as isize == NULL { return Err(errno_err(vm)); } } let mut overlapped: OVERLAPPED = unsafe { std::mem::zeroed() }; if event != NULL { - overlapped.hEvent = event; + overlapped.hEvent = event as _; } let inner = OverlappedInner { overlapped, - handle: NULL, + handle: NULL as _, error: 0, data: OverlappedData::None, }; @@ -292,29 +296,29 @@ mod _overlapped { #[pyfunction] fn CreateIoCompletionPort( - handle: HANDLE, - port: HANDLE, + handle: isize, + port: isize, key: usize, concurrency: u32, vm: &VirtualMachine, - ) -> PyResult { + ) -> PyResult { let r = unsafe { - windows_sys::Win32::System::IO::CreateIoCompletionPort(handle, port, key, concurrency) + windows_sys::Win32::System::IO::CreateIoCompletionPort(handle as _, port as _, key, concurrency) as isize }; - if r == 0 { + if r as usize == 0 { return Err(errno_err(vm)); } Ok(r) } #[pyfunction] - fn GetQueuedCompletionStatus(port: HANDLE, msecs: u32, vm: &VirtualMachine) -> PyResult { + fn GetQueuedCompletionStatus(port: isize, msecs: u32, vm: &VirtualMachine) -> PyResult { let mut bytes_transferred = 0; let mut completion_key = 0; let mut overlapped: *mut OVERLAPPED = std::ptr::null_mut(); let ret = unsafe { windows_sys::Win32::System::IO::GetQueuedCompletionStatus( - port, + port as _, &mut bytes_transferred, &mut completion_key, &mut overlapped, diff --git a/stdlib/src/socket.rs b/stdlib/src/socket.rs index 149408a6f3..25d8dcf5aa 100644 --- a/stdlib/src/socket.rs +++ b/stdlib/src/socket.rs @@ -36,11 +36,7 @@ mod _socket { #[cfg(windows)] mod c { pub use windows_sys::Win32::NetworkManagement::IpHelper::{if_indextoname, if_nametoindex}; - - pub const INADDR_ANY: u32 = 0x00000000; - pub const INADDR_LOOPBACK: u32 = 0x7f000001; - pub const INADDR_BROADCAST: u32 = 0xffffffff; - pub const INADDR_NONE: u32 = 0xffffffff; + pub use windows_sys::Win32::Networking::WinSock::{INADDR_ANY, INADDR_LOOPBACK, INADDR_BROADCAST, INADDR_NONE}; pub use windows_sys::Win32::Networking::WinSock::{ AF_APPLETALK, AF_DECnet, AF_IPX, AF_LINK, AI_ADDRCONFIG, AI_ALL, AI_CANONNAME, @@ -2135,7 +2131,7 @@ mod _socket { #[cfg(all(unix, not(target_os = "redox")))] type IfIndex = c::c_uint; #[cfg(windows)] - type IfIndex = u32; // NET_IFINDEX but windows-sys 0.59 doesn't have it + type IfIndex = u32; // NET_IFINDEX but windows-sys 0.59 doesn't have it #[cfg(not(target_os = "redox"))] #[pyfunction] diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 330a2beab5..a0e357baa9 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -144,6 +144,7 @@ features = [ "Win32_System_SystemInformation", "Win32_System_SystemServices", "Win32_System_Threading", + "Win32_System_WindowsProgramming", "Win32_UI_Shell", "Win32_UI_WindowsAndMessaging", ] diff --git a/vm/src/stdlib/nt.rs b/vm/src/stdlib/nt.rs index 34fa8792d5..bf7dca8b05 100644 --- a/vm/src/stdlib/nt.rs +++ b/vm/src/stdlib/nt.rs @@ -150,7 +150,7 @@ pub(crate) mod module { } let h = unsafe { Threading::OpenProcess(Threading::PROCESS_ALL_ACCESS, 0, pid) }; - if h == 0 { + if h == std::ptr::null_mut() { return Err(errno_err(vm)); } let ret = unsafe { Threading::TerminateProcess(h, sig) }; @@ -172,7 +172,7 @@ pub(crate) mod module { _ => return Err(vm.new_value_error("bad file descriptor".to_owned())), }; let h = unsafe { Console::GetStdHandle(stdhandle) }; - if h == 0 { + if h == std::ptr::null_mut() { return Err(vm.new_os_error("handle cannot be retrieved".to_owned())); } if h == INVALID_HANDLE_VALUE { diff --git a/vm/src/stdlib/winapi.rs b/vm/src/stdlib/winapi.rs index c1edb2739e..4a87a0f70f 100644 --- a/vm/src/stdlib/winapi.rs +++ b/vm/src/stdlib/winapi.rs @@ -79,7 +79,7 @@ mod _winapi { #[pyfunction] fn CloseHandle(handle: HANDLE) -> WindowsSysResult { - WindowsSysResult(unsafe { windows_sys::Win32::Foundation::CloseHandle(handle.0) }) + WindowsSysResult(unsafe { windows_sys::Win32::Foundation::CloseHandle(handle.0 as _) }) } #[pyfunction] @@ -99,8 +99,8 @@ mod _winapi { let mut read = std::mem::MaybeUninit::::uninit(); let mut write = std::mem::MaybeUninit::::uninit(); WindowsSysResult(windows_sys::Win32::System::Pipes::CreatePipe( - read.as_mut_ptr(), - write.as_mut_ptr(), + read.as_mut_ptr() as _, + write.as_mut_ptr() as _, std::ptr::null(), size, )) @@ -122,10 +122,10 @@ mod _winapi { let target = unsafe { let mut target = std::mem::MaybeUninit::::uninit(); WindowsSysResult(windows_sys::Win32::Foundation::DuplicateHandle( - src_process.0, - src.0, - target_process.0, - target.as_mut_ptr(), + src_process.0 as _, + src.0 as _, + target_process.0 as _, + target.as_mut_ptr() as _, access, inherit, options.unwrap_or(0), @@ -151,7 +151,7 @@ mod _winapi { h: HANDLE, vm: &VirtualMachine, ) -> PyResult { - let file_type = unsafe { windows_sys::Win32::Storage::FileSystem::GetFileType(h.0) }; + let file_type = unsafe { windows_sys::Win32::Storage::FileSystem::GetFileType(h.0 as _) }; if file_type == 0 && unsafe { windows_sys::Win32::Foundation::GetLastError() } != 0 { Err(errno_err(vm)) } else { @@ -274,8 +274,8 @@ mod _winapi { }; Ok(( - HANDLE(procinfo.hProcess), - HANDLE(procinfo.hThread), + HANDLE(procinfo.hProcess as _), + HANDLE(procinfo.hThread as _), procinfo.dwProcessId, procinfo.dwThreadId, )) @@ -286,13 +286,13 @@ mod _winapi { desired_access: u32, inherit_handle: bool, process_id: u32, - ) -> windows_sys::Win32::Foundation::HANDLE { + ) -> isize { unsafe { windows_sys::Win32::System::Threading::OpenProcess( desired_access, BOOL::from(inherit_handle), process_id, - ) + ) as _ } } @@ -438,7 +438,7 @@ mod _winapi { #[pyfunction] fn WaitForSingleObject(h: HANDLE, ms: u32, vm: &VirtualMachine) -> PyResult { - let ret = unsafe { windows_sys::Win32::System::Threading::WaitForSingleObject(h.0, ms) }; + let ret = unsafe { windows_sys::Win32::System::Threading::WaitForSingleObject(h.0 as _, ms) }; if ret == windows_sys::Win32::Foundation::WAIT_FAILED { Err(errno_err(vm)) } else { @@ -451,7 +451,7 @@ mod _winapi { unsafe { let mut ec = std::mem::MaybeUninit::uninit(); WindowsSysResult(windows_sys::Win32::System::Threading::GetExitCodeProcess( - h.0, + h.0 as _, ec.as_mut_ptr(), )) .to_pyresult(vm)?; @@ -462,7 +462,7 @@ mod _winapi { #[pyfunction] fn TerminateProcess(h: HANDLE, exit_code: u32) -> WindowsSysResult { WindowsSysResult(unsafe { - windows_sys::Win32::System::Threading::TerminateProcess(h.0, exit_code) + windows_sys::Win32::System::Threading::TerminateProcess(h.0 as _, exit_code) }) } @@ -507,11 +507,11 @@ mod _winapi { // if handle.is_invalid() { // return Err(errno_err(vm)); // } - Ok(handle) + Ok(handle as _) } #[pyfunction] fn ReleaseMutex(handle: isize) -> WindowsSysResult { - WindowsSysResult(unsafe { windows_sys::Win32::System::Threading::ReleaseMutex(handle) }) + WindowsSysResult(unsafe { windows_sys::Win32::System::Threading::ReleaseMutex(handle as _) }) } } diff --git a/vm/src/windows.rs b/vm/src/windows.rs index f4f4dad0b3..a14216768e 100644 --- a/vm/src/windows.rs +++ b/vm/src/windows.rs @@ -23,7 +23,7 @@ impl WindowsSysResultValue for RAW_HANDLE { *self == INVALID_HANDLE_VALUE } fn into_ok(self) -> Self::Ok { - HANDLE(self) + HANDLE(self as _) } } From ddf2e591c6f2933db520f265ec727c87769f76ec Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Wed, 5 Mar 2025 10:52:31 -0800 Subject: [PATCH 068/295] resolve comments --- stdlib/src/socket.rs | 33 +++++++-------------------------- 1 file changed, 7 insertions(+), 26 deletions(-) diff --git a/stdlib/src/socket.rs b/stdlib/src/socket.rs index 25d8dcf5aa..39c21da544 100644 --- a/stdlib/src/socket.rs +++ b/stdlib/src/socket.rs @@ -1757,15 +1757,12 @@ mod _socket { .map(|s| s.to_cstring(vm)) .transpose()?; let cstr_proto = cstr_opt_as_ptr(&cstr_proto); - #[cfg(windows)] let serv = unsafe { c::getservbyname( - cstr_name.as_ptr() as windows_sys::core::PCSTR, - cstr_proto as windows_sys::core::PCSTR, + cstr_name.as_ptr() as _, + cstr_proto as _, ) }; - #[cfg(not(windows))] - let serv = unsafe { c::getservbyname(cstr_name.as_ptr(), cstr_proto) }; if serv.is_null() { return Err(vm.new_os_error("service/proto not found".to_owned())); } @@ -1787,18 +1784,11 @@ mod _socket { .map(|s| s.to_cstring(vm)) .transpose()?; let cstr_proto = cstr_opt_as_ptr(&cstr_proto); - #[cfg(windows)] - let serv = - unsafe { c::getservbyport(port.to_be() as _, cstr_proto as windows_sys::core::PCSTR) }; - #[cfg(not(windows))] - let serv = unsafe { c::getservbyport(port.to_be() as _, cstr_proto) }; + let serv = unsafe { c::getservbyport(port.to_be() as _, cstr_proto as _) }; if serv.is_null() { return Err(vm.new_os_error("port/proto not found".to_owned())); } - #[cfg(windows)] - let s = unsafe { ffi::CStr::from_ptr((*serv).s_name as *const i8) }; - #[cfg(not(windows))] - let s = unsafe { ffi::CStr::from_ptr((*serv).s_name) }; + let s = unsafe { ffi::CStr::from_ptr((*serv).s_name as _) }; Ok(s.to_string_lossy().into_owned()) } @@ -2050,10 +2040,7 @@ mod _socket { #[pyfunction] fn getprotobyname(name: PyStrRef, vm: &VirtualMachine) -> PyResult { let cstr = name.to_cstring(vm)?; - #[cfg(windows)] - let proto = unsafe { c::getprotobyname(cstr.as_ptr() as *const u8) }; - #[cfg(not(windows))] - let proto = unsafe { c::getprotobyname(cstr.as_ptr()) }; + let proto = unsafe { c::getprotobyname(cstr.as_ptr() as _) }; if proto.is_null() { return Err(vm.new_os_error("protocol not found".to_owned())); } @@ -2138,10 +2125,7 @@ mod _socket { fn if_nametoindex(name: FsPath, vm: &VirtualMachine) -> PyResult { let name = name.to_cstring(vm)?; - #[cfg(windows)] - let ret = unsafe { c::if_nametoindex(name.as_ptr() as *const u8) }; - #[cfg(not(windows))] - let ret = unsafe { c::if_nametoindex(name.as_ptr()) }; + let ret = unsafe { c::if_nametoindex(name.as_ptr() as _) }; if ret == 0 { Err(vm.new_os_error("no interface with this name".to_owned())) } else { @@ -2157,10 +2141,7 @@ mod _socket { if ret.is_null() { Err(crate::vm::stdlib::os::errno_err(vm)) } else { - #[cfg(windows)] - let buf = unsafe { ffi::CStr::from_ptr(buf.as_ptr() as *const i8) }; - #[cfg(not(windows))] - let buf = unsafe { ffi::CStr::from_ptr(buf.as_ptr()) }; + let buf = unsafe { ffi::CStr::from_ptr(buf.as_ptr() as _) }; Ok(buf.to_string_lossy().into_owned()) } } From b4929d258da5bbe8d74873b978c552b39702851d Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Wed, 5 Mar 2025 11:07:20 -0800 Subject: [PATCH 069/295] formatting --- common/src/fileutils.rs | 2 +- stdlib/src/overlapped.rs | 7 ++++++- stdlib/src/socket.rs | 11 ++++------- vm/src/stdlib/winapi.rs | 15 +++++++-------- 4 files changed, 18 insertions(+), 17 deletions(-) diff --git a/common/src/fileutils.rs b/common/src/fileutils.rs index ce4aa8250a..cabeb833c1 100644 --- a/common/src/fileutils.rs +++ b/common/src/fileutils.rs @@ -153,7 +153,7 @@ pub mod windows { let p_id_info = if unsafe { GetFileInformationByHandleEx( - h as _, + h as _, FileIdInfo, &mut id_info as *mut _ as *mut _, std::mem::size_of_val(&id_info) as u32, diff --git a/stdlib/src/overlapped.rs b/stdlib/src/overlapped.rs index d3c46cad0a..8877e5befb 100644 --- a/stdlib/src/overlapped.rs +++ b/stdlib/src/overlapped.rs @@ -303,7 +303,12 @@ mod _overlapped { vm: &VirtualMachine, ) -> PyResult { let r = unsafe { - windows_sys::Win32::System::IO::CreateIoCompletionPort(handle as _, port as _, key, concurrency) as isize + windows_sys::Win32::System::IO::CreateIoCompletionPort( + handle as _, + port as _, + key, + concurrency, + ) as isize }; if r as usize == 0 { return Err(errno_err(vm)); diff --git a/stdlib/src/socket.rs b/stdlib/src/socket.rs index 39c21da544..39bfde4bee 100644 --- a/stdlib/src/socket.rs +++ b/stdlib/src/socket.rs @@ -36,7 +36,9 @@ mod _socket { #[cfg(windows)] mod c { pub use windows_sys::Win32::NetworkManagement::IpHelper::{if_indextoname, if_nametoindex}; - pub use windows_sys::Win32::Networking::WinSock::{INADDR_ANY, INADDR_LOOPBACK, INADDR_BROADCAST, INADDR_NONE}; + pub use windows_sys::Win32::Networking::WinSock::{ + INADDR_ANY, INADDR_BROADCAST, INADDR_LOOPBACK, INADDR_NONE, + }; pub use windows_sys::Win32::Networking::WinSock::{ AF_APPLETALK, AF_DECnet, AF_IPX, AF_LINK, AI_ADDRCONFIG, AI_ALL, AI_CANONNAME, @@ -1757,12 +1759,7 @@ mod _socket { .map(|s| s.to_cstring(vm)) .transpose()?; let cstr_proto = cstr_opt_as_ptr(&cstr_proto); - let serv = unsafe { - c::getservbyname( - cstr_name.as_ptr() as _, - cstr_proto as _, - ) - }; + let serv = unsafe { c::getservbyname(cstr_name.as_ptr() as _, cstr_proto as _) }; if serv.is_null() { return Err(vm.new_os_error("service/proto not found".to_owned())); } diff --git a/vm/src/stdlib/winapi.rs b/vm/src/stdlib/winapi.rs index 4a87a0f70f..7ffc4227e6 100644 --- a/vm/src/stdlib/winapi.rs +++ b/vm/src/stdlib/winapi.rs @@ -275,18 +275,14 @@ mod _winapi { Ok(( HANDLE(procinfo.hProcess as _), - HANDLE(procinfo.hThread as _), + HANDLE(procinfo.hThread as _), procinfo.dwProcessId, procinfo.dwThreadId, )) } #[pyfunction] - fn OpenProcess( - desired_access: u32, - inherit_handle: bool, - process_id: u32, - ) -> isize { + fn OpenProcess(desired_access: u32, inherit_handle: bool, process_id: u32) -> isize { unsafe { windows_sys::Win32::System::Threading::OpenProcess( desired_access, @@ -438,7 +434,8 @@ mod _winapi { #[pyfunction] fn WaitForSingleObject(h: HANDLE, ms: u32, vm: &VirtualMachine) -> PyResult { - let ret = unsafe { windows_sys::Win32::System::Threading::WaitForSingleObject(h.0 as _, ms) }; + let ret = + unsafe { windows_sys::Win32::System::Threading::WaitForSingleObject(h.0 as _, ms) }; if ret == windows_sys::Win32::Foundation::WAIT_FAILED { Err(errno_err(vm)) } else { @@ -512,6 +509,8 @@ mod _winapi { #[pyfunction] fn ReleaseMutex(handle: isize) -> WindowsSysResult { - WindowsSysResult(unsafe { windows_sys::Win32::System::Threading::ReleaseMutex(handle as _) }) + WindowsSysResult(unsafe { + windows_sys::Win32::System::Threading::ReleaseMutex(handle as _) + }) } } From d2bf31724f05ad1f6b32c2fa623ab2e111cc89ea Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Wed, 5 Mar 2025 11:15:42 -0800 Subject: [PATCH 070/295] fix clippy --- common/src/fileutils.rs | 2 +- stdlib/src/overlapped.rs | 6 +++--- vm/src/stdlib/nt.rs | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/common/src/fileutils.rs b/common/src/fileutils.rs index cabeb833c1..67713c0148 100644 --- a/common/src/fileutils.rs +++ b/common/src/fileutils.rs @@ -320,7 +320,7 @@ pub mod windows { .get_or_init(|| { let library_name = OsString::from("api-ms-win-core-file-l2-1-4").to_wide_with_nul(); let module = unsafe { LoadLibraryW(library_name.as_ptr()) }; - if module == std::ptr::null_mut() { + if module.is_null() { return None; } let name = CString::new("GetFileInformationByName").unwrap(); diff --git a/stdlib/src/overlapped.rs b/stdlib/src/overlapped.rs index 8877e5befb..234e3be53a 100644 --- a/stdlib/src/overlapped.rs +++ b/stdlib/src/overlapped.rs @@ -130,7 +130,7 @@ mod _overlapped { fn mark_as_completed(ov: &mut OVERLAPPED) { ov.Internal = 0; - if ov.hEvent != std::ptr::null_mut() { + if !ov.hEvent.is_null() { unsafe { windows_sys::Win32::System::Threading::SetEvent(ov.hEvent) }; } } @@ -263,7 +263,7 @@ mod _overlapped { type Args = (isize,); fn py_new(cls: PyTypeRef, (mut event,): Self::Args, vm: &VirtualMachine) -> PyResult { - if event as isize == INVALID_HANDLE_VALUE as isize { + if event == INVALID_HANDLE_VALUE(vm) { event = unsafe { windows_sys::Win32::System::Threading::CreateEventA( std::ptr::null(), @@ -272,7 +272,7 @@ mod _overlapped { std::ptr::null(), ) as isize }; - if event as isize == NULL { + if event == NULL { return Err(errno_err(vm)); } } diff --git a/vm/src/stdlib/nt.rs b/vm/src/stdlib/nt.rs index bf7dca8b05..ecc63e0aa6 100644 --- a/vm/src/stdlib/nt.rs +++ b/vm/src/stdlib/nt.rs @@ -150,7 +150,7 @@ pub(crate) mod module { } let h = unsafe { Threading::OpenProcess(Threading::PROCESS_ALL_ACCESS, 0, pid) }; - if h == std::ptr::null_mut() { + if h.is_null() { return Err(errno_err(vm)); } let ret = unsafe { Threading::TerminateProcess(h, sig) }; @@ -172,7 +172,7 @@ pub(crate) mod module { _ => return Err(vm.new_value_error("bad file descriptor".to_owned())), }; let h = unsafe { Console::GetStdHandle(stdhandle) }; - if h == std::ptr::null_mut() { + if h.is_null() { return Err(vm.new_os_error("handle cannot be retrieved".to_owned())); } if h == INVALID_HANDLE_VALUE { From 7fea1e1b4a92bec3a9addc4e2e64e161a8b6a0c7 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Fri, 28 Feb 2025 21:46:28 -0800 Subject: [PATCH 071/295] fix what is left data upload to website and trigger cron-ci on workflow update --- .github/workflows/cron-ci.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cron-ci.yaml b/.github/workflows/cron-ci.yaml index 4b8e701ced..b5590f145f 100644 --- a/.github/workflows/cron-ci.yaml +++ b/.github/workflows/cron-ci.yaml @@ -2,6 +2,9 @@ on: schedule: - cron: '0 0 * * 6' workflow_dispatch: + push: + paths: + - .github/workflows/cron-ci.yaml name: Periodic checks/tasks @@ -97,8 +100,8 @@ jobs: cd website [ -f ./_data/whats_left.temp ] && cp ./_data/whats_left.temp ./_data/whats_left_lastrun.temp cp ../whats_left.temp ./_data/whats_left.temp - rm _data/whats_left/modules.csv - cat _data/whats_left.temp | grep "(entire module)" | cut -d ' ' -f 1 | sort >> ../_data/whats_left/modules.csv + rm ./_data/whats_left/modules.csv + cat ./_data/whats_left.temp | grep "(entire module)" | cut -d ' ' -f 1 | sort >> ./_data/whats_left/modules.csv git add -A if git -c user.name="Github Actions" -c user.email="actions@github.com" commit -m "Update what is left results" --author="$GITHUB_ACTOR"; then From 58ebf04bac4a29359ecccbec930072e4146bd4d9 Mon Sep 17 00:00:00 2001 From: Daniel O'Hear <149127239+dohear@users.noreply.github.com> Date: Wed, 5 Mar 2025 20:41:45 +0000 Subject: [PATCH 072/295] Add JIT compilation support for integer multiplication, division, and exponents (#5561) * Initial commit for power function * Float power jit developed * Addded support for Floats and Ints * Integration Testing for JITPower implementation. * Update instructions.rs cranelift more like painlift * Update instructions.rs * Update instructions.rs * initial commit for making stable PR ready features * fixed final edge case for compile_ipow * fixed final edge case for compile_ipow * commenting out compile_ipow * fixed spelling errors * removed unused tests * forgot to run clippy --------- Co-authored-by: Nicholas Paulick Co-authored-by: Nick Co-authored-by: JoeLoparco Co-authored-by: Nathan Rusch Co-authored-by: dohear --- jit/src/instructions.rs | 163 ++++++++++++++++++++++++++++++++++++++++ jit/tests/int_tests.rs | 70 ++++++++++++++++- 2 files changed, 232 insertions(+), 1 deletion(-) diff --git a/jit/src/instructions.rs b/jit/src/instructions.rs index b495c6e1f6..1b74760dc0 100644 --- a/jit/src/instructions.rs +++ b/jit/src/instructions.rs @@ -425,12 +425,25 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> { (BinaryOperator::Subtract, JitValue::Int(a), JitValue::Int(b)) => { JitValue::Int(self.compile_sub(a, b)) } + (BinaryOperator::Multiply, JitValue::Int(a), JitValue::Int(b)) => { + JitValue::Int(self.builder.ins().imul(a, b)) + } (BinaryOperator::FloorDivide, JitValue::Int(a), JitValue::Int(b)) => { JitValue::Int(self.builder.ins().sdiv(a, b)) } + (BinaryOperator::Divide, JitValue::Int(a), JitValue::Int(b)) => { + // Convert to float for regular division + let a_float = self.builder.ins().fcvt_from_sint(types::F64, a); + let b_float = self.builder.ins().fcvt_from_sint(types::F64, b); + JitValue::Float(self.builder.ins().fdiv(a_float, b_float)) + } (BinaryOperator::Modulo, JitValue::Int(a), JitValue::Int(b)) => { JitValue::Int(self.builder.ins().srem(a, b)) } + // Todo: This should return int when possible + (BinaryOperator::Power, JitValue::Int(a), JitValue::Int(b)) => { + JitValue::Float(self.compile_ipow(a, b)) + } ( BinaryOperator::Lshift | BinaryOperator::Rshift, JitValue::Int(a), @@ -562,4 +575,154 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> { .trapif(IntCC::Overflow, carry, TrapCode::IntegerOverflow); out } + fn compile_ipow(&mut self, a: Value, b: Value) -> Value { + // Convert base to float since result might not always be a Int + let float_base = self.builder.ins().fcvt_from_sint(types::F64, a); + + // Create code blocks + let check_block1 = self.builder.create_block(); + let check_block2 = self.builder.create_block(); + let check_block3 = self.builder.create_block(); + let handle_neg_exp = self.builder.create_block(); + let loop_block = self.builder.create_block(); + let continue_block = self.builder.create_block(); + let exit_block = self.builder.create_block(); + + // Set code block params + // Set code block params + self.builder.append_block_param(check_block1, types::F64); + self.builder.append_block_param(check_block1, types::I64); + + self.builder.append_block_param(check_block2, types::F64); + self.builder.append_block_param(check_block2, types::I64); + + self.builder.append_block_param(check_block3, types::F64); + self.builder.append_block_param(check_block3, types::I64); + + self.builder.append_block_param(handle_neg_exp, types::F64); + self.builder.append_block_param(handle_neg_exp, types::I64); + + self.builder.append_block_param(loop_block, types::F64); //base + self.builder.append_block_param(loop_block, types::F64); //result + self.builder.append_block_param(loop_block, types::I64); //exponent + + self.builder.append_block_param(continue_block, types::F64); //base + self.builder.append_block_param(continue_block, types::F64); //result + self.builder.append_block_param(continue_block, types::I64); //exponent + + self.builder.append_block_param(exit_block, types::F64); + + // Begin evaluating by jumping to first check block + self.builder.ins().jump(check_block1, &[float_base, b]); + + // Check block one: + // Checks if input is O ** n where n > 0 + // Jumps to exit_block as 0 if true + self.builder.switch_to_block(check_block1); + let paramsc1 = self.builder.block_params(check_block1); + let basec1 = paramsc1[0]; + let expc1 = paramsc1[1]; + let zero_f64 = self.builder.ins().f64const(0.0); + let zero_i64 = self.builder.ins().iconst(types::I64, 0); + let is_base_zero = self.builder.ins().fcmp(FloatCC::Equal, zero_f64, basec1); + let is_exp_positive = self + .builder + .ins() + .icmp(IntCC::SignedGreaterThan, expc1, zero_i64); + let is_zero_to_positive = self.builder.ins().band(is_base_zero, is_exp_positive); + self.builder + .ins() + .brnz(is_zero_to_positive, exit_block, &[zero_f64]); + self.builder.ins().jump(check_block2, &[basec1, expc1]); + + // Check block two: + // Checks if exponent is negative + // Jumps to a special handle_neg_exponent block if true + self.builder.switch_to_block(check_block2); + let paramsc2 = self.builder.block_params(check_block2); + let basec2 = paramsc2[0]; + let expc2 = paramsc2[1]; + let zero_i64 = self.builder.ins().iconst(types::I64, 0); + let is_neg = self + .builder + .ins() + .icmp(IntCC::SignedLessThan, expc2, zero_i64); + self.builder + .ins() + .brnz(is_neg, handle_neg_exp, &[basec2, expc2]); + self.builder.ins().jump(check_block3, &[basec2, expc2]); + + // Check block three: + // Checks if exponent is one + // jumps to exit block with the base of the exponents value + self.builder.switch_to_block(check_block3); + let paramsc3 = self.builder.block_params(check_block3); + let basec3 = paramsc3[0]; + let expc3 = paramsc3[1]; + let resc3 = self.builder.ins().f64const(1.0); + let one_i64 = self.builder.ins().iconst(types::I64, 1); + let is_one = self.builder.ins().icmp(IntCC::Equal, expc3, one_i64); + self.builder.ins().brnz(is_one, exit_block, &[basec3]); + self.builder.ins().jump(loop_block, &[basec3, resc3, expc3]); + + // Handles negative Exponents + // calculates x^(-n) = (1/x)^n + // then proceeds to the loop to evaluate + self.builder.switch_to_block(handle_neg_exp); + let paramshn = self.builder.block_params(handle_neg_exp); + let basehn = paramshn[0]; + let exphn = paramshn[1]; + let one_f64 = self.builder.ins().f64const(1.0); + let base_inverse = self.builder.ins().fdiv(one_f64, basehn); + let pos_exp = self.builder.ins().ineg(exphn); + self.builder + .ins() + .jump(loop_block, &[base_inverse, one_f64, pos_exp]); + + // Main loop block + // checks loop condition (exp > 0) + // Jumps to continue block if true, exit block if false + self.builder.switch_to_block(loop_block); + let paramslb = self.builder.block_params(loop_block); + let baselb = paramslb[0]; + let reslb = paramslb[1]; + let explb = paramslb[2]; + let zero = self.builder.ins().iconst(types::I64, 0); + let is_zero = self.builder.ins().icmp(IntCC::Equal, explb, zero); + self.builder.ins().brnz(is_zero, exit_block, &[reslb]); + self.builder + .ins() + .jump(continue_block, &[baselb, reslb, explb]); + + // Continue block + // Main math logic + // Always jumps back to loob_block + self.builder.switch_to_block(continue_block); + let paramscb = self.builder.block_params(continue_block); + let basecb = paramscb[0]; + let rescb = paramscb[1]; + let expcb = paramscb[2]; + let is_odd = self.builder.ins().band_imm(expcb, 1); + let is_odd = self.builder.ins().icmp_imm(IntCC::Equal, is_odd, 1); + let mul_result = self.builder.ins().fmul(rescb, basecb); + let new_result = self.builder.ins().select(is_odd, mul_result, rescb); + let squared_base = self.builder.ins().fmul(basecb, basecb); + let new_exp = self.builder.ins().sshr_imm(expcb, 1); + self.builder + .ins() + .jump(loop_block, &[squared_base, new_result, new_exp]); + + self.builder.switch_to_block(exit_block); + let result = self.builder.block_params(exit_block)[0]; + + self.builder.seal_block(check_block1); + self.builder.seal_block(check_block2); + self.builder.seal_block(check_block3); + self.builder.seal_block(handle_neg_exp); + self.builder.seal_block(loop_block); + self.builder.seal_block(continue_block); + self.builder.seal_block(exit_block); + + result + } } diff --git a/jit/tests/int_tests.rs b/jit/tests/int_tests.rs index 9ce3f3b4a6..353052df00 100644 --- a/jit/tests/int_tests.rs +++ b/jit/tests/int_tests.rs @@ -1,3 +1,5 @@ +use core::f64; + #[test] fn test_add() { let add = jit_function! { add(a:i64, b:i64) -> i64 => r##" @@ -23,6 +25,51 @@ fn test_sub() { assert_eq!(sub(-3, -10), Ok(7)); } +#[test] +fn test_mul() { + let mul = jit_function! { mul(a:i64, b:i64) -> i64 => r##" + def mul(a: int, b: int): + return a * b + "## }; + + assert_eq!(mul(5, 10), Ok(50)); + assert_eq!(mul(0, 5), Ok(0)); + assert_eq!(mul(5, 0), Ok(0)); + assert_eq!(mul(0, 0), Ok(0)); + assert_eq!(mul(-5, 10), Ok(-50)); + assert_eq!(mul(5, -10), Ok(-50)); + assert_eq!(mul(-5, -10), Ok(50)); + assert_eq!(mul(999999, 999999), Ok(999998000001)); + assert_eq!(mul(i64::MAX, 1), Ok(i64::MAX)); + assert_eq!(mul(1, i64::MAX), Ok(i64::MAX)); +} + +#[test] + +fn test_div() { + let div = jit_function! { div(a:i64, b:i64) -> f64 => r##" + def div(a: int, b: int): + return a / b + "## }; + + assert_eq!(div(0, 1), Ok(0.0)); + assert_eq!(div(5, 1), Ok(5.0)); + assert_eq!(div(5, 10), Ok(0.5)); + assert_eq!(div(5, 2), Ok(2.5)); + assert_eq!(div(12, 10), Ok(1.2)); + assert_eq!(div(7, 10), Ok(0.7)); + assert_eq!(div(-3, -1), Ok(3.0)); + assert_eq!(div(-3, 1), Ok(-3.0)); + assert_eq!(div(1, 1000), Ok(0.001)); + assert_eq!(div(1, 100000), Ok(0.00001)); + assert_eq!(div(2, 3), Ok(0.6666666666666666)); + assert_eq!(div(1, 3), Ok(0.3333333333333333)); + assert_eq!(div(i64::MAX, 2), Ok(4611686018427387904.0)); + assert_eq!(div(i64::MIN, 2), Ok(-4611686018427387904.0)); + assert_eq!(div(i64::MIN, -1), Ok(9223372036854775808.0)); // Overflow case + assert_eq!(div(i64::MIN, i64::MAX), Ok(-1.0)); +} + #[test] fn test_floor_div() { let floor_div = jit_function! { floor_div(a:i64, b:i64) -> i64 => r##" @@ -35,7 +82,28 @@ fn test_floor_div() { assert_eq!(floor_div(12, 10), Ok(1)); assert_eq!(floor_div(7, 10), Ok(0)); assert_eq!(floor_div(-3, -1), Ok(3)); - assert_eq!(floor_div(-3, 1), Ok(-3)); +} + +#[test] + +fn test_exp() { + let exp = jit_function! { exp(a: i64, b: i64) -> f64 => r##" + def exp(a: int, b: int): + return a ** b + "## }; + + assert_eq!(exp(2, 3), Ok(8.0)); + assert_eq!(exp(3, 2), Ok(9.0)); + assert_eq!(exp(5, 0), Ok(1.0)); + assert_eq!(exp(0, 0), Ok(1.0)); + assert_eq!(exp(-5, 0), Ok(1.0)); + assert_eq!(exp(0, 1), Ok(0.0)); + assert_eq!(exp(0, 5), Ok(0.0)); + assert_eq!(exp(-2, 2), Ok(4.0)); + assert_eq!(exp(-3, 4), Ok(81.0)); + assert_eq!(exp(-2, 3), Ok(-8.0)); + assert_eq!(exp(-3, 3), Ok(-27.0)); + assert_eq!(exp(1000, 2), Ok(1000000.0)); } #[test] From cc0a1ce9e269062da91086e15089cff53d422ecf Mon Sep 17 00:00:00 2001 From: Noa Date: Wed, 5 Mar 2025 16:15:14 -0600 Subject: [PATCH 073/295] Update webpack (#5585) * Update webpack * Build demo before notebook * Use with instead of env for actions-gh-pages --- .github/workflows/release.yml | 19 +++++++++++++------ wasm/notebook/package.json | 20 +++++++++----------- wasm/notebook/webpack.config.js | 12 +++++++----- 3 files changed, 29 insertions(+), 22 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 98b7de4823..4be6b4eaf0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -109,6 +109,13 @@ jobs: - uses: actions/setup-node@v4 - uses: mwilliamson/setup-wabt-action@v3 with: { wabt-version: "1.0.30" } + - name: build demo + run: | + npm install + npm run dist + env: + NODE_OPTIONS: "--openssl-legacy-provider" + working-directory: ./wasm/demo - name: build notebook demo run: | npm install @@ -119,11 +126,11 @@ jobs: working-directory: ./wasm/notebook - name: Deploy demo to Github Pages uses: peaceiris/actions-gh-pages@v4 - env: - ACTIONS_DEPLOY_KEY: ${{ secrets.ACTIONS_DEMO_DEPLOY_KEY }} - PUBLISH_DIR: ./wasm/demo/dist - EXTERNAL_REPOSITORY: RustPython/demo - PUBLISH_BRANCH: master + with: + deploy_key: ${{ secrets.ACTIONS_DEMO_DEPLOY_KEY }} + publish_dir: ./wasm/demo/dist + external_repository: RustPython/demo + publish_branch: master release: runs-on: ubuntu-latest @@ -161,4 +168,4 @@ jobs: --target="$tag" \ --generate-notes \ $PRERELEASE_ARG \ - bin/rustpython-release-* \ No newline at end of file + bin/rustpython-release-* diff --git a/wasm/notebook/package.json b/wasm/notebook/package.json index 2a730258cd..64517331c4 100644 --- a/wasm/notebook/package.json +++ b/wasm/notebook/package.json @@ -12,19 +12,17 @@ "xterm": "^3.8.0" }, "devDependencies": { - "@wasm-tool/wasm-pack-plugin": "^1.1.0", - "clean-webpack-plugin": "^3.0.0", - "css-loader": "^3.4.1", - "html-webpack-plugin": "^3.2.0", - "mini-css-extract-plugin": "^0.9.0", - "raw-loader": "^4.0.0", - "serve": "^11.0.2", - "webpack": "^4.16.3", - "webpack-cli": "^3.1.0", - "webpack-dev-server": "^3.1.5" + "@wasm-tool/wasm-pack-plugin": "^1.7.0", + "css-loader": "^7.1.2", + "html-webpack-plugin": "^5.6.3", + "lezer-loader": "^0.3.0", + "mini-css-extract-plugin": "^2.9.2", + "webpack": "^5.97.1", + "webpack-cli": "^6.0.1", + "webpack-dev-server": "^5.2.0" }, "scripts": { - "dev": "webpack-dev-server -d", + "dev": "webpack serve", "build": "webpack", "dist": "webpack --mode production", "test": "webpack --mode production && cd ../tests && pytest" diff --git a/wasm/notebook/webpack.config.js b/wasm/notebook/webpack.config.js index 9fda3cf4aa..ca19f3384a 100644 --- a/wasm/notebook/webpack.config.js +++ b/wasm/notebook/webpack.config.js @@ -1,7 +1,6 @@ const HtmlWebpackPlugin = require('html-webpack-plugin'); const MiniCssExtractPlugin = require('mini-css-extract-plugin'); const WasmPackPlugin = require('@wasm-tool/wasm-pack-plugin'); -const { CleanWebpackPlugin } = require('clean-webpack-plugin'); const path = require('path'); const fs = require('fs'); @@ -12,6 +11,7 @@ module.exports = (env = {}) => { output: { path: path.join(__dirname, 'dist'), filename: 'index.js', + clean: true, }, mode: 'development', resolve: { @@ -30,15 +30,14 @@ module.exports = (env = {}) => { }, { test: /\.(woff(2)?|ttf)$/, - use: { - loader: 'file-loader', - options: { name: 'fonts/[name].[ext]' }, + type: 'asset/resource', + generator: { + filename: 'fonts/[name].[ext]', }, }, ], }, plugins: [ - new CleanWebpackPlugin(), new HtmlWebpackPlugin({ filename: 'index.html', template: 'src/index.ejs', @@ -58,6 +57,9 @@ module.exports = (env = {}) => { filename: 'styles.css', }), ], + experiments: { + asyncWebAssembly: true, + }, }; if (!env.noWasmPack) { config.plugins.push( From 97853bf0f1edbef92fc758badf102564653ed73a Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Wed, 5 Mar 2025 20:20:04 -0800 Subject: [PATCH 074/295] Fix module.csv generation in cron ci (#5586) --- .github/workflows/cron-ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cron-ci.yaml b/.github/workflows/cron-ci.yaml index b5590f145f..5c14502643 100644 --- a/.github/workflows/cron-ci.yaml +++ b/.github/workflows/cron-ci.yaml @@ -101,8 +101,8 @@ jobs: [ -f ./_data/whats_left.temp ] && cp ./_data/whats_left.temp ./_data/whats_left_lastrun.temp cp ../whats_left.temp ./_data/whats_left.temp rm ./_data/whats_left/modules.csv + echo -e "modules\n" > ./_data/whats_left/modules.csv cat ./_data/whats_left.temp | grep "(entire module)" | cut -d ' ' -f 1 | sort >> ./_data/whats_left/modules.csv - git add -A if git -c user.name="Github Actions" -c user.email="actions@github.com" commit -m "Update what is left results" --author="$GITHUB_ACTOR"; then git push From 87fae150daafd8ab02bd7223f21d076d70ceaaae Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 9 Mar 2025 11:03:22 +0900 Subject: [PATCH 075/295] Replace pyattr(once) to constant --- stdlib/src/overlapped.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/stdlib/src/overlapped.rs b/stdlib/src/overlapped.rs index 234e3be53a..629461abe9 100644 --- a/stdlib/src/overlapped.rs +++ b/stdlib/src/overlapped.rs @@ -31,10 +31,11 @@ mod _overlapped { }, System::Threading::INFINITE, }; - #[pyattr(once)] - fn INVALID_HANDLE_VALUE(_vm: &VirtualMachine) -> isize { - windows_sys::Win32::Foundation::INVALID_HANDLE_VALUE as isize - } + + #[pyattr] + const INVALID_HANDLE_VALUE: isize = + unsafe { std::mem::transmute(windows_sys::Win32::Foundation::INVALID_HANDLE_VALUE) }; + #[pyattr] const NULL: isize = 0; @@ -263,7 +264,7 @@ mod _overlapped { type Args = (isize,); fn py_new(cls: PyTypeRef, (mut event,): Self::Args, vm: &VirtualMachine) -> PyResult { - if event == INVALID_HANDLE_VALUE(vm) { + if event == INVALID_HANDLE_VALUE { event = unsafe { windows_sys::Win32::System::Threading::CreateEventA( std::ptr::null(), From bae0ad3aeb2cccb86132ddf34436e346bd6ba53d Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sun, 9 Mar 2025 19:43:26 -0700 Subject: [PATCH 076/295] Fix extra newline in module.csv generation in cron ci (#5591) --- .github/workflows/cron-ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cron-ci.yaml b/.github/workflows/cron-ci.yaml index 5c14502643..6ae118cb56 100644 --- a/.github/workflows/cron-ci.yaml +++ b/.github/workflows/cron-ci.yaml @@ -101,7 +101,7 @@ jobs: [ -f ./_data/whats_left.temp ] && cp ./_data/whats_left.temp ./_data/whats_left_lastrun.temp cp ../whats_left.temp ./_data/whats_left.temp rm ./_data/whats_left/modules.csv - echo -e "modules\n" > ./_data/whats_left/modules.csv + echo -e "modules" > ./_data/whats_left/modules.csv cat ./_data/whats_left.temp | grep "(entire module)" | cut -d ' ' -f 1 | sort >> ./_data/whats_left/modules.csv git add -A if git -c user.name="Github Actions" -c user.email="actions@github.com" commit -m "Update what is left results" --author="$GITHUB_ACTOR"; then From bf28152a3251e6ebe9ccbbe8a4858c51c3c97e27 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sun, 9 Mar 2025 16:55:03 -0700 Subject: [PATCH 077/295] add os support modules --- Lib/_android_support.py | 181 ++++++++++++++++++++++++++++++++++++++++ Lib/_apple_support.py | 66 +++++++++++++++ Lib/_ios_support.py | 71 ++++++++++++++++ 3 files changed, 318 insertions(+) create mode 100644 Lib/_android_support.py create mode 100644 Lib/_apple_support.py create mode 100644 Lib/_ios_support.py diff --git a/Lib/_android_support.py b/Lib/_android_support.py new file mode 100644 index 0000000000..ae506f6a4b --- /dev/null +++ b/Lib/_android_support.py @@ -0,0 +1,181 @@ +import io +import sys +from threading import RLock +from time import sleep, time + +# The maximum length of a log message in bytes, including the level marker and +# tag, is defined as LOGGER_ENTRY_MAX_PAYLOAD at +# https://cs.android.com/android/platform/superproject/+/android-14.0.0_r1:system/logging/liblog/include/log/log.h;l=71. +# Messages longer than this will be truncated by logcat. This limit has already +# been reduced at least once in the history of Android (from 4076 to 4068 between +# API level 23 and 26), so leave some headroom. +MAX_BYTES_PER_WRITE = 4000 + +# UTF-8 uses a maximum of 4 bytes per character, so limiting text writes to this +# size ensures that we can always avoid exceeding MAX_BYTES_PER_WRITE. +# However, if the actual number of bytes per character is smaller than that, +# then we may still join multiple consecutive text writes into binary +# writes containing a larger number of characters. +MAX_CHARS_PER_WRITE = MAX_BYTES_PER_WRITE // 4 + + +# When embedded in an app on current versions of Android, there's no easy way to +# monitor the C-level stdout and stderr. The testbed comes with a .c file to +# redirect them to the system log using a pipe, but that wouldn't be convenient +# or appropriate for all apps. So we redirect at the Python level instead. +def init_streams(android_log_write, stdout_prio, stderr_prio): + if sys.executable: + return # Not embedded in an app. + + global logcat + logcat = Logcat(android_log_write) + + sys.stdout = TextLogStream( + stdout_prio, "python.stdout", sys.stdout.fileno()) + sys.stderr = TextLogStream( + stderr_prio, "python.stderr", sys.stderr.fileno()) + + +class TextLogStream(io.TextIOWrapper): + def __init__(self, prio, tag, fileno=None, **kwargs): + # The default is surrogateescape for stdout and backslashreplace for + # stderr, but in the context of an Android log, readability is more + # important than reversibility. + kwargs.setdefault("encoding", "UTF-8") + kwargs.setdefault("errors", "backslashreplace") + + super().__init__(BinaryLogStream(prio, tag, fileno), **kwargs) + self._lock = RLock() + self._pending_bytes = [] + self._pending_bytes_count = 0 + + def __repr__(self): + return f"" + + def write(self, s): + if not isinstance(s, str): + raise TypeError( + f"write() argument must be str, not {type(s).__name__}") + + # In case `s` is a str subclass that writes itself to stdout or stderr + # when we call its methods, convert it to an actual str. + s = str.__str__(s) + + # We want to emit one log message per line wherever possible, so split + # the string into lines first. Note that "".splitlines() == [], so + # nothing will be logged for an empty string. + with self._lock: + for line in s.splitlines(keepends=True): + while line: + chunk = line[:MAX_CHARS_PER_WRITE] + line = line[MAX_CHARS_PER_WRITE:] + self._write_chunk(chunk) + + return len(s) + + # The size and behavior of TextIOWrapper's buffer is not part of its public + # API, so we handle buffering ourselves to avoid truncation. + def _write_chunk(self, s): + b = s.encode(self.encoding, self.errors) + if self._pending_bytes_count + len(b) > MAX_BYTES_PER_WRITE: + self.flush() + + self._pending_bytes.append(b) + self._pending_bytes_count += len(b) + if ( + self.write_through + or b.endswith(b"\n") + or self._pending_bytes_count > MAX_BYTES_PER_WRITE + ): + self.flush() + + def flush(self): + with self._lock: + self.buffer.write(b"".join(self._pending_bytes)) + self._pending_bytes.clear() + self._pending_bytes_count = 0 + + # Since this is a line-based logging system, line buffering cannot be turned + # off, i.e. a newline always causes a flush. + @property + def line_buffering(self): + return True + + +class BinaryLogStream(io.RawIOBase): + def __init__(self, prio, tag, fileno=None): + self.prio = prio + self.tag = tag + self._fileno = fileno + + def __repr__(self): + return f"" + + def writable(self): + return True + + def write(self, b): + if type(b) is not bytes: + try: + b = bytes(memoryview(b)) + except TypeError: + raise TypeError( + f"write() argument must be bytes-like, not {type(b).__name__}" + ) from None + + # Writing an empty string to the stream should have no effect. + if b: + logcat.write(self.prio, self.tag, b) + return len(b) + + # This is needed by the test suite --timeout option, which uses faulthandler. + def fileno(self): + if self._fileno is None: + raise io.UnsupportedOperation("fileno") + return self._fileno + + +# When a large volume of data is written to logcat at once, e.g. when a test +# module fails in --verbose3 mode, there's a risk of overflowing logcat's own +# buffer and losing messages. We avoid this by imposing a rate limit using the +# token bucket algorithm, based on a conservative estimate of how fast `adb +# logcat` can consume data. +MAX_BYTES_PER_SECOND = 1024 * 1024 + +# The logcat buffer size of a device can be determined by running `logcat -g`. +# We set the token bucket size to half of the buffer size of our current minimum +# API level, because other things on the system will be producing messages as +# well. +BUCKET_SIZE = 128 * 1024 + +# https://cs.android.com/android/platform/superproject/+/android-14.0.0_r1:system/logging/liblog/include/log/log_read.h;l=39 +PER_MESSAGE_OVERHEAD = 28 + + +class Logcat: + def __init__(self, android_log_write): + self.android_log_write = android_log_write + self._lock = RLock() + self._bucket_level = 0 + self._prev_write_time = time() + + def write(self, prio, tag, message): + # Encode null bytes using "modified UTF-8" to avoid them truncating the + # message. + message = message.replace(b"\x00", b"\xc0\x80") + + with self._lock: + now = time() + self._bucket_level += ( + (now - self._prev_write_time) * MAX_BYTES_PER_SECOND) + + # If the bucket level is still below zero, the clock must have gone + # backwards, so reset it to zero and continue. + self._bucket_level = max(0, min(self._bucket_level, BUCKET_SIZE)) + self._prev_write_time = now + + self._bucket_level -= PER_MESSAGE_OVERHEAD + len(tag) + len(message) + if self._bucket_level < 0: + sleep(-self._bucket_level / MAX_BYTES_PER_SECOND) + + self.android_log_write(prio, tag, message) diff --git a/Lib/_apple_support.py b/Lib/_apple_support.py new file mode 100644 index 0000000000..92febdcf58 --- /dev/null +++ b/Lib/_apple_support.py @@ -0,0 +1,66 @@ +import io +import sys + + +def init_streams(log_write, stdout_level, stderr_level): + # Redirect stdout and stderr to the Apple system log. This method is + # invoked by init_apple_streams() (initconfig.c) if config->use_system_logger + # is enabled. + sys.stdout = SystemLog(log_write, stdout_level, errors=sys.stderr.errors) + sys.stderr = SystemLog(log_write, stderr_level, errors=sys.stderr.errors) + + +class SystemLog(io.TextIOWrapper): + def __init__(self, log_write, level, **kwargs): + kwargs.setdefault("encoding", "UTF-8") + kwargs.setdefault("line_buffering", True) + super().__init__(LogStream(log_write, level), **kwargs) + + def __repr__(self): + return f"" + + def write(self, s): + if not isinstance(s, str): + raise TypeError( + f"write() argument must be str, not {type(s).__name__}") + + # In case `s` is a str subclass that writes itself to stdout or stderr + # when we call its methods, convert it to an actual str. + s = str.__str__(s) + + # We want to emit one log message per line, so split + # the string before sending it to the superclass. + for line in s.splitlines(keepends=True): + super().write(line) + + return len(s) + + +class LogStream(io.RawIOBase): + def __init__(self, log_write, level): + self.log_write = log_write + self.level = level + + def __repr__(self): + return f"" + + def writable(self): + return True + + def write(self, b): + if type(b) is not bytes: + try: + b = bytes(memoryview(b)) + except TypeError: + raise TypeError( + f"write() argument must be bytes-like, not {type(b).__name__}" + ) from None + + # Writing an empty string to the stream should have no effect. + if b: + # Encode null bytes using "modified UTF-8" to avoid truncating the + # message. This should not affect the return value, as the caller + # may be expecting it to match the length of the input. + self.log_write(self.level, b.replace(b"\x00", b"\xc0\x80")) + + return len(b) diff --git a/Lib/_ios_support.py b/Lib/_ios_support.py new file mode 100644 index 0000000000..20467a7c2b --- /dev/null +++ b/Lib/_ios_support.py @@ -0,0 +1,71 @@ +import sys +try: + from ctypes import cdll, c_void_p, c_char_p, util +except ImportError: + # ctypes is an optional module. If it's not present, we're limited in what + # we can tell about the system, but we don't want to prevent the module + # from working. + print("ctypes isn't available; iOS system calls will not be available", file=sys.stderr) + objc = None +else: + # ctypes is available. Load the ObjC library, and wrap the objc_getClass, + # sel_registerName methods + lib = util.find_library("objc") + if lib is None: + # Failed to load the objc library + raise ImportError("ObjC runtime library couldn't be loaded") + + objc = cdll.LoadLibrary(lib) + objc.objc_getClass.restype = c_void_p + objc.objc_getClass.argtypes = [c_char_p] + objc.sel_registerName.restype = c_void_p + objc.sel_registerName.argtypes = [c_char_p] + + +def get_platform_ios(): + # Determine if this is a simulator using the multiarch value + is_simulator = sys.implementation._multiarch.endswith("simulator") + + # We can't use ctypes; abort + if not objc: + return None + + # Most of the methods return ObjC objects + objc.objc_msgSend.restype = c_void_p + # All the methods used have no arguments. + objc.objc_msgSend.argtypes = [c_void_p, c_void_p] + + # Equivalent of: + # device = [UIDevice currentDevice] + UIDevice = objc.objc_getClass(b"UIDevice") + SEL_currentDevice = objc.sel_registerName(b"currentDevice") + device = objc.objc_msgSend(UIDevice, SEL_currentDevice) + + # Equivalent of: + # device_systemVersion = [device systemVersion] + SEL_systemVersion = objc.sel_registerName(b"systemVersion") + device_systemVersion = objc.objc_msgSend(device, SEL_systemVersion) + + # Equivalent of: + # device_systemName = [device systemName] + SEL_systemName = objc.sel_registerName(b"systemName") + device_systemName = objc.objc_msgSend(device, SEL_systemName) + + # Equivalent of: + # device_model = [device model] + SEL_model = objc.sel_registerName(b"model") + device_model = objc.objc_msgSend(device, SEL_model) + + # UTF8String returns a const char*; + SEL_UTF8String = objc.sel_registerName(b"UTF8String") + objc.objc_msgSend.restype = c_char_p + + # Equivalent of: + # system = [device_systemName UTF8String] + # release = [device_systemVersion UTF8String] + # model = [device_model UTF8String] + system = objc.objc_msgSend(device_systemName, SEL_UTF8String).decode() + release = objc.objc_msgSend(device_systemVersion, SEL_UTF8String).decode() + model = objc.objc_msgSend(device_model, SEL_UTF8String).decode() + + return system, release, model, is_simulator From 4308321f395dd6ecce228007332d0979e030733f Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Mon, 10 Mar 2025 23:13:32 -0700 Subject: [PATCH 078/295] _osx_support update to 3.13 --- Lib/_osx_support.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Lib/_osx_support.py b/Lib/_osx_support.py index aa66c8b9f4..0cb064fcd7 100644 --- a/Lib/_osx_support.py +++ b/Lib/_osx_support.py @@ -507,6 +507,11 @@ def get_platform_osx(_config_vars, osname, release, machine): # MACOSX_DEPLOYMENT_TARGET. macver = _config_vars.get('MACOSX_DEPLOYMENT_TARGET', '') + if macver and '.' not in macver: + # Ensure that the version includes at least a major + # and minor version, even if MACOSX_DEPLOYMENT_TARGET + # is set to a single-label version like "14". + macver += '.0' macrelease = _get_system_version() or macver macver = macver or macrelease From cbbadf562fe8af01111f23cbbcf9001791c2fd5a Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Mon, 10 Mar 2025 23:27:05 -0700 Subject: [PATCH 079/295] Fixed whats left --- .github/workflows/cron-ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cron-ci.yaml b/.github/workflows/cron-ci.yaml index 6ae118cb56..4e88d749fb 100644 --- a/.github/workflows/cron-ci.yaml +++ b/.github/workflows/cron-ci.yaml @@ -101,7 +101,7 @@ jobs: [ -f ./_data/whats_left.temp ] && cp ./_data/whats_left.temp ./_data/whats_left_lastrun.temp cp ../whats_left.temp ./_data/whats_left.temp rm ./_data/whats_left/modules.csv - echo -e "modules" > ./_data/whats_left/modules.csv + echo -e "module" > ./_data/whats_left/modules.csv cat ./_data/whats_left.temp | grep "(entire module)" | cut -d ' ' -f 1 | sort >> ./_data/whats_left/modules.csv git add -A if git -c user.name="Github Actions" -c user.email="actions@github.com" commit -m "Update what is left results" --author="$GITHUB_ACTOR"; then From ff970b0e1c8f51d2ccc292e4c405511c0ea3eb69 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Tue, 11 Mar 2025 06:37:26 -0700 Subject: [PATCH 080/295] Update statistics to 3.13.2 (#5592) * statistics to 3.13.2 * set flaky test --- Lib/statistics.py | 899 ++++++++++++++++++++++++++++-------- Lib/test/test_statistics.py | 530 ++++++++++++++++++--- 2 files changed, 1178 insertions(+), 251 deletions(-) diff --git a/Lib/statistics.py b/Lib/statistics.py index f66245380a..5f0a6e67d4 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -11,7 +11,7 @@ Function Description ================== ================================================== mean Arithmetic mean (average) of data. -fmean Fast, floating point arithmetic mean. +fmean Fast, floating-point arithmetic mean. geometric_mean Geometric mean of data. harmonic_mean Harmonic mean of data. median Median (middle value) of data. @@ -112,6 +112,8 @@ 'fmean', 'geometric_mean', 'harmonic_mean', + 'kde', + 'kde_random', 'linear_regression', 'mean', 'median', @@ -130,14 +132,20 @@ import math import numbers import random +import sys from fractions import Fraction from decimal import Decimal -from itertools import groupby, repeat +from itertools import count, groupby, repeat from bisect import bisect_left, bisect_right -from math import hypot, sqrt, fabs, exp, erf, tau, log, fsum +from math import hypot, sqrt, fabs, exp, erf, tau, log, fsum, sumprod +from math import isfinite, isinf, pi, cos, sin, tan, cosh, asin, atan, acos +from functools import reduce from operator import itemgetter -from collections import Counter, namedtuple +from collections import Counter, namedtuple, defaultdict + +_SQRT2 = sqrt(2.0) +_random = random # === Exceptions === @@ -180,11 +188,12 @@ def _sum(data): allowed. """ count = 0 + types = set() + types_add = types.add partials = {} partials_get = partials.get - T = int for typ, values in groupby(data, type): - T = _coerce(T, typ) # or raise TypeError + types_add(typ) for n, d in map(_exact_ratio, values): count += 1 partials[d] = partials_get(d, 0) + n @@ -196,9 +205,51 @@ def _sum(data): else: # Sum all the partial sums using builtin sum. total = sum(Fraction(n, d) for d, n in partials.items()) + T = reduce(_coerce, types, int) # or raise TypeError return (T, total, count) +def _ss(data, c=None): + """Return the exact mean and sum of square deviations of sequence data. + + Calculations are done in a single pass, allowing the input to be an iterator. + + If given *c* is used the mean; otherwise, it is calculated from the data. + Use the *c* argument with care, as it can lead to garbage results. + + """ + if c is not None: + T, ssd, count = _sum((d := x - c) * d for x in data) + return (T, ssd, c, count) + count = 0 + types = set() + types_add = types.add + sx_partials = defaultdict(int) + sxx_partials = defaultdict(int) + for typ, values in groupby(data, type): + types_add(typ) + for n, d in map(_exact_ratio, values): + count += 1 + sx_partials[d] += n + sxx_partials[d] += n * n + if not count: + ssd = c = Fraction(0) + elif None in sx_partials: + # The sum will be a NAN or INF. We can ignore all the finite + # partials, and just look at this special one. + ssd = c = sx_partials[None] + assert not _isfinite(ssd) + else: + sx = sum(Fraction(n, d) for d, n in sx_partials.items()) + sxx = sum(Fraction(n, d*d) for d, n in sxx_partials.items()) + # This formula has poor numeric properties for floats, + # but with fractions it is exact. + ssd = (count * sxx - sx * sx) / count + c = sx / count + T = reduce(_coerce, types, int) # or raise TypeError + return (T, ssd, c, count) + + def _isfinite(x): try: return x.is_finite() # Likely a Decimal. @@ -245,6 +296,28 @@ def _exact_ratio(x): x is expected to be an int, Fraction, Decimal or float. """ + + # XXX We should revisit whether using fractions to accumulate exact + # ratios is the right way to go. + + # The integer ratios for binary floats can have numerators or + # denominators with over 300 decimal digits. The problem is more + # acute with decimal floats where the default decimal context + # supports a huge range of exponents from Emin=-999999 to + # Emax=999999. When expanded with as_integer_ratio(), numbers like + # Decimal('3.14E+5000') and Decimal('3.14E-5000') have large + # numerators or denominators that will slow computation. + + # When the integer ratios are accumulated as fractions, the size + # grows to cover the full range from the smallest magnitude to the + # largest. For example, Fraction(3.14E+300) + Fraction(3.14E-300), + # has a 616 digit numerator. Likewise, + # Fraction(Decimal('3.14E+5000')) + Fraction(Decimal('3.14E-5000')) + # has 10,003 digit numerator. + + # This doesn't seem to have been problem in practice, but it is a + # potential pitfall. + try: return x.as_integer_ratio() except AttributeError: @@ -279,22 +352,6 @@ def _convert(value, T): raise -def _find_lteq(a, x): - 'Locate the leftmost value exactly equal to x' - i = bisect_left(a, x) - if i != len(a) and a[i] == x: - return i - raise ValueError - - -def _find_rteq(a, l, x): - 'Locate the rightmost value exactly equal to x' - i = bisect_right(a, x, lo=l) - if i != (len(a) + 1) and a[i - 1] == x: - return i - 1 - raise ValueError - - def _fail_neg(values, errmsg='negative value'): """Iterate over values, failing if any are less than zero.""" for x in values: @@ -303,6 +360,113 @@ def _fail_neg(values, errmsg='negative value'): yield x +def _rank(data, /, *, key=None, reverse=False, ties='average', start=1) -> list[float]: + """Rank order a dataset. The lowest value has rank 1. + + Ties are averaged so that equal values receive the same rank: + + >>> data = [31, 56, 31, 25, 75, 18] + >>> _rank(data) + [3.5, 5.0, 3.5, 2.0, 6.0, 1.0] + + The operation is idempotent: + + >>> _rank([3.5, 5.0, 3.5, 2.0, 6.0, 1.0]) + [3.5, 5.0, 3.5, 2.0, 6.0, 1.0] + + It is possible to rank the data in reverse order so that the + highest value has rank 1. Also, a key-function can extract + the field to be ranked: + + >>> goals = [('eagles', 45), ('bears', 48), ('lions', 44)] + >>> _rank(goals, key=itemgetter(1), reverse=True) + [2.0, 1.0, 3.0] + + Ranks are conventionally numbered starting from one; however, + setting *start* to zero allows the ranks to be used as array indices: + + >>> prize = ['Gold', 'Silver', 'Bronze', 'Certificate'] + >>> scores = [8.1, 7.3, 9.4, 8.3] + >>> [prize[int(i)] for i in _rank(scores, start=0, reverse=True)] + ['Bronze', 'Certificate', 'Gold', 'Silver'] + + """ + # If this function becomes public at some point, more thought + # needs to be given to the signature. A list of ints is + # plausible when ties is "min" or "max". When ties is "average", + # either list[float] or list[Fraction] is plausible. + + # Default handling of ties matches scipy.stats.mstats.spearmanr. + if ties != 'average': + raise ValueError(f'Unknown tie resolution method: {ties!r}') + if key is not None: + data = map(key, data) + val_pos = sorted(zip(data, count()), reverse=reverse) + i = start - 1 + result = [0] * len(val_pos) + for _, g in groupby(val_pos, key=itemgetter(0)): + group = list(g) + size = len(group) + rank = i + (size + 1) / 2 + for value, orig_pos in group: + result[orig_pos] = rank + i += size + return result + + +def _integer_sqrt_of_frac_rto(n: int, m: int) -> int: + """Square root of n/m, rounded to the nearest integer using round-to-odd.""" + # Reference: https://www.lri.fr/~melquion/doc/05-imacs17_1-expose.pdf + a = math.isqrt(n // m) + return a | (a*a*m != n) + + +# For 53 bit precision floats, the bit width used in +# _float_sqrt_of_frac() is 109. +_sqrt_bit_width: int = 2 * sys.float_info.mant_dig + 3 + + +def _float_sqrt_of_frac(n: int, m: int) -> float: + """Square root of n/m as a float, correctly rounded.""" + # See principle and proof sketch at: https://bugs.python.org/msg407078 + q = (n.bit_length() - m.bit_length() - _sqrt_bit_width) // 2 + if q >= 0: + numerator = _integer_sqrt_of_frac_rto(n, m << 2 * q) << q + denominator = 1 + else: + numerator = _integer_sqrt_of_frac_rto(n << -2 * q, m) + denominator = 1 << -q + return numerator / denominator # Convert to float + + +def _decimal_sqrt_of_frac(n: int, m: int) -> Decimal: + """Square root of n/m as a Decimal, correctly rounded.""" + # Premise: For decimal, computing (n/m).sqrt() can be off + # by 1 ulp from the correctly rounded result. + # Method: Check the result, moving up or down a step if needed. + if n <= 0: + if not n: + return Decimal('0.0') + n, m = -n, -m + + root = (Decimal(n) / Decimal(m)).sqrt() + nr, dr = root.as_integer_ratio() + + plus = root.next_plus() + np, dp = plus.as_integer_ratio() + # test: n / m > ((root + plus) / 2) ** 2 + if 4 * n * (dr*dp)**2 > m * (dr*np + dp*nr)**2: + return plus + + minus = root.next_minus() + nm, dm = minus.as_integer_ratio() + # test: n / m < ((root + minus) / 2) ** 2 + if 4 * n * (dr*dm)**2 < m * (dr*nm + dm*nr)**2: + return minus + + return root + + # === Measures of central tendency (averages) === def mean(data): @@ -321,17 +485,13 @@ def mean(data): If ``data`` is empty, StatisticsError will be raised. """ - if iter(data) is data: - data = list(data) - n = len(data) + T, total, n = _sum(data) if n < 1: raise StatisticsError('mean requires at least one data point') - T, total, count = _sum(data) - assert count == n return _convert(total / n, T) -def fmean(data): +def fmean(data, weights=None): """Convert data to floats and compute the arithmetic mean. This runs faster than the mean() function and it always returns a float. @@ -340,29 +500,40 @@ def fmean(data): >>> fmean([3.5, 4.0, 5.25]) 4.25 """ - try: - n = len(data) - except TypeError: - # Handle iterators that do not define __len__(). - n = 0 - def count(iterable): - nonlocal n - for n, x in enumerate(iterable, start=1): - yield x - total = fsum(count(data)) - else: + if weights is None: + try: + n = len(data) + except TypeError: + # Handle iterators that do not define __len__(). + n = 0 + def count(iterable): + nonlocal n + for n, x in enumerate(iterable, start=1): + yield x + data = count(data) total = fsum(data) - try: + if not n: + raise StatisticsError('fmean requires at least one data point') return total / n - except ZeroDivisionError: - raise StatisticsError('fmean requires at least one data point') from None + if not isinstance(weights, (list, tuple)): + weights = list(weights) + try: + num = sumprod(data, weights) + except ValueError: + raise StatisticsError('data and weights must be the same length') + den = fsum(weights) + if not den: + raise StatisticsError('sum of weights must be non-zero') + return num / den def geometric_mean(data): """Convert data to floats and compute the geometric mean. - Raises a StatisticsError if the input dataset is empty, - if it contains a zero, or if it contains a negative value. + Raises a StatisticsError if the input dataset is empty + or if it contains a negative value. + + Returns zero if the product of inputs is zero. No special efforts are made to achieve exact results. (However, this may change in the future.) @@ -370,11 +541,25 @@ def geometric_mean(data): >>> round(geometric_mean([54, 24, 36]), 9) 36.0 """ - try: - return exp(fmean(map(log, data))) - except ValueError: - raise StatisticsError('geometric mean requires a non-empty dataset ' - 'containing positive numbers') from None + n = 0 + found_zero = False + def count_positive(iterable): + nonlocal n, found_zero + for n, x in enumerate(iterable, start=1): + if x > 0.0 or math.isnan(x): + yield x + elif x == 0.0: + found_zero = True + else: + raise StatisticsError('No negative inputs allowed', x) + total = fsum(map(log, count_positive(data))) + if not n: + raise StatisticsError('Must have a non-empty dataset') + if math.isnan(total): + return math.nan + if found_zero: + return math.nan if total == math.inf else 0.0 + return exp(total / n) def harmonic_mean(data, weights=None): @@ -498,58 +683,75 @@ def median_high(data): return data[n // 2] -def median_grouped(data, interval=1): - """Return the 50th percentile (median) of grouped continuous data. +def median_grouped(data, interval=1.0): + """Estimates the median for numeric data binned around the midpoints + of consecutive, fixed-width intervals. + + The *data* can be any iterable of numeric data with each value being + exactly the midpoint of a bin. At least one value must be present. - >>> median_grouped([1, 2, 2, 3, 4, 4, 4, 4, 4, 5]) - 3.7 - >>> median_grouped([52, 52, 53, 54]) - 52.5 + The *interval* is width of each bin. - This calculates the median as the 50th percentile, and should be - used when your data is continuous and grouped. In the above example, - the values 1, 2, 3, etc. actually represent the midpoint of classes - 0.5-1.5, 1.5-2.5, 2.5-3.5, etc. The middle value falls somewhere in - class 3.5-4.5, and interpolation is used to estimate it. + For example, demographic information may have been summarized into + consecutive ten-year age groups with each group being represented + by the 5-year midpoints of the intervals: - Optional argument ``interval`` represents the class interval, and - defaults to 1. Changing the class interval naturally will change the - interpolated 50th percentile value: + >>> demographics = Counter({ + ... 25: 172, # 20 to 30 years old + ... 35: 484, # 30 to 40 years old + ... 45: 387, # 40 to 50 years old + ... 55: 22, # 50 to 60 years old + ... 65: 6, # 60 to 70 years old + ... }) - >>> median_grouped([1, 3, 3, 5, 7], interval=1) - 3.25 - >>> median_grouped([1, 3, 3, 5, 7], interval=2) - 3.5 + The 50th percentile (median) is the 536th person out of the 1071 + member cohort. That person is in the 30 to 40 year old age group. + + The regular median() function would assume that everyone in the + tricenarian age group was exactly 35 years old. A more tenable + assumption is that the 484 members of that age group are evenly + distributed between 30 and 40. For that, we use median_grouped(). + + >>> data = list(demographics.elements()) + >>> median(data) + 35 + >>> round(median_grouped(data, interval=10), 1) + 37.5 + + The caller is responsible for making sure the data points are separated + by exact multiples of *interval*. This is essential for getting a + correct result. The function does not check this precondition. + + Inputs may be any numeric type that can be coerced to a float during + the interpolation step. - This function does not check whether the data points are at least - ``interval`` apart. """ data = sorted(data) n = len(data) - if n == 0: + if not n: raise StatisticsError("no median for empty data") - elif n == 1: - return data[0] + # Find the value at the midpoint. Remember this corresponds to the - # centre of the class interval. + # midpoint of the class interval. x = data[n // 2] - for obj in (x, interval): - if isinstance(obj, (str, bytes)): - raise TypeError('expected number but got %r' % obj) + + # Using O(log n) bisection, find where all the x values occur in the data. + # All x will lie within data[i:j]. + i = bisect_left(data, x) + j = bisect_right(data, x, lo=i) + + # Coerce to floats, raising a TypeError if not possible try: - L = x - interval / 2 # The lower limit of the median interval. - except TypeError: - # Mixed type. For now we just coerce to float. - L = float(x) - float(interval) / 2 - - # Uses bisection search to search for x in data with log(n) time complexity - # Find the position of leftmost occurrence of x in data - l1 = _find_lteq(data, x) - # Find the position of rightmost occurrence of x in data[l1...len(data)] - # Assuming always l1 <= l2 - l2 = _find_rteq(data, l1, x) - cf = l1 - f = l2 - l1 + 1 + interval = float(interval) + x = float(x) + except ValueError: + raise TypeError(f'Value cannot be converted to a float') + + # Interpolate the median using the formula found at: + # https://www.cuemath.com/data/median-of-grouped-data/ + L = x - interval / 2.0 # Lower limit of the median interval + cf = i # Cumulative frequency of the preceding interval + f = j - i # Number of elements in the median internal return L + interval * (n / 2 - cf) / f @@ -596,9 +798,212 @@ def multimode(data): >>> multimode('') [] """ - counts = Counter(iter(data)).most_common() - maxcount, mode_items = next(groupby(counts, key=itemgetter(1)), (0, [])) - return list(map(itemgetter(0), mode_items)) + counts = Counter(iter(data)) + if not counts: + return [] + maxcount = max(counts.values()) + return [value for value, count in counts.items() if count == maxcount] + + +def kde(data, h, kernel='normal', *, cumulative=False): + """Kernel Density Estimation: Create a continuous probability density + function or cumulative distribution function from discrete samples. + + The basic idea is to smooth the data using a kernel function + to help draw inferences about a population from a sample. + + The degree of smoothing is controlled by the scaling parameter h + which is called the bandwidth. Smaller values emphasize local + features while larger values give smoother results. + + The kernel determines the relative weights of the sample data + points. Generally, the choice of kernel shape does not matter + as much as the more influential bandwidth smoothing parameter. + + Kernels that give some weight to every sample point: + + normal (gauss) + logistic + sigmoid + + Kernels that only give weight to sample points within + the bandwidth: + + rectangular (uniform) + triangular + parabolic (epanechnikov) + quartic (biweight) + triweight + cosine + + If *cumulative* is true, will return a cumulative distribution function. + + A StatisticsError will be raised if the data sequence is empty. + + Example + ------- + + Given a sample of six data points, construct a continuous + function that estimates the underlying probability density: + + >>> sample = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] + >>> f_hat = kde(sample, h=1.5) + + Compute the area under the curve: + + >>> area = sum(f_hat(x) for x in range(-20, 20)) + >>> round(area, 4) + 1.0 + + Plot the estimated probability density function at + evenly spaced points from -6 to 10: + + >>> for x in range(-6, 11): + ... density = f_hat(x) + ... plot = ' ' * int(density * 400) + 'x' + ... print(f'{x:2}: {density:.3f} {plot}') + ... + -6: 0.002 x + -5: 0.009 x + -4: 0.031 x + -3: 0.070 x + -2: 0.111 x + -1: 0.125 x + 0: 0.110 x + 1: 0.086 x + 2: 0.068 x + 3: 0.059 x + 4: 0.066 x + 5: 0.082 x + 6: 0.082 x + 7: 0.058 x + 8: 0.028 x + 9: 0.009 x + 10: 0.002 x + + Estimate P(4.5 < X <= 7.5), the probability that a new sample value + will be between 4.5 and 7.5: + + >>> cdf = kde(sample, h=1.5, cumulative=True) + >>> round(cdf(7.5) - cdf(4.5), 2) + 0.22 + + References + ---------- + + Kernel density estimation and its application: + https://www.itm-conferences.org/articles/itmconf/pdf/2018/08/itmconf_sam2018_00037.pdf + + Kernel functions in common use: + https://en.wikipedia.org/wiki/Kernel_(statistics)#kernel_functions_in_common_use + + Interactive graphical demonstration and exploration: + https://demonstrations.wolfram.com/KernelDensityEstimation/ + + Kernel estimation of cumulative distribution function of a random variable with bounded support + https://www.econstor.eu/bitstream/10419/207829/1/10.21307_stattrans-2016-037.pdf + + """ + + n = len(data) + if not n: + raise StatisticsError('Empty data sequence') + + if not isinstance(data[0], (int, float)): + raise TypeError('Data sequence must contain ints or floats') + + if h <= 0.0: + raise StatisticsError(f'Bandwidth h must be positive, not {h=!r}') + + if kernel == "normal" or kernel == "gauss": + sqrt2pi = sqrt(2 * pi) + sqrt2 = sqrt(2) + K = lambda t: exp(-1/2 * t * t) / sqrt2pi + W = lambda t: 1/2 * (1.0 + erf(t / sqrt2)) + support = None + elif kernel == "logistic": + # 1.0 / (exp(t) + 2.0 + exp(-t)) + K = lambda t: 1/2 / (1.0 + cosh(t)) + W = lambda t: 1.0 - 1.0 / (exp(t) + 1.0) + support = None + elif kernel == "sigmoid": + # (2/pi) / (exp(t) + exp(-t)) + c1 = 1 / pi + c2 = 2 / pi + K = lambda t: c1 / cosh(t) + W = lambda t: c2 * atan(exp(t)) + support = None + elif kernel == "rectangular" or kernel == "uniform": + K = lambda t: 1/2 + W = lambda t: 1/2 * t + 1/2 + support = 1.0 + elif kernel == "triangular": + K = lambda t: 1.0 - abs(t) + W = lambda t: t*t * (1/2 if t < 0.0 else -1/2) + t + 1/2 + support = 1.0 + elif kernel == "parabolic" or kernel == "epanechnikov": + K = lambda t: 3/4 * (1.0 - t * t) + W = lambda t: -1/4 * t**3 + 3/4 * t + 1/2 + support = 1. + elif kernel == "quartic" or kernel == "biweight": + K = lambda t: 15/16 * (1.0 - t * t) ** 2 + W = lambda t: 3/16 * t**5 - 5/8 * t**3 + 15/16 * t + 1/2 + support = 1.0 + elif kernel == "triweight": + K = lambda t: 35/32 * (1.0 - t * t) ** 3 + W = lambda t: 35/32 * (-1/7*t**7 + 3/5*t**5 - t**3 + t) + 1/2 + support = 1.0 + elif kernel == "cosine": + c1 = pi / 4 + c2 = pi / 2 + K = lambda t: c1 * cos(c2 * t) + W = lambda t: 1/2 * sin(c2 * t) + 1/2 + support = 1.0 + else: + raise StatisticsError(f'Unknown kernel name: {kernel!r}') + + if support is None: + + def pdf(x): + n = len(data) + return sum(K((x - x_i) / h) for x_i in data) / (n * h) + + def cdf(x): + n = len(data) + return sum(W((x - x_i) / h) for x_i in data) / n + + else: + + sample = sorted(data) + bandwidth = h * support + + def pdf(x): + nonlocal n, sample + if len(data) != n: + sample = sorted(data) + n = len(data) + i = bisect_left(sample, x - bandwidth) + j = bisect_right(sample, x + bandwidth) + supported = sample[i : j] + return sum(K((x - x_i) / h) for x_i in supported) / (n * h) + + def cdf(x): + nonlocal n, sample + if len(data) != n: + sample = sorted(data) + n = len(data) + i = bisect_left(sample, x - bandwidth) + j = bisect_right(sample, x + bandwidth) + supported = sample[i : j] + return sum((W((x - x_i) / h) for x_i in supported), i) / n + + if cumulative: + cdf.__doc__ = f'CDF estimate with {h=!r} and {kernel=!r}' + return cdf + + else: + pdf.__doc__ = f'PDF estimate with {h=!r} and {kernel=!r}' + return pdf # Notes on methods for computing quantiles @@ -659,7 +1064,10 @@ def quantiles(data, *, n=4, method='exclusive'): data = sorted(data) ld = len(data) if ld < 2: - raise StatisticsError('must have at least two data points') + if ld == 1: + return data * (n - 1) + raise StatisticsError('must have at least one data point') + if method == 'inclusive': m = ld - 1 result = [] @@ -668,6 +1076,7 @@ def quantiles(data, *, n=4, method='exclusive'): interpolated = (data[j] * (n - delta) + data[j + 1] * delta) / n result.append(interpolated) return result + if method == 'exclusive': m = ld + 1 result = [] @@ -678,6 +1087,7 @@ def quantiles(data, *, n=4, method='exclusive'): interpolated = (data[j - 1] * (n - delta) + data[j] * delta) / n result.append(interpolated) return result + raise ValueError(f'Unknown method: {method!r}') @@ -685,41 +1095,6 @@ def quantiles(data, *, n=4, method='exclusive'): # See http://mathworld.wolfram.com/Variance.html # http://mathworld.wolfram.com/SampleVariance.html -# http://en.wikipedia.org/wiki/Algorithms_for_calculating_variance -# -# Under no circumstances use the so-called "computational formula for -# variance", as that is only suitable for hand calculations with a small -# amount of low-precision data. It has terrible numeric properties. -# -# See a comparison of three computational methods here: -# http://www.johndcook.com/blog/2008/09/26/comparing-three-methods-of-computing-standard-deviation/ - -def _ss(data, c=None): - """Return sum of square deviations of sequence data. - - If ``c`` is None, the mean is calculated in one pass, and the deviations - from the mean are calculated in a second pass. Otherwise, deviations are - calculated from ``c`` as given. Use the second case with care, as it can - lead to garbage results. - """ - if c is not None: - T, total, count = _sum((x-c)**2 for x in data) - return (T, total) - T, total, count = _sum(data) - mean_n, mean_d = (total / count).as_integer_ratio() - partials = Counter() - for n, d in map(_exact_ratio, data): - diff_n = n * mean_d - d * mean_n - diff_d = d * mean_d - partials[diff_d * diff_d] += diff_n * diff_n - if None in partials: - # The sum will be a NAN or INF. We can ignore all the finite - # partials, and just look at this special one. - total = partials[None] - assert not _isfinite(total) - else: - total = sum(Fraction(n, d) for d, n in partials.items()) - return (T, total) def variance(data, xbar=None): @@ -760,12 +1135,9 @@ def variance(data, xbar=None): Fraction(67, 108) """ - if iter(data) is data: - data = list(data) - n = len(data) + T, ss, c, n = _ss(data, xbar) if n < 2: raise StatisticsError('variance requires at least two data points') - T, ss = _ss(data, xbar) return _convert(ss / (n - 1), T) @@ -804,12 +1176,9 @@ def pvariance(data, mu=None): Fraction(13, 72) """ - if iter(data) is data: - data = list(data) - n = len(data) + T, ss, c, n = _ss(data, mu) if n < 1: raise StatisticsError('pvariance requires at least one data point') - T, ss = _ss(data, mu) return _convert(ss / n, T) @@ -822,14 +1191,13 @@ def stdev(data, xbar=None): 1.0810874155219827 """ - # Fixme: Despite the exact sum of squared deviations, some inaccuracy - # remain because there are two rounding steps. The first occurs in - # the _convert() step for variance(), the second occurs in math.sqrt(). - var = variance(data, xbar) - try: - return var.sqrt() - except AttributeError: - return math.sqrt(var) + T, ss, c, n = _ss(data, xbar) + if n < 2: + raise StatisticsError('stdev requires at least two data points') + mss = ss / (n - 1) + if issubclass(T, Decimal): + return _decimal_sqrt_of_frac(mss.numerator, mss.denominator) + return _float_sqrt_of_frac(mss.numerator, mss.denominator) def pstdev(data, mu=None): @@ -841,14 +1209,47 @@ def pstdev(data, mu=None): 0.986893273527251 """ - # Fixme: Despite the exact sum of squared deviations, some inaccuracy - # remain because there are two rounding steps. The first occurs in - # the _convert() step for pvariance(), the second occurs in math.sqrt(). - var = pvariance(data, mu) + T, ss, c, n = _ss(data, mu) + if n < 1: + raise StatisticsError('pstdev requires at least one data point') + mss = ss / n + if issubclass(T, Decimal): + return _decimal_sqrt_of_frac(mss.numerator, mss.denominator) + return _float_sqrt_of_frac(mss.numerator, mss.denominator) + + +def _mean_stdev(data): + """In one pass, compute the mean and sample standard deviation as floats.""" + T, ss, xbar, n = _ss(data) + if n < 2: + raise StatisticsError('stdev requires at least two data points') + mss = ss / (n - 1) try: - return var.sqrt() + return float(xbar), _float_sqrt_of_frac(mss.numerator, mss.denominator) except AttributeError: - return math.sqrt(var) + # Handle Nans and Infs gracefully + return float(xbar), float(xbar) / float(ss) + +def _sqrtprod(x: float, y: float) -> float: + "Return sqrt(x * y) computed with improved accuracy and without overflow/underflow." + h = sqrt(x * y) + if not isfinite(h): + if isinf(h) and not isinf(x) and not isinf(y): + # Finite inputs overflowed, so scale down, and recompute. + scale = 2.0 ** -512 # sqrt(1 / sys.float_info.max) + return _sqrtprod(scale * x, scale * y) / scale + return h + if not h: + if x and y: + # Non-zero inputs underflowed, so scale up, and recompute. + # Scale: 1 / sqrt(sys.float_info.min * sys.float_info.epsilon) + scale = 2.0 ** 537 + return _sqrtprod(scale * x, scale * y) / scale + return h + # Improve accuracy with a differential correction. + # https://www.wolframalpha.com/input/?i=Maclaurin+series+sqrt%28h**2+%2B+x%29+at+x%3D0 + d = sumprod((x, h), (y, -h)) + return h + d / (2.0 * h) # === Statistics for relations between two inputs === @@ -882,18 +1283,16 @@ def covariance(x, y, /): raise StatisticsError('covariance requires at least two data points') xbar = fsum(x) / n ybar = fsum(y) / n - sxy = fsum((xi - xbar) * (yi - ybar) for xi, yi in zip(x, y)) + sxy = sumprod((xi - xbar for xi in x), (yi - ybar for yi in y)) return sxy / (n - 1) -def correlation(x, y, /): +def correlation(x, y, /, *, method='linear'): """Pearson's correlation coefficient Return the Pearson's correlation coefficient for two inputs. Pearson's - correlation coefficient *r* takes values between -1 and +1. It measures the - strength and direction of the linear relationship, where +1 means very - strong, positive linear relationship, -1 very strong, negative linear - relationship, and 0 no linear relationship. + correlation coefficient *r* takes values between -1 and +1. It measures + the strength and direction of a linear relationship. >>> x = [1, 2, 3, 4, 5, 6, 7, 8, 9] >>> y = [9, 8, 7, 6, 5, 4, 3, 2, 1] @@ -902,19 +1301,36 @@ def correlation(x, y, /): >>> correlation(x, y) -1.0 + If *method* is "ranked", computes Spearman's rank correlation coefficient + for two inputs. The data is replaced by ranks. Ties are averaged + so that equal values receive the same rank. The resulting coefficient + measures the strength of a monotonic relationship. + + Spearman's rank correlation coefficient is appropriate for ordinal + data or for continuous data that doesn't meet the linear proportion + requirement for Pearson's correlation coefficient. """ n = len(x) if len(y) != n: raise StatisticsError('correlation requires that both inputs have same number of data points') if n < 2: raise StatisticsError('correlation requires at least two data points') - xbar = fsum(x) / n - ybar = fsum(y) / n - sxy = fsum((xi - xbar) * (yi - ybar) for xi, yi in zip(x, y)) - sxx = fsum((xi - xbar) ** 2.0 for xi in x) - syy = fsum((yi - ybar) ** 2.0 for yi in y) + if method not in {'linear', 'ranked'}: + raise ValueError(f'Unknown method: {method!r}') + if method == 'ranked': + start = (n - 1) / -2 # Center rankings around zero + x = _rank(x, start=start) + y = _rank(y, start=start) + else: + xbar = fsum(x) / n + ybar = fsum(y) / n + x = [xi - xbar for xi in x] + y = [yi - ybar for yi in y] + sxy = sumprod(x, y) + sxx = sumprod(x, x) + syy = sumprod(y, y) try: - return sxy / sqrt(sxx * syy) + return sxy / _sqrtprod(sxx, syy) except ZeroDivisionError: raise StatisticsError('at least one of the inputs is constant') @@ -922,13 +1338,13 @@ def correlation(x, y, /): LinearRegression = namedtuple('LinearRegression', ('slope', 'intercept')) -def linear_regression(x, y, /): +def linear_regression(x, y, /, *, proportional=False): """Slope and intercept for simple linear regression. Return the slope and intercept of simple linear regression parameters estimated using ordinary least squares. Simple linear regression describes relationship between an independent variable - *x* and a dependent variable *y* in terms of linear function: + *x* and a dependent variable *y* in terms of a linear function: y = slope * x + intercept + noise @@ -944,7 +1360,20 @@ def linear_regression(x, y, /): >>> noise = NormalDist().samples(5, seed=42) >>> y = [3 * x[i] + 2 + noise[i] for i in range(5)] >>> linear_regression(x, y) #doctest: +ELLIPSIS - LinearRegression(slope=3.09078914170..., intercept=1.75684970486...) + LinearRegression(slope=3.17495..., intercept=1.00925...) + + If *proportional* is true, the independent variable *x* and the + dependent variable *y* are assumed to be directly proportional. + The data is fit to a line passing through the origin. + + Since the *intercept* will always be 0.0, the underlying linear + function simplifies to: + + y = slope * x + noise + + >>> y = [3 * x[i] + noise[i] for i in range(5)] + >>> linear_regression(x, y, proportional=True) #doctest: +ELLIPSIS + LinearRegression(slope=2.90475..., intercept=0.0) """ n = len(x) @@ -952,15 +1381,18 @@ def linear_regression(x, y, /): raise StatisticsError('linear regression requires that both inputs have same number of data points') if n < 2: raise StatisticsError('linear regression requires at least two data points') - xbar = fsum(x) / n - ybar = fsum(y) / n - sxy = fsum((xi - xbar) * (yi - ybar) for xi, yi in zip(x, y)) - sxx = fsum((xi - xbar) ** 2.0 for xi in x) + if not proportional: + xbar = fsum(x) / n + ybar = fsum(y) / n + x = [xi - xbar for xi in x] # List because used three times below + y = (yi - ybar for yi in y) # Generator because only used once below + sxy = sumprod(x, y) + 0.0 # Add zero to coerce result to a float + sxx = sumprod(x, x) try: slope = sxy / sxx # equivalent to: covariance(x, y) / variance(x) except ZeroDivisionError: raise StatisticsError('x is constant') - intercept = ybar - slope * xbar + intercept = 0.0 if proportional else ybar - slope * xbar return LinearRegression(slope=slope, intercept=intercept) @@ -1068,29 +1500,29 @@ def __init__(self, mu=0.0, sigma=1.0): @classmethod def from_samples(cls, data): "Make a normal distribution instance from sample data." - if not isinstance(data, (list, tuple)): - data = list(data) - xbar = fmean(data) - return cls(xbar, stdev(data, xbar)) + return cls(*_mean_stdev(data)) def samples(self, n, *, seed=None): "Generate *n* samples for a given mean and standard deviation." - gauss = random.gauss if seed is None else random.Random(seed).gauss - mu, sigma = self._mu, self._sigma - return [gauss(mu, sigma) for i in range(n)] + rnd = random.random if seed is None else random.Random(seed).random + inv_cdf = _normal_dist_inv_cdf + mu = self._mu + sigma = self._sigma + return [inv_cdf(rnd(), mu, sigma) for _ in repeat(None, n)] def pdf(self, x): "Probability density function. P(x <= X < x+dx) / dx" - variance = self._sigma ** 2.0 + variance = self._sigma * self._sigma if not variance: raise StatisticsError('pdf() not defined when sigma is zero') - return exp((x - self._mu)**2.0 / (-2.0*variance)) / sqrt(tau*variance) + diff = x - self._mu + return exp(diff * diff / (-2.0 * variance)) / sqrt(tau * variance) def cdf(self, x): "Cumulative distribution function. P(X <= x)" if not self._sigma: raise StatisticsError('cdf() not defined when sigma is zero') - return 0.5 * (1.0 + erf((x - self._mu) / (self._sigma * sqrt(2.0)))) + return 0.5 * (1.0 + erf((x - self._mu) / (self._sigma * _SQRT2))) def inv_cdf(self, p): """Inverse cumulative distribution function. x : P(X <= x) = p @@ -1104,8 +1536,6 @@ def inv_cdf(self, p): """ if p <= 0.0 or p >= 1.0: raise StatisticsError('p must be in the range 0.0 < p < 1.0') - if self._sigma <= 0.0: - raise StatisticsError('cdf() not defined when sigma at or below zero') return _normal_dist_inv_cdf(p, self._mu, self._sigma) def quantiles(self, n=4): @@ -1146,9 +1576,9 @@ def overlap(self, other): dv = Y_var - X_var dm = fabs(Y._mu - X._mu) if not dv: - return 1.0 - erf(dm / (2.0 * X._sigma * sqrt(2.0))) + return 1.0 - erf(dm / (2.0 * X._sigma * _SQRT2)) a = X._mu * Y_var - Y._mu * X_var - b = X._sigma * Y._sigma * sqrt(dm**2.0 + dv * log(Y_var / X_var)) + b = X._sigma * Y._sigma * sqrt(dm * dm + dv * log(Y_var / X_var)) x1 = (a + b) / dv x2 = (a - b) / dv return 1.0 - (fabs(Y.cdf(x1) - X.cdf(x1)) + fabs(Y.cdf(x2) - X.cdf(x2))) @@ -1191,7 +1621,7 @@ def stdev(self): @property def variance(self): "Square of the standard deviation." - return self._sigma ** 2.0 + return self._sigma * self._sigma def __add__(x1, x2): """Add a constant or another NormalDist instance. @@ -1265,3 +1695,102 @@ def __hash__(self): def __repr__(self): return f'{type(self).__name__}(mu={self._mu!r}, sigma={self._sigma!r})' + + def __getstate__(self): + return self._mu, self._sigma + + def __setstate__(self, state): + self._mu, self._sigma = state + + +## kde_random() ############################################################## + +def _newton_raphson(f_inv_estimate, f, f_prime, tolerance=1e-12): + def f_inv(y): + "Return x such that f(x) ≈ y within the specified tolerance." + x = f_inv_estimate(y) + while abs(diff := f(x) - y) > tolerance: + x -= diff / f_prime(x) + return x + return f_inv + +def _quartic_invcdf_estimate(p): + sign, p = (1.0, p) if p <= 1/2 else (-1.0, 1.0 - p) + x = (2.0 * p) ** 0.4258865685331 - 1.0 + if p >= 0.004 < 0.499: + x += 0.026818732 * sin(7.101753784 * p + 2.73230839482953) + return x * sign + +_quartic_invcdf = _newton_raphson( + f_inv_estimate = _quartic_invcdf_estimate, + f = lambda t: 3/16 * t**5 - 5/8 * t**3 + 15/16 * t + 1/2, + f_prime = lambda t: 15/16 * (1.0 - t * t) ** 2) + +def _triweight_invcdf_estimate(p): + sign, p = (1.0, p) if p <= 1/2 else (-1.0, 1.0 - p) + x = (2.0 * p) ** 0.3400218741872791 - 1.0 + return x * sign + +_triweight_invcdf = _newton_raphson( + f_inv_estimate = _triweight_invcdf_estimate, + f = lambda t: 35/32 * (-1/7*t**7 + 3/5*t**5 - t**3 + t) + 1/2, + f_prime = lambda t: 35/32 * (1.0 - t * t) ** 3) + +_kernel_invcdfs = { + 'normal': NormalDist().inv_cdf, + 'logistic': lambda p: log(p / (1 - p)), + 'sigmoid': lambda p: log(tan(p * pi/2)), + 'rectangular': lambda p: 2*p - 1, + 'parabolic': lambda p: 2 * cos((acos(2*p-1) + pi) / 3), + 'quartic': _quartic_invcdf, + 'triweight': _triweight_invcdf, + 'triangular': lambda p: sqrt(2*p) - 1 if p < 1/2 else 1 - sqrt(2 - 2*p), + 'cosine': lambda p: 2 * asin(2*p - 1) / pi, +} +_kernel_invcdfs['gauss'] = _kernel_invcdfs['normal'] +_kernel_invcdfs['uniform'] = _kernel_invcdfs['rectangular'] +_kernel_invcdfs['epanechnikov'] = _kernel_invcdfs['parabolic'] +_kernel_invcdfs['biweight'] = _kernel_invcdfs['quartic'] + +def kde_random(data, h, kernel='normal', *, seed=None): + """Return a function that makes a random selection from the estimated + probability density function created by kde(data, h, kernel). + + Providing a *seed* allows reproducible selections within a single + thread. The seed may be an integer, float, str, or bytes. + + A StatisticsError will be raised if the *data* sequence is empty. + + Example: + + >>> data = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] + >>> rand = kde_random(data, h=1.5, seed=8675309) + >>> new_selections = [rand() for i in range(10)] + >>> [round(x, 1) for x in new_selections] + [0.7, 6.2, 1.2, 6.9, 7.0, 1.8, 2.5, -0.5, -1.8, 5.6] + + """ + n = len(data) + if not n: + raise StatisticsError('Empty data sequence') + + if not isinstance(data[0], (int, float)): + raise TypeError('Data sequence must contain ints or floats') + + if h <= 0.0: + raise StatisticsError(f'Bandwidth h must be positive, not {h=!r}') + + kernel_invcdf = _kernel_invcdfs.get(kernel) + if kernel_invcdf is None: + raise StatisticsError(f'Unknown kernel name: {kernel!r}') + + prng = _random.Random(seed) + random = prng.random + choice = prng.choice + + def rand(): + return choice(data) + h * kernel_invcdf(random()) + + rand.__doc__ = f'Random KDE selection with {h=!r} and {kernel=!r}' + + return rand diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index 8fcbcf3540..7b0dfd05e0 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -1,4 +1,4 @@ -"""Test suite for statistics module, including helper NumericTestCase and +x = """Test suite for statistics module, including helper NumericTestCase and approx_equal function. """ @@ -9,13 +9,14 @@ import copy import decimal import doctest +import itertools import math import pickle import random import sys import unittest from test import support -from test.support import import_helper +from test.support import import_helper, requires_IEEE_754 from decimal import Decimal from fractions import Fraction @@ -27,6 +28,12 @@ # === Helper functions and class === +# Test copied from Lib/test/test_math.py +# detect evidence of double-rounding: fsum is not always correctly +# rounded on machines that suffer from double rounding. +x, y = 1e16, 2.9999 # use temporary values to defeat peephole optimizer +HAVE_DOUBLE_ROUNDING = (x + y == 1e16 + 4) + def sign(x): """Return -1.0 for negatives, including -0.0, otherwise +1.0.""" return math.copysign(1, x) @@ -691,14 +698,6 @@ def test_check_all(self): 'missing name "%s" in __all__' % name) -class DocTests(unittest.TestCase): - @unittest.skipIf(sys.flags.optimize >= 2, - "Docstrings are omitted with -OO and above") - def test_doc_tests(self): - failed, tried = doctest.testmod(statistics, optionflags=doctest.ELLIPSIS) - self.assertGreater(tried, 0) - self.assertEqual(failed, 0) - class StatisticsErrorTest(unittest.TestCase): def test_has_exception(self): errmsg = ( @@ -1039,50 +1038,6 @@ def test_error_msg(self): self.assertEqual(errmsg, msg) -class FindLteqTest(unittest.TestCase): - # Test _find_lteq private function. - - def test_invalid_input_values(self): - for a, x in [ - ([], 1), - ([1, 2], 3), - ([1, 3], 2) - ]: - with self.subTest(a=a, x=x): - with self.assertRaises(ValueError): - statistics._find_lteq(a, x) - - def test_locate_successfully(self): - for a, x, expected_i in [ - ([1, 1, 1, 2, 3], 1, 0), - ([0, 1, 1, 1, 2, 3], 1, 1), - ([1, 2, 3, 3, 3], 3, 2) - ]: - with self.subTest(a=a, x=x): - self.assertEqual(expected_i, statistics._find_lteq(a, x)) - - -class FindRteqTest(unittest.TestCase): - # Test _find_rteq private function. - - def test_invalid_input_values(self): - for a, l, x in [ - ([1], 2, 1), - ([1, 3], 0, 2) - ]: - with self.assertRaises(ValueError): - statistics._find_rteq(a, l, x) - - def test_locate_successfully(self): - for a, l, x, expected_i in [ - ([1, 1, 1, 2, 3], 0, 1, 2), - ([0, 1, 1, 1, 2, 3], 0, 1, 3), - ([1, 2, 3, 3, 3], 0, 3, 4) - ]: - with self.subTest(a=a, l=l, x=x): - self.assertEqual(expected_i, statistics._find_rteq(a, l, x)) - - # === Tests for public functions === class UnivariateCommonMixin: @@ -1117,7 +1072,7 @@ def test_no_inplace_modifications(self): def test_order_doesnt_matter(self): # Test that the order of data points doesn't change the result. - # CAUTION: due to floating point rounding errors, the result actually + # CAUTION: due to floating-point rounding errors, the result actually # may depend on the order. Consider this test representing an ideal. # To avoid this test failing, only test with exact values such as ints # or Fractions. @@ -1210,6 +1165,9 @@ def __pow__(self, other): def __add__(self, other): return type(self)(super().__add__(other)) __radd__ = __add__ + def __mul__(self, other): + return type(self)(super().__mul__(other)) + __rmul__ = __mul__ return (float, Decimal, Fraction, MyFloat) def test_types_conserved(self): @@ -1782,6 +1740,12 @@ def test_repeated_single_value(self): data = [x]*count self.assertEqual(self.func(data), float(x)) + def test_single_value(self): + # Override method from AverageMixin. + # Average of a single value is the value as a float. + for x in (23, 42.5, 1.3e15, Fraction(15, 19), Decimal('0.28')): + self.assertEqual(self.func([x]), float(x)) + def test_odd_fractions(self): # Test median_grouped works with an odd number of Fractions. F = Fraction @@ -1961,6 +1925,27 @@ def test_special_values(self): with self.assertRaises(ValueError): fmean([Inf, -Inf]) + def test_weights(self): + fmean = statistics.fmean + StatisticsError = statistics.StatisticsError + self.assertEqual( + fmean([10, 10, 10, 50], [0.25] * 4), + fmean([10, 10, 10, 50])) + self.assertEqual( + fmean([10, 10, 20], [0.25, 0.25, 0.50]), + fmean([10, 10, 20, 20])) + self.assertEqual( # inputs are iterators + fmean(iter([10, 10, 20]), iter([0.25, 0.25, 0.50])), + fmean([10, 10, 20, 20])) + with self.assertRaises(StatisticsError): + fmean([10, 20, 30], [1, 2]) # unequal lengths + with self.assertRaises(StatisticsError): + fmean(iter([10, 20, 30]), iter([1, 2])) # unequal lengths + with self.assertRaises(StatisticsError): + fmean([10, 20], [-1, 1]) # sum of weights is zero + with self.assertRaises(StatisticsError): + fmean(iter([10, 20]), iter([-1, 1])) # sum of weights is zero + # === Tests for variances and standard deviations === @@ -2137,6 +2122,104 @@ def test_center_not_at_mean(self): self.assertEqual(self.func(data), 2.5) self.assertEqual(self.func(data, mu=0.5), 6.5) +class TestSqrtHelpers(unittest.TestCase): + + def test_integer_sqrt_of_frac_rto(self): + for n, m in itertools.product(range(100), range(1, 1000)): + r = statistics._integer_sqrt_of_frac_rto(n, m) + self.assertIsInstance(r, int) + if r*r*m == n: + # Root is exact + continue + # Inexact, so the root should be odd + self.assertEqual(r&1, 1) + # Verify correct rounding + self.assertTrue(m * (r - 1)**2 < n < m * (r + 1)**2) + + @requires_IEEE_754 + @support.requires_resource('cpu') + def test_float_sqrt_of_frac(self): + + def is_root_correctly_rounded(x: Fraction, root: float) -> bool: + if not x: + return root == 0.0 + + # Extract adjacent representable floats + r_up: float = math.nextafter(root, math.inf) + r_down: float = math.nextafter(root, -math.inf) + assert r_down < root < r_up + + # Convert to fractions for exact arithmetic + frac_root: Fraction = Fraction(root) + half_way_up: Fraction = (frac_root + Fraction(r_up)) / 2 + half_way_down: Fraction = (frac_root + Fraction(r_down)) / 2 + + # Check a closed interval. + # Does not test for a midpoint rounding rule. + return half_way_down ** 2 <= x <= half_way_up ** 2 + + randrange = random.randrange + + for i in range(60_000): + numerator: int = randrange(10 ** randrange(50)) + denonimator: int = randrange(10 ** randrange(50)) + 1 + with self.subTest(numerator=numerator, denonimator=denonimator): + x: Fraction = Fraction(numerator, denonimator) + root: float = statistics._float_sqrt_of_frac(numerator, denonimator) + self.assertTrue(is_root_correctly_rounded(x, root)) + + # Verify that corner cases and error handling match math.sqrt() + self.assertEqual(statistics._float_sqrt_of_frac(0, 1), 0.0) + with self.assertRaises(ValueError): + statistics._float_sqrt_of_frac(-1, 1) + with self.assertRaises(ValueError): + statistics._float_sqrt_of_frac(1, -1) + + # Error handling for zero denominator matches that for Fraction(1, 0) + with self.assertRaises(ZeroDivisionError): + statistics._float_sqrt_of_frac(1, 0) + + # The result is well defined if both inputs are negative + self.assertEqual(statistics._float_sqrt_of_frac(-2, -1), statistics._float_sqrt_of_frac(2, 1)) + + def test_decimal_sqrt_of_frac(self): + root: Decimal + numerator: int + denominator: int + + for root, numerator, denominator in [ + (Decimal('0.4481904599041192673635338663'), 200874688349065940678243576378, 1000000000000000000000000000000), # No adj + (Decimal('0.7924949131383786609961759598'), 628048187350206338833590574929, 1000000000000000000000000000000), # Adj up + (Decimal('0.8500554152289934068192208727'), 722594208960136395984391238251, 1000000000000000000000000000000), # Adj down + ]: + with decimal.localcontext(decimal.DefaultContext): + self.assertEqual(statistics._decimal_sqrt_of_frac(numerator, denominator), root) + + # Confirm expected root with a quad precision decimal computation + with decimal.localcontext(decimal.DefaultContext) as ctx: + ctx.prec *= 4 + high_prec_ratio = Decimal(numerator) / Decimal(denominator) + ctx.rounding = decimal.ROUND_05UP + high_prec_root = high_prec_ratio.sqrt() + with decimal.localcontext(decimal.DefaultContext): + target_root = +high_prec_root + self.assertEqual(root, target_root) + + # Verify that corner cases and error handling match Decimal.sqrt() + self.assertEqual(statistics._decimal_sqrt_of_frac(0, 1), 0.0) + with self.assertRaises(decimal.InvalidOperation): + statistics._decimal_sqrt_of_frac(-1, 1) + with self.assertRaises(decimal.InvalidOperation): + statistics._decimal_sqrt_of_frac(1, -1) + + # Error handling for zero denominator matches that for Fraction(1, 0) + with self.assertRaises(ZeroDivisionError): + statistics._decimal_sqrt_of_frac(1, 0) + + # The result is well defined if both inputs are negative + self.assertEqual(statistics._decimal_sqrt_of_frac(-2, -1), statistics._decimal_sqrt_of_frac(2, 1)) + + class TestStdev(VarianceStdevMixin, NumericTestCase): # Tests for sample standard deviation. def setUp(self): @@ -2151,7 +2234,7 @@ def test_compare_to_variance(self): # Test that stdev is, in fact, the square root of variance. data = [random.uniform(-2, 9) for _ in range(1000)] expected = math.sqrt(statistics.variance(data)) - self.assertEqual(self.func(data), expected) + self.assertAlmostEqual(self.func(data), expected) def test_center_not_at_mean(self): data = (1.0, 2.0) @@ -2219,10 +2302,12 @@ def test_error_cases(self): StatisticsError = statistics.StatisticsError with self.assertRaises(StatisticsError): geometric_mean([]) # empty input - with self.assertRaises(StatisticsError): - geometric_mean([3.5, 0.0, 5.25]) # zero input with self.assertRaises(StatisticsError): geometric_mean([3.5, -4.0, 5.25]) # negative input + with self.assertRaises(StatisticsError): + geometric_mean([0.0, -4.0, 5.25]) # negative input with zero + with self.assertRaises(StatisticsError): + geometric_mean([3.5, -math.inf, 5.25]) # negative infinity with self.assertRaises(StatisticsError): geometric_mean(iter([])) # empty iterator with self.assertRaises(TypeError): @@ -2245,6 +2330,200 @@ def test_special_values(self): with self.assertRaises(ValueError): geometric_mean([Inf, -Inf]) + # Cases with zero + self.assertEqual(geometric_mean([3, 0.0, 5]), 0.0) # Any zero gives a zero + self.assertEqual(geometric_mean([3, -0.0, 5]), 0.0) # Negative zero allowed + self.assertTrue(math.isnan(geometric_mean([0, NaN]))) # NaN beats zero + self.assertTrue(math.isnan(geometric_mean([0, Inf]))) # Because 0.0 * Inf -> NaN + + def test_mixed_int_and_float(self): + # Regression test for b.p.o. issue #28327 + geometric_mean = statistics.geometric_mean + expected_mean = 3.80675409583932 + values = [ + [2, 3, 5, 7], + [2, 3, 5, 7.0], + [2, 3, 5.0, 7.0], + [2, 3.0, 5.0, 7.0], + [2.0, 3.0, 5.0, 7.0], + ] + for v in values: + with self.subTest(v=v): + actual_mean = geometric_mean(v) + self.assertAlmostEqual(actual_mean, expected_mean, places=5) + + +class TestKDE(unittest.TestCase): + + def test_kde(self): + kde = statistics.kde + StatisticsError = statistics.StatisticsError + + kernels = ['normal', 'gauss', 'logistic', 'sigmoid', 'rectangular', + 'uniform', 'triangular', 'parabolic', 'epanechnikov', + 'quartic', 'biweight', 'triweight', 'cosine'] + + sample = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] + + # The approximate integral of a PDF should be close to 1.0 + + def integrate(func, low, high, steps=10_000): + "Numeric approximation of a definite function integral." + dx = (high - low) / steps + midpoints = (low + (i + 1/2) * dx for i in range(steps)) + return sum(map(func, midpoints)) * dx + + for kernel in kernels: + with self.subTest(kernel=kernel): + f_hat = kde(sample, h=1.5, kernel=kernel) + area = integrate(f_hat, -20, 20) + self.assertAlmostEqual(area, 1.0, places=4) + + # Check CDF against an integral of the PDF + + data = [3, 5, 10, 12] + h = 2.3 + x = 10.5 + for kernel in kernels: + with self.subTest(kernel=kernel): + cdf = kde(data, h, kernel, cumulative=True) + f_hat = kde(data, h, kernel) + area = integrate(f_hat, -20, x, 100_000) + self.assertAlmostEqual(cdf(x), area, places=4) + + # Check error cases + + with self.assertRaises(StatisticsError): + kde([], h=1.0) # Empty dataset + with self.assertRaises(TypeError): + kde(['abc', 'def'], 1.5) # Non-numeric data + with self.assertRaises(TypeError): + kde(iter(sample), 1.5) # Data is not a sequence + with self.assertRaises(StatisticsError): + kde(sample, h=0.0) # Zero bandwidth + with self.assertRaises(StatisticsError): + kde(sample, h=-1.0) # Negative bandwidth + with self.assertRaises(TypeError): + kde(sample, h='str') # Wrong bandwidth type + with self.assertRaises(StatisticsError): + kde(sample, h=1.0, kernel='bogus') # Invalid kernel + with self.assertRaises(TypeError): + kde(sample, 1.0, 'gauss', True) # Positional cumulative argument + + # Test name and docstring of the generated function + + h = 1.5 + kernel = 'cosine' + f_hat = kde(sample, h, kernel) + self.assertEqual(f_hat.__name__, 'pdf') + self.assertIn(kernel, f_hat.__doc__) + self.assertIn(repr(h), f_hat.__doc__) + + # Test closed interval for the support boundaries. + # In particular, 'uniform' should non-zero at the boundaries. + + f_hat = kde([0], 1.0, 'uniform') + self.assertEqual(f_hat(-1.0), 1/2) + self.assertEqual(f_hat(1.0), 1/2) + + # Test online updates to data + + data = [1, 2] + f_hat = kde(data, 5.0, 'triangular') + self.assertEqual(f_hat(100), 0.0) + data.append(100) + self.assertGreater(f_hat(100), 0.0) + + def test_kde_kernel_invcdfs(self): + kernel_invcdfs = statistics._kernel_invcdfs + kde = statistics.kde + + # Verify that cdf / invcdf will round trip + xarr = [i/100 for i in range(-100, 101)] + for kernel, invcdf in kernel_invcdfs.items(): + with self.subTest(kernel=kernel): + cdf = kde([0.0], h=1.0, kernel=kernel, cumulative=True) + for x in xarr: + self.assertAlmostEqual(invcdf(cdf(x)), x, places=5) + + @support.requires_resource('cpu') + def test_kde_random(self): + kde_random = statistics.kde_random + StatisticsError = statistics.StatisticsError + kernels = ['normal', 'gauss', 'logistic', 'sigmoid', 'rectangular', + 'uniform', 'triangular', 'parabolic', 'epanechnikov', + 'quartic', 'biweight', 'triweight', 'cosine'] + sample = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] + + # Smoke test + + for kernel in kernels: + with self.subTest(kernel=kernel): + rand = kde_random(sample, h=1.5, kernel=kernel) + selections = [rand() for i in range(10)] + + # Check error cases + + with self.assertRaises(StatisticsError): + kde_random([], h=1.0) # Empty dataset + with self.assertRaises(TypeError): + kde_random(['abc', 'def'], 1.5) # Non-numeric data + with self.assertRaises(TypeError): + kde_random(iter(sample), 1.5) # Data is not a sequence + with self.assertRaises(StatisticsError): + kde_random(sample, h=-1.0) # Zero bandwidth + with self.assertRaises(StatisticsError): + kde_random(sample, h=0.0) # Negative bandwidth + with self.assertRaises(TypeError): + kde_random(sample, h='str') # Wrong bandwidth type + with self.assertRaises(StatisticsError): + kde_random(sample, h=1.0, kernel='bogus') # Invalid kernel + + # Test name and docstring of the generated function + + h = 1.5 + kernel = 'cosine' + rand = kde_random(sample, h, kernel) + self.assertEqual(rand.__name__, 'rand') + self.assertIn(kernel, rand.__doc__) + self.assertIn(repr(h), rand.__doc__) + + # Approximate distribution test: Compare a random sample to the expected distribution + + data = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2, 7.8, 14.3, 15.1, 15.3, 15.8, 17.0] + xarr = [x / 10 for x in range(-100, 250)] + n = 1_000_000 + h = 1.75 + dx = 0.1 + + def p_observed(x): + # P(x <= X < x+dx) + i = bisect.bisect_left(big_sample, x) + j = bisect.bisect_left(big_sample, x + dx) + return (j - i) / len(big_sample) + + def p_expected(x): + # P(x <= X < x+dx) + return F_hat(x + dx) - F_hat(x) + + for kernel in kernels: + with self.subTest(kernel=kernel): + + rand = kde_random(data, h, kernel, seed=8675309**2) + big_sample = sorted([rand() for i in range(n)]) + F_hat = statistics.kde(data, h, kernel, cumulative=True) + + for x in xarr: + self.assertTrue(math.isclose(p_observed(x), p_expected(x), abs_tol=0.0005)) + + # Test online updates to data + + data = [1, 2] + rand = kde_random(data, 5, 'triangular') + self.assertLess(max([rand() for i in range(5000)]), 10) + data.append(100) + self.assertGreater(max(rand() for i in range(5000)), 10) + class TestQuantiles(unittest.TestCase): @@ -2355,6 +2634,11 @@ def f(x): data = random.choices(range(100), k=k) q1, q2, q3 = quantiles(data, method='inclusive') self.assertEqual(q2, statistics.median(data)) + # Base case with a single data point: When estimating quantiles from + # a sample, we want to be able to add one sample point at a time, + # getting increasingly better estimates. + self.assertEqual(quantiles([10], n=4), [10.0, 10.0, 10.0]) + self.assertEqual(quantiles([10], n=4, method='exclusive'), [10.0, 10.0, 10.0]) def test_equal_inputs(self): quantiles = statistics.quantiles @@ -2405,7 +2689,7 @@ def test_error_cases(self): with self.assertRaises(ValueError): quantiles([10, 20, 30], method='X') # method is unknown with self.assertRaises(StatisticsError): - quantiles([10], n=4) # not enough data points + quantiles([], n=4) # not enough data points with self.assertRaises(TypeError): quantiles([10, None, 30], n=4) # data is non-numeric @@ -2464,6 +2748,95 @@ def test_different_scales(self): self.assertAlmostEqual(statistics.correlation(x, y), 1) self.assertAlmostEqual(statistics.covariance(x, y), 0.1) + def test_sqrtprod_helper_function_fundamentals(self): + # Verify that results are close to sqrt(x * y) + for i in range(100): + x = random.expovariate() + y = random.expovariate() + expected = math.sqrt(x * y) + actual = statistics._sqrtprod(x, y) + with self.subTest(x=x, y=y, expected=expected, actual=actual): + self.assertAlmostEqual(expected, actual) + + x, y, target = 0.8035720646477457, 0.7957468097636939, 0.7996498651651661 + self.assertEqual(statistics._sqrtprod(x, y), target) + self.assertNotEqual(math.sqrt(x * y), target) + + # Test that range extremes avoid underflow and overflow + smallest = sys.float_info.min * sys.float_info.epsilon + self.assertEqual(statistics._sqrtprod(smallest, smallest), smallest) + biggest = sys.float_info.max + self.assertEqual(statistics._sqrtprod(biggest, biggest), biggest) + + # Check special values and the sign of the result + special_values = [0.0, -0.0, 1.0, -1.0, 4.0, -4.0, + math.nan, -math.nan, math.inf, -math.inf] + for x, y in itertools.product(special_values, repeat=2): + try: + expected = math.sqrt(x * y) + except ValueError: + expected = 'ValueError' + try: + actual = statistics._sqrtprod(x, y) + except ValueError: + actual = 'ValueError' + with self.subTest(x=x, y=y, expected=expected, actual=actual): + if isinstance(expected, str) and expected == 'ValueError': + self.assertEqual(actual, 'ValueError') + continue + self.assertIsInstance(actual, float) + if math.isnan(expected): + self.assertTrue(math.isnan(actual)) + continue + self.assertEqual(actual, expected) + self.assertEqual(sign(actual), sign(expected)) + + @requires_IEEE_754 + @unittest.skipIf(HAVE_DOUBLE_ROUNDING, + "accuracy not guaranteed on machines with double rounding") + @support.cpython_only # Allow for a weaker sumprod() implmentation + def test_sqrtprod_helper_function_improved_accuracy(self): + # Test a known example where accuracy is improved + x, y, target = 0.8035720646477457, 0.7957468097636939, 0.7996498651651661 + self.assertEqual(statistics._sqrtprod(x, y), target) + self.assertNotEqual(math.sqrt(x * y), target) + + def reference_value(x: float, y: float) -> float: + x = decimal.Decimal(x) + y = decimal.Decimal(y) + with decimal.localcontext() as ctx: + ctx.prec = 200 + return float((x * y).sqrt()) + + # Verify that the new function with improved accuracy + # agrees with a reference value more often than old version. + new_agreements = 0 + old_agreements = 0 + for i in range(10_000): + x = random.expovariate() + y = random.expovariate() + new = statistics._sqrtprod(x, y) + old = math.sqrt(x * y) + ref = reference_value(x, y) + new_agreements += (new == ref) + old_agreements += (old == ref) + self.assertGreater(new_agreements, old_agreements) + + def test_correlation_spearman(self): + # https://statistics.laerd.com/statistical-guides/spearmans-rank-order-correlation-statistical-guide-2.php + # Compare with: + # >>> import scipy.stats.mstats + # >>> scipy.stats.mstats.spearmanr(reading, mathematics) + # SpearmanrResult(correlation=0.6686960980480712, pvalue=0.03450954165178532) + # And Wolfram Alpha gives: 0.668696 + # https://www.wolframalpha.com/input?i=SpearmanRho%5B%7B56%2C+75%2C+45%2C+71%2C+61%2C+64%2C+58%2C+80%2C+76%2C+61%7D%2C+%7B66%2C+70%2C+40%2C+60%2C+65%2C+56%2C+59%2C+77%2C+67%2C+63%7D%5D + reading = [56, 75, 45, 71, 61, 64, 58, 80, 76, 61] + mathematics = [66, 70, 40, 60, 65, 56, 59, 77, 67, 63] + self.assertAlmostEqual(statistics.correlation(reading, mathematics, method='ranked'), + 0.6686960980480712) + + with self.assertRaises(ValueError): + statistics.correlation(reading, mathematics, method='bad_method') class TestLinearRegression(unittest.TestCase): @@ -2487,6 +2860,22 @@ def test_results(self): self.assertAlmostEqual(intercept, true_intercept) self.assertAlmostEqual(slope, true_slope) + def test_proportional(self): + x = [10, 20, 30, 40] + y = [180, 398, 610, 799] + slope, intercept = statistics.linear_regression(x, y, proportional=True) + self.assertAlmostEqual(slope, 20 + 1/150) + self.assertEqual(intercept, 0.0) + + def test_float_output(self): + x = [Fraction(2, 3), Fraction(3, 4)] + y = [Fraction(4, 5), Fraction(5, 6)] + slope, intercept = statistics.linear_regression(x, y) + self.assertTrue(isinstance(slope, float)) + self.assertTrue(isinstance(intercept, float)) + slope, intercept = statistics.linear_regression(x, y, proportional=True) + self.assertTrue(isinstance(slope, float)) + self.assertTrue(isinstance(intercept, float)) class TestNormalDist: @@ -2640,6 +3029,8 @@ def test_cdf(self): self.assertTrue(math.isnan(X.cdf(float('NaN')))) @support.skip_if_pgo_task + @support.requires_resource('cpu') + @unittest.skip("TODO: RUSTPYTHON Flaky") def test_inv_cdf(self): NormalDist = self.module.NormalDist @@ -2697,9 +3088,10 @@ def test_inv_cdf(self): iq.inv_cdf(1.0) # p is one with self.assertRaises(self.module.StatisticsError): iq.inv_cdf(1.1) # p over one - with self.assertRaises(self.module.StatisticsError): - iq = NormalDist(100, 0) # sigma is zero - iq.inv_cdf(0.5) + + # Supported case: + iq = NormalDist(100, 0) # sigma is zero + self.assertEqual(iq.inv_cdf(0.5), 100) # Special values self.assertTrue(math.isnan(Z.inv_cdf(float('NaN')))) @@ -2882,14 +3274,19 @@ def __init__(self, mu, sigma): nd = NormalDist(100, 15) self.assertNotEqual(nd, lnd) - def test_pickle_and_copy(self): + def test_copy(self): nd = self.module.NormalDist(37.5, 5.625) nd1 = copy.copy(nd) self.assertEqual(nd, nd1) nd2 = copy.deepcopy(nd) self.assertEqual(nd, nd2) - nd3 = pickle.loads(pickle.dumps(nd)) - self.assertEqual(nd, nd3) + + def test_pickle(self): + nd = self.module.NormalDist(37.5, 5.625) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + pickled = pickle.loads(pickle.dumps(nd, protocol=proto)) + self.assertEqual(nd, pickled) def test_hashability(self): ND = self.module.NormalDist @@ -2911,7 +3308,7 @@ def setUp(self): def tearDown(self): sys.modules['statistics'] = statistics - + @unittest.skipUnless(c_statistics, 'requires _statistics') class TestNormalDistC(unittest.TestCase, TestNormalDist): @@ -2928,6 +3325,7 @@ def tearDown(self): def load_tests(loader, tests, ignore): """Used for doctest/unittest integration.""" tests.addTests(doctest.DocTestSuite()) + tests.addTests(doctest.DocTestSuite(statistics)) return tests From 8484bfa2e0da627d2ce0c2652a52bef431c662be Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Tue, 11 Mar 2025 17:47:34 -0700 Subject: [PATCH 081/295] Remove cgi module (#5597) * no cgi * mark failing test --- Lib/cgi.py | 1012 ---------------------------------- Lib/test/test_cgi.py | 645 ---------------------- Lib/test/test_httpservers.py | 1 + 3 files changed, 1 insertion(+), 1657 deletions(-) delete mode 100755 Lib/cgi.py delete mode 100644 Lib/test/test_cgi.py diff --git a/Lib/cgi.py b/Lib/cgi.py deleted file mode 100755 index 8787567be7..0000000000 --- a/Lib/cgi.py +++ /dev/null @@ -1,1012 +0,0 @@ -#! /usr/local/bin/python - -# NOTE: the above "/usr/local/bin/python" is NOT a mistake. It is -# intentionally NOT "/usr/bin/env python". On many systems -# (e.g. Solaris), /usr/local/bin is not in $PATH as passed to CGI -# scripts, and /usr/local/bin is the default directory where Python is -# installed, so /usr/bin/env would be unable to find python. Granted, -# binary installations by Linux vendors often install Python in -# /usr/bin. So let those vendors patch cgi.py to match their choice -# of installation. - -"""Support module for CGI (Common Gateway Interface) scripts. - -This module defines a number of utilities for use by CGI scripts -written in Python. - -The global variable maxlen can be set to an integer indicating the maximum size -of a POST request. POST requests larger than this size will result in a -ValueError being raised during parsing. The default value of this variable is 0, -meaning the request size is unlimited. -""" - -# History -# ------- -# -# Michael McLay started this module. Steve Majewski changed the -# interface to SvFormContentDict and FormContentDict. The multipart -# parsing was inspired by code submitted by Andreas Paepcke. Guido van -# Rossum rewrote, reformatted and documented the module and is currently -# responsible for its maintenance. -# - -__version__ = "2.6" - - -# Imports -# ======= - -from io import StringIO, BytesIO, TextIOWrapper -from collections.abc import Mapping -import sys -import os -import urllib.parse -from email.parser import FeedParser -from email.message import Message -import html -import locale -import tempfile -import warnings - -__all__ = ["MiniFieldStorage", "FieldStorage", "parse", "parse_multipart", - "parse_header", "test", "print_exception", "print_environ", - "print_form", "print_directory", "print_arguments", - "print_environ_usage"] - - -warnings._deprecated(__name__, remove=(3,13)) - -# Logging support -# =============== - -logfile = "" # Filename to log to, if not empty -logfp = None # File object to log to, if not None - -def initlog(*allargs): - """Write a log message, if there is a log file. - - Even though this function is called initlog(), you should always - use log(); log is a variable that is set either to initlog - (initially), to dolog (once the log file has been opened), or to - nolog (when logging is disabled). - - The first argument is a format string; the remaining arguments (if - any) are arguments to the % operator, so e.g. - log("%s: %s", "a", "b") - will write "a: b" to the log file, followed by a newline. - - If the global logfp is not None, it should be a file object to - which log data is written. - - If the global logfp is None, the global logfile may be a string - giving a filename to open, in append mode. This file should be - world writable!!! If the file can't be opened, logging is - silently disabled (since there is no safe place where we could - send an error message). - - """ - global log, logfile, logfp - warnings.warn("cgi.log() is deprecated as of 3.10. Use logging instead", - DeprecationWarning, stacklevel=2) - if logfile and not logfp: - try: - logfp = open(logfile, "a", encoding="locale") - except OSError: - pass - if not logfp: - log = nolog - else: - log = dolog - log(*allargs) - -def dolog(fmt, *args): - """Write a log message to the log file. See initlog() for docs.""" - logfp.write(fmt%args + "\n") - -def nolog(*allargs): - """Dummy function, assigned to log when logging is disabled.""" - pass - -def closelog(): - """Close the log file.""" - global log, logfile, logfp - logfile = '' - if logfp: - logfp.close() - logfp = None - log = initlog - -log = initlog # The current logging function - - -# Parsing functions -# ================= - -# Maximum input we will accept when REQUEST_METHOD is POST -# 0 ==> unlimited input -maxlen = 0 - -def parse(fp=None, environ=os.environ, keep_blank_values=0, - strict_parsing=0, separator='&'): - """Parse a query in the environment or from a file (default stdin) - - Arguments, all optional: - - fp : file pointer; default: sys.stdin.buffer - - environ : environment dictionary; default: os.environ - - keep_blank_values: flag indicating whether blank values in - percent-encoded forms should be treated as blank strings. - A true value indicates that blanks should be retained as - blank strings. The default false value indicates that - blank values are to be ignored and treated as if they were - not included. - - strict_parsing: flag indicating what to do with parsing errors. - If false (the default), errors are silently ignored. - If true, errors raise a ValueError exception. - - separator: str. The symbol to use for separating the query arguments. - Defaults to &. - """ - if fp is None: - fp = sys.stdin - - # field keys and values (except for files) are returned as strings - # an encoding is required to decode the bytes read from self.fp - if hasattr(fp,'encoding'): - encoding = fp.encoding - else: - encoding = 'latin-1' - - # fp.read() must return bytes - if isinstance(fp, TextIOWrapper): - fp = fp.buffer - - if not 'REQUEST_METHOD' in environ: - environ['REQUEST_METHOD'] = 'GET' # For testing stand-alone - if environ['REQUEST_METHOD'] == 'POST': - ctype, pdict = parse_header(environ['CONTENT_TYPE']) - if ctype == 'multipart/form-data': - return parse_multipart(fp, pdict, separator=separator) - elif ctype == 'application/x-www-form-urlencoded': - clength = int(environ['CONTENT_LENGTH']) - if maxlen and clength > maxlen: - raise ValueError('Maximum content length exceeded') - qs = fp.read(clength).decode(encoding) - else: - qs = '' # Unknown content-type - if 'QUERY_STRING' in environ: - if qs: qs = qs + '&' - qs = qs + environ['QUERY_STRING'] - elif sys.argv[1:]: - if qs: qs = qs + '&' - qs = qs + sys.argv[1] - environ['QUERY_STRING'] = qs # XXX Shouldn't, really - elif 'QUERY_STRING' in environ: - qs = environ['QUERY_STRING'] - else: - if sys.argv[1:]: - qs = sys.argv[1] - else: - qs = "" - environ['QUERY_STRING'] = qs # XXX Shouldn't, really - return urllib.parse.parse_qs(qs, keep_blank_values, strict_parsing, - encoding=encoding, separator=separator) - - -def parse_multipart(fp, pdict, encoding="utf-8", errors="replace", separator='&'): - """Parse multipart input. - - Arguments: - fp : input file - pdict: dictionary containing other parameters of content-type header - encoding, errors: request encoding and error handler, passed to - FieldStorage - - Returns a dictionary just like parse_qs(): keys are the field names, each - value is a list of values for that field. For non-file fields, the value - is a list of strings. - """ - # RFC 2046, Section 5.1 : The "multipart" boundary delimiters are always - # represented as 7bit US-ASCII. - boundary = pdict['boundary'].decode('ascii') - ctype = "multipart/form-data; boundary={}".format(boundary) - headers = Message() - headers.set_type(ctype) - try: - headers['Content-Length'] = pdict['CONTENT-LENGTH'] - except KeyError: - pass - fs = FieldStorage(fp, headers=headers, encoding=encoding, errors=errors, - environ={'REQUEST_METHOD': 'POST'}, separator=separator) - return {k: fs.getlist(k) for k in fs} - -def _parseparam(s): - while s[:1] == ';': - s = s[1:] - end = s.find(';') - while end > 0 and (s.count('"', 0, end) - s.count('\\"', 0, end)) % 2: - end = s.find(';', end + 1) - if end < 0: - end = len(s) - f = s[:end] - yield f.strip() - s = s[end:] - -def parse_header(line): - """Parse a Content-type like header. - - Return the main content-type and a dictionary of options. - - """ - parts = _parseparam(';' + line) - key = parts.__next__() - pdict = {} - for p in parts: - i = p.find('=') - if i >= 0: - name = p[:i].strip().lower() - value = p[i+1:].strip() - if len(value) >= 2 and value[0] == value[-1] == '"': - value = value[1:-1] - value = value.replace('\\\\', '\\').replace('\\"', '"') - pdict[name] = value - return key, pdict - - -# Classes for field storage -# ========================= - -class MiniFieldStorage: - - """Like FieldStorage, for use when no file uploads are possible.""" - - # Dummy attributes - filename = None - list = None - type = None - file = None - type_options = {} - disposition = None - disposition_options = {} - headers = {} - - def __init__(self, name, value): - """Constructor from field name and value.""" - self.name = name - self.value = value - # self.file = StringIO(value) - - def __repr__(self): - """Return printable representation.""" - return "MiniFieldStorage(%r, %r)" % (self.name, self.value) - - -class FieldStorage: - - """Store a sequence of fields, reading multipart/form-data. - - This class provides naming, typing, files stored on disk, and - more. At the top level, it is accessible like a dictionary, whose - keys are the field names. (Note: None can occur as a field name.) - The items are either a Python list (if there's multiple values) or - another FieldStorage or MiniFieldStorage object. If it's a single - object, it has the following attributes: - - name: the field name, if specified; otherwise None - - filename: the filename, if specified; otherwise None; this is the - client side filename, *not* the file name on which it is - stored (that's a temporary file you don't deal with) - - value: the value as a *string*; for file uploads, this - transparently reads the file every time you request the value - and returns *bytes* - - file: the file(-like) object from which you can read the data *as - bytes* ; None if the data is stored a simple string - - type: the content-type, or None if not specified - - type_options: dictionary of options specified on the content-type - line - - disposition: content-disposition, or None if not specified - - disposition_options: dictionary of corresponding options - - headers: a dictionary(-like) object (sometimes email.message.Message or a - subclass thereof) containing *all* headers - - The class is subclassable, mostly for the purpose of overriding - the make_file() method, which is called internally to come up with - a file open for reading and writing. This makes it possible to - override the default choice of storing all files in a temporary - directory and unlinking them as soon as they have been opened. - - """ - def __init__(self, fp=None, headers=None, outerboundary=b'', - environ=os.environ, keep_blank_values=0, strict_parsing=0, - limit=None, encoding='utf-8', errors='replace', - max_num_fields=None, separator='&'): - """Constructor. Read multipart/* until last part. - - Arguments, all optional: - - fp : file pointer; default: sys.stdin.buffer - (not used when the request method is GET) - Can be : - 1. a TextIOWrapper object - 2. an object whose read() and readline() methods return bytes - - headers : header dictionary-like object; default: - taken from environ as per CGI spec - - outerboundary : terminating multipart boundary - (for internal use only) - - environ : environment dictionary; default: os.environ - - keep_blank_values: flag indicating whether blank values in - percent-encoded forms should be treated as blank strings. - A true value indicates that blanks should be retained as - blank strings. The default false value indicates that - blank values are to be ignored and treated as if they were - not included. - - strict_parsing: flag indicating what to do with parsing errors. - If false (the default), errors are silently ignored. - If true, errors raise a ValueError exception. - - limit : used internally to read parts of multipart/form-data forms, - to exit from the reading loop when reached. It is the difference - between the form content-length and the number of bytes already - read - - encoding, errors : the encoding and error handler used to decode the - binary stream to strings. Must be the same as the charset defined - for the page sending the form (content-type : meta http-equiv or - header) - - max_num_fields: int. If set, then __init__ throws a ValueError - if there are more than n fields read by parse_qsl(). - - """ - method = 'GET' - self.keep_blank_values = keep_blank_values - self.strict_parsing = strict_parsing - self.max_num_fields = max_num_fields - self.separator = separator - if 'REQUEST_METHOD' in environ: - method = environ['REQUEST_METHOD'].upper() - self.qs_on_post = None - if method == 'GET' or method == 'HEAD': - if 'QUERY_STRING' in environ: - qs = environ['QUERY_STRING'] - elif sys.argv[1:]: - qs = sys.argv[1] - else: - qs = "" - qs = qs.encode(locale.getpreferredencoding(), 'surrogateescape') - fp = BytesIO(qs) - if headers is None: - headers = {'content-type': - "application/x-www-form-urlencoded"} - if headers is None: - headers = {} - if method == 'POST': - # Set default content-type for POST to what's traditional - headers['content-type'] = "application/x-www-form-urlencoded" - if 'CONTENT_TYPE' in environ: - headers['content-type'] = environ['CONTENT_TYPE'] - if 'QUERY_STRING' in environ: - self.qs_on_post = environ['QUERY_STRING'] - if 'CONTENT_LENGTH' in environ: - headers['content-length'] = environ['CONTENT_LENGTH'] - else: - if not (isinstance(headers, (Mapping, Message))): - raise TypeError("headers must be mapping or an instance of " - "email.message.Message") - self.headers = headers - if fp is None: - self.fp = sys.stdin.buffer - # self.fp.read() must return bytes - elif isinstance(fp, TextIOWrapper): - self.fp = fp.buffer - else: - if not (hasattr(fp, 'read') and hasattr(fp, 'readline')): - raise TypeError("fp must be file pointer") - self.fp = fp - - self.encoding = encoding - self.errors = errors - - if not isinstance(outerboundary, bytes): - raise TypeError('outerboundary must be bytes, not %s' - % type(outerboundary).__name__) - self.outerboundary = outerboundary - - self.bytes_read = 0 - self.limit = limit - - # Process content-disposition header - cdisp, pdict = "", {} - if 'content-disposition' in self.headers: - cdisp, pdict = parse_header(self.headers['content-disposition']) - self.disposition = cdisp - self.disposition_options = pdict - self.name = None - if 'name' in pdict: - self.name = pdict['name'] - self.filename = None - if 'filename' in pdict: - self.filename = pdict['filename'] - self._binary_file = self.filename is not None - - # Process content-type header - # - # Honor any existing content-type header. But if there is no - # content-type header, use some sensible defaults. Assume - # outerboundary is "" at the outer level, but something non-false - # inside a multi-part. The default for an inner part is text/plain, - # but for an outer part it should be urlencoded. This should catch - # bogus clients which erroneously forget to include a content-type - # header. - # - # See below for what we do if there does exist a content-type header, - # but it happens to be something we don't understand. - if 'content-type' in self.headers: - ctype, pdict = parse_header(self.headers['content-type']) - elif self.outerboundary or method != 'POST': - ctype, pdict = "text/plain", {} - else: - ctype, pdict = 'application/x-www-form-urlencoded', {} - self.type = ctype - self.type_options = pdict - if 'boundary' in pdict: - self.innerboundary = pdict['boundary'].encode(self.encoding, - self.errors) - else: - self.innerboundary = b"" - - clen = -1 - if 'content-length' in self.headers: - try: - clen = int(self.headers['content-length']) - except ValueError: - pass - if maxlen and clen > maxlen: - raise ValueError('Maximum content length exceeded') - self.length = clen - if self.limit is None and clen >= 0: - self.limit = clen - - self.list = self.file = None - self.done = 0 - if ctype == 'application/x-www-form-urlencoded': - self.read_urlencoded() - elif ctype[:10] == 'multipart/': - self.read_multi(environ, keep_blank_values, strict_parsing) - else: - self.read_single() - - def __del__(self): - try: - self.file.close() - except AttributeError: - pass - - def __enter__(self): - return self - - def __exit__(self, *args): - self.file.close() - - def __repr__(self): - """Return a printable representation.""" - return "FieldStorage(%r, %r, %r)" % ( - self.name, self.filename, self.value) - - def __iter__(self): - return iter(self.keys()) - - def __getattr__(self, name): - if name != 'value': - raise AttributeError(name) - if self.file: - self.file.seek(0) - value = self.file.read() - self.file.seek(0) - elif self.list is not None: - value = self.list - else: - value = None - return value - - def __getitem__(self, key): - """Dictionary style indexing.""" - if self.list is None: - raise TypeError("not indexable") - found = [] - for item in self.list: - if item.name == key: found.append(item) - if not found: - raise KeyError(key) - if len(found) == 1: - return found[0] - else: - return found - - def getvalue(self, key, default=None): - """Dictionary style get() method, including 'value' lookup.""" - if key in self: - value = self[key] - if isinstance(value, list): - return [x.value for x in value] - else: - return value.value - else: - return default - - def getfirst(self, key, default=None): - """ Return the first value received.""" - if key in self: - value = self[key] - if isinstance(value, list): - return value[0].value - else: - return value.value - else: - return default - - def getlist(self, key): - """ Return list of received values.""" - if key in self: - value = self[key] - if isinstance(value, list): - return [x.value for x in value] - else: - return [value.value] - else: - return [] - - def keys(self): - """Dictionary style keys() method.""" - if self.list is None: - raise TypeError("not indexable") - return list(set(item.name for item in self.list)) - - def __contains__(self, key): - """Dictionary style __contains__ method.""" - if self.list is None: - raise TypeError("not indexable") - return any(item.name == key for item in self.list) - - def __len__(self): - """Dictionary style len(x) support.""" - return len(self.keys()) - - def __bool__(self): - if self.list is None: - raise TypeError("Cannot be converted to bool.") - return bool(self.list) - - def read_urlencoded(self): - """Internal: read data in query string format.""" - qs = self.fp.read(self.length) - if not isinstance(qs, bytes): - raise ValueError("%s should return bytes, got %s" \ - % (self.fp, type(qs).__name__)) - qs = qs.decode(self.encoding, self.errors) - if self.qs_on_post: - qs += '&' + self.qs_on_post - query = urllib.parse.parse_qsl( - qs, self.keep_blank_values, self.strict_parsing, - encoding=self.encoding, errors=self.errors, - max_num_fields=self.max_num_fields, separator=self.separator) - self.list = [MiniFieldStorage(key, value) for key, value in query] - self.skip_lines() - - FieldStorageClass = None - - def read_multi(self, environ, keep_blank_values, strict_parsing): - """Internal: read a part that is itself multipart.""" - ib = self.innerboundary - if not valid_boundary(ib): - raise ValueError('Invalid boundary in multipart form: %r' % (ib,)) - self.list = [] - if self.qs_on_post: - query = urllib.parse.parse_qsl( - self.qs_on_post, self.keep_blank_values, self.strict_parsing, - encoding=self.encoding, errors=self.errors, - max_num_fields=self.max_num_fields, separator=self.separator) - self.list.extend(MiniFieldStorage(key, value) for key, value in query) - - klass = self.FieldStorageClass or self.__class__ - first_line = self.fp.readline() # bytes - if not isinstance(first_line, bytes): - raise ValueError("%s should return bytes, got %s" \ - % (self.fp, type(first_line).__name__)) - self.bytes_read += len(first_line) - - # Ensure that we consume the file until we've hit our inner boundary - while (first_line.strip() != (b"--" + self.innerboundary) and - first_line): - first_line = self.fp.readline() - self.bytes_read += len(first_line) - - # Propagate max_num_fields into the sub class appropriately - max_num_fields = self.max_num_fields - if max_num_fields is not None: - max_num_fields -= len(self.list) - - while True: - parser = FeedParser() - hdr_text = b"" - while True: - data = self.fp.readline() - hdr_text += data - if not data.strip(): - break - if not hdr_text: - break - # parser takes strings, not bytes - self.bytes_read += len(hdr_text) - parser.feed(hdr_text.decode(self.encoding, self.errors)) - headers = parser.close() - - # Some clients add Content-Length for part headers, ignore them - if 'content-length' in headers: - del headers['content-length'] - - limit = None if self.limit is None \ - else self.limit - self.bytes_read - part = klass(self.fp, headers, ib, environ, keep_blank_values, - strict_parsing, limit, - self.encoding, self.errors, max_num_fields, self.separator) - - if max_num_fields is not None: - max_num_fields -= 1 - if part.list: - max_num_fields -= len(part.list) - if max_num_fields < 0: - raise ValueError('Max number of fields exceeded') - - self.bytes_read += part.bytes_read - self.list.append(part) - if part.done or self.bytes_read >= self.length > 0: - break - self.skip_lines() - - def read_single(self): - """Internal: read an atomic part.""" - if self.length >= 0: - self.read_binary() - self.skip_lines() - else: - self.read_lines() - self.file.seek(0) - - bufsize = 8*1024 # I/O buffering size for copy to file - - def read_binary(self): - """Internal: read binary data.""" - self.file = self.make_file() - todo = self.length - if todo >= 0: - while todo > 0: - data = self.fp.read(min(todo, self.bufsize)) # bytes - if not isinstance(data, bytes): - raise ValueError("%s should return bytes, got %s" - % (self.fp, type(data).__name__)) - self.bytes_read += len(data) - if not data: - self.done = -1 - break - self.file.write(data) - todo = todo - len(data) - - def read_lines(self): - """Internal: read lines until EOF or outerboundary.""" - if self._binary_file: - self.file = self.__file = BytesIO() # store data as bytes for files - else: - self.file = self.__file = StringIO() # as strings for other fields - if self.outerboundary: - self.read_lines_to_outerboundary() - else: - self.read_lines_to_eof() - - def __write(self, line): - """line is always bytes, not string""" - if self.__file is not None: - if self.__file.tell() + len(line) > 1000: - self.file = self.make_file() - data = self.__file.getvalue() - self.file.write(data) - self.__file = None - if self._binary_file: - # keep bytes - self.file.write(line) - else: - # decode to string - self.file.write(line.decode(self.encoding, self.errors)) - - def read_lines_to_eof(self): - """Internal: read lines until EOF.""" - while 1: - line = self.fp.readline(1<<16) # bytes - self.bytes_read += len(line) - if not line: - self.done = -1 - break - self.__write(line) - - def read_lines_to_outerboundary(self): - """Internal: read lines until outerboundary. - Data is read as bytes: boundaries and line ends must be converted - to bytes for comparisons. - """ - next_boundary = b"--" + self.outerboundary - last_boundary = next_boundary + b"--" - delim = b"" - last_line_lfend = True - _read = 0 - while 1: - - if self.limit is not None and 0 <= self.limit <= _read: - break - line = self.fp.readline(1<<16) # bytes - self.bytes_read += len(line) - _read += len(line) - if not line: - self.done = -1 - break - if delim == b"\r": - line = delim + line - delim = b"" - if line.startswith(b"--") and last_line_lfend: - strippedline = line.rstrip() - if strippedline == next_boundary: - break - if strippedline == last_boundary: - self.done = 1 - break - odelim = delim - if line.endswith(b"\r\n"): - delim = b"\r\n" - line = line[:-2] - last_line_lfend = True - elif line.endswith(b"\n"): - delim = b"\n" - line = line[:-1] - last_line_lfend = True - elif line.endswith(b"\r"): - # We may interrupt \r\n sequences if they span the 2**16 - # byte boundary - delim = b"\r" - line = line[:-1] - last_line_lfend = False - else: - delim = b"" - last_line_lfend = False - self.__write(odelim + line) - - def skip_lines(self): - """Internal: skip lines until outer boundary if defined.""" - if not self.outerboundary or self.done: - return - next_boundary = b"--" + self.outerboundary - last_boundary = next_boundary + b"--" - last_line_lfend = True - while True: - line = self.fp.readline(1<<16) - self.bytes_read += len(line) - if not line: - self.done = -1 - break - if line.endswith(b"--") and last_line_lfend: - strippedline = line.strip() - if strippedline == next_boundary: - break - if strippedline == last_boundary: - self.done = 1 - break - last_line_lfend = line.endswith(b'\n') - - def make_file(self): - """Overridable: return a readable & writable file. - - The file will be used as follows: - - data is written to it - - seek(0) - - data is read from it - - The file is opened in binary mode for files, in text mode - for other fields - - This version opens a temporary file for reading and writing, - and immediately deletes (unlinks) it. The trick (on Unix!) is - that the file can still be used, but it can't be opened by - another process, and it will automatically be deleted when it - is closed or when the current process terminates. - - If you want a more permanent file, you derive a class which - overrides this method. If you want a visible temporary file - that is nevertheless automatically deleted when the script - terminates, try defining a __del__ method in a derived class - which unlinks the temporary files you have created. - - """ - if self._binary_file: - return tempfile.TemporaryFile("wb+") - else: - return tempfile.TemporaryFile("w+", - encoding=self.encoding, newline = '\n') - - -# Test/debug code -# =============== - -def test(environ=os.environ): - """Robust test CGI script, usable as main program. - - Write minimal HTTP headers and dump all information provided to - the script in HTML form. - - """ - print("Content-type: text/html") - print() - sys.stderr = sys.stdout - try: - form = FieldStorage() # Replace with other classes to test those - print_directory() - print_arguments() - print_form(form) - print_environ(environ) - print_environ_usage() - def f(): - exec("testing print_exception() -- italics?") - def g(f=f): - f() - print("

What follows is a test, not an actual exception:

") - g() - except: - print_exception() - - print("

Second try with a small maxlen...

") - - global maxlen - maxlen = 50 - try: - form = FieldStorage() # Replace with other classes to test those - print_directory() - print_arguments() - print_form(form) - print_environ(environ) - except: - print_exception() - -def print_exception(type=None, value=None, tb=None, limit=None): - if type is None: - type, value, tb = sys.exc_info() - import traceback - print() - print("

Traceback (most recent call last):

") - list = traceback.format_tb(tb, limit) + \ - traceback.format_exception_only(type, value) - print("
%s%s
" % ( - html.escape("".join(list[:-1])), - html.escape(list[-1]), - )) - del tb - -def print_environ(environ=os.environ): - """Dump the shell environment as HTML.""" - keys = sorted(environ.keys()) - print() - print("

Shell Environment:

") - print("
") - for key in keys: - print("
", html.escape(key), "
", html.escape(environ[key])) - print("
") - print() - -def print_form(form): - """Dump the contents of a form as HTML.""" - keys = sorted(form.keys()) - print() - print("

Form Contents:

") - if not keys: - print("

No form fields.") - print("

") - for key in keys: - print("
" + html.escape(key) + ":", end=' ') - value = form[key] - print("" + html.escape(repr(type(value))) + "") - print("
" + html.escape(repr(value))) - print("
") - print() - -def print_directory(): - """Dump the current directory as HTML.""" - print() - print("

Current Working Directory:

") - try: - pwd = os.getcwd() - except OSError as msg: - print("OSError:", html.escape(str(msg))) - else: - print(html.escape(pwd)) - print() - -def print_arguments(): - print() - print("

Command Line Arguments:

") - print() - print(sys.argv) - print() - -def print_environ_usage(): - """Dump a list of environment variables used by CGI as HTML.""" - print(""" -

These environment variables could have been set:

-
    -
  • AUTH_TYPE -
  • CONTENT_LENGTH -
  • CONTENT_TYPE -
  • DATE_GMT -
  • DATE_LOCAL -
  • DOCUMENT_NAME -
  • DOCUMENT_ROOT -
  • DOCUMENT_URI -
  • GATEWAY_INTERFACE -
  • LAST_MODIFIED -
  • PATH -
  • PATH_INFO -
  • PATH_TRANSLATED -
  • QUERY_STRING -
  • REMOTE_ADDR -
  • REMOTE_HOST -
  • REMOTE_IDENT -
  • REMOTE_USER -
  • REQUEST_METHOD -
  • SCRIPT_NAME -
  • SERVER_NAME -
  • SERVER_PORT -
  • SERVER_PROTOCOL -
  • SERVER_ROOT -
  • SERVER_SOFTWARE -
-In addition, HTTP headers sent by the server may be passed in the -environment as well. Here are some common variable names: -
    -
  • HTTP_ACCEPT -
  • HTTP_CONNECTION -
  • HTTP_HOST -
  • HTTP_PRAGMA -
  • HTTP_REFERER -
  • HTTP_USER_AGENT -
-""") - - -# Utilities -# ========= - -def valid_boundary(s): - import re - if isinstance(s, bytes): - _vb_pattern = b"^[ -~]{0,200}[!-~]$" - else: - _vb_pattern = "^[ -~]{0,200}[!-~]$" - return re.match(_vb_pattern, s) - -# Invoke mainline -# =============== - -# Call test() when this file is run as a script (not imported as a module) -if __name__ == '__main__': - test() diff --git a/Lib/test/test_cgi.py b/Lib/test/test_cgi.py deleted file mode 100644 index 43164cff31..0000000000 --- a/Lib/test/test_cgi.py +++ /dev/null @@ -1,645 +0,0 @@ -import os -import sys -import tempfile -import unittest -from collections import namedtuple -from io import StringIO, BytesIO -from test import support -from test.support import warnings_helper - -cgi = warnings_helper.import_deprecated("cgi") - - -class HackedSysModule: - # The regression test will have real values in sys.argv, which - # will completely confuse the test of the cgi module - argv = [] - stdin = sys.stdin - -cgi.sys = HackedSysModule() - -class ComparableException: - def __init__(self, err): - self.err = err - - def __str__(self): - return str(self.err) - - def __eq__(self, anExc): - if not isinstance(anExc, Exception): - return NotImplemented - return (self.err.__class__ == anExc.__class__ and - self.err.args == anExc.args) - - def __getattr__(self, attr): - return getattr(self.err, attr) - -def do_test(buf, method): - env = {} - if method == "GET": - fp = None - env['REQUEST_METHOD'] = 'GET' - env['QUERY_STRING'] = buf - elif method == "POST": - fp = BytesIO(buf.encode('latin-1')) # FieldStorage expects bytes - env['REQUEST_METHOD'] = 'POST' - env['CONTENT_TYPE'] = 'application/x-www-form-urlencoded' - env['CONTENT_LENGTH'] = str(len(buf)) - else: - raise ValueError("unknown method: %s" % method) - try: - return cgi.parse(fp, env, strict_parsing=1) - except Exception as err: - return ComparableException(err) - -parse_strict_test_cases = [ - ("", {}), - ("&", ValueError("bad query field: ''")), - ("&&", ValueError("bad query field: ''")), - # Should the next few really be valid? - ("=", {}), - ("=&=", {}), - # This rest seem to make sense - ("=a", {'': ['a']}), - ("&=a", ValueError("bad query field: ''")), - ("=a&", ValueError("bad query field: ''")), - ("=&a", ValueError("bad query field: 'a'")), - ("b=a", {'b': ['a']}), - ("b+=a", {'b ': ['a']}), - ("a=b=a", {'a': ['b=a']}), - ("a=+b=a", {'a': [' b=a']}), - ("&b=a", ValueError("bad query field: ''")), - ("b&=a", ValueError("bad query field: 'b'")), - ("a=a+b&b=b+c", {'a': ['a b'], 'b': ['b c']}), - ("a=a+b&a=b+a", {'a': ['a b', 'b a']}), - ("x=1&y=2.0&z=2-3.%2b0", {'x': ['1'], 'y': ['2.0'], 'z': ['2-3.+0']}), - ("Hbc5161168c542333633315dee1182227:key_store_seqid=400006&cuyer=r&view=bustomer&order_id=0bb2e248638833d48cb7fed300000f1b&expire=964546263&lobale=en-US&kid=130003.300038&ss=env", - {'Hbc5161168c542333633315dee1182227:key_store_seqid': ['400006'], - 'cuyer': ['r'], - 'expire': ['964546263'], - 'kid': ['130003.300038'], - 'lobale': ['en-US'], - 'order_id': ['0bb2e248638833d48cb7fed300000f1b'], - 'ss': ['env'], - 'view': ['bustomer'], - }), - - ("group_id=5470&set=custom&_assigned_to=31392&_status=1&_category=100&SUBMIT=Browse", - {'SUBMIT': ['Browse'], - '_assigned_to': ['31392'], - '_category': ['100'], - '_status': ['1'], - 'group_id': ['5470'], - 'set': ['custom'], - }) - ] - -def norm(seq): - return sorted(seq, key=repr) - -def first_elts(list): - return [p[0] for p in list] - -def first_second_elts(list): - return [(p[0], p[1][0]) for p in list] - -def gen_result(data, environ): - encoding = 'latin-1' - fake_stdin = BytesIO(data.encode(encoding)) - fake_stdin.seek(0) - form = cgi.FieldStorage(fp=fake_stdin, environ=environ, encoding=encoding) - - result = {} - for k, v in dict(form).items(): - result[k] = isinstance(v, list) and form.getlist(k) or v.value - - return result - -class CgiTests(unittest.TestCase): - - def test_parse_multipart(self): - fp = BytesIO(POSTDATA.encode('latin1')) - env = {'boundary': BOUNDARY.encode('latin1'), - 'CONTENT-LENGTH': '558'} - result = cgi.parse_multipart(fp, env) - expected = {'submit': [' Add '], 'id': ['1234'], - 'file': [b'Testing 123.\n'], 'title': ['']} - self.assertEqual(result, expected) - - def test_parse_multipart_without_content_length(self): - POSTDATA = '''--JfISa01 -Content-Disposition: form-data; name="submit-name" - -just a string - ---JfISa01-- -''' - fp = BytesIO(POSTDATA.encode('latin1')) - env = {'boundary': 'JfISa01'.encode('latin1')} - result = cgi.parse_multipart(fp, env) - expected = {'submit-name': ['just a string\n']} - self.assertEqual(result, expected) - - # TODO RUSTPYTHON - see https://github.com/RustPython/RustPython/issues/935 - @unittest.expectedFailure - def test_parse_multipart_invalid_encoding(self): - BOUNDARY = "JfISa01" - POSTDATA = """--JfISa01 -Content-Disposition: form-data; name="submit-name" -Content-Length: 3 - -\u2603 ---JfISa01""" - fp = BytesIO(POSTDATA.encode('utf8')) - env = {'boundary': BOUNDARY.encode('latin1'), - 'CONTENT-LENGTH': str(len(POSTDATA.encode('utf8')))} - result = cgi.parse_multipart(fp, env, encoding="ascii", - errors="surrogateescape") - expected = {'submit-name': ["\udce2\udc98\udc83"]} - self.assertEqual(result, expected) - self.assertEqual("\u2603".encode('utf8'), - result["submit-name"][0].encode('utf8', 'surrogateescape')) - - def test_fieldstorage_properties(self): - fs = cgi.FieldStorage() - self.assertFalse(fs) - self.assertIn("FieldStorage", repr(fs)) - self.assertEqual(list(fs), list(fs.keys())) - fs.list.append(namedtuple('MockFieldStorage', 'name')('fieldvalue')) - self.assertTrue(fs) - - def test_fieldstorage_invalid(self): - self.assertRaises(TypeError, cgi.FieldStorage, "not-a-file-obj", - environ={"REQUEST_METHOD":"PUT"}) - self.assertRaises(TypeError, cgi.FieldStorage, "foo", "bar") - fs = cgi.FieldStorage(headers={'content-type':'text/plain'}) - self.assertRaises(TypeError, bool, fs) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_strict(self): - for orig, expect in parse_strict_test_cases: - # Test basic parsing - d = do_test(orig, "GET") - self.assertEqual(d, expect, "Error parsing %s method GET" % repr(orig)) - d = do_test(orig, "POST") - self.assertEqual(d, expect, "Error parsing %s method POST" % repr(orig)) - - env = {'QUERY_STRING': orig} - fs = cgi.FieldStorage(environ=env) - if isinstance(expect, dict): - # test dict interface - self.assertEqual(len(expect), len(fs)) - self.assertCountEqual(expect.keys(), fs.keys()) - ##self.assertEqual(norm(expect.values()), norm(fs.values())) - ##self.assertEqual(norm(expect.items()), norm(fs.items())) - self.assertEqual(fs.getvalue("nonexistent field", "default"), "default") - # test individual fields - for key in expect.keys(): - expect_val = expect[key] - self.assertIn(key, fs) - if len(expect_val) > 1: - self.assertEqual(fs.getvalue(key), expect_val) - else: - self.assertEqual(fs.getvalue(key), expect_val[0]) - - def test_separator(self): - parse_semicolon = [ - ("x=1;y=2.0", {'x': ['1'], 'y': ['2.0']}), - ("x=1;y=2.0;z=2-3.%2b0", {'x': ['1'], 'y': ['2.0'], 'z': ['2-3.+0']}), - (";", ValueError("bad query field: ''")), - (";;", ValueError("bad query field: ''")), - ("=;a", ValueError("bad query field: 'a'")), - (";b=a", ValueError("bad query field: ''")), - ("b;=a", ValueError("bad query field: 'b'")), - ("a=a+b;b=b+c", {'a': ['a b'], 'b': ['b c']}), - ("a=a+b;a=b+a", {'a': ['a b', 'b a']}), - ] - for orig, expect in parse_semicolon: - env = {'QUERY_STRING': orig} - fs = cgi.FieldStorage(separator=';', environ=env) - if isinstance(expect, dict): - for key in expect.keys(): - expect_val = expect[key] - self.assertIn(key, fs) - if len(expect_val) > 1: - self.assertEqual(fs.getvalue(key), expect_val) - else: - self.assertEqual(fs.getvalue(key), expect_val[0]) - - @warnings_helper.ignore_warnings(category=DeprecationWarning) - def test_log(self): - cgi.log("Testing") - - cgi.logfp = StringIO() - cgi.initlog("%s", "Testing initlog 1") - cgi.log("%s", "Testing log 2") - self.assertEqual(cgi.logfp.getvalue(), "Testing initlog 1\nTesting log 2\n") - if os.path.exists(os.devnull): - cgi.logfp = None - cgi.logfile = os.devnull - cgi.initlog("%s", "Testing log 3") - self.addCleanup(cgi.closelog) - cgi.log("Testing log 4") - - def test_fieldstorage_readline(self): - # FieldStorage uses readline, which has the capacity to read all - # contents of the input file into memory; we use readline's size argument - # to prevent that for files that do not contain any newlines in - # non-GET/HEAD requests - class TestReadlineFile: - def __init__(self, file): - self.file = file - self.numcalls = 0 - - def readline(self, size=None): - self.numcalls += 1 - if size: - return self.file.readline(size) - else: - return self.file.readline() - - def __getattr__(self, name): - file = self.__dict__['file'] - a = getattr(file, name) - if not isinstance(a, int): - setattr(self, name, a) - return a - - f = TestReadlineFile(tempfile.TemporaryFile("wb+")) - self.addCleanup(f.close) - f.write(b'x' * 256 * 1024) - f.seek(0) - env = {'REQUEST_METHOD':'PUT'} - fs = cgi.FieldStorage(fp=f, environ=env) - self.addCleanup(fs.file.close) - # if we're not chunking properly, readline is only called twice - # (by read_binary); if we are chunking properly, it will be called 5 times - # as long as the chunksize is 1 << 16. - self.assertGreater(f.numcalls, 2) - f.close() - - def test_fieldstorage_multipart(self): - #Test basic FieldStorage multipart parsing - env = { - 'REQUEST_METHOD': 'POST', - 'CONTENT_TYPE': 'multipart/form-data; boundary={}'.format(BOUNDARY), - 'CONTENT_LENGTH': '558'} - fp = BytesIO(POSTDATA.encode('latin-1')) - fs = cgi.FieldStorage(fp, environ=env, encoding="latin-1") - self.assertEqual(len(fs.list), 4) - expect = [{'name':'id', 'filename':None, 'value':'1234'}, - {'name':'title', 'filename':None, 'value':''}, - {'name':'file', 'filename':'test.txt', 'value':b'Testing 123.\n'}, - {'name':'submit', 'filename':None, 'value':' Add '}] - for x in range(len(fs.list)): - for k, exp in expect[x].items(): - got = getattr(fs.list[x], k) - self.assertEqual(got, exp) - - def test_fieldstorage_multipart_leading_whitespace(self): - env = { - 'REQUEST_METHOD': 'POST', - 'CONTENT_TYPE': 'multipart/form-data; boundary={}'.format(BOUNDARY), - 'CONTENT_LENGTH': '560'} - # Add some leading whitespace to our post data that will cause the - # first line to not be the innerboundary. - fp = BytesIO(b"\r\n" + POSTDATA.encode('latin-1')) - fs = cgi.FieldStorage(fp, environ=env, encoding="latin-1") - self.assertEqual(len(fs.list), 4) - expect = [{'name':'id', 'filename':None, 'value':'1234'}, - {'name':'title', 'filename':None, 'value':''}, - {'name':'file', 'filename':'test.txt', 'value':b'Testing 123.\n'}, - {'name':'submit', 'filename':None, 'value':' Add '}] - for x in range(len(fs.list)): - for k, exp in expect[x].items(): - got = getattr(fs.list[x], k) - self.assertEqual(got, exp) - - def test_fieldstorage_multipart_non_ascii(self): - #Test basic FieldStorage multipart parsing - env = {'REQUEST_METHOD':'POST', - 'CONTENT_TYPE': 'multipart/form-data; boundary={}'.format(BOUNDARY), - 'CONTENT_LENGTH':'558'} - for encoding in ['iso-8859-1','utf-8']: - fp = BytesIO(POSTDATA_NON_ASCII.encode(encoding)) - fs = cgi.FieldStorage(fp, environ=env,encoding=encoding) - self.assertEqual(len(fs.list), 1) - expect = [{'name':'id', 'filename':None, 'value':'\xe7\xf1\x80'}] - for x in range(len(fs.list)): - for k, exp in expect[x].items(): - got = getattr(fs.list[x], k) - self.assertEqual(got, exp) - - def test_fieldstorage_multipart_maxline(self): - # Issue #18167 - maxline = 1 << 16 - self.maxDiff = None - def check(content): - data = """---123 -Content-Disposition: form-data; name="upload"; filename="fake.txt" -Content-Type: text/plain - -%s ----123-- -""".replace('\n', '\r\n') % content - environ = { - 'CONTENT_LENGTH': str(len(data)), - 'CONTENT_TYPE': 'multipart/form-data; boundary=-123', - 'REQUEST_METHOD': 'POST', - } - self.assertEqual(gen_result(data, environ), - {'upload': content.encode('latin1')}) - check('x' * (maxline - 1)) - check('x' * (maxline - 1) + '\r') - check('x' * (maxline - 1) + '\r' + 'y' * (maxline - 1)) - - def test_fieldstorage_multipart_w3c(self): - # Test basic FieldStorage multipart parsing (W3C sample) - env = { - 'REQUEST_METHOD': 'POST', - 'CONTENT_TYPE': 'multipart/form-data; boundary={}'.format(BOUNDARY_W3), - 'CONTENT_LENGTH': str(len(POSTDATA_W3))} - fp = BytesIO(POSTDATA_W3.encode('latin-1')) - fs = cgi.FieldStorage(fp, environ=env, encoding="latin-1") - self.assertEqual(len(fs.list), 2) - self.assertEqual(fs.list[0].name, 'submit-name') - self.assertEqual(fs.list[0].value, 'Larry') - self.assertEqual(fs.list[1].name, 'files') - files = fs.list[1].value - self.assertEqual(len(files), 2) - expect = [{'name': None, 'filename': 'file1.txt', 'value': b'... contents of file1.txt ...'}, - {'name': None, 'filename': 'file2.gif', 'value': b'...contents of file2.gif...'}] - for x in range(len(files)): - for k, exp in expect[x].items(): - got = getattr(files[x], k) - self.assertEqual(got, exp) - - def test_fieldstorage_part_content_length(self): - BOUNDARY = "JfISa01" - POSTDATA = """--JfISa01 -Content-Disposition: form-data; name="submit-name" -Content-Length: 5 - -Larry ---JfISa01""" - env = { - 'REQUEST_METHOD': 'POST', - 'CONTENT_TYPE': 'multipart/form-data; boundary={}'.format(BOUNDARY), - 'CONTENT_LENGTH': str(len(POSTDATA))} - fp = BytesIO(POSTDATA.encode('latin-1')) - fs = cgi.FieldStorage(fp, environ=env, encoding="latin-1") - self.assertEqual(len(fs.list), 1) - self.assertEqual(fs.list[0].name, 'submit-name') - self.assertEqual(fs.list[0].value, 'Larry') - - def test_field_storage_multipart_no_content_length(self): - fp = BytesIO(b"""--MyBoundary -Content-Disposition: form-data; name="my-arg"; filename="foo" - -Test - ---MyBoundary-- -""") - env = { - "REQUEST_METHOD": "POST", - "CONTENT_TYPE": "multipart/form-data; boundary=MyBoundary", - "wsgi.input": fp, - } - fields = cgi.FieldStorage(fp, environ=env) - - self.assertEqual(len(fields["my-arg"].file.read()), 5) - - def test_fieldstorage_as_context_manager(self): - fp = BytesIO(b'x' * 10) - env = {'REQUEST_METHOD': 'PUT'} - with cgi.FieldStorage(fp=fp, environ=env) as fs: - content = fs.file.read() - self.assertFalse(fs.file.closed) - self.assertTrue(fs.file.closed) - self.assertEqual(content, 'x' * 10) - with self.assertRaisesRegex(ValueError, 'I/O operation on closed file'): - fs.file.read() - - _qs_result = { - 'key1': 'value1', - 'key2': ['value2x', 'value2y'], - 'key3': 'value3', - 'key4': 'value4' - } - def testQSAndUrlEncode(self): - data = "key2=value2x&key3=value3&key4=value4" - environ = { - 'CONTENT_LENGTH': str(len(data)), - 'CONTENT_TYPE': 'application/x-www-form-urlencoded', - 'QUERY_STRING': 'key1=value1&key2=value2y', - 'REQUEST_METHOD': 'POST', - } - v = gen_result(data, environ) - self.assertEqual(self._qs_result, v) - - def test_max_num_fields(self): - # For application/x-www-form-urlencoded - data = '&'.join(['a=a']*11) - environ = { - 'CONTENT_LENGTH': str(len(data)), - 'CONTENT_TYPE': 'application/x-www-form-urlencoded', - 'REQUEST_METHOD': 'POST', - } - - with self.assertRaises(ValueError): - cgi.FieldStorage( - fp=BytesIO(data.encode()), - environ=environ, - max_num_fields=10, - ) - - # For multipart/form-data - data = """---123 -Content-Disposition: form-data; name="a" - -3 ----123 -Content-Type: application/x-www-form-urlencoded - -a=4 ----123 -Content-Type: application/x-www-form-urlencoded - -a=5 ----123-- -""" - environ = { - 'CONTENT_LENGTH': str(len(data)), - 'CONTENT_TYPE': 'multipart/form-data; boundary=-123', - 'QUERY_STRING': 'a=1&a=2', - 'REQUEST_METHOD': 'POST', - } - - # 2 GET entities - # 1 top level POST entities - # 1 entity within the second POST entity - # 1 entity within the third POST entity - with self.assertRaises(ValueError): - cgi.FieldStorage( - fp=BytesIO(data.encode()), - environ=environ, - max_num_fields=4, - ) - cgi.FieldStorage( - fp=BytesIO(data.encode()), - environ=environ, - max_num_fields=5, - ) - - def testQSAndFormData(self): - data = """---123 -Content-Disposition: form-data; name="key2" - -value2y ----123 -Content-Disposition: form-data; name="key3" - -value3 ----123 -Content-Disposition: form-data; name="key4" - -value4 ----123-- -""" - environ = { - 'CONTENT_LENGTH': str(len(data)), - 'CONTENT_TYPE': 'multipart/form-data; boundary=-123', - 'QUERY_STRING': 'key1=value1&key2=value2x', - 'REQUEST_METHOD': 'POST', - } - v = gen_result(data, environ) - self.assertEqual(self._qs_result, v) - - def testQSAndFormDataFile(self): - data = """---123 -Content-Disposition: form-data; name="key2" - -value2y ----123 -Content-Disposition: form-data; name="key3" - -value3 ----123 -Content-Disposition: form-data; name="key4" - -value4 ----123 -Content-Disposition: form-data; name="upload"; filename="fake.txt" -Content-Type: text/plain - -this is the content of the fake file - ----123-- -""" - environ = { - 'CONTENT_LENGTH': str(len(data)), - 'CONTENT_TYPE': 'multipart/form-data; boundary=-123', - 'QUERY_STRING': 'key1=value1&key2=value2x', - 'REQUEST_METHOD': 'POST', - } - result = self._qs_result.copy() - result.update({ - 'upload': b'this is the content of the fake file\n' - }) - v = gen_result(data, environ) - self.assertEqual(result, v) - - def test_parse_header(self): - self.assertEqual( - cgi.parse_header("text/plain"), - ("text/plain", {})) - self.assertEqual( - cgi.parse_header("text/vnd.just.made.this.up ; "), - ("text/vnd.just.made.this.up", {})) - self.assertEqual( - cgi.parse_header("text/plain;charset=us-ascii"), - ("text/plain", {"charset": "us-ascii"})) - self.assertEqual( - cgi.parse_header('text/plain ; charset="us-ascii"'), - ("text/plain", {"charset": "us-ascii"})) - self.assertEqual( - cgi.parse_header('text/plain ; charset="us-ascii"; another=opt'), - ("text/plain", {"charset": "us-ascii", "another": "opt"})) - self.assertEqual( - cgi.parse_header('attachment; filename="silly.txt"'), - ("attachment", {"filename": "silly.txt"})) - self.assertEqual( - cgi.parse_header('attachment; filename="strange;name"'), - ("attachment", {"filename": "strange;name"})) - self.assertEqual( - cgi.parse_header('attachment; filename="strange;name";size=123;'), - ("attachment", {"filename": "strange;name", "size": "123"})) - self.assertEqual( - cgi.parse_header('form-data; name="files"; filename="fo\\"o;bar"'), - ("form-data", {"name": "files", "filename": 'fo"o;bar'})) - - def test_all(self): - not_exported = { - "logfile", "logfp", "initlog", "dolog", "nolog", "closelog", "log", - "maxlen", "valid_boundary"} - support.check__all__(self, cgi, not_exported=not_exported) - - -BOUNDARY = "---------------------------721837373350705526688164684" - -POSTDATA = """-----------------------------721837373350705526688164684 -Content-Disposition: form-data; name="id" - -1234 ------------------------------721837373350705526688164684 -Content-Disposition: form-data; name="title" - - ------------------------------721837373350705526688164684 -Content-Disposition: form-data; name="file"; filename="test.txt" -Content-Type: text/plain - -Testing 123. - ------------------------------721837373350705526688164684 -Content-Disposition: form-data; name="submit" - - Add\x20 ------------------------------721837373350705526688164684-- -""" - -POSTDATA_NON_ASCII = """-----------------------------721837373350705526688164684 -Content-Disposition: form-data; name="id" - -\xe7\xf1\x80 ------------------------------721837373350705526688164684 -""" - -# http://www.w3.org/TR/html401/interact/forms.html#h-17.13.4 -BOUNDARY_W3 = "AaB03x" -POSTDATA_W3 = """--AaB03x -Content-Disposition: form-data; name="submit-name" - -Larry ---AaB03x -Content-Disposition: form-data; name="files" -Content-Type: multipart/mixed; boundary=BbC04y - ---BbC04y -Content-Disposition: file; filename="file1.txt" -Content-Type: text/plain - -... contents of file1.txt ... ---BbC04y -Content-Disposition: file; filename="file2.gif" -Content-Type: image/gif -Content-Transfer-Encoding: binary - -...contents of file2.gif... ---BbC04y-- ---AaB03x-- -""" - -if __name__ == '__main__': - unittest.main() diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index 59700ac791..cd689492ca 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -778,6 +778,7 @@ def test_issue19435(self): # TODO: RUSTPYTHON @unittest.skipIf(sys.platform != 'win32', "TODO: RUSTPYTHON; works only on windows") + @unittest.expectedFailure def test_post(self): params = urllib.parse.urlencode( {'spam' : 1, 'eggs' : 'python', 'bacon' : 123456}) From 8da66978bf8adf6e088854b82dd236ace36a2ce0 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Thu, 13 Mar 2025 09:49:10 -0700 Subject: [PATCH 082/295] Remove uu.py and test_uu.py --- Lib/test/test_uu.py | 258 -------------------------------------------- Lib/uu.py | 199 ---------------------------------- 2 files changed, 457 deletions(-) delete mode 100644 Lib/test/test_uu.py delete mode 100755 Lib/uu.py diff --git a/Lib/test/test_uu.py b/Lib/test/test_uu.py deleted file mode 100644 index f71d877365..0000000000 --- a/Lib/test/test_uu.py +++ /dev/null @@ -1,258 +0,0 @@ -""" -Tests for uu module. -Nick Mathewson -""" - -import unittest -from test.support import os_helper - -import os -import stat -import sys -import uu -import io - -plaintext = b"The symbols on top of your keyboard are !@#$%^&*()_+|~\n" - -encodedtext = b"""\ -M5&AE('-Y;6)O;',@;VX@=&]P(&]F('EO=7(@:V5Y8F]A 0: - out_file.write(binascii.b2a_uu(data)) - data = in_file.read(45) - out_file.write(b' \nend\n') - finally: - for f in opened_files: - f.close() - - -def decode(in_file, out_file=None, mode=None, quiet=False): - """Decode uuencoded file""" - # - # Open the input file, if needed. - # - opened_files = [] - if in_file == '-': - in_file = sys.stdin.buffer - elif isinstance(in_file, str): - in_file = open(in_file, 'rb') - opened_files.append(in_file) - - try: - # - # Read until a begin is encountered or we've exhausted the file - # - while True: - hdr = in_file.readline() - if not hdr: - raise Error('No valid begin line found in input file') - if not hdr.startswith(b'begin'): - continue - hdrfields = hdr.split(b' ', 2) - if len(hdrfields) == 3 and hdrfields[0] == b'begin': - try: - int(hdrfields[1], 8) - break - except ValueError: - pass - if out_file is None: - # If the filename isn't ASCII, what's up with that?!? - out_file = hdrfields[2].rstrip(b' \t\r\n\f').decode("ascii") - if os.path.exists(out_file): - raise Error('Cannot overwrite existing file: %s' % out_file) - if mode is None: - mode = int(hdrfields[1], 8) - # - # Open the output file - # - if out_file == '-': - out_file = sys.stdout.buffer - elif isinstance(out_file, str): - fp = open(out_file, 'wb') - try: - os.path.chmod(out_file, mode) - except AttributeError: - pass - out_file = fp - opened_files.append(out_file) - # - # Main decoding loop - # - s = in_file.readline() - while s and s.strip(b' \t\r\n\f') != b'end': - try: - data = binascii.a2b_uu(s) - except binascii.Error as v: - # Workaround for broken uuencoders by /Fredrik Lundh - nbytes = (((s[0]-32) & 63) * 4 + 5) // 3 - data = binascii.a2b_uu(s[:nbytes]) - if not quiet: - sys.stderr.write("Warning: %s\n" % v) - out_file.write(data) - s = in_file.readline() - if not s: - raise Error('Truncated input file') - finally: - for f in opened_files: - f.close() - -def test(): - """uuencode/uudecode main program""" - - import optparse - parser = optparse.OptionParser(usage='usage: %prog [-d] [-t] [input [output]]') - parser.add_option('-d', '--decode', dest='decode', help='Decode (instead of encode)?', default=False, action='store_true') - parser.add_option('-t', '--text', dest='text', help='data is text, encoded format unix-compatible text?', default=False, action='store_true') - - (options, args) = parser.parse_args() - if len(args) > 2: - parser.error('incorrect number of arguments') - sys.exit(1) - - # Use the binary streams underlying stdin/stdout - input = sys.stdin.buffer - output = sys.stdout.buffer - if len(args) > 0: - input = args[0] - if len(args) > 1: - output = args[1] - - if options.decode: - if options.text: - if isinstance(output, str): - output = open(output, 'wb') - else: - print(sys.argv[0], ': cannot do -t to stdout') - sys.exit(1) - decode(input, output) - else: - if options.text: - if isinstance(input, str): - input = open(input, 'rb') - else: - print(sys.argv[0], ': cannot do -t from stdin') - sys.exit(1) - encode(input, output) - -if __name__ == '__main__': - test() From 7546ea91a98475a9d0efd081abc469a3c0c22775 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Thu, 13 Mar 2025 10:16:51 -0700 Subject: [PATCH 083/295] patch email.message --- Lib/email/message.py | 39 ++++++++++++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/Lib/email/message.py b/Lib/email/message.py index b6512f2198..f932186875 100644 --- a/Lib/email/message.py +++ b/Lib/email/message.py @@ -7,7 +7,6 @@ __all__ = ['Message', 'EmailMessage'] import re -import uu import quopri from io import BytesIO, StringIO @@ -101,6 +100,35 @@ def _unquotevalue(value): return utils.unquote(value) +def _decode_uu(encoded): + """Decode uuencoded data.""" + decoded_lines = [] + encoded_lines_iter = iter(encoded.splitlines()) + for line in encoded_lines_iter: + if line.startswith(b"begin "): + mode, _, path = line.removeprefix(b"begin ").partition(b" ") + try: + int(mode, base=8) + except ValueError: + continue + else: + break + else: + raise ValueError("`begin` line not found") + for line in encoded_lines_iter: + if not line: + raise ValueError("Truncated input") + elif line.strip(b' \t\r\n\f') == b'end': + break + try: + decoded_line = binascii.a2b_uu(line) + except binascii.Error: + # Workaround for broken uuencoders by /Fredrik Lundh + nbytes = (((line[0]-32) & 63) * 4 + 5) // 3 + decoded_line = binascii.a2b_uu(line[:nbytes]) + decoded_lines.append(decoded_line) + + return b''.join(decoded_lines) class Message: """Basic message object. @@ -288,13 +316,10 @@ def get_payload(self, i=None, decode=False): self.policy.handle_defect(self, defect) return value elif cte in ('x-uuencode', 'uuencode', 'uue', 'x-uue'): - in_file = BytesIO(bpayload) - out_file = BytesIO() try: - uu.decode(in_file, out_file, quiet=True) - return out_file.getvalue() - except uu.Error: - # Some decoding problem + return _decode_uu(bpayload) + except ValueError: + # Some decoding problem. return bpayload if isinstance(payload, str): return bpayload From 8e22c399df2948b0a736572b2eb5db1b5e701864 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Thu, 13 Mar 2025 19:38:35 -0700 Subject: [PATCH 084/295] partially fix sys.getwindowsversion() (#5595) --- extra_tests/snippets/stdlib_sys.py | 6 +- vm/src/stdlib/sys.rs | 96 +++++++++++++++++++++++++++--- 2 files changed, 91 insertions(+), 11 deletions(-) diff --git a/extra_tests/snippets/stdlib_sys.py b/extra_tests/snippets/stdlib_sys.py index 5d8859ac8e..d4a2d2cdca 100644 --- a/extra_tests/snippets/stdlib_sys.py +++ b/extra_tests/snippets/stdlib_sys.py @@ -81,7 +81,7 @@ def recursive_call(n): 0x00000100 | 0x00000001 | 0x00000020 | 0x00002000 | 0x00000010 | 0x00008000 | 0x00020000 # We really can't test if the results are correct, so it just checks for meaningful value - assert winver.major > 0 + assert winver.major > 6 assert winver.minor >= 0 assert winver.build > 0 assert winver.platform == 2 @@ -91,8 +91,8 @@ def recursive_call(n): # XXX if platform_version is implemented correctly, this'll break on compatiblity mode or a build without manifest # these fields can mismatch in CPython - # assert winver.major == winver.platform_version[0] - # assert winver.minor == winver.platform_version[1] + assert winver.major == winver.platform_version[0] + assert winver.minor == winver.platform_version[1] # assert winver.build == winver.platform_version[2] # test int_max_str_digits getter and setter diff --git a/vm/src/stdlib/sys.rs b/vm/src/stdlib/sys.rs index dfaab20f2a..39c803a01b 100644 --- a/vm/src/stdlib/sys.rs +++ b/vm/src/stdlib/sys.rs @@ -22,12 +22,23 @@ mod sys { vm::{Settings, VirtualMachine}, }; use num_traits::ToPrimitive; + #[cfg(windows)] + use std::os::windows::ffi::OsStrExt; use std::{ env::{self, VarError}, path, sync::atomic::Ordering, }; + #[cfg(windows)] + use windows_sys::Win32::{ + Foundation::MAX_PATH, + Storage::FileSystem::{ + GetFileVersionInfoSizeW, GetFileVersionInfoW, VS_FIXEDFILEINFO, VerQueryValueW, + }, + System::LibraryLoader::{GetModuleFileNameW, GetModuleHandleW}, + }; + // not the same as CPython (e.g. rust's x86_x64-unknown-linux-gnu is just x86_64-linux-gnu) // but hopefully that's just an implementation detail? TODO: copy CPython's multiarch exactly, // https://github.com/python/cpython/blob/3.8/configure.ac#L725 @@ -485,6 +496,78 @@ mod sys { vm.trace_func.borrow().clone() } + #[cfg(windows)] + fn get_kernel32_version() -> std::io::Result<(u32, u32, u32)> { + unsafe { + // Create a wide string for "kernel32.dll" + let module_name: Vec = std::ffi::OsStr::new("kernel32.dll") + .encode_wide() + .chain(Some(0)) + .collect(); + let h_kernel32 = GetModuleHandleW(module_name.as_ptr()); + if h_kernel32.is_null() { + return Err(std::io::Error::last_os_error()); + } + + // Prepare a buffer for the module file path + let mut kernel32_path = [0u16; MAX_PATH as usize]; + let len = GetModuleFileNameW( + h_kernel32, + kernel32_path.as_mut_ptr(), + kernel32_path.len() as u32, + ); + if len == 0 { + return Err(std::io::Error::last_os_error()); + } + + // Get the size of the version information block + let verblock_size = + GetFileVersionInfoSizeW(kernel32_path.as_ptr(), std::ptr::null_mut()); + if verblock_size == 0 { + return Err(std::io::Error::last_os_error()); + } + + // Allocate a buffer to hold the version information + let mut verblock = vec![0u8; verblock_size as usize]; + if GetFileVersionInfoW( + kernel32_path.as_ptr(), + 0, + verblock_size, + verblock.as_mut_ptr() as *mut _, + ) == 0 + { + return Err(std::io::Error::last_os_error()); + } + + // Prepare an empty sub-block string (L"") as required by VerQueryValueW + let sub_block: Vec = std::ffi::OsStr::new("") + .encode_wide() + .chain(Some(0)) + .collect(); + + let mut ffi_ptr: *mut VS_FIXEDFILEINFO = std::ptr::null_mut(); + let mut ffi_len: u32 = 0; + if VerQueryValueW( + verblock.as_ptr() as *const _, + sub_block.as_ptr(), + &mut ffi_ptr as *mut *mut VS_FIXEDFILEINFO as *mut *mut _, + &mut ffi_len as *mut u32, + ) == 0 + || ffi_ptr.is_null() + { + return Err(std::io::Error::last_os_error()); + } + + // Extract the version numbers from the VS_FIXEDFILEINFO structure. + let ffi = *ffi_ptr; + let real_major = (ffi.dwProductVersionMS >> 16) & 0xFFFF; + let real_minor = ffi.dwProductVersionMS & 0xFFFF; + let real_build = (ffi.dwProductVersionLS >> 16) & 0xFFFF; + + Ok((real_major, real_minor, real_build)) + } + } + #[cfg(windows)] #[pyfunction] fn getwindowsversion(vm: &VirtualMachine) -> PyResult { @@ -519,21 +602,18 @@ mod sys { sp.into_string() .map_err(|_| vm.new_os_error("service pack is not ASCII".to_owned()))? }; + let real_version = get_kernel32_version().map_err(|e| vm.new_os_error(e.to_string()))?; Ok(WindowsVersion { - major: version.dwMajorVersion, - minor: version.dwMinorVersion, - build: version.dwBuildNumber, + major: real_version.0, + minor: real_version.1, + build: real_version.2, platform: version.dwPlatformId, service_pack, service_pack_major: version.wServicePackMajor, service_pack_minor: version.wServicePackMinor, suite_mask: version.wSuiteMask, product_type: version.wProductType, - platform_version: ( - version.dwMajorVersion, - version.dwMinorVersion, - version.dwBuildNumber, - ), // TODO Provide accurate version, like CPython impl + platform_version: (real_version.0, real_version.1, real_version.2), // TODO Provide accurate version, like CPython impl } .into_struct_sequence(vm)) } From 7fab64ed9c534ded298e4c41239b5286cf42efcf Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Thu, 13 Mar 2025 19:41:14 -0700 Subject: [PATCH 085/295] Revert "Update statistics to 3.13.2 (#5592)" (#5606) This reverts commit ff970b0e1c8f51d2ccc292e4c405511c0ea3eb69. --- Lib/statistics.py | 899 ++++++++---------------------------- Lib/test/test_statistics.py | 530 +++------------------ 2 files changed, 251 insertions(+), 1178 deletions(-) diff --git a/Lib/statistics.py b/Lib/statistics.py index 5f0a6e67d4..f66245380a 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -11,7 +11,7 @@ Function Description ================== ================================================== mean Arithmetic mean (average) of data. -fmean Fast, floating-point arithmetic mean. +fmean Fast, floating point arithmetic mean. geometric_mean Geometric mean of data. harmonic_mean Harmonic mean of data. median Median (middle value) of data. @@ -112,8 +112,6 @@ 'fmean', 'geometric_mean', 'harmonic_mean', - 'kde', - 'kde_random', 'linear_regression', 'mean', 'median', @@ -132,20 +130,14 @@ import math import numbers import random -import sys from fractions import Fraction from decimal import Decimal -from itertools import count, groupby, repeat +from itertools import groupby, repeat from bisect import bisect_left, bisect_right -from math import hypot, sqrt, fabs, exp, erf, tau, log, fsum, sumprod -from math import isfinite, isinf, pi, cos, sin, tan, cosh, asin, atan, acos -from functools import reduce +from math import hypot, sqrt, fabs, exp, erf, tau, log, fsum from operator import itemgetter -from collections import Counter, namedtuple, defaultdict - -_SQRT2 = sqrt(2.0) -_random = random +from collections import Counter, namedtuple # === Exceptions === @@ -188,12 +180,11 @@ def _sum(data): allowed. """ count = 0 - types = set() - types_add = types.add partials = {} partials_get = partials.get + T = int for typ, values in groupby(data, type): - types_add(typ) + T = _coerce(T, typ) # or raise TypeError for n, d in map(_exact_ratio, values): count += 1 partials[d] = partials_get(d, 0) + n @@ -205,51 +196,9 @@ def _sum(data): else: # Sum all the partial sums using builtin sum. total = sum(Fraction(n, d) for d, n in partials.items()) - T = reduce(_coerce, types, int) # or raise TypeError return (T, total, count) -def _ss(data, c=None): - """Return the exact mean and sum of square deviations of sequence data. - - Calculations are done in a single pass, allowing the input to be an iterator. - - If given *c* is used the mean; otherwise, it is calculated from the data. - Use the *c* argument with care, as it can lead to garbage results. - - """ - if c is not None: - T, ssd, count = _sum((d := x - c) * d for x in data) - return (T, ssd, c, count) - count = 0 - types = set() - types_add = types.add - sx_partials = defaultdict(int) - sxx_partials = defaultdict(int) - for typ, values in groupby(data, type): - types_add(typ) - for n, d in map(_exact_ratio, values): - count += 1 - sx_partials[d] += n - sxx_partials[d] += n * n - if not count: - ssd = c = Fraction(0) - elif None in sx_partials: - # The sum will be a NAN or INF. We can ignore all the finite - # partials, and just look at this special one. - ssd = c = sx_partials[None] - assert not _isfinite(ssd) - else: - sx = sum(Fraction(n, d) for d, n in sx_partials.items()) - sxx = sum(Fraction(n, d*d) for d, n in sxx_partials.items()) - # This formula has poor numeric properties for floats, - # but with fractions it is exact. - ssd = (count * sxx - sx * sx) / count - c = sx / count - T = reduce(_coerce, types, int) # or raise TypeError - return (T, ssd, c, count) - - def _isfinite(x): try: return x.is_finite() # Likely a Decimal. @@ -296,28 +245,6 @@ def _exact_ratio(x): x is expected to be an int, Fraction, Decimal or float. """ - - # XXX We should revisit whether using fractions to accumulate exact - # ratios is the right way to go. - - # The integer ratios for binary floats can have numerators or - # denominators with over 300 decimal digits. The problem is more - # acute with decimal floats where the default decimal context - # supports a huge range of exponents from Emin=-999999 to - # Emax=999999. When expanded with as_integer_ratio(), numbers like - # Decimal('3.14E+5000') and Decimal('3.14E-5000') have large - # numerators or denominators that will slow computation. - - # When the integer ratios are accumulated as fractions, the size - # grows to cover the full range from the smallest magnitude to the - # largest. For example, Fraction(3.14E+300) + Fraction(3.14E-300), - # has a 616 digit numerator. Likewise, - # Fraction(Decimal('3.14E+5000')) + Fraction(Decimal('3.14E-5000')) - # has 10,003 digit numerator. - - # This doesn't seem to have been problem in practice, but it is a - # potential pitfall. - try: return x.as_integer_ratio() except AttributeError: @@ -352,6 +279,22 @@ def _convert(value, T): raise +def _find_lteq(a, x): + 'Locate the leftmost value exactly equal to x' + i = bisect_left(a, x) + if i != len(a) and a[i] == x: + return i + raise ValueError + + +def _find_rteq(a, l, x): + 'Locate the rightmost value exactly equal to x' + i = bisect_right(a, x, lo=l) + if i != (len(a) + 1) and a[i - 1] == x: + return i - 1 + raise ValueError + + def _fail_neg(values, errmsg='negative value'): """Iterate over values, failing if any are less than zero.""" for x in values: @@ -360,113 +303,6 @@ def _fail_neg(values, errmsg='negative value'): yield x -def _rank(data, /, *, key=None, reverse=False, ties='average', start=1) -> list[float]: - """Rank order a dataset. The lowest value has rank 1. - - Ties are averaged so that equal values receive the same rank: - - >>> data = [31, 56, 31, 25, 75, 18] - >>> _rank(data) - [3.5, 5.0, 3.5, 2.0, 6.0, 1.0] - - The operation is idempotent: - - >>> _rank([3.5, 5.0, 3.5, 2.0, 6.0, 1.0]) - [3.5, 5.0, 3.5, 2.0, 6.0, 1.0] - - It is possible to rank the data in reverse order so that the - highest value has rank 1. Also, a key-function can extract - the field to be ranked: - - >>> goals = [('eagles', 45), ('bears', 48), ('lions', 44)] - >>> _rank(goals, key=itemgetter(1), reverse=True) - [2.0, 1.0, 3.0] - - Ranks are conventionally numbered starting from one; however, - setting *start* to zero allows the ranks to be used as array indices: - - >>> prize = ['Gold', 'Silver', 'Bronze', 'Certificate'] - >>> scores = [8.1, 7.3, 9.4, 8.3] - >>> [prize[int(i)] for i in _rank(scores, start=0, reverse=True)] - ['Bronze', 'Certificate', 'Gold', 'Silver'] - - """ - # If this function becomes public at some point, more thought - # needs to be given to the signature. A list of ints is - # plausible when ties is "min" or "max". When ties is "average", - # either list[float] or list[Fraction] is plausible. - - # Default handling of ties matches scipy.stats.mstats.spearmanr. - if ties != 'average': - raise ValueError(f'Unknown tie resolution method: {ties!r}') - if key is not None: - data = map(key, data) - val_pos = sorted(zip(data, count()), reverse=reverse) - i = start - 1 - result = [0] * len(val_pos) - for _, g in groupby(val_pos, key=itemgetter(0)): - group = list(g) - size = len(group) - rank = i + (size + 1) / 2 - for value, orig_pos in group: - result[orig_pos] = rank - i += size - return result - - -def _integer_sqrt_of_frac_rto(n: int, m: int) -> int: - """Square root of n/m, rounded to the nearest integer using round-to-odd.""" - # Reference: https://www.lri.fr/~melquion/doc/05-imacs17_1-expose.pdf - a = math.isqrt(n // m) - return a | (a*a*m != n) - - -# For 53 bit precision floats, the bit width used in -# _float_sqrt_of_frac() is 109. -_sqrt_bit_width: int = 2 * sys.float_info.mant_dig + 3 - - -def _float_sqrt_of_frac(n: int, m: int) -> float: - """Square root of n/m as a float, correctly rounded.""" - # See principle and proof sketch at: https://bugs.python.org/msg407078 - q = (n.bit_length() - m.bit_length() - _sqrt_bit_width) // 2 - if q >= 0: - numerator = _integer_sqrt_of_frac_rto(n, m << 2 * q) << q - denominator = 1 - else: - numerator = _integer_sqrt_of_frac_rto(n << -2 * q, m) - denominator = 1 << -q - return numerator / denominator # Convert to float - - -def _decimal_sqrt_of_frac(n: int, m: int) -> Decimal: - """Square root of n/m as a Decimal, correctly rounded.""" - # Premise: For decimal, computing (n/m).sqrt() can be off - # by 1 ulp from the correctly rounded result. - # Method: Check the result, moving up or down a step if needed. - if n <= 0: - if not n: - return Decimal('0.0') - n, m = -n, -m - - root = (Decimal(n) / Decimal(m)).sqrt() - nr, dr = root.as_integer_ratio() - - plus = root.next_plus() - np, dp = plus.as_integer_ratio() - # test: n / m > ((root + plus) / 2) ** 2 - if 4 * n * (dr*dp)**2 > m * (dr*np + dp*nr)**2: - return plus - - minus = root.next_minus() - nm, dm = minus.as_integer_ratio() - # test: n / m < ((root + minus) / 2) ** 2 - if 4 * n * (dr*dm)**2 < m * (dr*nm + dm*nr)**2: - return minus - - return root - - # === Measures of central tendency (averages) === def mean(data): @@ -485,13 +321,17 @@ def mean(data): If ``data`` is empty, StatisticsError will be raised. """ - T, total, n = _sum(data) + if iter(data) is data: + data = list(data) + n = len(data) if n < 1: raise StatisticsError('mean requires at least one data point') + T, total, count = _sum(data) + assert count == n return _convert(total / n, T) -def fmean(data, weights=None): +def fmean(data): """Convert data to floats and compute the arithmetic mean. This runs faster than the mean() function and it always returns a float. @@ -500,40 +340,29 @@ def fmean(data, weights=None): >>> fmean([3.5, 4.0, 5.25]) 4.25 """ - if weights is None: - try: - n = len(data) - except TypeError: - # Handle iterators that do not define __len__(). - n = 0 - def count(iterable): - nonlocal n - for n, x in enumerate(iterable, start=1): - yield x - data = count(data) + try: + n = len(data) + except TypeError: + # Handle iterators that do not define __len__(). + n = 0 + def count(iterable): + nonlocal n + for n, x in enumerate(iterable, start=1): + yield x + total = fsum(count(data)) + else: total = fsum(data) - if not n: - raise StatisticsError('fmean requires at least one data point') - return total / n - if not isinstance(weights, (list, tuple)): - weights = list(weights) try: - num = sumprod(data, weights) - except ValueError: - raise StatisticsError('data and weights must be the same length') - den = fsum(weights) - if not den: - raise StatisticsError('sum of weights must be non-zero') - return num / den + return total / n + except ZeroDivisionError: + raise StatisticsError('fmean requires at least one data point') from None def geometric_mean(data): """Convert data to floats and compute the geometric mean. - Raises a StatisticsError if the input dataset is empty - or if it contains a negative value. - - Returns zero if the product of inputs is zero. + Raises a StatisticsError if the input dataset is empty, + if it contains a zero, or if it contains a negative value. No special efforts are made to achieve exact results. (However, this may change in the future.) @@ -541,25 +370,11 @@ def geometric_mean(data): >>> round(geometric_mean([54, 24, 36]), 9) 36.0 """ - n = 0 - found_zero = False - def count_positive(iterable): - nonlocal n, found_zero - for n, x in enumerate(iterable, start=1): - if x > 0.0 or math.isnan(x): - yield x - elif x == 0.0: - found_zero = True - else: - raise StatisticsError('No negative inputs allowed', x) - total = fsum(map(log, count_positive(data))) - if not n: - raise StatisticsError('Must have a non-empty dataset') - if math.isnan(total): - return math.nan - if found_zero: - return math.nan if total == math.inf else 0.0 - return exp(total / n) + try: + return exp(fmean(map(log, data))) + except ValueError: + raise StatisticsError('geometric mean requires a non-empty dataset ' + 'containing positive numbers') from None def harmonic_mean(data, weights=None): @@ -683,75 +498,58 @@ def median_high(data): return data[n // 2] -def median_grouped(data, interval=1.0): - """Estimates the median for numeric data binned around the midpoints - of consecutive, fixed-width intervals. - - The *data* can be any iterable of numeric data with each value being - exactly the midpoint of a bin. At least one value must be present. +def median_grouped(data, interval=1): + """Return the 50th percentile (median) of grouped continuous data. - The *interval* is width of each bin. + >>> median_grouped([1, 2, 2, 3, 4, 4, 4, 4, 4, 5]) + 3.7 + >>> median_grouped([52, 52, 53, 54]) + 52.5 - For example, demographic information may have been summarized into - consecutive ten-year age groups with each group being represented - by the 5-year midpoints of the intervals: + This calculates the median as the 50th percentile, and should be + used when your data is continuous and grouped. In the above example, + the values 1, 2, 3, etc. actually represent the midpoint of classes + 0.5-1.5, 1.5-2.5, 2.5-3.5, etc. The middle value falls somewhere in + class 3.5-4.5, and interpolation is used to estimate it. - >>> demographics = Counter({ - ... 25: 172, # 20 to 30 years old - ... 35: 484, # 30 to 40 years old - ... 45: 387, # 40 to 50 years old - ... 55: 22, # 50 to 60 years old - ... 65: 6, # 60 to 70 years old - ... }) + Optional argument ``interval`` represents the class interval, and + defaults to 1. Changing the class interval naturally will change the + interpolated 50th percentile value: - The 50th percentile (median) is the 536th person out of the 1071 - member cohort. That person is in the 30 to 40 year old age group. - - The regular median() function would assume that everyone in the - tricenarian age group was exactly 35 years old. A more tenable - assumption is that the 484 members of that age group are evenly - distributed between 30 and 40. For that, we use median_grouped(). - - >>> data = list(demographics.elements()) - >>> median(data) - 35 - >>> round(median_grouped(data, interval=10), 1) - 37.5 - - The caller is responsible for making sure the data points are separated - by exact multiples of *interval*. This is essential for getting a - correct result. The function does not check this precondition. - - Inputs may be any numeric type that can be coerced to a float during - the interpolation step. + >>> median_grouped([1, 3, 3, 5, 7], interval=1) + 3.25 + >>> median_grouped([1, 3, 3, 5, 7], interval=2) + 3.5 + This function does not check whether the data points are at least + ``interval`` apart. """ data = sorted(data) n = len(data) - if not n: + if n == 0: raise StatisticsError("no median for empty data") - + elif n == 1: + return data[0] # Find the value at the midpoint. Remember this corresponds to the - # midpoint of the class interval. + # centre of the class interval. x = data[n // 2] - - # Using O(log n) bisection, find where all the x values occur in the data. - # All x will lie within data[i:j]. - i = bisect_left(data, x) - j = bisect_right(data, x, lo=i) - - # Coerce to floats, raising a TypeError if not possible + for obj in (x, interval): + if isinstance(obj, (str, bytes)): + raise TypeError('expected number but got %r' % obj) try: - interval = float(interval) - x = float(x) - except ValueError: - raise TypeError(f'Value cannot be converted to a float') - - # Interpolate the median using the formula found at: - # https://www.cuemath.com/data/median-of-grouped-data/ - L = x - interval / 2.0 # Lower limit of the median interval - cf = i # Cumulative frequency of the preceding interval - f = j - i # Number of elements in the median internal + L = x - interval / 2 # The lower limit of the median interval. + except TypeError: + # Mixed type. For now we just coerce to float. + L = float(x) - float(interval) / 2 + + # Uses bisection search to search for x in data with log(n) time complexity + # Find the position of leftmost occurrence of x in data + l1 = _find_lteq(data, x) + # Find the position of rightmost occurrence of x in data[l1...len(data)] + # Assuming always l1 <= l2 + l2 = _find_rteq(data, l1, x) + cf = l1 + f = l2 - l1 + 1 return L + interval * (n / 2 - cf) / f @@ -798,212 +596,9 @@ def multimode(data): >>> multimode('') [] """ - counts = Counter(iter(data)) - if not counts: - return [] - maxcount = max(counts.values()) - return [value for value, count in counts.items() if count == maxcount] - - -def kde(data, h, kernel='normal', *, cumulative=False): - """Kernel Density Estimation: Create a continuous probability density - function or cumulative distribution function from discrete samples. - - The basic idea is to smooth the data using a kernel function - to help draw inferences about a population from a sample. - - The degree of smoothing is controlled by the scaling parameter h - which is called the bandwidth. Smaller values emphasize local - features while larger values give smoother results. - - The kernel determines the relative weights of the sample data - points. Generally, the choice of kernel shape does not matter - as much as the more influential bandwidth smoothing parameter. - - Kernels that give some weight to every sample point: - - normal (gauss) - logistic - sigmoid - - Kernels that only give weight to sample points within - the bandwidth: - - rectangular (uniform) - triangular - parabolic (epanechnikov) - quartic (biweight) - triweight - cosine - - If *cumulative* is true, will return a cumulative distribution function. - - A StatisticsError will be raised if the data sequence is empty. - - Example - ------- - - Given a sample of six data points, construct a continuous - function that estimates the underlying probability density: - - >>> sample = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] - >>> f_hat = kde(sample, h=1.5) - - Compute the area under the curve: - - >>> area = sum(f_hat(x) for x in range(-20, 20)) - >>> round(area, 4) - 1.0 - - Plot the estimated probability density function at - evenly spaced points from -6 to 10: - - >>> for x in range(-6, 11): - ... density = f_hat(x) - ... plot = ' ' * int(density * 400) + 'x' - ... print(f'{x:2}: {density:.3f} {plot}') - ... - -6: 0.002 x - -5: 0.009 x - -4: 0.031 x - -3: 0.070 x - -2: 0.111 x - -1: 0.125 x - 0: 0.110 x - 1: 0.086 x - 2: 0.068 x - 3: 0.059 x - 4: 0.066 x - 5: 0.082 x - 6: 0.082 x - 7: 0.058 x - 8: 0.028 x - 9: 0.009 x - 10: 0.002 x - - Estimate P(4.5 < X <= 7.5), the probability that a new sample value - will be between 4.5 and 7.5: - - >>> cdf = kde(sample, h=1.5, cumulative=True) - >>> round(cdf(7.5) - cdf(4.5), 2) - 0.22 - - References - ---------- - - Kernel density estimation and its application: - https://www.itm-conferences.org/articles/itmconf/pdf/2018/08/itmconf_sam2018_00037.pdf - - Kernel functions in common use: - https://en.wikipedia.org/wiki/Kernel_(statistics)#kernel_functions_in_common_use - - Interactive graphical demonstration and exploration: - https://demonstrations.wolfram.com/KernelDensityEstimation/ - - Kernel estimation of cumulative distribution function of a random variable with bounded support - https://www.econstor.eu/bitstream/10419/207829/1/10.21307_stattrans-2016-037.pdf - - """ - - n = len(data) - if not n: - raise StatisticsError('Empty data sequence') - - if not isinstance(data[0], (int, float)): - raise TypeError('Data sequence must contain ints or floats') - - if h <= 0.0: - raise StatisticsError(f'Bandwidth h must be positive, not {h=!r}') - - if kernel == "normal" or kernel == "gauss": - sqrt2pi = sqrt(2 * pi) - sqrt2 = sqrt(2) - K = lambda t: exp(-1/2 * t * t) / sqrt2pi - W = lambda t: 1/2 * (1.0 + erf(t / sqrt2)) - support = None - elif kernel == "logistic": - # 1.0 / (exp(t) + 2.0 + exp(-t)) - K = lambda t: 1/2 / (1.0 + cosh(t)) - W = lambda t: 1.0 - 1.0 / (exp(t) + 1.0) - support = None - elif kernel == "sigmoid": - # (2/pi) / (exp(t) + exp(-t)) - c1 = 1 / pi - c2 = 2 / pi - K = lambda t: c1 / cosh(t) - W = lambda t: c2 * atan(exp(t)) - support = None - elif kernel == "rectangular" or kernel == "uniform": - K = lambda t: 1/2 - W = lambda t: 1/2 * t + 1/2 - support = 1.0 - elif kernel == "triangular": - K = lambda t: 1.0 - abs(t) - W = lambda t: t*t * (1/2 if t < 0.0 else -1/2) + t + 1/2 - support = 1.0 - elif kernel == "parabolic" or kernel == "epanechnikov": - K = lambda t: 3/4 * (1.0 - t * t) - W = lambda t: -1/4 * t**3 + 3/4 * t + 1/2 - support = 1. - elif kernel == "quartic" or kernel == "biweight": - K = lambda t: 15/16 * (1.0 - t * t) ** 2 - W = lambda t: 3/16 * t**5 - 5/8 * t**3 + 15/16 * t + 1/2 - support = 1.0 - elif kernel == "triweight": - K = lambda t: 35/32 * (1.0 - t * t) ** 3 - W = lambda t: 35/32 * (-1/7*t**7 + 3/5*t**5 - t**3 + t) + 1/2 - support = 1.0 - elif kernel == "cosine": - c1 = pi / 4 - c2 = pi / 2 - K = lambda t: c1 * cos(c2 * t) - W = lambda t: 1/2 * sin(c2 * t) + 1/2 - support = 1.0 - else: - raise StatisticsError(f'Unknown kernel name: {kernel!r}') - - if support is None: - - def pdf(x): - n = len(data) - return sum(K((x - x_i) / h) for x_i in data) / (n * h) - - def cdf(x): - n = len(data) - return sum(W((x - x_i) / h) for x_i in data) / n - - else: - - sample = sorted(data) - bandwidth = h * support - - def pdf(x): - nonlocal n, sample - if len(data) != n: - sample = sorted(data) - n = len(data) - i = bisect_left(sample, x - bandwidth) - j = bisect_right(sample, x + bandwidth) - supported = sample[i : j] - return sum(K((x - x_i) / h) for x_i in supported) / (n * h) - - def cdf(x): - nonlocal n, sample - if len(data) != n: - sample = sorted(data) - n = len(data) - i = bisect_left(sample, x - bandwidth) - j = bisect_right(sample, x + bandwidth) - supported = sample[i : j] - return sum((W((x - x_i) / h) for x_i in supported), i) / n - - if cumulative: - cdf.__doc__ = f'CDF estimate with {h=!r} and {kernel=!r}' - return cdf - - else: - pdf.__doc__ = f'PDF estimate with {h=!r} and {kernel=!r}' - return pdf + counts = Counter(iter(data)).most_common() + maxcount, mode_items = next(groupby(counts, key=itemgetter(1)), (0, [])) + return list(map(itemgetter(0), mode_items)) # Notes on methods for computing quantiles @@ -1064,10 +659,7 @@ def quantiles(data, *, n=4, method='exclusive'): data = sorted(data) ld = len(data) if ld < 2: - if ld == 1: - return data * (n - 1) - raise StatisticsError('must have at least one data point') - + raise StatisticsError('must have at least two data points') if method == 'inclusive': m = ld - 1 result = [] @@ -1076,7 +668,6 @@ def quantiles(data, *, n=4, method='exclusive'): interpolated = (data[j] * (n - delta) + data[j + 1] * delta) / n result.append(interpolated) return result - if method == 'exclusive': m = ld + 1 result = [] @@ -1087,7 +678,6 @@ def quantiles(data, *, n=4, method='exclusive'): interpolated = (data[j - 1] * (n - delta) + data[j] * delta) / n result.append(interpolated) return result - raise ValueError(f'Unknown method: {method!r}') @@ -1095,6 +685,41 @@ def quantiles(data, *, n=4, method='exclusive'): # See http://mathworld.wolfram.com/Variance.html # http://mathworld.wolfram.com/SampleVariance.html +# http://en.wikipedia.org/wiki/Algorithms_for_calculating_variance +# +# Under no circumstances use the so-called "computational formula for +# variance", as that is only suitable for hand calculations with a small +# amount of low-precision data. It has terrible numeric properties. +# +# See a comparison of three computational methods here: +# http://www.johndcook.com/blog/2008/09/26/comparing-three-methods-of-computing-standard-deviation/ + +def _ss(data, c=None): + """Return sum of square deviations of sequence data. + + If ``c`` is None, the mean is calculated in one pass, and the deviations + from the mean are calculated in a second pass. Otherwise, deviations are + calculated from ``c`` as given. Use the second case with care, as it can + lead to garbage results. + """ + if c is not None: + T, total, count = _sum((x-c)**2 for x in data) + return (T, total) + T, total, count = _sum(data) + mean_n, mean_d = (total / count).as_integer_ratio() + partials = Counter() + for n, d in map(_exact_ratio, data): + diff_n = n * mean_d - d * mean_n + diff_d = d * mean_d + partials[diff_d * diff_d] += diff_n * diff_n + if None in partials: + # The sum will be a NAN or INF. We can ignore all the finite + # partials, and just look at this special one. + total = partials[None] + assert not _isfinite(total) + else: + total = sum(Fraction(n, d) for d, n in partials.items()) + return (T, total) def variance(data, xbar=None): @@ -1135,9 +760,12 @@ def variance(data, xbar=None): Fraction(67, 108) """ - T, ss, c, n = _ss(data, xbar) + if iter(data) is data: + data = list(data) + n = len(data) if n < 2: raise StatisticsError('variance requires at least two data points') + T, ss = _ss(data, xbar) return _convert(ss / (n - 1), T) @@ -1176,9 +804,12 @@ def pvariance(data, mu=None): Fraction(13, 72) """ - T, ss, c, n = _ss(data, mu) + if iter(data) is data: + data = list(data) + n = len(data) if n < 1: raise StatisticsError('pvariance requires at least one data point') + T, ss = _ss(data, mu) return _convert(ss / n, T) @@ -1191,13 +822,14 @@ def stdev(data, xbar=None): 1.0810874155219827 """ - T, ss, c, n = _ss(data, xbar) - if n < 2: - raise StatisticsError('stdev requires at least two data points') - mss = ss / (n - 1) - if issubclass(T, Decimal): - return _decimal_sqrt_of_frac(mss.numerator, mss.denominator) - return _float_sqrt_of_frac(mss.numerator, mss.denominator) + # Fixme: Despite the exact sum of squared deviations, some inaccuracy + # remain because there are two rounding steps. The first occurs in + # the _convert() step for variance(), the second occurs in math.sqrt(). + var = variance(data, xbar) + try: + return var.sqrt() + except AttributeError: + return math.sqrt(var) def pstdev(data, mu=None): @@ -1209,47 +841,14 @@ def pstdev(data, mu=None): 0.986893273527251 """ - T, ss, c, n = _ss(data, mu) - if n < 1: - raise StatisticsError('pstdev requires at least one data point') - mss = ss / n - if issubclass(T, Decimal): - return _decimal_sqrt_of_frac(mss.numerator, mss.denominator) - return _float_sqrt_of_frac(mss.numerator, mss.denominator) - - -def _mean_stdev(data): - """In one pass, compute the mean and sample standard deviation as floats.""" - T, ss, xbar, n = _ss(data) - if n < 2: - raise StatisticsError('stdev requires at least two data points') - mss = ss / (n - 1) + # Fixme: Despite the exact sum of squared deviations, some inaccuracy + # remain because there are two rounding steps. The first occurs in + # the _convert() step for pvariance(), the second occurs in math.sqrt(). + var = pvariance(data, mu) try: - return float(xbar), _float_sqrt_of_frac(mss.numerator, mss.denominator) + return var.sqrt() except AttributeError: - # Handle Nans and Infs gracefully - return float(xbar), float(xbar) / float(ss) - -def _sqrtprod(x: float, y: float) -> float: - "Return sqrt(x * y) computed with improved accuracy and without overflow/underflow." - h = sqrt(x * y) - if not isfinite(h): - if isinf(h) and not isinf(x) and not isinf(y): - # Finite inputs overflowed, so scale down, and recompute. - scale = 2.0 ** -512 # sqrt(1 / sys.float_info.max) - return _sqrtprod(scale * x, scale * y) / scale - return h - if not h: - if x and y: - # Non-zero inputs underflowed, so scale up, and recompute. - # Scale: 1 / sqrt(sys.float_info.min * sys.float_info.epsilon) - scale = 2.0 ** 537 - return _sqrtprod(scale * x, scale * y) / scale - return h - # Improve accuracy with a differential correction. - # https://www.wolframalpha.com/input/?i=Maclaurin+series+sqrt%28h**2+%2B+x%29+at+x%3D0 - d = sumprod((x, h), (y, -h)) - return h + d / (2.0 * h) + return math.sqrt(var) # === Statistics for relations between two inputs === @@ -1283,16 +882,18 @@ def covariance(x, y, /): raise StatisticsError('covariance requires at least two data points') xbar = fsum(x) / n ybar = fsum(y) / n - sxy = sumprod((xi - xbar for xi in x), (yi - ybar for yi in y)) + sxy = fsum((xi - xbar) * (yi - ybar) for xi, yi in zip(x, y)) return sxy / (n - 1) -def correlation(x, y, /, *, method='linear'): +def correlation(x, y, /): """Pearson's correlation coefficient Return the Pearson's correlation coefficient for two inputs. Pearson's - correlation coefficient *r* takes values between -1 and +1. It measures - the strength and direction of a linear relationship. + correlation coefficient *r* takes values between -1 and +1. It measures the + strength and direction of the linear relationship, where +1 means very + strong, positive linear relationship, -1 very strong, negative linear + relationship, and 0 no linear relationship. >>> x = [1, 2, 3, 4, 5, 6, 7, 8, 9] >>> y = [9, 8, 7, 6, 5, 4, 3, 2, 1] @@ -1301,36 +902,19 @@ def correlation(x, y, /, *, method='linear'): >>> correlation(x, y) -1.0 - If *method* is "ranked", computes Spearman's rank correlation coefficient - for two inputs. The data is replaced by ranks. Ties are averaged - so that equal values receive the same rank. The resulting coefficient - measures the strength of a monotonic relationship. - - Spearman's rank correlation coefficient is appropriate for ordinal - data or for continuous data that doesn't meet the linear proportion - requirement for Pearson's correlation coefficient. """ n = len(x) if len(y) != n: raise StatisticsError('correlation requires that both inputs have same number of data points') if n < 2: raise StatisticsError('correlation requires at least two data points') - if method not in {'linear', 'ranked'}: - raise ValueError(f'Unknown method: {method!r}') - if method == 'ranked': - start = (n - 1) / -2 # Center rankings around zero - x = _rank(x, start=start) - y = _rank(y, start=start) - else: - xbar = fsum(x) / n - ybar = fsum(y) / n - x = [xi - xbar for xi in x] - y = [yi - ybar for yi in y] - sxy = sumprod(x, y) - sxx = sumprod(x, x) - syy = sumprod(y, y) + xbar = fsum(x) / n + ybar = fsum(y) / n + sxy = fsum((xi - xbar) * (yi - ybar) for xi, yi in zip(x, y)) + sxx = fsum((xi - xbar) ** 2.0 for xi in x) + syy = fsum((yi - ybar) ** 2.0 for yi in y) try: - return sxy / _sqrtprod(sxx, syy) + return sxy / sqrt(sxx * syy) except ZeroDivisionError: raise StatisticsError('at least one of the inputs is constant') @@ -1338,13 +922,13 @@ def correlation(x, y, /, *, method='linear'): LinearRegression = namedtuple('LinearRegression', ('slope', 'intercept')) -def linear_regression(x, y, /, *, proportional=False): +def linear_regression(x, y, /): """Slope and intercept for simple linear regression. Return the slope and intercept of simple linear regression parameters estimated using ordinary least squares. Simple linear regression describes relationship between an independent variable - *x* and a dependent variable *y* in terms of a linear function: + *x* and a dependent variable *y* in terms of linear function: y = slope * x + intercept + noise @@ -1360,20 +944,7 @@ def linear_regression(x, y, /, *, proportional=False): >>> noise = NormalDist().samples(5, seed=42) >>> y = [3 * x[i] + 2 + noise[i] for i in range(5)] >>> linear_regression(x, y) #doctest: +ELLIPSIS - LinearRegression(slope=3.17495..., intercept=1.00925...) - - If *proportional* is true, the independent variable *x* and the - dependent variable *y* are assumed to be directly proportional. - The data is fit to a line passing through the origin. - - Since the *intercept* will always be 0.0, the underlying linear - function simplifies to: - - y = slope * x + noise - - >>> y = [3 * x[i] + noise[i] for i in range(5)] - >>> linear_regression(x, y, proportional=True) #doctest: +ELLIPSIS - LinearRegression(slope=2.90475..., intercept=0.0) + LinearRegression(slope=3.09078914170..., intercept=1.75684970486...) """ n = len(x) @@ -1381,18 +952,15 @@ def linear_regression(x, y, /, *, proportional=False): raise StatisticsError('linear regression requires that both inputs have same number of data points') if n < 2: raise StatisticsError('linear regression requires at least two data points') - if not proportional: - xbar = fsum(x) / n - ybar = fsum(y) / n - x = [xi - xbar for xi in x] # List because used three times below - y = (yi - ybar for yi in y) # Generator because only used once below - sxy = sumprod(x, y) + 0.0 # Add zero to coerce result to a float - sxx = sumprod(x, x) + xbar = fsum(x) / n + ybar = fsum(y) / n + sxy = fsum((xi - xbar) * (yi - ybar) for xi, yi in zip(x, y)) + sxx = fsum((xi - xbar) ** 2.0 for xi in x) try: slope = sxy / sxx # equivalent to: covariance(x, y) / variance(x) except ZeroDivisionError: raise StatisticsError('x is constant') - intercept = 0.0 if proportional else ybar - slope * xbar + intercept = ybar - slope * xbar return LinearRegression(slope=slope, intercept=intercept) @@ -1500,29 +1068,29 @@ def __init__(self, mu=0.0, sigma=1.0): @classmethod def from_samples(cls, data): "Make a normal distribution instance from sample data." - return cls(*_mean_stdev(data)) + if not isinstance(data, (list, tuple)): + data = list(data) + xbar = fmean(data) + return cls(xbar, stdev(data, xbar)) def samples(self, n, *, seed=None): "Generate *n* samples for a given mean and standard deviation." - rnd = random.random if seed is None else random.Random(seed).random - inv_cdf = _normal_dist_inv_cdf - mu = self._mu - sigma = self._sigma - return [inv_cdf(rnd(), mu, sigma) for _ in repeat(None, n)] + gauss = random.gauss if seed is None else random.Random(seed).gauss + mu, sigma = self._mu, self._sigma + return [gauss(mu, sigma) for i in range(n)] def pdf(self, x): "Probability density function. P(x <= X < x+dx) / dx" - variance = self._sigma * self._sigma + variance = self._sigma ** 2.0 if not variance: raise StatisticsError('pdf() not defined when sigma is zero') - diff = x - self._mu - return exp(diff * diff / (-2.0 * variance)) / sqrt(tau * variance) + return exp((x - self._mu)**2.0 / (-2.0*variance)) / sqrt(tau*variance) def cdf(self, x): "Cumulative distribution function. P(X <= x)" if not self._sigma: raise StatisticsError('cdf() not defined when sigma is zero') - return 0.5 * (1.0 + erf((x - self._mu) / (self._sigma * _SQRT2))) + return 0.5 * (1.0 + erf((x - self._mu) / (self._sigma * sqrt(2.0)))) def inv_cdf(self, p): """Inverse cumulative distribution function. x : P(X <= x) = p @@ -1536,6 +1104,8 @@ def inv_cdf(self, p): """ if p <= 0.0 or p >= 1.0: raise StatisticsError('p must be in the range 0.0 < p < 1.0') + if self._sigma <= 0.0: + raise StatisticsError('cdf() not defined when sigma at or below zero') return _normal_dist_inv_cdf(p, self._mu, self._sigma) def quantiles(self, n=4): @@ -1576,9 +1146,9 @@ def overlap(self, other): dv = Y_var - X_var dm = fabs(Y._mu - X._mu) if not dv: - return 1.0 - erf(dm / (2.0 * X._sigma * _SQRT2)) + return 1.0 - erf(dm / (2.0 * X._sigma * sqrt(2.0))) a = X._mu * Y_var - Y._mu * X_var - b = X._sigma * Y._sigma * sqrt(dm * dm + dv * log(Y_var / X_var)) + b = X._sigma * Y._sigma * sqrt(dm**2.0 + dv * log(Y_var / X_var)) x1 = (a + b) / dv x2 = (a - b) / dv return 1.0 - (fabs(Y.cdf(x1) - X.cdf(x1)) + fabs(Y.cdf(x2) - X.cdf(x2))) @@ -1621,7 +1191,7 @@ def stdev(self): @property def variance(self): "Square of the standard deviation." - return self._sigma * self._sigma + return self._sigma ** 2.0 def __add__(x1, x2): """Add a constant or another NormalDist instance. @@ -1695,102 +1265,3 @@ def __hash__(self): def __repr__(self): return f'{type(self).__name__}(mu={self._mu!r}, sigma={self._sigma!r})' - - def __getstate__(self): - return self._mu, self._sigma - - def __setstate__(self, state): - self._mu, self._sigma = state - - -## kde_random() ############################################################## - -def _newton_raphson(f_inv_estimate, f, f_prime, tolerance=1e-12): - def f_inv(y): - "Return x such that f(x) ≈ y within the specified tolerance." - x = f_inv_estimate(y) - while abs(diff := f(x) - y) > tolerance: - x -= diff / f_prime(x) - return x - return f_inv - -def _quartic_invcdf_estimate(p): - sign, p = (1.0, p) if p <= 1/2 else (-1.0, 1.0 - p) - x = (2.0 * p) ** 0.4258865685331 - 1.0 - if p >= 0.004 < 0.499: - x += 0.026818732 * sin(7.101753784 * p + 2.73230839482953) - return x * sign - -_quartic_invcdf = _newton_raphson( - f_inv_estimate = _quartic_invcdf_estimate, - f = lambda t: 3/16 * t**5 - 5/8 * t**3 + 15/16 * t + 1/2, - f_prime = lambda t: 15/16 * (1.0 - t * t) ** 2) - -def _triweight_invcdf_estimate(p): - sign, p = (1.0, p) if p <= 1/2 else (-1.0, 1.0 - p) - x = (2.0 * p) ** 0.3400218741872791 - 1.0 - return x * sign - -_triweight_invcdf = _newton_raphson( - f_inv_estimate = _triweight_invcdf_estimate, - f = lambda t: 35/32 * (-1/7*t**7 + 3/5*t**5 - t**3 + t) + 1/2, - f_prime = lambda t: 35/32 * (1.0 - t * t) ** 3) - -_kernel_invcdfs = { - 'normal': NormalDist().inv_cdf, - 'logistic': lambda p: log(p / (1 - p)), - 'sigmoid': lambda p: log(tan(p * pi/2)), - 'rectangular': lambda p: 2*p - 1, - 'parabolic': lambda p: 2 * cos((acos(2*p-1) + pi) / 3), - 'quartic': _quartic_invcdf, - 'triweight': _triweight_invcdf, - 'triangular': lambda p: sqrt(2*p) - 1 if p < 1/2 else 1 - sqrt(2 - 2*p), - 'cosine': lambda p: 2 * asin(2*p - 1) / pi, -} -_kernel_invcdfs['gauss'] = _kernel_invcdfs['normal'] -_kernel_invcdfs['uniform'] = _kernel_invcdfs['rectangular'] -_kernel_invcdfs['epanechnikov'] = _kernel_invcdfs['parabolic'] -_kernel_invcdfs['biweight'] = _kernel_invcdfs['quartic'] - -def kde_random(data, h, kernel='normal', *, seed=None): - """Return a function that makes a random selection from the estimated - probability density function created by kde(data, h, kernel). - - Providing a *seed* allows reproducible selections within a single - thread. The seed may be an integer, float, str, or bytes. - - A StatisticsError will be raised if the *data* sequence is empty. - - Example: - - >>> data = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] - >>> rand = kde_random(data, h=1.5, seed=8675309) - >>> new_selections = [rand() for i in range(10)] - >>> [round(x, 1) for x in new_selections] - [0.7, 6.2, 1.2, 6.9, 7.0, 1.8, 2.5, -0.5, -1.8, 5.6] - - """ - n = len(data) - if not n: - raise StatisticsError('Empty data sequence') - - if not isinstance(data[0], (int, float)): - raise TypeError('Data sequence must contain ints or floats') - - if h <= 0.0: - raise StatisticsError(f'Bandwidth h must be positive, not {h=!r}') - - kernel_invcdf = _kernel_invcdfs.get(kernel) - if kernel_invcdf is None: - raise StatisticsError(f'Unknown kernel name: {kernel!r}') - - prng = _random.Random(seed) - random = prng.random - choice = prng.choice - - def rand(): - return choice(data) + h * kernel_invcdf(random()) - - rand.__doc__ = f'Random KDE selection with {h=!r} and {kernel=!r}' - - return rand diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index 7b0dfd05e0..8fcbcf3540 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -1,4 +1,4 @@ -x = """Test suite for statistics module, including helper NumericTestCase and +"""Test suite for statistics module, including helper NumericTestCase and approx_equal function. """ @@ -9,14 +9,13 @@ import copy import decimal import doctest -import itertools import math import pickle import random import sys import unittest from test import support -from test.support import import_helper, requires_IEEE_754 +from test.support import import_helper from decimal import Decimal from fractions import Fraction @@ -28,12 +27,6 @@ # === Helper functions and class === -# Test copied from Lib/test/test_math.py -# detect evidence of double-rounding: fsum is not always correctly -# rounded on machines that suffer from double rounding. -x, y = 1e16, 2.9999 # use temporary values to defeat peephole optimizer -HAVE_DOUBLE_ROUNDING = (x + y == 1e16 + 4) - def sign(x): """Return -1.0 for negatives, including -0.0, otherwise +1.0.""" return math.copysign(1, x) @@ -698,6 +691,14 @@ def test_check_all(self): 'missing name "%s" in __all__' % name) +class DocTests(unittest.TestCase): + @unittest.skipIf(sys.flags.optimize >= 2, + "Docstrings are omitted with -OO and above") + def test_doc_tests(self): + failed, tried = doctest.testmod(statistics, optionflags=doctest.ELLIPSIS) + self.assertGreater(tried, 0) + self.assertEqual(failed, 0) + class StatisticsErrorTest(unittest.TestCase): def test_has_exception(self): errmsg = ( @@ -1038,6 +1039,50 @@ def test_error_msg(self): self.assertEqual(errmsg, msg) +class FindLteqTest(unittest.TestCase): + # Test _find_lteq private function. + + def test_invalid_input_values(self): + for a, x in [ + ([], 1), + ([1, 2], 3), + ([1, 3], 2) + ]: + with self.subTest(a=a, x=x): + with self.assertRaises(ValueError): + statistics._find_lteq(a, x) + + def test_locate_successfully(self): + for a, x, expected_i in [ + ([1, 1, 1, 2, 3], 1, 0), + ([0, 1, 1, 1, 2, 3], 1, 1), + ([1, 2, 3, 3, 3], 3, 2) + ]: + with self.subTest(a=a, x=x): + self.assertEqual(expected_i, statistics._find_lteq(a, x)) + + +class FindRteqTest(unittest.TestCase): + # Test _find_rteq private function. + + def test_invalid_input_values(self): + for a, l, x in [ + ([1], 2, 1), + ([1, 3], 0, 2) + ]: + with self.assertRaises(ValueError): + statistics._find_rteq(a, l, x) + + def test_locate_successfully(self): + for a, l, x, expected_i in [ + ([1, 1, 1, 2, 3], 0, 1, 2), + ([0, 1, 1, 1, 2, 3], 0, 1, 3), + ([1, 2, 3, 3, 3], 0, 3, 4) + ]: + with self.subTest(a=a, l=l, x=x): + self.assertEqual(expected_i, statistics._find_rteq(a, l, x)) + + # === Tests for public functions === class UnivariateCommonMixin: @@ -1072,7 +1117,7 @@ def test_no_inplace_modifications(self): def test_order_doesnt_matter(self): # Test that the order of data points doesn't change the result. - # CAUTION: due to floating-point rounding errors, the result actually + # CAUTION: due to floating point rounding errors, the result actually # may depend on the order. Consider this test representing an ideal. # To avoid this test failing, only test with exact values such as ints # or Fractions. @@ -1165,9 +1210,6 @@ def __pow__(self, other): def __add__(self, other): return type(self)(super().__add__(other)) __radd__ = __add__ - def __mul__(self, other): - return type(self)(super().__mul__(other)) - __rmul__ = __mul__ return (float, Decimal, Fraction, MyFloat) def test_types_conserved(self): @@ -1740,12 +1782,6 @@ def test_repeated_single_value(self): data = [x]*count self.assertEqual(self.func(data), float(x)) - def test_single_value(self): - # Override method from AverageMixin. - # Average of a single value is the value as a float. - for x in (23, 42.5, 1.3e15, Fraction(15, 19), Decimal('0.28')): - self.assertEqual(self.func([x]), float(x)) - def test_odd_fractions(self): # Test median_grouped works with an odd number of Fractions. F = Fraction @@ -1925,27 +1961,6 @@ def test_special_values(self): with self.assertRaises(ValueError): fmean([Inf, -Inf]) - def test_weights(self): - fmean = statistics.fmean - StatisticsError = statistics.StatisticsError - self.assertEqual( - fmean([10, 10, 10, 50], [0.25] * 4), - fmean([10, 10, 10, 50])) - self.assertEqual( - fmean([10, 10, 20], [0.25, 0.25, 0.50]), - fmean([10, 10, 20, 20])) - self.assertEqual( # inputs are iterators - fmean(iter([10, 10, 20]), iter([0.25, 0.25, 0.50])), - fmean([10, 10, 20, 20])) - with self.assertRaises(StatisticsError): - fmean([10, 20, 30], [1, 2]) # unequal lengths - with self.assertRaises(StatisticsError): - fmean(iter([10, 20, 30]), iter([1, 2])) # unequal lengths - with self.assertRaises(StatisticsError): - fmean([10, 20], [-1, 1]) # sum of weights is zero - with self.assertRaises(StatisticsError): - fmean(iter([10, 20]), iter([-1, 1])) # sum of weights is zero - # === Tests for variances and standard deviations === @@ -2122,104 +2137,6 @@ def test_center_not_at_mean(self): self.assertEqual(self.func(data), 2.5) self.assertEqual(self.func(data, mu=0.5), 6.5) -class TestSqrtHelpers(unittest.TestCase): - - def test_integer_sqrt_of_frac_rto(self): - for n, m in itertools.product(range(100), range(1, 1000)): - r = statistics._integer_sqrt_of_frac_rto(n, m) - self.assertIsInstance(r, int) - if r*r*m == n: - # Root is exact - continue - # Inexact, so the root should be odd - self.assertEqual(r&1, 1) - # Verify correct rounding - self.assertTrue(m * (r - 1)**2 < n < m * (r + 1)**2) - - @requires_IEEE_754 - @support.requires_resource('cpu') - def test_float_sqrt_of_frac(self): - - def is_root_correctly_rounded(x: Fraction, root: float) -> bool: - if not x: - return root == 0.0 - - # Extract adjacent representable floats - r_up: float = math.nextafter(root, math.inf) - r_down: float = math.nextafter(root, -math.inf) - assert r_down < root < r_up - - # Convert to fractions for exact arithmetic - frac_root: Fraction = Fraction(root) - half_way_up: Fraction = (frac_root + Fraction(r_up)) / 2 - half_way_down: Fraction = (frac_root + Fraction(r_down)) / 2 - - # Check a closed interval. - # Does not test for a midpoint rounding rule. - return half_way_down ** 2 <= x <= half_way_up ** 2 - - randrange = random.randrange - - for i in range(60_000): - numerator: int = randrange(10 ** randrange(50)) - denonimator: int = randrange(10 ** randrange(50)) + 1 - with self.subTest(numerator=numerator, denonimator=denonimator): - x: Fraction = Fraction(numerator, denonimator) - root: float = statistics._float_sqrt_of_frac(numerator, denonimator) - self.assertTrue(is_root_correctly_rounded(x, root)) - - # Verify that corner cases and error handling match math.sqrt() - self.assertEqual(statistics._float_sqrt_of_frac(0, 1), 0.0) - with self.assertRaises(ValueError): - statistics._float_sqrt_of_frac(-1, 1) - with self.assertRaises(ValueError): - statistics._float_sqrt_of_frac(1, -1) - - # Error handling for zero denominator matches that for Fraction(1, 0) - with self.assertRaises(ZeroDivisionError): - statistics._float_sqrt_of_frac(1, 0) - - # The result is well defined if both inputs are negative - self.assertEqual(statistics._float_sqrt_of_frac(-2, -1), statistics._float_sqrt_of_frac(2, 1)) - - def test_decimal_sqrt_of_frac(self): - root: Decimal - numerator: int - denominator: int - - for root, numerator, denominator in [ - (Decimal('0.4481904599041192673635338663'), 200874688349065940678243576378, 1000000000000000000000000000000), # No adj - (Decimal('0.7924949131383786609961759598'), 628048187350206338833590574929, 1000000000000000000000000000000), # Adj up - (Decimal('0.8500554152289934068192208727'), 722594208960136395984391238251, 1000000000000000000000000000000), # Adj down - ]: - with decimal.localcontext(decimal.DefaultContext): - self.assertEqual(statistics._decimal_sqrt_of_frac(numerator, denominator), root) - - # Confirm expected root with a quad precision decimal computation - with decimal.localcontext(decimal.DefaultContext) as ctx: - ctx.prec *= 4 - high_prec_ratio = Decimal(numerator) / Decimal(denominator) - ctx.rounding = decimal.ROUND_05UP - high_prec_root = high_prec_ratio.sqrt() - with decimal.localcontext(decimal.DefaultContext): - target_root = +high_prec_root - self.assertEqual(root, target_root) - - # Verify that corner cases and error handling match Decimal.sqrt() - self.assertEqual(statistics._decimal_sqrt_of_frac(0, 1), 0.0) - with self.assertRaises(decimal.InvalidOperation): - statistics._decimal_sqrt_of_frac(-1, 1) - with self.assertRaises(decimal.InvalidOperation): - statistics._decimal_sqrt_of_frac(1, -1) - - # Error handling for zero denominator matches that for Fraction(1, 0) - with self.assertRaises(ZeroDivisionError): - statistics._decimal_sqrt_of_frac(1, 0) - - # The result is well defined if both inputs are negative - self.assertEqual(statistics._decimal_sqrt_of_frac(-2, -1), statistics._decimal_sqrt_of_frac(2, 1)) - - class TestStdev(VarianceStdevMixin, NumericTestCase): # Tests for sample standard deviation. def setUp(self): @@ -2234,7 +2151,7 @@ def test_compare_to_variance(self): # Test that stdev is, in fact, the square root of variance. data = [random.uniform(-2, 9) for _ in range(1000)] expected = math.sqrt(statistics.variance(data)) - self.assertAlmostEqual(self.func(data), expected) + self.assertEqual(self.func(data), expected) def test_center_not_at_mean(self): data = (1.0, 2.0) @@ -2303,11 +2220,9 @@ def test_error_cases(self): with self.assertRaises(StatisticsError): geometric_mean([]) # empty input with self.assertRaises(StatisticsError): - geometric_mean([3.5, -4.0, 5.25]) # negative input - with self.assertRaises(StatisticsError): - geometric_mean([0.0, -4.0, 5.25]) # negative input with zero + geometric_mean([3.5, 0.0, 5.25]) # zero input with self.assertRaises(StatisticsError): - geometric_mean([3.5, -math.inf, 5.25]) # negative infinity + geometric_mean([3.5, -4.0, 5.25]) # negative input with self.assertRaises(StatisticsError): geometric_mean(iter([])) # empty iterator with self.assertRaises(TypeError): @@ -2330,200 +2245,6 @@ def test_special_values(self): with self.assertRaises(ValueError): geometric_mean([Inf, -Inf]) - # Cases with zero - self.assertEqual(geometric_mean([3, 0.0, 5]), 0.0) # Any zero gives a zero - self.assertEqual(geometric_mean([3, -0.0, 5]), 0.0) # Negative zero allowed - self.assertTrue(math.isnan(geometric_mean([0, NaN]))) # NaN beats zero - self.assertTrue(math.isnan(geometric_mean([0, Inf]))) # Because 0.0 * Inf -> NaN - - def test_mixed_int_and_float(self): - # Regression test for b.p.o. issue #28327 - geometric_mean = statistics.geometric_mean - expected_mean = 3.80675409583932 - values = [ - [2, 3, 5, 7], - [2, 3, 5, 7.0], - [2, 3, 5.0, 7.0], - [2, 3.0, 5.0, 7.0], - [2.0, 3.0, 5.0, 7.0], - ] - for v in values: - with self.subTest(v=v): - actual_mean = geometric_mean(v) - self.assertAlmostEqual(actual_mean, expected_mean, places=5) - - -class TestKDE(unittest.TestCase): - - def test_kde(self): - kde = statistics.kde - StatisticsError = statistics.StatisticsError - - kernels = ['normal', 'gauss', 'logistic', 'sigmoid', 'rectangular', - 'uniform', 'triangular', 'parabolic', 'epanechnikov', - 'quartic', 'biweight', 'triweight', 'cosine'] - - sample = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] - - # The approximate integral of a PDF should be close to 1.0 - - def integrate(func, low, high, steps=10_000): - "Numeric approximation of a definite function integral." - dx = (high - low) / steps - midpoints = (low + (i + 1/2) * dx for i in range(steps)) - return sum(map(func, midpoints)) * dx - - for kernel in kernels: - with self.subTest(kernel=kernel): - f_hat = kde(sample, h=1.5, kernel=kernel) - area = integrate(f_hat, -20, 20) - self.assertAlmostEqual(area, 1.0, places=4) - - # Check CDF against an integral of the PDF - - data = [3, 5, 10, 12] - h = 2.3 - x = 10.5 - for kernel in kernels: - with self.subTest(kernel=kernel): - cdf = kde(data, h, kernel, cumulative=True) - f_hat = kde(data, h, kernel) - area = integrate(f_hat, -20, x, 100_000) - self.assertAlmostEqual(cdf(x), area, places=4) - - # Check error cases - - with self.assertRaises(StatisticsError): - kde([], h=1.0) # Empty dataset - with self.assertRaises(TypeError): - kde(['abc', 'def'], 1.5) # Non-numeric data - with self.assertRaises(TypeError): - kde(iter(sample), 1.5) # Data is not a sequence - with self.assertRaises(StatisticsError): - kde(sample, h=0.0) # Zero bandwidth - with self.assertRaises(StatisticsError): - kde(sample, h=-1.0) # Negative bandwidth - with self.assertRaises(TypeError): - kde(sample, h='str') # Wrong bandwidth type - with self.assertRaises(StatisticsError): - kde(sample, h=1.0, kernel='bogus') # Invalid kernel - with self.assertRaises(TypeError): - kde(sample, 1.0, 'gauss', True) # Positional cumulative argument - - # Test name and docstring of the generated function - - h = 1.5 - kernel = 'cosine' - f_hat = kde(sample, h, kernel) - self.assertEqual(f_hat.__name__, 'pdf') - self.assertIn(kernel, f_hat.__doc__) - self.assertIn(repr(h), f_hat.__doc__) - - # Test closed interval for the support boundaries. - # In particular, 'uniform' should non-zero at the boundaries. - - f_hat = kde([0], 1.0, 'uniform') - self.assertEqual(f_hat(-1.0), 1/2) - self.assertEqual(f_hat(1.0), 1/2) - - # Test online updates to data - - data = [1, 2] - f_hat = kde(data, 5.0, 'triangular') - self.assertEqual(f_hat(100), 0.0) - data.append(100) - self.assertGreater(f_hat(100), 0.0) - - def test_kde_kernel_invcdfs(self): - kernel_invcdfs = statistics._kernel_invcdfs - kde = statistics.kde - - # Verify that cdf / invcdf will round trip - xarr = [i/100 for i in range(-100, 101)] - for kernel, invcdf in kernel_invcdfs.items(): - with self.subTest(kernel=kernel): - cdf = kde([0.0], h=1.0, kernel=kernel, cumulative=True) - for x in xarr: - self.assertAlmostEqual(invcdf(cdf(x)), x, places=5) - - @support.requires_resource('cpu') - def test_kde_random(self): - kde_random = statistics.kde_random - StatisticsError = statistics.StatisticsError - kernels = ['normal', 'gauss', 'logistic', 'sigmoid', 'rectangular', - 'uniform', 'triangular', 'parabolic', 'epanechnikov', - 'quartic', 'biweight', 'triweight', 'cosine'] - sample = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] - - # Smoke test - - for kernel in kernels: - with self.subTest(kernel=kernel): - rand = kde_random(sample, h=1.5, kernel=kernel) - selections = [rand() for i in range(10)] - - # Check error cases - - with self.assertRaises(StatisticsError): - kde_random([], h=1.0) # Empty dataset - with self.assertRaises(TypeError): - kde_random(['abc', 'def'], 1.5) # Non-numeric data - with self.assertRaises(TypeError): - kde_random(iter(sample), 1.5) # Data is not a sequence - with self.assertRaises(StatisticsError): - kde_random(sample, h=-1.0) # Zero bandwidth - with self.assertRaises(StatisticsError): - kde_random(sample, h=0.0) # Negative bandwidth - with self.assertRaises(TypeError): - kde_random(sample, h='str') # Wrong bandwidth type - with self.assertRaises(StatisticsError): - kde_random(sample, h=1.0, kernel='bogus') # Invalid kernel - - # Test name and docstring of the generated function - - h = 1.5 - kernel = 'cosine' - rand = kde_random(sample, h, kernel) - self.assertEqual(rand.__name__, 'rand') - self.assertIn(kernel, rand.__doc__) - self.assertIn(repr(h), rand.__doc__) - - # Approximate distribution test: Compare a random sample to the expected distribution - - data = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2, 7.8, 14.3, 15.1, 15.3, 15.8, 17.0] - xarr = [x / 10 for x in range(-100, 250)] - n = 1_000_000 - h = 1.75 - dx = 0.1 - - def p_observed(x): - # P(x <= X < x+dx) - i = bisect.bisect_left(big_sample, x) - j = bisect.bisect_left(big_sample, x + dx) - return (j - i) / len(big_sample) - - def p_expected(x): - # P(x <= X < x+dx) - return F_hat(x + dx) - F_hat(x) - - for kernel in kernels: - with self.subTest(kernel=kernel): - - rand = kde_random(data, h, kernel, seed=8675309**2) - big_sample = sorted([rand() for i in range(n)]) - F_hat = statistics.kde(data, h, kernel, cumulative=True) - - for x in xarr: - self.assertTrue(math.isclose(p_observed(x), p_expected(x), abs_tol=0.0005)) - - # Test online updates to data - - data = [1, 2] - rand = kde_random(data, 5, 'triangular') - self.assertLess(max([rand() for i in range(5000)]), 10) - data.append(100) - self.assertGreater(max(rand() for i in range(5000)), 10) - class TestQuantiles(unittest.TestCase): @@ -2634,11 +2355,6 @@ def f(x): data = random.choices(range(100), k=k) q1, q2, q3 = quantiles(data, method='inclusive') self.assertEqual(q2, statistics.median(data)) - # Base case with a single data point: When estimating quantiles from - # a sample, we want to be able to add one sample point at a time, - # getting increasingly better estimates. - self.assertEqual(quantiles([10], n=4), [10.0, 10.0, 10.0]) - self.assertEqual(quantiles([10], n=4, method='exclusive'), [10.0, 10.0, 10.0]) def test_equal_inputs(self): quantiles = statistics.quantiles @@ -2689,7 +2405,7 @@ def test_error_cases(self): with self.assertRaises(ValueError): quantiles([10, 20, 30], method='X') # method is unknown with self.assertRaises(StatisticsError): - quantiles([], n=4) # not enough data points + quantiles([10], n=4) # not enough data points with self.assertRaises(TypeError): quantiles([10, None, 30], n=4) # data is non-numeric @@ -2748,95 +2464,6 @@ def test_different_scales(self): self.assertAlmostEqual(statistics.correlation(x, y), 1) self.assertAlmostEqual(statistics.covariance(x, y), 0.1) - def test_sqrtprod_helper_function_fundamentals(self): - # Verify that results are close to sqrt(x * y) - for i in range(100): - x = random.expovariate() - y = random.expovariate() - expected = math.sqrt(x * y) - actual = statistics._sqrtprod(x, y) - with self.subTest(x=x, y=y, expected=expected, actual=actual): - self.assertAlmostEqual(expected, actual) - - x, y, target = 0.8035720646477457, 0.7957468097636939, 0.7996498651651661 - self.assertEqual(statistics._sqrtprod(x, y), target) - self.assertNotEqual(math.sqrt(x * y), target) - - # Test that range extremes avoid underflow and overflow - smallest = sys.float_info.min * sys.float_info.epsilon - self.assertEqual(statistics._sqrtprod(smallest, smallest), smallest) - biggest = sys.float_info.max - self.assertEqual(statistics._sqrtprod(biggest, biggest), biggest) - - # Check special values and the sign of the result - special_values = [0.0, -0.0, 1.0, -1.0, 4.0, -4.0, - math.nan, -math.nan, math.inf, -math.inf] - for x, y in itertools.product(special_values, repeat=2): - try: - expected = math.sqrt(x * y) - except ValueError: - expected = 'ValueError' - try: - actual = statistics._sqrtprod(x, y) - except ValueError: - actual = 'ValueError' - with self.subTest(x=x, y=y, expected=expected, actual=actual): - if isinstance(expected, str) and expected == 'ValueError': - self.assertEqual(actual, 'ValueError') - continue - self.assertIsInstance(actual, float) - if math.isnan(expected): - self.assertTrue(math.isnan(actual)) - continue - self.assertEqual(actual, expected) - self.assertEqual(sign(actual), sign(expected)) - - @requires_IEEE_754 - @unittest.skipIf(HAVE_DOUBLE_ROUNDING, - "accuracy not guaranteed on machines with double rounding") - @support.cpython_only # Allow for a weaker sumprod() implmentation - def test_sqrtprod_helper_function_improved_accuracy(self): - # Test a known example where accuracy is improved - x, y, target = 0.8035720646477457, 0.7957468097636939, 0.7996498651651661 - self.assertEqual(statistics._sqrtprod(x, y), target) - self.assertNotEqual(math.sqrt(x * y), target) - - def reference_value(x: float, y: float) -> float: - x = decimal.Decimal(x) - y = decimal.Decimal(y) - with decimal.localcontext() as ctx: - ctx.prec = 200 - return float((x * y).sqrt()) - - # Verify that the new function with improved accuracy - # agrees with a reference value more often than old version. - new_agreements = 0 - old_agreements = 0 - for i in range(10_000): - x = random.expovariate() - y = random.expovariate() - new = statistics._sqrtprod(x, y) - old = math.sqrt(x * y) - ref = reference_value(x, y) - new_agreements += (new == ref) - old_agreements += (old == ref) - self.assertGreater(new_agreements, old_agreements) - - def test_correlation_spearman(self): - # https://statistics.laerd.com/statistical-guides/spearmans-rank-order-correlation-statistical-guide-2.php - # Compare with: - # >>> import scipy.stats.mstats - # >>> scipy.stats.mstats.spearmanr(reading, mathematics) - # SpearmanrResult(correlation=0.6686960980480712, pvalue=0.03450954165178532) - # And Wolfram Alpha gives: 0.668696 - # https://www.wolframalpha.com/input?i=SpearmanRho%5B%7B56%2C+75%2C+45%2C+71%2C+61%2C+64%2C+58%2C+80%2C+76%2C+61%7D%2C+%7B66%2C+70%2C+40%2C+60%2C+65%2C+56%2C+59%2C+77%2C+67%2C+63%7D%5D - reading = [56, 75, 45, 71, 61, 64, 58, 80, 76, 61] - mathematics = [66, 70, 40, 60, 65, 56, 59, 77, 67, 63] - self.assertAlmostEqual(statistics.correlation(reading, mathematics, method='ranked'), - 0.6686960980480712) - - with self.assertRaises(ValueError): - statistics.correlation(reading, mathematics, method='bad_method') class TestLinearRegression(unittest.TestCase): @@ -2860,22 +2487,6 @@ def test_results(self): self.assertAlmostEqual(intercept, true_intercept) self.assertAlmostEqual(slope, true_slope) - def test_proportional(self): - x = [10, 20, 30, 40] - y = [180, 398, 610, 799] - slope, intercept = statistics.linear_regression(x, y, proportional=True) - self.assertAlmostEqual(slope, 20 + 1/150) - self.assertEqual(intercept, 0.0) - - def test_float_output(self): - x = [Fraction(2, 3), Fraction(3, 4)] - y = [Fraction(4, 5), Fraction(5, 6)] - slope, intercept = statistics.linear_regression(x, y) - self.assertTrue(isinstance(slope, float)) - self.assertTrue(isinstance(intercept, float)) - slope, intercept = statistics.linear_regression(x, y, proportional=True) - self.assertTrue(isinstance(slope, float)) - self.assertTrue(isinstance(intercept, float)) class TestNormalDist: @@ -3029,8 +2640,6 @@ def test_cdf(self): self.assertTrue(math.isnan(X.cdf(float('NaN')))) @support.skip_if_pgo_task - @support.requires_resource('cpu') - @unittest.skip("TODO: RUSTPYTHON Flaky") def test_inv_cdf(self): NormalDist = self.module.NormalDist @@ -3088,10 +2697,9 @@ def test_inv_cdf(self): iq.inv_cdf(1.0) # p is one with self.assertRaises(self.module.StatisticsError): iq.inv_cdf(1.1) # p over one - - # Supported case: - iq = NormalDist(100, 0) # sigma is zero - self.assertEqual(iq.inv_cdf(0.5), 100) + with self.assertRaises(self.module.StatisticsError): + iq = NormalDist(100, 0) # sigma is zero + iq.inv_cdf(0.5) # Special values self.assertTrue(math.isnan(Z.inv_cdf(float('NaN')))) @@ -3274,19 +2882,14 @@ def __init__(self, mu, sigma): nd = NormalDist(100, 15) self.assertNotEqual(nd, lnd) - def test_copy(self): + def test_pickle_and_copy(self): nd = self.module.NormalDist(37.5, 5.625) nd1 = copy.copy(nd) self.assertEqual(nd, nd1) nd2 = copy.deepcopy(nd) self.assertEqual(nd, nd2) - - def test_pickle(self): - nd = self.module.NormalDist(37.5, 5.625) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - with self.subTest(proto=proto): - pickled = pickle.loads(pickle.dumps(nd, protocol=proto)) - self.assertEqual(nd, pickled) + nd3 = pickle.loads(pickle.dumps(nd)) + self.assertEqual(nd, nd3) def test_hashability(self): ND = self.module.NormalDist @@ -3308,7 +2911,7 @@ def setUp(self): def tearDown(self): sys.modules['statistics'] = statistics - + @unittest.skipUnless(c_statistics, 'requires _statistics') class TestNormalDistC(unittest.TestCase, TestNormalDist): @@ -3325,7 +2928,6 @@ def tearDown(self): def load_tests(loader, tests, ignore): """Used for doctest/unittest integration.""" tests.addTests(doctest.DocTestSuite()) - tests.addTests(doctest.DocTestSuite(statistics)) return tests From a596568151a9a7dd4f9f7a5135b2516f9c336cd4 Mon Sep 17 00:00:00 2001 From: Noa Date: Wed, 19 Mar 2025 12:28:58 -0500 Subject: [PATCH 086/295] Use lexopt for argument parsing (#5602) --- Cargo.lock | 33 +-- Cargo.toml | 2 +- Lib/test/test_cmd_line.py | 8 - examples/dis.rs | 86 +++--- examples/parse_folder.rs | 24 +- src/lib.rs | 13 +- src/settings.rs | 553 ++++++++++++++++++-------------------- vm/src/version.rs | 3 +- 8 files changed, 328 insertions(+), 394 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1c428c2d73..e1f720698e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -51,15 +51,6 @@ dependencies = [ "libc", ] -[[package]] -name = "ansi_term" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" -dependencies = [ - "winapi", -] - [[package]] name = "anyhow" version = "1.0.96" @@ -255,13 +246,9 @@ version = "2.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" dependencies = [ - "ansi_term", - "atty", "bitflags 1.3.2", - "strsim", "textwrap 0.11.0", "unicode-width 0.1.14", - "vec_map", ] [[package]] @@ -1074,6 +1061,12 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "lexopt" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baff4b617f7df3d896f97fe922b64817f6cd9a756bb81d40f8883f2f66dcb401" + [[package]] name = "libc" version = "0.2.169" @@ -1952,12 +1945,12 @@ name = "rustpython" version = "0.4.0" dependencies = [ "cfg-if", - "clap", "criterion", "dirs-next", "env_logger", "flame", "flamescope", + "lexopt", "libc", "log", "pyo3", @@ -2557,12 +2550,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" -[[package]] -name = "strsim" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" - [[package]] name = "strum" version = "0.27.1" @@ -3023,12 +3010,6 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" -[[package]] -name = "vec_map" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" - [[package]] name = "version_check" version = "0.9.5" diff --git a/Cargo.toml b/Cargo.toml index 14e66b39eb..2d66f9775a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,7 +34,7 @@ cfg-if = { workspace = true } log = { workspace = true } flame = { workspace = true, optional = true } -clap = "2.34" +lexopt = "0.3" dirs = { package = "dirs-next", version = "2.0.0" } env_logger = { version = "0.9.0", default-features = false, features = ["atty", "termcolor"] } flamescope = { version = "0.1.2", optional = true } diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index f7247705d0..da53f085a5 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -38,8 +38,6 @@ def verify_valid_flag(self, cmd_line): self.assertNotIn(b'Traceback', err) return out - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_help(self): self.verify_valid_flag('-h') self.verify_valid_flag('-?') @@ -82,8 +80,6 @@ def test_optimize(self): def test_site_flag(self): self.verify_valid_flag('-S') - # NOTE: RUSTPYTHON version never starts with Python - @unittest.expectedFailure def test_version(self): version = ('Python %d.%d' % sys.version_info[:2]).encode("ascii") for switch in '-V', '--version', '-VV': @@ -550,8 +546,6 @@ def test_no_stderr(self): def test_no_std_streams(self): self._test_no_stdio(['stdin', 'stdout', 'stderr']) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_hash_randomization(self): # Verify that -R enables hash randomization: self.verify_valid_flag('-R') @@ -966,8 +960,6 @@ def test_ignore_PYTHONPATH(self): self.run_ignoring_vars("'{}' not in sys.path".format(path), PYTHONPATH=path) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_ignore_PYTHONHASHSEED(self): self.run_ignoring_vars("sys.flags.hash_randomization == 1", PYTHONHASHSEED="0") diff --git a/examples/dis.rs b/examples/dis.rs index 0b1e7c0d3d..ee424f7b34 100644 --- a/examples/dis.rs +++ b/examples/dis.rs @@ -6,67 +6,59 @@ /// example usage: /// $ cargo run --release --example dis demo*.py -#[macro_use] -extern crate clap; -extern crate env_logger; #[macro_use] extern crate log; -use clap::{App, Arg}; +use lexopt::ValueExt; use rustpython_compiler as compiler; use std::error::Error; use std::fs; -use std::path::Path; +use std::path::{Path, PathBuf}; -fn main() { +fn main() -> Result<(), lexopt::Error> { env_logger::init(); - let app = App::new("dis") - .version(crate_version!()) - .author(crate_authors!()) - .about("Compiles and disassembles python script files for viewing their bytecode.") - .arg( - Arg::with_name("scripts") - .help("Scripts to scan") - .multiple(true) - .required(true), - ) - .arg( - Arg::with_name("mode") - .help("The mode to compile the scripts in") - .long("mode") - .short("m") - .default_value("exec") - .possible_values(&["exec", "single", "eval"]) - .takes_value(true), - ) - .arg( - Arg::with_name("no_expand") - .help( - "Don't expand CodeObject LoadConst instructions to show \ - the instructions inside", - ) - .long("no-expand") - .short("x"), - ) - .arg( - Arg::with_name("optimize") - .help("The amount of optimization to apply to the compiled bytecode") - .short("O") - .multiple(true), - ); - let matches = app.get_matches(); - let mode = matches.value_of_lossy("mode").unwrap().parse().unwrap(); - let expand_code_objects = !matches.is_present("no_expand"); - let optimize = matches.occurrences_of("optimize") as u8; - let scripts = matches.values_of_os("scripts").unwrap(); + let mut scripts = vec![]; + let mut mode = compiler::Mode::Exec; + let mut expand_code_objects = true; + let mut optimize = 0; + + let mut parser = lexopt::Parser::from_env(); + while let Some(arg) = parser.next()? { + use lexopt::Arg::*; + match arg { + Long("help") | Short('h') => { + let bin_name = parser.bin_name().unwrap_or("dis"); + println!( + "usage: {bin_name} [-m,--mode=exec|single|eval] [-x,--no-expand] [-O]" + ); + println!( + "Compiles and disassembles python script files for viewing their bytecode." + ); + return Ok(()); + } + Value(x) => scripts.push(PathBuf::from(x)), + Long("mode") | Short('m') => { + mode = parser + .value()? + .parse_with(|s| s.parse::().map_err(|e| e.to_string()))? + } + Long("no-expand") | Short('x') => expand_code_objects = false, + Short('O') => optimize += 1, + _ => return Err(arg.unexpected()), + } + } + + if scripts.is_empty() { + return Err("expected at least one argument".into()); + } let opts = compiler::CompileOpts { optimize, ..Default::default() }; - for script in scripts.map(Path::new) { + for script in &scripts { if script.exists() && script.is_file() { let res = display_script(script, mode, opts.clone(), expand_code_objects); if let Err(e) = res { @@ -76,6 +68,8 @@ fn main() { eprintln!("{script:?} is not a file."); } } + + Ok(()) } fn display_script( diff --git a/examples/parse_folder.rs b/examples/parse_folder.rs index f54be635c8..97a1eaf036 100644 --- a/examples/parse_folder.rs +++ b/examples/parse_folder.rs @@ -4,38 +4,28 @@ /// /// example usage: /// $ RUST_LOG=info cargo run --release parse_folder /usr/lib/python3.7 - -#[macro_use] -extern crate clap; extern crate env_logger; #[macro_use] extern crate log; -use clap::{App, Arg}; use rustpython_parser::{Parse, ast}; use std::{ - path::Path, + path::{Path, PathBuf}, time::{Duration, Instant}, }; fn main() { env_logger::init(); - let app = App::new("parse_folders") - .version(crate_version!()) - .author(crate_authors!()) - .about("Walks over all .py files in a folder, and parses them.") - .arg( - Arg::with_name("folder") - .help("Folder to scan") - .required(true), - ); - let matches = app.get_matches(); - let folder = Path::new(matches.value_of("folder").unwrap()); + let folder: PathBuf = std::env::args_os() + .nth(1) + .expect("please pass a path argument") + .into(); + if folder.exists() && folder.is_dir() { println!("Parsing folder of python code: {folder:?}"); let t1 = Instant::now(); - let parsed_files = parse_folder(folder).unwrap(); + let parsed_files = parse_folder(&folder).unwrap(); let t2 = Instant::now(); let results = ScanResult { t1, diff --git a/src/lib.rs b/src/lib.rs index b0a176acf2..67a2a16eef 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -37,9 +37,6 @@ //! it will have your modules loaded into the vm. #![allow(clippy::needless_doctest_main)] -#[macro_use] -extern crate clap; -extern crate env_logger; #[macro_use] extern crate log; @@ -57,7 +54,7 @@ use std::process::ExitCode; pub use interpreter::InterpreterConfig; pub use rustpython_vm as vm; -pub use settings::{InstallPipMode, RunMode, opts_with_clap}; +pub use settings::{InstallPipMode, RunMode, parse_opts}; pub use shell::run_shell; /// The main cli of the `rustpython` interpreter. This function will return `std::process::ExitCode` @@ -73,7 +70,13 @@ pub fn run(init: impl FnOnce(&mut VirtualMachine) + 'static) -> ExitCode { }; } - let (settings, run_mode) = opts_with_clap(); + let (settings, run_mode) = match parse_opts() { + Ok(x) => x, + Err(e) => { + println!("{e}"); + return ExitCode::FAILURE; + } + }; // don't translate newlines (\r\n <=> \n) #[cfg(windows)] diff --git a/src/settings.rs b/src/settings.rs index 35114374c8..bc279ba287 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -1,6 +1,8 @@ -use clap::{App, AppSettings, Arg, ArgMatches}; -use rustpython_vm::Settings; -use std::env; +use lexopt::Arg::*; +use lexopt::ValueExt; +use rustpython_vm::{Settings, vm::CheckHashPycsMode}; +use std::str::FromStr; +use std::{cmp, env}; pub enum RunMode { Script(String), @@ -15,192 +17,200 @@ pub enum InstallPipMode { GetPip, } -pub fn opts_with_clap() -> (Settings, RunMode) { - let app = App::new("RustPython"); - let matches = parse_arguments(app); - settings_from(&matches) +impl FromStr for InstallPipMode { + type Err = &'static str; + + fn from_str(s: &str) -> Result { + match s { + "ensurepip" => Ok(InstallPipMode::Ensurepip), + "get-pip" => Ok(InstallPipMode::GetPip), + _ => Err("--install-pip takes ensurepip or get-pip as first argument"), + } + } } -fn parse_arguments<'a>(app: App<'a, '_>) -> ArgMatches<'a> { - let app = app - .setting(AppSettings::TrailingVarArg) - .version(crate_version!()) - .author(crate_authors!()) - .about("Rust implementation of the Python language") - .usage("rustpython [OPTIONS] [-c CMD | -m MODULE | FILE] [PYARGS]...") - .arg( - Arg::with_name("script") - .required(false) - .allow_hyphen_values(true) - .multiple(true) - .value_name("script, args") - .min_values(1), - ) - .arg( - Arg::with_name("c") - .short("c") - .takes_value(true) - .allow_hyphen_values(true) - .multiple(true) - .value_name("cmd, args") - .min_values(1) - .help("run the given string as a program"), - ) - .arg( - Arg::with_name("m") - .short("m") - .takes_value(true) - .allow_hyphen_values(true) - .multiple(true) - .value_name("module, args") - .min_values(1) - .help("run library module as script"), - ) - .arg( - Arg::with_name("install_pip") - .long("install-pip") - .takes_value(true) - .allow_hyphen_values(true) - .multiple(true) - .value_name("get-pip args") - .min_values(0) - .help( - "install the pip package manager for rustpython; \ - requires rustpython be build with the ssl feature enabled.", - ), - ) - .arg( - Arg::with_name("optimize") - .short("O") - .multiple(true) - .help("Optimize. Set __debug__ to false. Remove debug statements."), - ) - .arg( - Arg::with_name("verbose") - .short("v") - .multiple(true) - .help("Give the verbosity (can be applied multiple times)"), - ) - .arg( - Arg::with_name("debug") - .short("d") - .multiple(true) - .help("Debug the parser."), - ) - .arg( - Arg::with_name("quiet") - .short("q") - .multiple(true) - .help("Be quiet at startup."), - ) - .arg( - Arg::with_name("inspect") - .short("i") - .multiple(true) - .help("Inspect interactively after running the script."), - ) - .arg( - Arg::with_name("no-user-site") - .short("s") - .multiple(true) - .help("don't add user site directory to sys.path."), - ) - .arg( - Arg::with_name("no-site") - .short("S") - .multiple(true) - .help("don't imply 'import site' on initialization"), - ) - .arg( - Arg::with_name("dont-write-bytecode") - .short("B") - .multiple(true) - .help("don't write .pyc files on import"), - ) - .arg( - Arg::with_name("safe-path") - .short("P") - .multiple(true) - .help("don’t prepend a potentially unsafe path to sys.path"), - ) - .arg( - Arg::with_name("ignore-environment") - .short("E") - .multiple(true) - .help("Ignore environment variables PYTHON* such as PYTHONPATH"), - ) - .arg( - Arg::with_name("isolate") - .short("I") - .multiple(true) - .help("isolate Python from the user's environment (implies -E and -s)"), - ) - .arg( - Arg::with_name("implementation-option") - .short("X") - .takes_value(true) - .multiple(true) - .number_of_values(1) - .help("set implementation-specific option"), - ) - .arg( - Arg::with_name("warning-control") - .short("W") - .takes_value(true) - .multiple(true) - .number_of_values(1) - .help("warning control; arg is action:message:category:module:lineno"), - ) - .arg( - Arg::with_name("check-hash-based-pycs") - .long("check-hash-based-pycs") - .takes_value(true) - .number_of_values(1) - .possible_values(&["always", "default", "never"]) - .help("control how Python invalidates hash-based .pyc files"), - ) - .arg( - Arg::with_name("bytes-warning") - .short("b") - .multiple(true) - .help( - "issue warnings about using bytes where strings \ - are usually expected (-bb: issue errors)", - ), - ) - .arg(Arg::with_name("unbuffered").short("u").multiple(true).help( - "force the stdout and stderr streams to be unbuffered; \ - this option has no effect on stdin; also PYTHONUNBUFFERED=x", - )); +#[derive(Default)] +struct CliArgs { + bytes_warning: u8, + dont_write_bytecode: bool, + debug: u8, + ignore_environment: bool, + inspect: bool, + isolate: bool, + optimize: u8, + safe_path: bool, + quiet: bool, + random_hash_seed: bool, + no_user_site: bool, + no_site: bool, + unbuffered: bool, + verbose: u8, + warning_control: Vec, + implementation_option: Vec, + check_hash_based_pycs: CheckHashPycsMode, + + #[cfg(feature = "flame-it")] + profile_output: Option, #[cfg(feature = "flame-it")] - let app = app - .arg( - Arg::with_name("profile_output") - .long("profile-output") - .takes_value(true) - .help("the file to output the profiling information to"), - ) - .arg( - Arg::with_name("profile_format") - .long("profile-format") - .takes_value(true) - .help("the profile format to output the profiling information in"), - ); - app.get_matches() + profile_format: Option, +} + +const USAGE_STRING: &str = "\ +usage: {PROG} [option] ... [-c cmd | -m mod | file | -] [arg] ... +Options (and corresponding environment variables): +-b : issue warnings about converting bytes/bytearray to str and comparing + bytes/bytearray with str or bytes with int. (-bb: issue errors) +-B : don't write .pyc files on import; also PYTHONDONTWRITEBYTECODE=x +-c cmd : program passed in as string (terminates option list) +-d : turn on parser debugging output (for experts only, only works on + debug builds); also PYTHONDEBUG=x +-E : ignore PYTHON* environment variables (such as PYTHONPATH) +-h : print this help message and exit (also -? or --help) +-i : inspect interactively after running script; forces a prompt even + if stdin does not appear to be a terminal; also PYTHONINSPECT=x +-I : isolate Python from the user's environment (implies -E and -s) +-m mod : run library module as a script (terminates option list) +-O : remove assert and __debug__-dependent statements; add .opt-1 before + .pyc extension; also PYTHONOPTIMIZE=x +-OO : do -O changes and also discard docstrings; add .opt-2 before + .pyc extension +-P : don't prepend a potentially unsafe path to sys.path; also + PYTHONSAFEPATH +-q : don't print version and copyright messages on interactive startup +-s : don't add user site directory to sys.path; also PYTHONNOUSERSITE=x +-S : don't imply 'import site' on initialization +-u : force the stdout and stderr streams to be unbuffered; + this option has no effect on stdin; also PYTHONUNBUFFERED=x +-v : verbose (trace import statements); also PYTHONVERBOSE=x + can be supplied multiple times to increase verbosity +-V : print the Python version number and exit (also --version) + when given twice, print more information about the build +-W arg : warning control; arg is action:message:category:module:lineno + also PYTHONWARNINGS=arg +-x : skip first line of source, allowing use of non-Unix forms of #!cmd +-X opt : set implementation-specific option +--check-hash-based-pycs always|default|never: + control how Python invalidates hash-based .pyc files +--help-env: print help about Python environment variables and exit +--help-xoptions: print help about implementation-specific -X options and exit +--help-all: print complete help information and exit + +RustPython extensions: + + +Arguments: +file : program read from script file +- : program read from stdin (default; interactive mode if a tty) +arg ...: arguments passed to program in sys.argv[1:] +"; + +fn parse_args() -> Result<(CliArgs, RunMode, Vec), lexopt::Error> { + let mut args = CliArgs::default(); + let mut parser = lexopt::Parser::from_env(); + fn argv(argv0: String, mut parser: lexopt::Parser) -> Result, lexopt::Error> { + std::iter::once(Ok(argv0)) + .chain(parser.raw_args()?.map(|arg| arg.string())) + .collect() + } + while let Some(arg) = parser.next()? { + match arg { + Short('b') => args.bytes_warning += 1, + Short('B') => args.dont_write_bytecode = true, + Short('c') => { + let cmd = parser.value()?.string()?; + return Ok((args, RunMode::Command(cmd), argv("-c".to_owned(), parser)?)); + } + Short('d') => args.debug += 1, + Short('E') => args.ignore_environment = true, + Short('h' | '?') | Long("help") => help(parser), + Short('i') => args.inspect = true, + Short('I') => args.isolate = true, + Short('m') => { + let module = parser.value()?.string()?; + let argv = argv("PLACEHOLDER".to_owned(), parser)?; + return Ok((args, RunMode::Module(module), argv)); + } + Short('O') => args.optimize += 1, + Short('P') => args.safe_path = true, + Short('q') => args.quiet = true, + Short('R') => args.random_hash_seed = true, + Short('S') => args.no_site = true, + Short('s') => args.no_user_site = true, + Short('u') => args.unbuffered = true, + Short('v') => args.verbose += 1, + Short('V') | Long("version") => version(), + Short('W') => args.warning_control.push(parser.value()?.string()?), + // TODO: Short('x') => + Short('X') => args.implementation_option.push(parser.value()?.string()?), + + Long("check-hash-based-pycs") => { + args.check_hash_based_pycs = parser.value()?.parse()? + } + + // TODO: make these more specific + Long("help-env") => help(parser), + Long("help-xoptions") => help(parser), + Long("help-all") => help(parser), + + #[cfg(feature = "flame-it")] + Long("profile-output") => args.profile_output = Some(parser.value()?), + #[cfg(feature = "flame-it")] + Long("profile-format") => args.profile_format = Some(parser.value()?.string()?), + + Long("install-pip") => { + let (mode, argv) = if let Some(val) = parser.optional_value() { + (val.parse()?, vec![val.string()?]) + } else if let Ok(argv0) = parser.value() { + let mode = argv0.parse()?; + (mode, argv(argv0.string()?, parser)?) + } else { + ( + InstallPipMode::Ensurepip, + ["ensurepip", "--upgrade", "--default-pip"] + .map(str::to_owned) + .into(), + ) + }; + return Ok((args, RunMode::InstallPip(mode), argv)); + } + Value(script_name) => { + let script_name = script_name.string()?; + let mode = if script_name == "-" { + RunMode::Repl + } else { + RunMode::Script(script_name.clone()) + }; + return Ok((args, mode, argv(script_name, parser)?)); + } + _ => return Err(arg.unexpected()), + } + } + Ok((args, RunMode::Repl, vec![])) +} + +fn help(parser: lexopt::Parser) -> ! { + let usage = USAGE_STRING.replace("{PROG}", parser.bin_name().unwrap_or("rustpython")); + print!("{usage}"); + std::process::exit(0); +} + +fn version() -> ! { + println!("Python {}", rustpython_vm::version::get_version()); + std::process::exit(0); } /// Create settings by examining command line arguments and environment /// variables. -fn settings_from(matches: &ArgMatches) -> (Settings, RunMode) { +pub fn parse_opts() -> Result<(Settings, RunMode), lexopt::Error> { + let (args, mode, argv) = parse_args()?; + let mut settings = Settings::default(); - settings.isolated = matches.is_present("isolate"); - let ignore_environment = settings.isolated || matches.is_present("ignore-environment"); - settings.ignore_environment = ignore_environment; - settings.interactive = !matches.is_present("c") - && !matches.is_present("m") - && (!matches.is_present("script") || matches.is_present("inspect")); - settings.bytes_warning = matches.occurrences_of("bytes-warning"); - settings.import_site = !matches.is_present("no-site"); + settings.isolated = args.isolate; + settings.ignore_environment = settings.isolated || args.ignore_environment; + settings.bytes_warning = args.bytes_warning.into(); + settings.import_site = !args.no_site; + + let ignore_environment = settings.ignore_environment; if !ignore_environment { settings.path_list.extend(get_paths("RUSTPYTHONPATH")); @@ -209,34 +219,32 @@ fn settings_from(matches: &ArgMatches) -> (Settings, RunMode) { // Now process command line flags: - let count_flag = |arg, env| { - let mut val = matches.occurrences_of(arg) as u8; - if !ignore_environment { - if let Some(value) = get_env_var_value(env) { - val = std::cmp::max(val, value); - } - } - val + let get_env = |env| (!ignore_environment).then(|| env::var_os(env)).flatten(); + + let env_count = |env| { + get_env(env).filter(|v| !v.is_empty()).map_or(0, |val| { + val.to_str().and_then(|v| v.parse::().ok()).unwrap_or(1) + }) }; - settings.optimize = count_flag("optimize", "PYTHONOPTIMIZE"); - settings.verbose = count_flag("verbose", "PYTHONVERBOSE"); - settings.debug = count_flag("debug", "PYTHONDEBUG"); + settings.optimize = cmp::max(args.optimize, env_count("PYTHONOPTIMIZE")); + settings.verbose = cmp::max(args.verbose, env_count("PYTHONVERBOSE")); + settings.debug = cmp::max(args.debug, env_count("PYTHONDEBUG")); - let bool_env_var = |env| !ignore_environment && env::var_os(env).is_some_and(|v| !v.is_empty()); - let bool_flag = |arg, env| matches.is_present(arg) || bool_env_var(env); + let env_bool = |env| get_env(env).is_some_and(|v| !v.is_empty()); settings.user_site_directory = - !(settings.isolated || bool_flag("no-user-site", "PYTHONNOUSERSITE")); - settings.quiet = matches.is_present("quiet"); - settings.write_bytecode = !bool_flag("dont-write-bytecode", "PYTHONDONTWRITEBYTECODE"); - settings.safe_path = settings.isolated || bool_flag("safe-path", "PYTHONSAFEPATH"); - settings.inspect = bool_flag("inspect", "PYTHONINSPECT"); - settings.buffered_stdio = !bool_flag("unbuffered", "PYTHONUNBUFFERED"); - - if !ignore_environment && env::var_os("PYTHONINTMAXSTRDIGITS").is_some() { - settings.int_max_str_digits = match env::var("PYTHONINTMAXSTRDIGITS").unwrap().parse() { - Ok(digits @ (0 | 640..)) => digits, + !(settings.isolated || args.no_user_site || env_bool("PYTHONNOUSERSITE")); + settings.quiet = args.quiet; + settings.write_bytecode = !(args.dont_write_bytecode || env_bool("PYTHONDONTWRITEBYTECODE")); + settings.safe_path = settings.isolated || args.safe_path || env_bool("PYTHONSAFEPATH"); + settings.inspect = args.inspect || env_bool("PYTHONINSPECT"); + settings.interactive = args.inspect; + settings.buffered_stdio = !args.unbuffered; + + if let Some(val) = get_env("PYTHONINTMAXSTRDIGITS") { + settings.int_max_str_digits = match val.to_str().and_then(|s| s.parse().ok()) { + Some(digits @ (0 | 640..)) => digits, _ => { error!( "Fatal Python error: config_init_int_max_str_digits: PYTHONINTMAXSTRDIGITS: invalid limit; must be >= 640 or 0 for unlimited.\nPython runtime state: preinitialized" @@ -246,42 +254,39 @@ fn settings_from(matches: &ArgMatches) -> (Settings, RunMode) { }; } - settings.check_hash_pycs_mode = matches - .value_of("check-hash-based-pycs") - .map(|val| val.parse().unwrap()) - .unwrap_or_default(); - - let xopts = matches - .values_of("implementation-option") - .unwrap_or_default() - .map(|s| { - let (name, value) = s.split_once('=').unzip(); - let name = name.unwrap_or(s); - match name { - "dev" => settings.dev_mode = true, - "warn_default_encoding" => settings.warn_default_encoding = true, - "no_sig_int" => settings.install_signal_handlers = false, - "int_max_str_digits" => { - settings.int_max_str_digits = match value.unwrap().parse() { - Ok(digits) if digits == 0 || digits >= 640 => digits, - _ => { - error!( - "Fatal Python error: config_init_int_max_str_digits: \ - -X int_max_str_digits: \ - invalid limit; must be >= 640 or 0 for unlimited.\n\ - Python runtime state: preinitialized" - ); - std::process::exit(1); - } - }; - } - _ => {} + settings.check_hash_pycs_mode = args.check_hash_based_pycs; + + let xopts = args.implementation_option.into_iter().map(|s| { + let (name, value) = match s.split_once('=') { + Some((name, value)) => (name.to_owned(), Some(value)), + None => (s, None), + }; + match &*name { + "dev" => settings.dev_mode = true, + "warn_default_encoding" => settings.warn_default_encoding = true, + "no_sig_int" => settings.install_signal_handlers = false, + "int_max_str_digits" => { + settings.int_max_str_digits = match value.unwrap().parse() { + Ok(digits) if digits == 0 || digits >= 640 => digits, + _ => { + error!( + "Fatal Python error: config_init_int_max_str_digits: \ + -X int_max_str_digits: \ + invalid limit; must be >= 640 or 0 for unlimited.\n\ + Python runtime state: preinitialized" + ); + std::process::exit(1); + } + }; } - (name.to_owned(), value.map(str::to_owned)) - }); + _ => {} + } + (name, value.map(str::to_owned)) + }); settings.xoptions.extend(xopts); - settings.warn_default_encoding |= bool_env_var("PYTHONWARNDEFAULTENCODING"); + settings.warn_default_encoding = + settings.warn_default_encoding || env_bool("PYTHONWARNDEFAULTENCODING"); if settings.dev_mode { settings.warnoptions.push("default".to_owned()) @@ -294,66 +299,34 @@ fn settings_from(matches: &ArgMatches) -> (Settings, RunMode) { }; settings.warnoptions.push(warn.to_owned()); } - if let Some(warnings) = matches.values_of("warning-control") { - settings.warnoptions.extend(warnings.map(ToOwned::to_owned)); - } + settings.warnoptions.extend(args.warning_control); - let (mode, argv) = if let Some(mut cmd) = matches.values_of("c") { - let command = cmd.next().expect("clap ensure this exists"); - let argv = std::iter::once("-c".to_owned()) - .chain(cmd.map(ToOwned::to_owned)) - .collect(); - (RunMode::Command(command.to_owned()), argv) - } else if let Some(mut cmd) = matches.values_of("m") { - let module = cmd.next().expect("clap ensure this exists"); - let argv = std::iter::once("PLACEHOLDER".to_owned()) - .chain(cmd.map(ToOwned::to_owned)) - .collect(); - (RunMode::Module(module.to_owned()), argv) - } else if let Some(get_pip_args) = matches.values_of("install_pip") { - settings.isolated = true; - let mut args: Vec<_> = get_pip_args.map(ToOwned::to_owned).collect(); - if args.is_empty() { - args.extend(["ensurepip", "--upgrade", "--default-pip"].map(str::to_owned)); + settings.hash_seed = match (!args.random_hash_seed) + .then(|| get_env("PYTHONHASHSEED")) + .flatten() + { + Some(s) if s == "random" || s == "" => None, + Some(s) => { + let seed = s.parse_with(|s| { + s.parse::().map_err(|_| { + "Fatal Python init error: PYTHONHASHSEED must be \ + \"random\" or an integer in range [0; 4294967295]" + }) + })?; + Some(seed) } - let mode = match &*args[0] { - "ensurepip" => InstallPipMode::Ensurepip, - "get-pip" => InstallPipMode::GetPip, - _ => panic!("--install-pip takes ensurepip or get-pip as first argument"), - }; - (RunMode::InstallPip(mode), args) - } else if let Some(argv) = matches.values_of("script") { - let argv: Vec<_> = argv.map(ToOwned::to_owned).collect(); - let script = argv[0].clone(); - (RunMode::Script(script), argv) - } else { - (RunMode::Repl, vec!["".to_owned()]) + None => None, }; - let hash_seed = match env::var("PYTHONHASHSEED") { - Ok(s) if s == "random" => Some(None), - Ok(s) => s.parse::().ok().map(Some), - Err(_) => Some(None), - }; - settings.hash_seed = hash_seed.unwrap_or_else(|| { - error!("Fatal Python init error: PYTHONHASHSEED must be \"random\" or an integer in range [0; 4294967295]"); - // TODO: Need to change to ExitCode or Termination - std::process::exit(1) - }); - settings.argv = argv; - (settings, mode) -} + #[cfg(feature = "flame-it")] + { + settings.profile_output = args.profile_output; + settings.profile_format = args.profile_format; + } -/// Get environment variable and turn it into integer. -fn get_env_var_value(name: &str) -> Option { - env::var_os(name).filter(|v| !v.is_empty()).map(|value| { - value - .to_str() - .and_then(|v| v.parse::().ok()) - .unwrap_or(1) - }) + Ok((settings, mode)) } /// Helper function to retrieve a sequence of paths from an environment variable. diff --git a/vm/src/version.rs b/vm/src/version.rs index 8c42866a64..f2ac2354f8 100644 --- a/vm/src/version.rs +++ b/vm/src/version.rs @@ -17,9 +17,10 @@ pub const VERSION_HEX: usize = pub fn get_version() -> String { format!( - "{:.80} ({:.80}) \n[{:.80}]", // \n is PyPy convention + "{:.80} ({:.80}) \n[RustPython {} with {:.80}]", // \n is PyPy convention get_version_number(), get_build_info(), + env!("CARGO_PKG_VERSION"), get_compiler() ) } From 45c0fa0e7790143d175c32aaa122cdbc09cb1a1c Mon Sep 17 00:00:00 2001 From: Nicholas Paulick Date: Wed, 19 Mar 2025 13:50:42 -0500 Subject: [PATCH 087/295] Floating Point Power and While Loop JIT (#5614) * Initial commit for power function * Float power jit developed * Addded support for Floats and Ints * Integration Testing for JITPower implementation. * Update instructions.rs * Update int_tests.rs * Update float_tests.rs * Updated cranelift and power function to use magic bs * Updated the compile_fpow accuracy * updating while loop test * Update instructions.rs cranelift more like painlift * Update instructions.rs * Update instructions.rs * initial commit for making stable PR ready features * fixed final edge case for compile_ipow * fixed final edge case for compile_ipow * commenting out compile_ipow * fixed spelling errors * removed unused tests * forgot to run clippy * Cleaned the branch * While loop implementation for Cranelift 0.116.1 * Floating Point * Removed testing print statement * Resolved some formatting * Fixed cargo fmt warning * Fixed int div and int exp issues * Fixed formatting --------- Co-authored-by: Nick Co-authored-by: JoeLoparco Co-authored-by: Daniel O'Hear <149127239+dohear@users.noreply.github.com> Co-authored-by: Joseph Loparco <149088810+JoeLoparco@users.noreply.github.com> Co-authored-by: Nick Co-authored-by: dohear Co-authored-by: Nathan Rusch Co-authored-by: Nick Co-authored-by: Joseph Loparco <--global loparcoJoseph@gmail.com> Co-authored-by: Nick --- Cargo.lock | 231 +++++----- jit/Cargo.toml | 6 +- jit/src/instructions.rs | 893 +++++++++++++++++++++++++++++++-------- jit/src/lib.rs | 2 +- jit/tests/float_tests.rs | 111 +++++ jit/tests/int_tests.rs | 39 +- jit/tests/misc_tests.rs | 1 - 7 files changed, 983 insertions(+), 300 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e1f720698e..b3b8da765e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -36,6 +36,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "android-tzdata" version = "0.1.1" @@ -67,10 +73,10 @@ dependencies = [ ] [[package]] -name = "arrayvec" -version = "0.7.6" +name = "arbitrary" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223" [[package]] name = "ascii" @@ -156,6 +162,9 @@ name = "bumpalo" version = "3.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" +dependencies = [ + "allocator-api2", +] [[package]] name = "bytemuck" @@ -309,118 +318,145 @@ dependencies = [ [[package]] name = "cranelift" -version = "0.88.2" +version = "0.116.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea1b0c164043c16a8ece6813eef609ac2262a32a0bb0f5ed6eecf5d7bfb79ba8" +checksum = "a71de5e59f616d79d14d2c71aa2799ce898241d7f10f7e64a4997014b4000a28" dependencies = [ "cranelift-codegen", "cranelift-frontend", + "cranelift-module", ] [[package]] name = "cranelift-bforest" -version = "0.88.2" +version = "0.116.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52056f6d0584484b57fa6c1a65c1fcb15f3780d8b6a758426d9e3084169b2ddd" +checksum = "e15d04a0ce86cb36ead88ad68cf693ffd6cda47052b9e0ac114bc47fd9cd23c4" dependencies = [ "cranelift-entity", ] +[[package]] +name = "cranelift-bitset" +version = "0.116.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c6e3969a7ce267259ce244b7867c5d3bc9e65b0a87e81039588dfdeaede9f34" + [[package]] name = "cranelift-codegen" -version = "0.88.2" +version = "0.116.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18fed94c8770dc25d01154c3ffa64ed0b3ba9d583736f305fed7beebe5d9cf74" +checksum = "2c22032c4cb42558371cf516bb47f26cdad1819d3475c133e93c49f50ebf304e" dependencies = [ - "arrayvec", "bumpalo", "cranelift-bforest", + "cranelift-bitset", "cranelift-codegen-meta", "cranelift-codegen-shared", + "cranelift-control", "cranelift-entity", "cranelift-isle", + "gimli", + "hashbrown 0.14.5", "log", "regalloc2", + "rustc-hash 2.1.1", + "serde", "smallvec", - "target-lexicon", + "target-lexicon 0.13.2", ] [[package]] name = "cranelift-codegen-meta" -version = "0.88.2" +version = "0.116.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c451b81faf237d11c7e4f3165eeb6bac61112762c5cfe7b4c0fb7241474358f" +checksum = "c904bc71c61b27fc57827f4a1379f29de64fe95653b620a3db77d59655eee0b8" dependencies = [ "cranelift-codegen-shared", ] [[package]] name = "cranelift-codegen-shared" -version = "0.88.2" +version = "0.116.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40180f5497572f644ce88c255480981ae2ec1d7bb4d8e0c0136a13b87a2f2ceb" + +[[package]] +name = "cranelift-control" +version = "0.116.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7c940133198426d26128f08be2b40b0bd117b84771fd36798969c4d712d81fc" +checksum = "26d132c6d0bd8a489563472afc171759da0707804a65ece7ceb15a8c6d7dd5ef" +dependencies = [ + "arbitrary", +] [[package]] name = "cranelift-entity" -version = "0.88.2" +version = "0.116.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87a0f1b2fdc18776956370cf8d9b009ded3f855350c480c1c52142510961f352" +checksum = "4b2d0d9618275474fbf679dd018ac6e009acbd6ae6850f6a67be33fb3b00b323" +dependencies = [ + "cranelift-bitset", +] [[package]] name = "cranelift-frontend" -version = "0.88.2" +version = "0.116.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34897538b36b216cc8dd324e73263596d51b8cf610da6498322838b2546baf8a" +checksum = "4fac41e16729107393174b0c9e3730fb072866100e1e64e80a1a963b2e484d57" dependencies = [ "cranelift-codegen", "log", "smallvec", - "target-lexicon", + "target-lexicon 0.13.2", ] [[package]] name = "cranelift-isle" -version = "0.88.2" +version = "0.116.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b2629a569fae540f16a76b70afcc87ad7decb38dc28fa6c648ac73b51e78470" +checksum = "1ca20d576e5070044d0a72a9effc2deacf4d6aa650403189d8ea50126483944d" [[package]] name = "cranelift-jit" -version = "0.88.2" +version = "0.116.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "625be33ce54cf906c408f5ad9d08caa6e2a09e52d05fd0bd1bd95b132bfbba73" +checksum = "5e65c42755a719b09662b00c700daaf76cc35d5ace1f5c002ad404b591ff1978" dependencies = [ "anyhow", "cranelift-codegen", + "cranelift-control", "cranelift-entity", "cranelift-module", "cranelift-native", "libc", "log", "region", - "target-lexicon", - "windows-sys 0.36.1", + "target-lexicon 0.13.2", + "wasmtime-jit-icache-coherence", + "windows-sys 0.59.0", ] [[package]] name = "cranelift-module" -version = "0.88.2" +version = "0.116.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "883f8d42e07fd6b283941688f6c41a9e3b97fbf2b4ddcfb2756e675b86dc5edb" +checksum = "4d55612bebcf16ff7306c8a6f5bdb6d45662b8aa1ee058ecce8807ad87db719b" dependencies = [ "anyhow", "cranelift-codegen", + "cranelift-control", ] [[package]] name = "cranelift-native" -version = "0.88.2" +version = "0.116.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20937dab4e14d3e225c5adfc9c7106bafd4ac669bdb43027b911ff794c6fb318" +checksum = "b8dee82f3f1f2c4cba9177f1cc5e350fe98764379bcd29340caa7b01f85076c7" dependencies = [ "cranelift-codegen", "libc", - "target-lexicon", + "target-lexicon 0.13.2", ] [[package]] @@ -658,6 +694,12 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de853764b47027c2e862a995c34978ffa63c1501f2e15f987ba11bd4f9bba193" +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + [[package]] name = "fd-lock" version = "4.0.2" @@ -737,15 +779,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" -[[package]] -name = "fxhash" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" -dependencies = [ - "byteorder", -] - [[package]] name = "generic-array" version = "0.14.7" @@ -802,6 +835,17 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "gimli" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +dependencies = [ + "fallible-iterator", + "indexmap", + "stable_deref_trait", +] + [[package]] name = "glob" version = "0.3.2" @@ -1186,10 +1230,10 @@ dependencies = [ ] [[package]] -name = "mach" -version = "0.3.2" +name = "mach2" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa" +checksum = "19b955cdeb2a02b9117f121ce63aa52d08ade45de53e48fe6a38b39c10f6f709" dependencies = [ "libc", ] @@ -1668,7 +1712,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b14b5775b5ff446dd1056212d778012cbe8a0fbffd368029fd9e25b514479c38" dependencies = [ "once_cell", - "target-lexicon", + "target-lexicon 0.12.16", ] [[package]] @@ -1840,13 +1884,15 @@ dependencies = [ [[package]] name = "regalloc2" -version = "0.3.2" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d43a209257d978ef079f3d446331d0f1794f5e0fc19b306a199983857833a779" +checksum = "145c1c267e14f20fb0f88aa76a1c5ffec42d592c1d28b3cd9148ae35916158d3" dependencies = [ - "fxhash", + "allocator-api2", + "bumpalo", + "hashbrown 0.15.2", "log", - "slice-group-by", + "rustc-hash 2.1.1", "smallvec", ] @@ -1881,14 +1927,14 @@ checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "region" -version = "2.2.0" +version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877e54ea2adcd70d80e9179344c97f93ef0dffd6b03e1f4529e6e83ab2fa9ae0" +checksum = "e6b6ebd13bc009aef9cd476c1310d49ac354d36e240cf1bd753290f3dc7199a7" dependencies = [ "bitflags 1.3.2", "libc", - "mach", - "winapi", + "mach2", + "windows-sys 0.52.0", ] [[package]] @@ -1918,6 +1964,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +[[package]] +name = "rustc-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" + [[package]] name = "rustc_version" version = "0.4.1" @@ -2128,7 +2180,7 @@ dependencies = [ "num-traits", "phf", "phf_codegen", - "rustc-hash", + "rustc-hash 1.1.0", "rustpython-ast", "rustpython-parser-core", "tiny-keccak", @@ -2522,12 +2574,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" -[[package]] -name = "slice-group-by" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "826167069c09b99d56f31e9ae5c99049e932a98c9dc2dac47645b08dbbf76ba7" - [[package]] name = "smallvec" version = "1.14.0" @@ -2544,6 +2590,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "static_assertions" version = "1.1.0" @@ -2635,6 +2687,12 @@ version = "0.12.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" +[[package]] +name = "target-lexicon" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e502f78cdbb8ba4718f566c418c52bc729126ffd16baee5baa718cf25dd5a69a" + [[package]] name = "termcolor" version = "1.4.1" @@ -3118,6 +3176,18 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "wasmtime-jit-icache-coherence" +version = "29.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec5e8552e01692e6c2e5293171704fed8abdec79d1a6995a0870ab190e5747d1" +dependencies = [ + "anyhow", + "cfg-if", + "libc", + "windows-sys 0.59.0", +] + [[package]] name = "web-sys" version = "0.3.77" @@ -3196,19 +3266,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "windows-sys" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" -dependencies = [ - "windows_aarch64_msvc 0.36.1", - "windows_i686_gnu 0.36.1", - "windows_i686_msvc 0.36.1", - "windows_x86_64_gnu 0.36.1", - "windows_x86_64_msvc 0.36.1", -] - [[package]] name = "windows-sys" version = "0.48.0" @@ -3279,12 +3336,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_msvc" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" - [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -3297,12 +3348,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_i686_gnu" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" - [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -3321,12 +3366,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_msvc" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" - [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -3339,12 +3378,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_x86_64_gnu" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" - [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -3369,12 +3402,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_msvc" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" - [[package]] name = "windows_x86_64_msvc" version = "0.48.5" diff --git a/jit/Cargo.toml b/jit/Cargo.toml index cc26eb59a5..f59293a7ea 100644 --- a/jit/Cargo.toml +++ b/jit/Cargo.toml @@ -16,9 +16,9 @@ rustpython-compiler-core = { workspace = true } num-traits = { workspace = true } thiserror = { workspace = true } -cranelift = "0.88.0" -cranelift-jit = "0.88.0" -cranelift-module = "0.88.0" +cranelift = "0.116.1" +cranelift-jit = "0.116.1" +cranelift-module = "0.116.1" [dependencies.libffi] version = "3.1.0" diff --git a/jit/src/instructions.rs b/jit/src/instructions.rs index 1b74760dc0..bf30e51d74 100644 --- a/jit/src/instructions.rs +++ b/jit/src/instructions.rs @@ -11,7 +11,7 @@ use std::collections::HashMap; #[repr(u16)] enum CustomTrapCode { /// Raised when shifting by a negative number - NegativeShiftCount = 0, + NegativeShiftCount = 1, } #[derive(Clone)] @@ -56,6 +56,12 @@ impl JitValue { } } +#[derive(Clone)] +struct DDValue { + hi: Value, + lo: Value, +} + pub struct FunctionCompiler<'a, 'b> { builder: &'a mut FunctionBuilder<'b>, stack: Vec, @@ -123,14 +129,14 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> { fn boolean_val(&mut self, val: JitValue) -> Result { match val { JitValue::Float(val) => { - let zero = self.builder.ins().f64const(0); + let zero = self.builder.ins().f64const(0.0); let val = self.builder.ins().fcmp(FloatCC::NotEqual, val, zero); - Ok(self.builder.ins().bint(types::I8, val)) + Ok(val) } JitValue::Int(val) => { let zero = self.builder.ins().iconst(types::I64, 0); let val = self.builder.ins().icmp(IntCC::NotEqual, val, zero); - Ok(self.builder.ins().bint(types::I8, val)) + Ok(val) } JitValue::Bool(val) => Ok(val), JitValue::None => Ok(self.builder.ins().iconst(types::I8, 0)), @@ -151,38 +157,60 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> { func_ref: FuncRef, bytecode: &CodeObject, ) -> Result<(), JitCompileError> { - // TODO: figure out if this is sufficient -- previously individual labels were associated - // pretty much per-bytecode that uses them, or at least per "type" of block -- in theory an - // if block and a with block might jump to the same place. Now it's all "flattened", so - // there might be less distinction between different types of blocks going off - // label_targets alone let label_targets = bytecode.label_targets(); - let mut arg_state = OpArgState::default(); - for (offset, instruction) in bytecode.instructions.iter().enumerate() { - let (instruction, arg) = arg_state.get(*instruction); + + // Track whether we have "returned" in the current block + let mut in_unreachable_code = false; + + for (offset, &raw_instr) in bytecode.instructions.iter().enumerate() { let label = Label(offset as u32); + let (instruction, arg) = arg_state.get(raw_instr); + + // If this is a label that some earlier jump can target, + // treat it as the start of a new reachable block: if label_targets.contains(&label) { - let block = self.get_or_create_block(label); + // Create or get the block for this label: + let target_block = self.get_or_create_block(label); - // If the current block is not terminated/filled just jump - // into the new block. - if !self.builder.is_filled() { - self.builder.ins().jump(block, &[]); + // If the current block isn't terminated, jump: + if let Some(cur) = self.builder.current_block() { + if cur != target_block && self.builder.func.layout.last_inst(cur).is_none() { + self.builder.ins().jump(target_block, &[]); + } + } + // Switch to the target block + if self.builder.current_block() != Some(target_block) { + self.builder.switch_to_block(target_block); } - self.builder.switch_to_block(block); + // We are definitely reachable again at this label + in_unreachable_code = false; } - // Sometimes the bytecode contains instructions after a return - // just ignore those until we are at the next label - if self.builder.is_filled() { + // If we're in unreachable code, skip this instruction unless the label re-entered above. + if in_unreachable_code { continue; } + // Actually compile this instruction: self.add_instruction(func_ref, bytecode, instruction, arg)?; + + // If that was a return instruction, mark future instructions unreachable + match instruction { + Instruction::ReturnValue | Instruction::ReturnConst { .. } => { + in_unreachable_code = true; + } + _ => {} + } } + // After processing, if the current block is unterminated, insert a trap or fallthrough + if let Some(cur) = self.builder.current_block() { + if self.builder.func.layout.last_inst(cur).is_none() { + self.builder.ins().trap(TrapCode::user(0).unwrap()); + } + } Ok(()) } @@ -214,10 +242,12 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> { fn return_value(&mut self, val: JitValue) -> Result<(), JitCompileError> { if let Some(ref ty) = self.sig.ret { + // If the signature has a return type, enforce it if val.to_jit_type().as_ref() != Some(ty) { return Err(JitCompileError::NotSupported); } } else { + // First time we see a return, define it in the signature let ty = val.to_jit_type().ok_or(JitCompileError::NotSupported)?; self.sig.ret = Some(ty.clone()); self.builder @@ -226,7 +256,12 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> { .returns .push(AbiParam::new(ty.to_cranelift())); } - self.builder.ins().return_(&[val.into_value().unwrap()]); + + // If this is e.g. an Int, Float, or Bool we have a Cranelift `Value`. + // If we have JitValue::None or .Tuple(...) but can't handle that, error out (or handle differently). + let cr_val = val.into_value().ok_or(JitCompileError::NotSupported)?; + + self.builder.ins().return_(&[cr_val]); Ok(()) } @@ -241,34 +276,34 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> { Instruction::ExtendedArg => Ok(()), Instruction::JumpIfFalse { target } => { let cond = self.stack.pop().ok_or(JitCompileError::BadBytecode)?; - let val = self.boolean_val(cond)?; let then_block = self.get_or_create_block(target.get(arg)); - self.builder.ins().brz(val, then_block, &[]); + let else_block = self.builder.create_block(); - let block = self.builder.create_block(); - self.builder.ins().jump(block, &[]); - self.builder.switch_to_block(block); + self.builder + .ins() + .brif(val, else_block, &[], then_block, &[]); + self.builder.switch_to_block(else_block); Ok(()) } Instruction::JumpIfTrue { target } => { let cond = self.stack.pop().ok_or(JitCompileError::BadBytecode)?; - let val = self.boolean_val(cond)?; let then_block = self.get_or_create_block(target.get(arg)); - self.builder.ins().brnz(val, then_block, &[]); + let else_block = self.builder.create_block(); - let block = self.builder.create_block(); - self.builder.ins().jump(block, &[]); - self.builder.switch_to_block(block); + self.builder + .ins() + .brif(val, then_block, &[], else_block, &[]); + self.builder.switch_to_block(else_block); Ok(()) } + Instruction::Jump { target } => { let target_block = self.get_or_create_block(target.get(arg)); self.builder.ins().jump(target_block, &[]); - Ok(()) } Instruction::LoadFast(idx) => { @@ -354,9 +389,7 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> { }; let val = self.builder.ins().icmp(cond, operand_one, operand_two); - // TODO: Remove this `bint` in cranelift 0.90 as icmp now returns i8 - self.stack - .push(JitValue::Bool(self.builder.ins().bint(types::I8, val))); + self.stack.push(JitValue::Bool(val)); Ok(()) } (JitValue::Float(a), JitValue::Float(b)) => { @@ -370,9 +403,7 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> { }; let val = self.builder.ins().fcmp(cond, a, b); - // TODO: Remove this `bint` in cranelift 0.90 as fcmp now returns i8 - self.stack - .push(JitValue::Bool(self.builder.ins().bint(types::I8, val))); + self.stack.push(JitValue::Bool(val)); Ok(()) } _ => Err(JitCompileError::NotSupported), @@ -414,35 +445,34 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> { let val = match (op, a, b) { (BinaryOperator::Add, JitValue::Int(a), JitValue::Int(b)) => { - let (out, carry) = self.builder.ins().iadd_ifcout(a, b); - self.builder.ins().trapif( - IntCC::Overflow, - carry, - TrapCode::IntegerOverflow, - ); + let (out, carry) = self.builder.ins().sadd_overflow(a, b); + self.builder.ins().trapnz(carry, TrapCode::INTEGER_OVERFLOW); JitValue::Int(out) } (BinaryOperator::Subtract, JitValue::Int(a), JitValue::Int(b)) => { JitValue::Int(self.compile_sub(a, b)) } - (BinaryOperator::Multiply, JitValue::Int(a), JitValue::Int(b)) => { - JitValue::Int(self.builder.ins().imul(a, b)) - } (BinaryOperator::FloorDivide, JitValue::Int(a), JitValue::Int(b)) => { JitValue::Int(self.builder.ins().sdiv(a, b)) } (BinaryOperator::Divide, JitValue::Int(a), JitValue::Int(b)) => { - // Convert to float for regular division + // Check if b == 0, If so trap with a division by zero error + self.builder + .ins() + .trapz(b, TrapCode::INTEGER_DIVISION_BY_ZERO); + // Else convert to float and divide let a_float = self.builder.ins().fcvt_from_sint(types::F64, a); let b_float = self.builder.ins().fcvt_from_sint(types::F64, b); JitValue::Float(self.builder.ins().fdiv(a_float, b_float)) } + (BinaryOperator::Multiply, JitValue::Int(a), JitValue::Int(b)) => { + JitValue::Int(self.builder.ins().imul(a, b)) + } (BinaryOperator::Modulo, JitValue::Int(a), JitValue::Int(b)) => { JitValue::Int(self.builder.ins().srem(a, b)) } - // Todo: This should return int when possible (BinaryOperator::Power, JitValue::Int(a), JitValue::Int(b)) => { - JitValue::Float(self.compile_ipow(a, b)) + JitValue::Int(self.compile_ipow(a, b)) } ( BinaryOperator::Lshift | BinaryOperator::Rshift, @@ -454,7 +484,7 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> { let sign = self.builder.ins().ushr_imm(b, 63); self.builder.ins().trapnz( sign, - TrapCode::User(CustomTrapCode::NegativeShiftCount as u16), + TrapCode::user(CustomTrapCode::NegativeShiftCount as u8).unwrap(), ); let out = if op == BinaryOperator::Lshift { @@ -487,6 +517,9 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> { (BinaryOperator::Divide, JitValue::Float(a), JitValue::Float(b)) => { JitValue::Float(self.builder.ins().fdiv(a, b)) } + (BinaryOperator::Power, JitValue::Float(a), JitValue::Float(b)) => { + JitValue::Float(self.compile_fpow(a, b)) + } // Floats and Integers (_, JitValue::Int(a), JitValue::Float(b)) @@ -514,6 +547,9 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> { BinaryOperator::Divide => { JitValue::Float(self.builder.ins().fdiv(operand_one, operand_two)) } + BinaryOperator::Power => { + JitValue::Float(self.compile_fpow(operand_one, operand_two)) + } _ => return Err(JitCompileError::NotSupported), } } @@ -523,7 +559,13 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> { Ok(()) } - Instruction::SetupLoop { .. } | Instruction::PopBlock => { + Instruction::SetupLoop { .. } => { + let loop_head = self.builder.create_block(); + self.builder.ins().jump(loop_head, &[]); + self.builder.switch_to_block(loop_head); + Ok(()) + } + Instruction::PopBlock => { // TODO: block support Ok(()) } @@ -562,167 +604,660 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> { } fn compile_sub(&mut self, a: Value, b: Value) -> Value { - // TODO: this should be fine, but cranelift doesn't special-case isub_ifbout - // let (out, carry) = self.builder.ins().isub_ifbout(a, b); - // self.builder - // .ins() - // .trapif(IntCC::Overflow, carry, TrapCode::IntegerOverflow); - // TODO: this shouldn't wrap - let neg_b = self.builder.ins().ineg(b); - let (out, carry) = self.builder.ins().iadd_ifcout(a, neg_b); - self.builder - .ins() - .trapif(IntCC::Overflow, carry, TrapCode::IntegerOverflow); + let (out, carry) = self.builder.ins().ssub_overflow(a, b); + self.builder.ins().trapnz(carry, TrapCode::INTEGER_OVERFLOW); out } - fn compile_ipow(&mut self, a: Value, b: Value) -> Value { - // Convert base to float since result might not always be a Int - let float_base = self.builder.ins().fcvt_from_sint(types::F64, a); - - // Create code blocks - let check_block1 = self.builder.create_block(); - let check_block2 = self.builder.create_block(); - let check_block3 = self.builder.create_block(); - let handle_neg_exp = self.builder.create_block(); - let loop_block = self.builder.create_block(); - let continue_block = self.builder.create_block(); - let exit_block = self.builder.create_block(); - // Set code block params - // Set code block params - self.builder.append_block_param(check_block1, types::F64); - self.builder.append_block_param(check_block1, types::I64); + /// Creates a double–double (DDValue) from a regular f64 constant. + /// The high part is set to x and the low part is set to 0.0. + fn dd_from_f64(&mut self, x: f64) -> DDValue { + DDValue { + hi: self.builder.ins().f64const(x), + lo: self.builder.ins().f64const(0.0), + } + } - self.builder.append_block_param(check_block2, types::F64); - self.builder.append_block_param(check_block2, types::I64); + /// Creates a DDValue from a Value (assumed to represent an f64). + /// This function initializes the high part with x and the low part to 0.0. + fn dd_from_value(&mut self, x: Value) -> DDValue { + DDValue { + hi: x, + lo: self.builder.ins().f64const(0.0), + } + } + + /// Creates a DDValue from two f64 parts. + /// The 'hi' parameter sets the high part and 'lo' sets the low part. + fn dd_from_parts(&mut self, hi: f64, lo: f64) -> DDValue { + DDValue { + hi: self.builder.ins().f64const(hi), + lo: self.builder.ins().f64const(lo), + } + } + + /// Converts a DDValue back to a single f64 value by adding the high and low parts. + fn dd_to_f64(&mut self, dd: DDValue) -> Value { + self.builder.ins().fadd(dd.hi, dd.lo) + } + + /// Computes the negation of a DDValue. + /// It subtracts both the high and low parts from zero. + fn dd_neg(&mut self, dd: DDValue) -> DDValue { + let zero = self.builder.ins().f64const(0.0); + DDValue { + hi: self.builder.ins().fsub(zero, dd.hi), + lo: self.builder.ins().fsub(zero, dd.lo), + } + } + + /// Adds two DDValue numbers using error-free transformations to maintain extra precision. + /// It carefully adds the high parts, computes the rounding error, adds the low parts along with the error, + /// and then normalizes the result. + fn dd_add(&mut self, a: DDValue, b: DDValue) -> DDValue { + // Compute the sum of the high parts. + let s = self.builder.ins().fadd(a.hi, b.hi); + // Compute t = s - a.hi to capture part of the rounding error. + let t = self.builder.ins().fsub(s, a.hi); + // Compute the error e from the high part additions. + let s_minus_t = self.builder.ins().fsub(s, t); + let part1 = self.builder.ins().fsub(a.hi, s_minus_t); + let part2 = self.builder.ins().fsub(b.hi, t); + let e = self.builder.ins().fadd(part1, part2); + // Sum the low parts along with the error. + let lo = self.builder.ins().fadd(a.lo, b.lo); + let lo_sum = self.builder.ins().fadd(lo, e); + // Renormalize: add the low sum to s and compute a new low component. + let hi_new = self.builder.ins().fadd(s, lo_sum); + let hi_new_minus_s = self.builder.ins().fsub(hi_new, s); + let lo_new = self.builder.ins().fsub(lo_sum, hi_new_minus_s); + DDValue { + hi: hi_new, + lo: lo_new, + } + } - self.builder.append_block_param(check_block3, types::F64); - self.builder.append_block_param(check_block3, types::I64); + /// Subtracts DDValue b from DDValue a by negating b and then using the addition function. + fn dd_sub(&mut self, a: DDValue, b: DDValue) -> DDValue { + let neg_b = self.dd_neg(b); + self.dd_add(a, neg_b) + } - self.builder.append_block_param(handle_neg_exp, types::F64); - self.builder.append_block_param(handle_neg_exp, types::I64); + /// Multiplies two DDValue numbers using double–double arithmetic. + /// It calculates the high product, uses a fused multiply–add (FMA) to capture rounding error, + /// computes the cross products, and then normalizes the result. + fn dd_mul(&mut self, a: DDValue, b: DDValue) -> DDValue { + // p = a.hi * b.hi (primary product) + let p = self.builder.ins().fmul(a.hi, b.hi); + // err = fma(a.hi, b.hi, -p) recovers the rounding error. + let zero = self.builder.ins().f64const(0.0); + let neg_p = self.builder.ins().fsub(zero, p); + let err = self.builder.ins().fma(a.hi, b.hi, neg_p); + // Compute cross terms: a.hi*b.lo + a.lo*b.hi. + let a_hi_b_lo = self.builder.ins().fmul(a.hi, b.lo); + let a_lo_b_hi = self.builder.ins().fmul(a.lo, b.hi); + let cross = self.builder.ins().fadd(a_hi_b_lo, a_lo_b_hi); + // Sum p and the cross terms. + let s = self.builder.ins().fadd(p, cross); + // Isolate rounding error from the addition. + let t = self.builder.ins().fsub(s, p); + let s_minus_t = self.builder.ins().fsub(s, t); + let part1 = self.builder.ins().fsub(p, s_minus_t); + let part2 = self.builder.ins().fsub(cross, t); + let e = self.builder.ins().fadd(part1, part2); + // Include the error from the low parts multiplication. + let a_lo_b_lo = self.builder.ins().fmul(a.lo, b.lo); + let err_plus_e = self.builder.ins().fadd(err, e); + let lo_sum = self.builder.ins().fadd(err_plus_e, a_lo_b_lo); + // Renormalize the sum. + let hi_new = self.builder.ins().fadd(s, lo_sum); + let hi_new_minus_s = self.builder.ins().fsub(hi_new, s); + let lo_new = self.builder.ins().fsub(lo_sum, hi_new_minus_s); + DDValue { + hi: hi_new, + lo: lo_new, + } + } - self.builder.append_block_param(loop_block, types::F64); //base - self.builder.append_block_param(loop_block, types::F64); //result - self.builder.append_block_param(loop_block, types::I64); //exponent + /// Multiplies a DDValue by a regular f64 (Value) using similar techniques as dd_mul. + /// It multiplies both the high and low parts by b, computes the rounding error, + /// and then renormalizes the result. + fn dd_mul_f64(&mut self, a: DDValue, b: Value) -> DDValue { + // p = a.hi * b (primary product) + let p = self.builder.ins().fmul(a.hi, b); + // Compute the rounding error using fma. + let zero = self.builder.ins().f64const(0.0); + let neg_p = self.builder.ins().fsub(zero, p); + let err = self.builder.ins().fma(a.hi, b, neg_p); + // Multiply the low part. + let cross = self.builder.ins().fmul(a.lo, b); + // Sum the primary product and the low multiplication. + let s = self.builder.ins().fadd(p, cross); + // Capture rounding error from addition. + let t = self.builder.ins().fsub(s, p); + let s_minus_t = self.builder.ins().fsub(s, t); + let part1 = self.builder.ins().fsub(p, s_minus_t); + let part2 = self.builder.ins().fsub(cross, t); + let e = self.builder.ins().fadd(part1, part2); + // Combine the error components. + let lo_sum = self.builder.ins().fadd(err, e); + // Renormalize to form the final double–double number. + let hi_new = self.builder.ins().fadd(s, lo_sum); + let hi_new_minus_s = self.builder.ins().fsub(hi_new, s); + let lo_new = self.builder.ins().fsub(lo_sum, hi_new_minus_s); + DDValue { + hi: hi_new, + lo: lo_new, + } + } - self.builder.append_block_param(continue_block, types::F64); //base - self.builder.append_block_param(continue_block, types::F64); //result - self.builder.append_block_param(continue_block, types::I64); //exponent + /// Scales a DDValue by multiplying both its high and low parts by the given factor. + fn dd_scale(&mut self, dd: DDValue, factor: Value) -> DDValue { + DDValue { + hi: self.builder.ins().fmul(dd.hi, factor), + lo: self.builder.ins().fmul(dd.lo, factor), + } + } - self.builder.append_block_param(exit_block, types::F64); + /// Approximates ln(1+f) using its Taylor series expansion in double–double arithmetic. + /// It computes the series ∑ (-1)^(i-1) * f^i / i from i = 1 to 1000 for high precision. + fn dd_ln_1p_series(&mut self, f: Value) -> DDValue { + // Convert f to a DDValue and initialize the sum and term. + let f_dd = self.dd_from_value(f); + let mut sum = f_dd.clone(); + let mut term = f_dd; + // Alternating sign starts at -1 for the second term. + let mut sign = -1.0_f64; + let range = 1000; + + // Loop over terms from i = 2 to 1000. + for i in 2..=range { + // Compute f^i by multiplying the previous term by f. + term = self.dd_mul_f64(term, f); + // Divide the term by i. + let inv_i = 1.0 / (i as f64); + let c_inv_i = self.builder.ins().f64const(inv_i); + let term_div = self.dd_mul_f64(term.clone(), c_inv_i); + // Multiply by the alternating sign. + let dd_sign = self.dd_from_f64(sign); + let to_add = self.dd_mul(dd_sign, term_div); + // Add the term to the cumulative sum. + sum = self.dd_add(sum, to_add); + // Flip the sign for the next term. + sign = -sign; + } + sum + } - // Begin evaluating by jumping to first check block - self.builder.ins().jump(check_block1, &[float_base, b]); + /// Computes the natural logarithm ln(x) in double–double arithmetic. + /// It first checks for domain errors (x ≤ 0 or NaN), then extracts the exponent + /// and mantissa from the bit-level representation of x. It computes ln(mantissa) using + /// the ln(1+f) series and adds k*ln2 to obtain ln(x). + fn dd_ln(&mut self, x: Value) -> DDValue { + // (A) Prepare a DDValue representing NaN. + let dd_nan = self.dd_from_f64(f64::NAN); - // Check block one: - // Checks if input is O ** n where n > 0 - // Jumps to exit_block as 0 if true - self.builder.switch_to_block(check_block1); - let paramsc1 = self.builder.block_params(check_block1); - let basec1 = paramsc1[0]; - let expc1 = paramsc1[1]; + // Build a zero constant for comparisons. let zero_f64 = self.builder.ins().f64const(0.0); - let zero_i64 = self.builder.ins().iconst(types::I64, 0); - let is_base_zero = self.builder.ins().fcmp(FloatCC::Equal, zero_f64, basec1); - let is_exp_positive = self + + // Check if x is less than or equal to 0 or is NaN. + let cmp_le = self .builder .ins() - .icmp(IntCC::SignedGreaterThan, expc1, zero_i64); - let is_zero_to_positive = self.builder.ins().band(is_base_zero, is_exp_positive); - self.builder + .fcmp(FloatCC::LessThanOrEqual, x, zero_f64); + let cmp_nan = self.builder.ins().fcmp(FloatCC::Unordered, x, x); + let need_nan = self.builder.ins().bor(cmp_le, cmp_nan); + + // (B) Reinterpret the bits of x as an integer. + let bits = self.builder.ins().bitcast(types::I64, MemFlags::new(), x); + + // (C) Extract the exponent (top 11 bits) from the bit representation. + let shift_52 = self.builder.ins().ushr_imm(bits, 52); + let exponent_mask = self.builder.ins().iconst(types::I64, 0x7FF); + let exponent = self.builder.ins().band(shift_52, exponent_mask); + + // k = exponent - 1023 (unbias the exponent). + let bias = self.builder.ins().iconst(types::I64, 1023); + let k_i64 = self.builder.ins().isub(exponent, bias); + + // (D) Extract the fraction (mantissa) from the lower 52 bits. + let fraction_mask = self.builder.ins().iconst(types::I64, 0x000F_FFFF_FFFF_FFFF); + let fraction_part = self.builder.ins().band(bits, fraction_mask); + + // (E) For normal numbers (exponent ≠ 0), add the implicit leading 1. + let implicit_one = self.builder.ins().iconst(types::I64, 1 << 52); + let zero_exp = self.builder.ins().icmp_imm(IntCC::Equal, exponent, 0); + let frac_one_bor = self.builder.ins().bor(fraction_part, implicit_one); + let fraction_with_leading_one = self.builder.ins().select( + zero_exp, + fraction_part, // For subnormals, do not add the implicit 1. + frac_one_bor, + ); + + // (F) Force the exponent bits to 1023, yielding a mantissa m in [1, 2). + let new_exp = self.builder.ins().iconst(types::I64, 0x3FF0_0000_0000_0000); + let fraction_bits = self.builder.ins().bor(fraction_with_leading_one, new_exp); + let m = self + .builder + .ins() + .bitcast(types::F64, MemFlags::new(), fraction_bits); + + // (G) Compute ln(m) using the series ln(1+f) with f = m - 1. + let one_f64 = self.builder.ins().f64const(1.0); + let f_val = self.builder.ins().fsub(m, one_f64); + let dd_ln_m = self.dd_ln_1p_series(f_val); + + // (H) Compute k*ln2 in double–double arithmetic. + let ln2_dd = self.dd_from_parts( + f64::from_bits(0x3fe62e42fefa39ef), + f64::from_bits(0x3c7abc9e3b39803f), + ); + let k_f64 = self.builder.ins().fcvt_from_sint(types::F64, k_i64); + let dd_ln2_k = self.dd_mul_f64(ln2_dd, k_f64); + + // Add ln(m) and k*ln2 to get the final ln(x). + let normal_result = self.dd_add(dd_ln_m, dd_ln2_k); + + // (I) If x was nonpositive or NaN, return NaN; otherwise, return the computed result. + let final_hi = self + .builder .ins() - .brnz(is_zero_to_positive, exit_block, &[zero_f64]); - self.builder.ins().jump(check_block2, &[basec1, expc1]); - - // Check block two: - // Checks if exponent is negative - // Jumps to a special handle_neg_exponent block if true - self.builder.switch_to_block(check_block2); - let paramsc2 = self.builder.block_params(check_block2); - let basec2 = paramsc2[0]; - let expc2 = paramsc2[1]; - let zero_i64 = self.builder.ins().iconst(types::I64, 0); - let is_neg = self + .select(need_nan, dd_nan.hi, normal_result.hi); + let final_lo = self .builder .ins() - .icmp(IntCC::SignedLessThan, expc2, zero_i64); + .select(need_nan, dd_nan.lo, normal_result.lo); + + DDValue { + hi: final_hi, + lo: final_lo, + } + } + + /// Computes the exponential function exp(x) in double–double arithmetic. + /// It uses range reduction to write x = k*ln2 + r, computes exp(r) via a Taylor series, + /// scales the result by 2^k, and handles overflow by checking if k exceeds the maximum. + fn dd_exp(&mut self, dd: DDValue) -> DDValue { + // (A) Range reduction: Convert dd to a single f64 value. + let x = self.dd_to_f64(dd.clone()); + let ln2_f64 = self + .builder + .ins() + .f64const(f64::from_bits(0x3fe62e42fefa39ef)); + let div = self.builder.ins().fdiv(x, ln2_f64); + let half = self.builder.ins().f64const(0.5); + let div_plus_half = self.builder.ins().fadd(div, half); + // Rounding: floor(div + 0.5) gives the nearest integer k. + let k = self.builder.ins().fcvt_to_sint(types::I64, div_plus_half); + + // --- OVERFLOW CHECK --- + // Check if k is greater than the maximum exponent for finite doubles (1023). + let max_k = self.builder.ins().iconst(types::I64, 1023); + let is_overflow = self.builder.ins().icmp(IntCC::SignedGreaterThan, k, max_k); + + // Define infinity and zero for the overflow case. + let inf = self.builder.ins().f64const(f64::INFINITY); + let zero = self.builder.ins().f64const(0.0); + + // (B) Compute exp(x) normally when not overflowing. + // Compute k*ln2 in double–double arithmetic and subtract it from x. + let ln2_dd = self.dd_from_parts( + f64::from_bits(0x3fe62e42fefa39ef), + f64::from_bits(0x3c7abc9e3b39803f), + ); + let k_f64 = self.builder.ins().fcvt_from_sint(types::F64, k); + let k_ln2 = self.dd_mul_f64(ln2_dd, k_f64); + let r = self.dd_sub(dd, k_ln2); + + // Compute exp(r) using a Taylor series. + let mut sum = self.dd_from_f64(1.0); // Initialize sum to 1. + let mut term = self.dd_from_f64(1.0); // Initialize the first term to 1. + let n_terms = 1000; + for i in 1..=n_terms { + term = self.dd_mul(term, r.clone()); + let inv = 1.0 / (i as f64); + let inv_const = self.builder.ins().f64const(inv); + term = self.dd_mul_f64(term, inv_const); + sum = self.dd_add(sum, term.clone()); + } + + // Reconstruct the final result by scaling with 2^k. + let bias = self.builder.ins().iconst(types::I64, 1023); + let k_plus_bias = self.builder.ins().iadd(k, bias); + let shift_count = self.builder.ins().iconst(types::I64, 52); + let shifted = self.builder.ins().ishl(k_plus_bias, shift_count); + let two_to_k = self + .builder + .ins() + .bitcast(types::F64, MemFlags::new(), shifted); + let result = self.dd_scale(sum, two_to_k); + + // (C) If overflow was detected, return infinity; otherwise, return the computed value. + let final_hi = self.builder.ins().select(is_overflow, inf, result.hi); + let final_lo = self.builder.ins().select(is_overflow, zero, result.lo); + DDValue { + hi: final_hi, + lo: final_lo, + } + } + + /// Computes the power function a^b (f_pow) for f64 values using double–double arithmetic for high precision. + /// It handles different cases for the base 'a': + /// - For a > 0: Computes exp(b * ln(a)). + /// - For a == 0: Handles special cases for 0^b, including returning 0, 1, or a domain error. + /// - For a < 0: Allows only an integer exponent b and adjusts the sign if b is odd. + fn compile_fpow(&mut self, a: Value, b: Value) -> Value { + let f64_ty = types::F64; + let i64_ty = types::I64; + let zero_f = self.builder.ins().f64const(0.0); + let one_f = self.builder.ins().f64const(1.0); + let nan_f = self.builder.ins().f64const(f64::NAN); + let inf_f = self.builder.ins().f64const(f64::INFINITY); + let neg_inf_f = self.builder.ins().f64const(f64::NEG_INFINITY); + + // Merge block for final result. + let merge_block = self.builder.create_block(); + self.builder.append_block_param(merge_block, f64_ty); + + // --- Edge Case 1: b == 0.0 → return 1.0 + let cmp_b_zero = self.builder.ins().fcmp(FloatCC::Equal, b, zero_f); + let b_zero_block = self.builder.create_block(); + let continue_block = self.builder.create_block(); self.builder .ins() - .brnz(is_neg, handle_neg_exp, &[basec2, expc2]); - self.builder.ins().jump(check_block3, &[basec2, expc2]); - - // Check block three: - // Checks if exponent is one - // jumps to exit block with the base of the exponents value - self.builder.switch_to_block(check_block3); - let paramsc3 = self.builder.block_params(check_block3); - let basec3 = paramsc3[0]; - let expc3 = paramsc3[1]; - let resc3 = self.builder.ins().f64const(1.0); - let one_i64 = self.builder.ins().iconst(types::I64, 1); - let is_one = self.builder.ins().icmp(IntCC::Equal, expc3, one_i64); - self.builder.ins().brnz(is_one, exit_block, &[basec3]); - self.builder.ins().jump(loop_block, &[basec3, resc3, expc3]); - - // Handles negative Exponents - // calculates x^(-n) = (1/x)^n - // then proceeds to the loop to evaluate - self.builder.switch_to_block(handle_neg_exp); - let paramshn = self.builder.block_params(handle_neg_exp); - let basehn = paramshn[0]; - let exphn = paramshn[1]; - let one_f64 = self.builder.ins().f64const(1.0); - let base_inverse = self.builder.ins().fdiv(one_f64, basehn); - let pos_exp = self.builder.ins().ineg(exphn); + .brif(cmp_b_zero, b_zero_block, &[], continue_block, &[]); + self.builder.switch_to_block(b_zero_block); + self.builder.ins().jump(merge_block, &[one_f]); + self.builder.switch_to_block(continue_block); + + // --- Edge Case 2: b is NaN → return NaN + let cmp_b_nan = self.builder.ins().fcmp(FloatCC::Unordered, b, b); + let b_nan_block = self.builder.create_block(); + let continue_block2 = self.builder.create_block(); + self.builder + .ins() + .brif(cmp_b_nan, b_nan_block, &[], continue_block2, &[]); + self.builder.switch_to_block(b_nan_block); + self.builder.ins().jump(merge_block, &[nan_f]); + self.builder.switch_to_block(continue_block2); + + // --- Edge Case 3: a == 0.0 → return 0.0 + let cmp_a_zero = self.builder.ins().fcmp(FloatCC::Equal, a, zero_f); + let a_zero_block = self.builder.create_block(); + let continue_block3 = self.builder.create_block(); + self.builder + .ins() + .brif(cmp_a_zero, a_zero_block, &[], continue_block3, &[]); + self.builder.switch_to_block(a_zero_block); + self.builder.ins().jump(merge_block, &[zero_f]); + self.builder.switch_to_block(continue_block3); + + // --- Edge Case 4: a is NaN → return NaN + let cmp_a_nan = self.builder.ins().fcmp(FloatCC::Unordered, a, a); + let a_nan_block = self.builder.create_block(); + let continue_block4 = self.builder.create_block(); + self.builder + .ins() + .brif(cmp_a_nan, a_nan_block, &[], continue_block4, &[]); + self.builder.switch_to_block(a_nan_block); + self.builder.ins().jump(merge_block, &[nan_f]); + self.builder.switch_to_block(continue_block4); + + // --- Edge Case 5: b == +infinity → return +infinity + let cmp_b_inf = self.builder.ins().fcmp(FloatCC::Equal, b, inf_f); + let b_inf_block = self.builder.create_block(); + let continue_block5 = self.builder.create_block(); + self.builder + .ins() + .brif(cmp_b_inf, b_inf_block, &[], continue_block5, &[]); + self.builder.switch_to_block(b_inf_block); + self.builder.ins().jump(merge_block, &[inf_f]); + self.builder.switch_to_block(continue_block5); + + // --- Edge Case 6: b == -infinity → return 0.0 + let cmp_b_neg_inf = self.builder.ins().fcmp(FloatCC::Equal, b, neg_inf_f); + let b_neg_inf_block = self.builder.create_block(); + let continue_block6 = self.builder.create_block(); + self.builder + .ins() + .brif(cmp_b_neg_inf, b_neg_inf_block, &[], continue_block6, &[]); + self.builder.switch_to_block(b_neg_inf_block); + self.builder.ins().jump(merge_block, &[zero_f]); + self.builder.switch_to_block(continue_block6); + + // --- Edge Case 7: a == +infinity → return +infinity + let cmp_a_inf = self.builder.ins().fcmp(FloatCC::Equal, a, inf_f); + let a_inf_block = self.builder.create_block(); + let continue_block7 = self.builder.create_block(); + self.builder + .ins() + .brif(cmp_a_inf, a_inf_block, &[], continue_block7, &[]); + self.builder.switch_to_block(a_inf_block); + self.builder.ins().jump(merge_block, &[inf_f]); + self.builder.switch_to_block(continue_block7); + + // --- Edge Case 8: a == -infinity → check exponent parity + let cmp_a_neg_inf = self.builder.ins().fcmp(FloatCC::Equal, a, neg_inf_f); + let a_neg_inf_block = self.builder.create_block(); + let continue_block8 = self.builder.create_block(); + self.builder + .ins() + .brif(cmp_a_neg_inf, a_neg_inf_block, &[], continue_block8, &[]); + + self.builder.switch_to_block(a_neg_inf_block); + // a is -infinity here. First, ensure that b is an integer. + let b_floor = self.builder.ins().floor(b); + let cmp_int = self.builder.ins().fcmp(FloatCC::Equal, b_floor, b); + let domain_error_blk = self.builder.create_block(); + let continue_neg_inf = self.builder.create_block(); + self.builder + .ins() + .brif(cmp_int, continue_neg_inf, &[], domain_error_blk, &[]); + + self.builder.switch_to_block(domain_error_blk); + self.builder.ins().jump(merge_block, &[nan_f]); + + self.builder.switch_to_block(continue_neg_inf); + // b is an integer here; convert b_floor to an i64. + let b_i64 = self.builder.ins().fcvt_to_sint(i64_ty, b_floor); + let one_i = self.builder.ins().iconst(i64_ty, 1); + let remainder = self.builder.ins().band(b_i64, one_i); + let zero_i = self.builder.ins().iconst(i64_ty, 0); + let is_odd = self.builder.ins().icmp(IntCC::NotEqual, remainder, zero_i); + + // Create separate blocks for odd and even cases. + let odd_block = self.builder.create_block(); + let even_block = self.builder.create_block(); + self.builder.append_block_param(odd_block, f64_ty); + self.builder.append_block_param(even_block, f64_ty); self.builder .ins() - .jump(loop_block, &[base_inverse, one_f64, pos_exp]); + .brif(is_odd, odd_block, &[neg_inf_f], even_block, &[inf_f]); - // Main loop block - // checks loop condition (exp > 0) - // Jumps to continue block if true, exit block if false - self.builder.switch_to_block(loop_block); - let paramslb = self.builder.block_params(loop_block); - let baselb = paramslb[0]; - let reslb = paramslb[1]; - let explb = paramslb[2]; - let zero = self.builder.ins().iconst(types::I64, 0); - let is_zero = self.builder.ins().icmp(IntCC::Equal, explb, zero); - self.builder.ins().brnz(is_zero, exit_block, &[reslb]); + self.builder.switch_to_block(odd_block); + let phi_neg_inf = self.builder.block_params(odd_block)[0]; + self.builder.ins().jump(merge_block, &[phi_neg_inf]); + + self.builder.switch_to_block(even_block); + let phi_inf = self.builder.block_params(even_block)[0]; + self.builder.ins().jump(merge_block, &[phi_inf]); + + self.builder.switch_to_block(continue_block8); + + // --- Normal branch: neither a nor b hit the special cases. + // Here we branch based on the sign of a. + let cmp_lt = self.builder.ins().fcmp(FloatCC::LessThan, a, zero_f); + let a_neg_block = self.builder.create_block(); + let a_pos_block = self.builder.create_block(); + self.builder + .ins() + .brif(cmp_lt, a_neg_block, &[], a_pos_block, &[]); + + // ----- Case: a > 0: Compute a^b = exp(b * ln(a)) using double–double arithmetic. + self.builder.switch_to_block(a_pos_block); + let ln_a_dd = self.dd_ln(a); + let b_dd = self.dd_from_value(b); + let product_dd = self.dd_mul(ln_a_dd, b_dd); + let exp_dd = self.dd_exp(product_dd); + let pos_res = self.dd_to_f64(exp_dd); + self.builder.ins().jump(merge_block, &[pos_res]); + + // ----- Case: a < 0: Only allow an integral exponent. + self.builder.switch_to_block(a_neg_block); + let b_floor = self.builder.ins().floor(b); + let cmp_int = self.builder.ins().fcmp(FloatCC::Equal, b_floor, b); + let neg_int_block = self.builder.create_block(); + let domain_error_blk = self.builder.create_block(); + self.builder + .ins() + .brif(cmp_int, neg_int_block, &[], domain_error_blk, &[]); + + // Domain error: non-integer exponent for negative base + self.builder.switch_to_block(domain_error_blk); + self.builder.ins().jump(merge_block, &[nan_f]); + + // For negative base with an integer exponent: + self.builder.switch_to_block(neg_int_block); + let abs_a = self.builder.ins().fabs(a); + let ln_abs_dd = self.dd_ln(abs_a); + let b_dd = self.dd_from_value(b); + let product_dd = self.dd_mul(ln_abs_dd, b_dd); + let exp_dd = self.dd_exp(product_dd); + let mag_val = self.dd_to_f64(exp_dd); + + let b_i64 = self.builder.ins().fcvt_to_sint(i64_ty, b_floor); + let one_i = self.builder.ins().iconst(i64_ty, 1); + let remainder = self.builder.ins().band(b_i64, one_i); + let zero_i = self.builder.ins().iconst(i64_ty, 0); + let is_odd = self.builder.ins().icmp(IntCC::NotEqual, remainder, zero_i); + + let odd_block = self.builder.create_block(); + let even_block = self.builder.create_block(); + // Append block parameters for both branches: + self.builder.append_block_param(odd_block, f64_ty); + self.builder.append_block_param(even_block, f64_ty); + // Pass mag_val to both branches: self.builder .ins() - .jump(continue_block, &[baselb, reslb, explb]); + .brif(is_odd, odd_block, &[mag_val], even_block, &[mag_val]); + + self.builder.switch_to_block(odd_block); + let phi_mag_val = self.builder.block_params(odd_block)[0]; + let neg_val = self.builder.ins().fneg(phi_mag_val); + self.builder.ins().jump(merge_block, &[neg_val]); + + self.builder.switch_to_block(even_block); + let phi_mag_val_even = self.builder.block_params(even_block)[0]; + self.builder.ins().jump(merge_block, &[phi_mag_val_even]); + + // ----- Merge: Return the final result. + self.builder.switch_to_block(merge_block); + let final_val = self.builder.block_params(merge_block)[0]; + final_val + } + + fn compile_ipow(&mut self, a: Value, b: Value) -> Value { + let zero = self.builder.ins().iconst(types::I64, 0); + let one_i64 = self.builder.ins().iconst(types::I64, 1); - // Continue block - // Main math logic - // Always jumps back to loob_block + // Create required blocks + let check_negative = self.builder.create_block(); + let handle_negative = self.builder.create_block(); + let loop_block = self.builder.create_block(); + let continue_block = self.builder.create_block(); + let exit_block = self.builder.create_block(); + + // Set up block parameters + self.builder.append_block_param(check_negative, types::I64); // exponent + self.builder.append_block_param(check_negative, types::I64); // base + + self.builder.append_block_param(handle_negative, types::I64); // abs(exponent) + self.builder.append_block_param(handle_negative, types::I64); // base + + self.builder.append_block_param(loop_block, types::I64); // exponent + self.builder.append_block_param(loop_block, types::I64); // result + self.builder.append_block_param(loop_block, types::I64); // base + + self.builder.append_block_param(exit_block, types::I64); // final result + + // Set up parameters for continue_block + self.builder.append_block_param(continue_block, types::I64); // exponent + self.builder.append_block_param(continue_block, types::I64); // result + self.builder.append_block_param(continue_block, types::I64); // base + + // Initial jump to check if exponent is negative + self.builder.ins().jump(check_negative, &[b, a]); + + // Check if exponent is negative + self.builder.switch_to_block(check_negative); + let params = self.builder.block_params(check_negative); + let exp_check = params[0]; + let base_check = params[1]; + + let is_negative = self + .builder + .ins() + .icmp(IntCC::SignedLessThan, exp_check, zero); + self.builder.ins().brif( + is_negative, + handle_negative, + &[exp_check, base_check], + loop_block, + &[exp_check, one_i64, base_check], + ); + + // Handle negative exponent (return 0 for integer exponentiation) + self.builder.switch_to_block(handle_negative); + self.builder.ins().jump(exit_block, &[zero]); // Return 0 for negative exponents + + // Loop block logic (square-and-multiply algorithm) + self.builder.switch_to_block(loop_block); + let params = self.builder.block_params(loop_block); + let exp_phi = params[0]; + let result_phi = params[1]; + let base_phi = params[2]; + + // Check if exponent is zero + let is_zero = self.builder.ins().icmp(IntCC::Equal, exp_phi, zero); + self.builder.ins().brif( + is_zero, + exit_block, + &[result_phi], + continue_block, + &[exp_phi, result_phi, base_phi], + ); + + // Continue block for non-zero case self.builder.switch_to_block(continue_block); - let paramscb = self.builder.block_params(continue_block); - let basecb = paramscb[0]; - let rescb = paramscb[1]; - let expcb = paramscb[2]; - let is_odd = self.builder.ins().band_imm(expcb, 1); + let params = self.builder.block_params(continue_block); + let exp_phi = params[0]; + let result_phi = params[1]; + let base_phi = params[2]; + + // If exponent is odd, multiply result by base + let is_odd = self.builder.ins().band_imm(exp_phi, 1); let is_odd = self.builder.ins().icmp_imm(IntCC::Equal, is_odd, 1); - let mul_result = self.builder.ins().fmul(rescb, basecb); - let new_result = self.builder.ins().select(is_odd, mul_result, rescb); - let squared_base = self.builder.ins().fmul(basecb, basecb); - let new_exp = self.builder.ins().sshr_imm(expcb, 1); + let mul_result = self.builder.ins().imul(result_phi, base_phi); + let new_result = self.builder.ins().select(is_odd, mul_result, result_phi); + + // Square the base and divide exponent by 2 + let squared_base = self.builder.ins().imul(base_phi, base_phi); + let new_exp = self.builder.ins().sshr_imm(exp_phi, 1); self.builder .ins() - .jump(loop_block, &[squared_base, new_result, new_exp]); + .jump(loop_block, &[new_exp, new_result, squared_base]); + // Exit block self.builder.switch_to_block(exit_block); - let result = self.builder.block_params(exit_block)[0]; + let res = self.builder.block_params(exit_block)[0]; - self.builder.seal_block(check_block1); - self.builder.seal_block(check_block2); - self.builder.seal_block(check_block3); - self.builder.seal_block(handle_neg_exp); + // Seal all blocks + self.builder.seal_block(check_negative); + self.builder.seal_block(handle_negative); self.builder.seal_block(loop_block); self.builder.seal_block(continue_block); self.builder.seal_block(exit_block); - result + res } } diff --git a/jit/src/lib.rs b/jit/src/lib.rs index 37f1f2a3dd..33054b1c95 100644 --- a/jit/src/lib.rs +++ b/jit/src/lib.rs @@ -114,7 +114,7 @@ pub fn compile( let (id, sig) = jit.build_function(bytecode, args, ret)?; - jit.module.finalize_definitions(); + jit.module.finalize_definitions()?; let code = jit.module.get_finalized_function(id); Ok(CompiledCode { diff --git a/jit/tests/float_tests.rs b/jit/tests/float_tests.rs index 2ba7dec822..384d7b9468 100644 --- a/jit/tests/float_tests.rs +++ b/jit/tests/float_tests.rs @@ -110,6 +110,117 @@ fn test_mul_with_integer() { assert_bits_eq!(mul(-0.0, -1), Ok(0.0f64)); } +#[test] +fn test_power() { + let pow = jit_function! { pow(a:f64, b:f64) -> f64 => r##" + def pow(a:float, b: float): + return a**b + "##}; + // Test base cases + assert_approx_eq!(pow(0.0, 0.0), Ok(1.0)); + assert_approx_eq!(pow(0.0, 1.0), Ok(0.0)); + assert_approx_eq!(pow(1.0, 0.0), Ok(1.0)); + assert_approx_eq!(pow(1.0, 1.0), Ok(1.0)); + assert_approx_eq!(pow(1.0, -1.0), Ok(1.0)); + assert_approx_eq!(pow(-1.0, 0.0), Ok(1.0)); + assert_approx_eq!(pow(-1.0, 1.0), Ok(-1.0)); + assert_approx_eq!(pow(-1.0, -1.0), Ok(-1.0)); + + // NaN and Infinity cases + assert_approx_eq!(pow(f64::NAN, 0.0), Ok(1.0)); + //assert_approx_eq!(pow(f64::NAN, 1.0), Ok(f64::NAN)); // Return the correct answer but fails compare + //assert_approx_eq!(pow(0.0, f64::NAN), Ok(f64::NAN)); // Return the correct answer but fails compare + assert_approx_eq!(pow(f64::INFINITY, 0.0), Ok(1.0)); + assert_approx_eq!(pow(f64::INFINITY, 1.0), Ok(f64::INFINITY)); + assert_approx_eq!(pow(f64::INFINITY, f64::INFINITY), Ok(f64::INFINITY)); + // Negative infinity cases: + // For any exponent of 0.0, the result is 1.0. + assert_approx_eq!(pow(f64::NEG_INFINITY, 0.0), Ok(1.0)); + // For negative infinity base, when b is an odd integer, result is -infinity; + // when b is even, result is +infinity. + assert_approx_eq!(pow(f64::NEG_INFINITY, 1.0), Ok(f64::NEG_INFINITY)); + assert_approx_eq!(pow(f64::NEG_INFINITY, 2.0), Ok(f64::INFINITY)); + assert_approx_eq!(pow(f64::NEG_INFINITY, 3.0), Ok(f64::NEG_INFINITY)); + // Exponent -infinity gives 0.0. + assert_approx_eq!(pow(f64::NEG_INFINITY, f64::NEG_INFINITY), Ok(0.0)); + + // Test positive float base, positive float exponent + assert_approx_eq!(pow(2.0, 2.0), Ok(4.0)); + assert_approx_eq!(pow(3.0, 3.0), Ok(27.0)); + assert_approx_eq!(pow(4.0, 4.0), Ok(256.0)); + assert_approx_eq!(pow(2.0, 3.0), Ok(8.0)); + assert_approx_eq!(pow(2.0, 4.0), Ok(16.0)); + // Test negative float base, positive float exponent (integral exponents only) + assert_approx_eq!(pow(-2.0, 2.0), Ok(4.0)); + assert_approx_eq!(pow(-3.0, 3.0), Ok(-27.0)); + assert_approx_eq!(pow(-4.0, 4.0), Ok(256.0)); + assert_approx_eq!(pow(-2.0, 3.0), Ok(-8.0)); + assert_approx_eq!(pow(-2.0, 4.0), Ok(16.0)); + // Test positive float base, positive float exponent + assert_approx_eq!(pow(2.5, 2.0), Ok(6.25)); + assert_approx_eq!(pow(3.5, 3.0), Ok(42.875)); + assert_approx_eq!(pow(4.5, 4.0), Ok(410.0625)); + assert_approx_eq!(pow(2.5, 3.0), Ok(15.625)); + assert_approx_eq!(pow(2.5, 4.0), Ok(39.0625)); + // Test negative float base, positive float exponent (integral exponents only) + assert_approx_eq!(pow(-2.5, 2.0), Ok(6.25)); + assert_approx_eq!(pow(-3.5, 3.0), Ok(-42.875)); + assert_approx_eq!(pow(-4.5, 4.0), Ok(410.0625)); + assert_approx_eq!(pow(-2.5, 3.0), Ok(-15.625)); + assert_approx_eq!(pow(-2.5, 4.0), Ok(39.0625)); + // Test positive float base, positive float exponent with nonintegral exponents + assert_approx_eq!(pow(2.0, 2.5), Ok(5.656854249492381)); + assert_approx_eq!(pow(3.0, 3.5), Ok(46.76537180435969)); + assert_approx_eq!(pow(4.0, 4.5), Ok(512.0)); + assert_approx_eq!(pow(2.0, 3.5), Ok(11.313708498984761)); + assert_approx_eq!(pow(2.0, 4.5), Ok(22.627416997969522)); + // Test positive float base, negative float exponent + assert_approx_eq!(pow(2.0, -2.5), Ok(0.1767766952966369)); + assert_approx_eq!(pow(3.0, -3.5), Ok(0.021383343303319473)); + assert_approx_eq!(pow(4.0, -4.5), Ok(0.001953125)); + assert_approx_eq!(pow(2.0, -3.5), Ok(0.08838834764831845)); + assert_approx_eq!(pow(2.0, -4.5), Ok(0.04419417382415922)); + // Test negative float base, negative float exponent (integral exponents only) + assert_approx_eq!(pow(-2.0, -2.0), Ok(0.25)); + assert_approx_eq!(pow(-3.0, -3.0), Ok(-0.037037037037037035)); + assert_approx_eq!(pow(-4.0, -4.0), Ok(0.00390625)); + assert_approx_eq!(pow(-2.0, -3.0), Ok(-0.125)); + assert_approx_eq!(pow(-2.0, -4.0), Ok(0.0625)); + + // Currently negative float base with nonintegral exponent is not supported: + // assert_approx_eq!(pow(-2.0, 2.5), Ok(5.656854249492381)); + // assert_approx_eq!(pow(-3.0, 3.5), Ok(-46.76537180435969)); + // assert_approx_eq!(pow(-4.0, 4.5), Ok(512.0)); + // assert_approx_eq!(pow(-2.0, -2.5), Ok(0.1767766952966369)); + // assert_approx_eq!(pow(-3.0, -3.5), Ok(0.021383343303319473)); + // assert_approx_eq!(pow(-4.0, -4.5), Ok(0.001953125)); + + // Extra cases **NOTE** these are not all working: + // * If they are commented in then they work + // * If they are commented out with a number that is the current return value it throws vs the expected value + // * If they are commented out with a "fail to run" that means I couldn't get them to work, could add a case for really big or small values + // 1e308^2.0 + assert_approx_eq!(pow(1e308, 2.0), Ok(f64::INFINITY)); + // 1e308^(1e-2) + assert_approx_eq!(pow(1e308, 1e-2), Ok(1202.2644346174131)); + // 1e-308^2.0 + //assert_approx_eq!(pow(1e-308, 2.0), Ok(0.0)); // --8.403311421507407 + // 1e-308^-2.0 + assert_approx_eq!(pow(1e-308, -2.0), Ok(f64::INFINITY)); + // 1e100^(1e50) + //assert_approx_eq!(pow(1e100, 1e50), Ok(1.0000000000000002e+150)); // fail to run (Crashes as "illegal hardware instruction") + // 1e50^(1e-100) + assert_approx_eq!(pow(1e50, 1e-100), Ok(1.0)); + // 1e308^(-1e2) + //assert_approx_eq!(pow(1e308, -1e2), Ok(0.0)); // 2.961801792837933e25 + // 1e-308^(1e2) + //assert_approx_eq!(pow(1e-308, 1e2), Ok(f64::INFINITY)); // 1.6692559244043896e46 + // 1e308^(-1e308) + // assert_approx_eq!(pow(1e308, -1e308), Ok(0.0)); // fail to run (Crashes as "illegal hardware instruction") + // 1e-308^(1e308) + // assert_approx_eq!(pow(1e-308, 1e308), Ok(0.0)); // fail to run (Crashes as "illegal hardware instruction") +} + #[test] fn test_div() { let div = jit_function! { div(a:f64, b:f64) -> f64 => r##" diff --git a/jit/tests/int_tests.rs b/jit/tests/int_tests.rs index 353052df00..5ab2697e07 100644 --- a/jit/tests/int_tests.rs +++ b/jit/tests/int_tests.rs @@ -45,7 +45,6 @@ fn test_mul() { } #[test] - fn test_div() { let div = jit_function! { div(a:i64, b:i64) -> f64 => r##" def div(a: int, b: int): @@ -87,23 +86,23 @@ fn test_floor_div() { #[test] fn test_exp() { - let exp = jit_function! { exp(a: i64, b: i64) -> f64 => r##" + let exp = jit_function! { exp(a: i64, b: i64) -> i64 => r##" def exp(a: int, b: int): return a ** b "## }; - assert_eq!(exp(2, 3), Ok(8.0)); - assert_eq!(exp(3, 2), Ok(9.0)); - assert_eq!(exp(5, 0), Ok(1.0)); - assert_eq!(exp(0, 0), Ok(1.0)); - assert_eq!(exp(-5, 0), Ok(1.0)); - assert_eq!(exp(0, 1), Ok(0.0)); - assert_eq!(exp(0, 5), Ok(0.0)); - assert_eq!(exp(-2, 2), Ok(4.0)); - assert_eq!(exp(-3, 4), Ok(81.0)); - assert_eq!(exp(-2, 3), Ok(-8.0)); - assert_eq!(exp(-3, 3), Ok(-27.0)); - assert_eq!(exp(1000, 2), Ok(1000000.0)); + assert_eq!(exp(2, 3), Ok(8)); + assert_eq!(exp(3, 2), Ok(9)); + assert_eq!(exp(5, 0), Ok(1)); + assert_eq!(exp(0, 0), Ok(1)); + assert_eq!(exp(-5, 0), Ok(1)); + assert_eq!(exp(0, 1), Ok(0)); + assert_eq!(exp(0, 5), Ok(0)); + assert_eq!(exp(-2, 2), Ok(4)); + assert_eq!(exp(-3, 4), Ok(81)); + assert_eq!(exp(-2, 3), Ok(-8)); + assert_eq!(exp(-3, 3), Ok(-27)); + assert_eq!(exp(1000, 2), Ok(1000000)); } #[test] @@ -121,6 +120,18 @@ fn test_mod() { assert_eq!(modulo(-5, 10), Ok(-5)); } +#[test] +fn test_power() { + let power = jit_function! { power(a:i64, b:i64) -> i64 => r##" + def power(a: int, b: int): + return a ** b + "## + }; + assert_eq!(power(10, 2), Ok(100)); + assert_eq!(power(5, 1), Ok(5)); + assert_eq!(power(1, 0), Ok(1)); +} + #[test] fn test_lshift() { let lshift = jit_function! { lshift(a:i64, b:i64) -> i64 => r##" diff --git a/jit/tests/misc_tests.rs b/jit/tests/misc_tests.rs index 7e1174da4a..25d66c46c0 100644 --- a/jit/tests/misc_tests.rs +++ b/jit/tests/misc_tests.rs @@ -95,7 +95,6 @@ fn test_while_loop() { a -= 1 return b "## }; - assert_eq!(while_loop(0), Ok(0)); assert_eq!(while_loop(-1), Ok(0)); assert_eq!(while_loop(1), Ok(1)); From a6b4ef7f5dab1cced27699c55b75729146ce49f3 Mon Sep 17 00:00:00 2001 From: Stefan Lukas Date: Thu, 20 Mar 2025 03:06:03 +0100 Subject: [PATCH 088/295] Replace Python parser with ruff parser (#5494) * stage1 * compiler pass build * introduce rustpython-compiler-source * stage2 * fixup * pass compile * Fix hello world compiler test * Fix code generation for if-elif-else statement * Fix code generation for lambda expression * Fix code generation for integers * Fix code generation for fstrings * Fix code generation for if statement * Fix code generation for if statement * Fix code generation for if statement * Fix code generation for fstring * Fix code generation for class definition * Replace feature flags * Initialize frozen core modules * Allow __future__ import after module doc comment * Disable ast module * Commit remaining fixes for compile errors in examples * Fix some warnings * Update ast stdlib module * Update ast stdlib module * Update ast stdlib module * Update ast stdlib module * Update ast stdlib module * Split ast stdlib module into files * Fix codegen for positional arguments with defaults * Update ast stdlib module * Update ast stdlib module * Extract string and constant handling from expression.rs * Always add required fields to AST nodes * Compile doc strings correctly again * Enable "ast" Cargo feature by default * Refactor compilation of big integer literal * Update ast stdlib module * Update ast stdlib module * Update ast stdlib module * Reset barebones example * Fix some left-over warnings * Undo accidential change * Adapt shell to ruff parser * Pin parser to v0.4.10 * fix clippy * Add TODO about interactive mode * Fix compilation of complex number expression * Remove moved code * Update test case to ruff v0.4.10 * Apply suggestion Co-authored-by: Jeong, YunWon <69878+youknowone@users.noreply.github.com> * Apply suggestion Co-authored-by: Jeong, YunWon <69878+youknowone@users.noreply.github.com> * Fix compilation of fstring expression * Fix compilation of fstring expression * Fix wasm compile errors * Attach correct source locations to ast objects * Fix some more wasm compile errors * Consider compile mode and enable AST stdlib module again * Fix incorrect AST source location end column * Fix compile error if "compiler" feature is not enabled * Fix regrtests * Fix some test_ast tests * Add source range to type ignore * Fix incompatibility with Rust 2024 edition * Fix todos by implementing missing ast conversions and deleting unused code * Appease clippy * Fix remaining ast tests * Fix remaining ast tests * Mark/fix remaining tests * Fix more * Hacky windows fix --------- Co-authored-by: Kangzhi Shi Co-authored-by: Jeong YunWon Co-authored-by: Jeong, YunWon <69878+youknowone@users.noreply.github.com> Co-authored-by: Noa --- Cargo.lock | 216 +- Cargo.toml | 16 +- Lib/test/test_ast.py | 14 +- Lib/test/test_codeop.py | 2 + Lib/test/test_eof.py | 12 +- Lib/test/test_exceptions.py | 2 + Lib/test/test_fstring.py | 14 +- Lib/test/test_future_stmt/test_future.py | 10 +- Lib/test/test_global.py | 6 + Lib/test/test_grammar.py | 2 + Lib/test/test_keywordonlyarg.py | 2 - Lib/test/test_named_expressions.py | 34 +- Lib/test/test_positional_only_arg.py | 6 + Lib/test/test_syntax.py | 22 +- Lib/test/test_timeit.py | 2 - Lib/test/test_unicode_identifiers.py | 4 - compiler/Cargo.toml | 8 +- compiler/codegen/Cargo.toml | 16 +- compiler/codegen/src/compile.rs | 1419 +++--- compiler/codegen/src/error.rs | 19 +- compiler/codegen/src/ir.rs | 11 +- compiler/codegen/src/lib.rs | 46 + compiler/codegen/src/symboltable.rs | 463 +- compiler/core/Cargo.toml | 5 +- compiler/core/src/bytecode.rs | 6 +- compiler/core/src/marshal.rs | 14 +- compiler/core/src/mode.rs | 8 +- compiler/source/Cargo.toml | 16 + compiler/source/src/lib.rs | 42 + compiler/src/lib.rs | 307 +- derive-impl/Cargo.toml | 2 +- derive/src/lib.rs | 2 +- examples/dis.rs | 2 +- examples/parse_folder.rs | 20 +- src/shell.rs | 26 +- stdlib/src/faulthandler.rs | 2 +- vm/Cargo.toml | 21 +- vm/src/builtins/code.rs | 6 +- vm/src/builtins/frame.rs | 2 +- vm/src/builtins/traceback.rs | 4 +- vm/src/compiler.rs | 42 +- vm/src/exceptions.rs | 2 +- vm/src/frame.rs | 6 +- vm/src/lib.rs | 2 +- vm/src/stdlib/ast.rs | 479 +- vm/src/stdlib/ast/argument.rs | 161 + vm/src/stdlib/ast/basic.rs | 50 + vm/src/stdlib/ast/constant.rs | 396 ++ vm/src/stdlib/ast/elif_else_clause.rs | 88 + vm/src/stdlib/ast/exception.rs | 75 + vm/src/stdlib/ast/expression.rs | 1197 +++++ vm/src/stdlib/ast/gen.rs | 5496 ---------------------- vm/src/stdlib/ast/module.rs | 221 + vm/src/stdlib/ast/node.rs | 93 + vm/src/stdlib/ast/operator.rs | 185 + vm/src/stdlib/ast/other.rs | 131 + vm/src/stdlib/ast/parameter.rs | 401 ++ vm/src/stdlib/ast/pattern.rs | 515 ++ vm/src/stdlib/ast/pyast.rs | 2448 ++++++++++ vm/src/stdlib/ast/python.rs | 57 + vm/src/stdlib/ast/statement.rs | 1222 +++++ vm/src/stdlib/ast/string.rs | 354 ++ vm/src/stdlib/ast/type_ignore.rs | 73 + vm/src/stdlib/ast/type_parameters.rs | 197 + vm/src/stdlib/builtins.rs | 19 +- vm/src/stdlib/mod.rs | 8 +- vm/src/vm/compile.rs | 2 +- vm/src/vm/vm_new.rs | 65 +- wasm/lib/Cargo.toml | 2 +- wasm/lib/src/convert.rs | 21 +- 70 files changed, 9981 insertions(+), 6858 deletions(-) create mode 100644 compiler/source/Cargo.toml create mode 100644 compiler/source/src/lib.rs create mode 100644 vm/src/stdlib/ast/argument.rs create mode 100644 vm/src/stdlib/ast/basic.rs create mode 100644 vm/src/stdlib/ast/constant.rs create mode 100644 vm/src/stdlib/ast/elif_else_clause.rs create mode 100644 vm/src/stdlib/ast/exception.rs create mode 100644 vm/src/stdlib/ast/expression.rs delete mode 100644 vm/src/stdlib/ast/gen.rs create mode 100644 vm/src/stdlib/ast/module.rs create mode 100644 vm/src/stdlib/ast/node.rs create mode 100644 vm/src/stdlib/ast/operator.rs create mode 100644 vm/src/stdlib/ast/other.rs create mode 100644 vm/src/stdlib/ast/parameter.rs create mode 100644 vm/src/stdlib/ast/pattern.rs create mode 100644 vm/src/stdlib/ast/pyast.rs create mode 100644 vm/src/stdlib/ast/python.rs create mode 100644 vm/src/stdlib/ast/statement.rs create mode 100644 vm/src/stdlib/ast/string.rs create mode 100644 vm/src/stdlib/ast/type_ignore.rs create mode 100644 vm/src/stdlib/ast/type_parameters.rs diff --git a/Cargo.lock b/Cargo.lock index b3b8da765e..acf545efa8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1007,6 +1007,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.14.0" @@ -1051,12 +1060,6 @@ dependencies = [ "cpufeatures", ] -[[package]] -name = "lalrpop-util" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "507460a910eb7b32ee961886ff48539633b788a36b65692b95f225b844c82553" - [[package]] name = "lambert_w" version = "1.0.17" @@ -1958,6 +1961,90 @@ dependencies = [ "syn 2.0.98", ] +[[package]] +name = "ruff_python_ast" +version = "0.0.0" +source = "git+https://github.com/astral-sh/ruff.git?tag=v0.4.10#b54922fd7394c36cdc390fd21aaee99206ebc361" +dependencies = [ + "aho-corasick", + "bitflags 2.8.0", + "is-macro", + "itertools 0.13.0", + "once_cell", + "ruff_python_trivia", + "ruff_source_file", + "ruff_text_size", + "rustc-hash 1.1.0", +] + +[[package]] +name = "ruff_python_codegen" +version = "0.0.0" +source = "git+https://github.com/astral-sh/ruff.git?tag=v0.4.10#b54922fd7394c36cdc390fd21aaee99206ebc361" +dependencies = [ + "once_cell", + "ruff_python_ast", + "ruff_python_literal", + "ruff_python_parser", + "ruff_source_file", + "ruff_text_size", +] + +[[package]] +name = "ruff_python_literal" +version = "0.0.0" +source = "git+https://github.com/astral-sh/ruff.git?tag=v0.4.10#b54922fd7394c36cdc390fd21aaee99206ebc361" +dependencies = [ + "bitflags 2.8.0", + "itertools 0.13.0", + "ruff_python_ast", + "unic-ucd-category", +] + +[[package]] +name = "ruff_python_parser" +version = "0.0.0" +source = "git+https://github.com/astral-sh/ruff.git?tag=v0.4.10#b54922fd7394c36cdc390fd21aaee99206ebc361" +dependencies = [ + "bitflags 2.8.0", + "bstr", + "memchr", + "ruff_python_ast", + "ruff_python_trivia", + "ruff_text_size", + "rustc-hash 1.1.0", + "static_assertions", + "unicode-ident", + "unicode-normalization", + "unicode_names2", +] + +[[package]] +name = "ruff_python_trivia" +version = "0.0.0" +source = "git+https://github.com/astral-sh/ruff.git?tag=v0.4.10#b54922fd7394c36cdc390fd21aaee99206ebc361" +dependencies = [ + "itertools 0.13.0", + "ruff_source_file", + "ruff_text_size", + "unicode-ident", +] + +[[package]] +name = "ruff_source_file" +version = "0.0.0" +source = "git+https://github.com/astral-sh/ruff.git?tag=v0.4.10#b54922fd7394c36cdc390fd21aaee99206ebc361" +dependencies = [ + "memchr", + "once_cell", + "ruff_text_size", +] + +[[package]] +name = "ruff_text_size" +version = "0.0.0" +source = "git+https://github.com/astral-sh/ruff.git?tag=v0.4.10#b54922fd7394c36cdc390fd21aaee99206ebc361" + [[package]] name = "rustc-hash" version = "1.1.0" @@ -2006,26 +2093,14 @@ dependencies = [ "libc", "log", "pyo3", + "ruff_python_parser", "rustpython-compiler", - "rustpython-parser", "rustpython-pylib", "rustpython-stdlib", "rustpython-vm", "rustyline", ] -[[package]] -name = "rustpython-ast" -version = "0.4.0" -source = "git+https://github.com/RustPython/Parser.git?rev=d2f137b372ec08ce4a243564a80f8f9153c45a23#d2f137b372ec08ce4a243564a80f8f9153c45a23" -dependencies = [ - "is-macro", - "malachite-bigint", - "rustpython-literal", - "rustpython-parser-core", - "static_assertions", -] - [[package]] name = "rustpython-codegen" version = "0.4.0" @@ -2036,13 +2111,18 @@ dependencies = [ "insta", "itertools 0.14.0", "log", + "malachite-bigint", "num-complex", "num-traits", - "rustpython-ast", + "ruff_python_ast", + "ruff_python_codegen", + "ruff_python_parser", + "ruff_source_file", + "ruff_text_size", "rustpython-common", "rustpython-compiler-core", - "rustpython-parser", - "rustpython-parser-core", + "rustpython-compiler-source", + "thiserror 2.0.11", ] [[package]] @@ -2075,9 +2155,14 @@ dependencies = [ name = "rustpython-compiler" version = "0.4.0" dependencies = [ + "ruff_python_ast", + "ruff_python_parser", + "ruff_source_file", + "ruff_text_size", "rustpython-codegen", "rustpython-compiler-core", - "rustpython-parser", + "rustpython-compiler-source", + "thiserror 2.0.11", ] [[package]] @@ -2089,10 +2174,20 @@ dependencies = [ "lz4_flex", "malachite-bigint", "num-complex", - "rustpython-parser-core", + "ruff_python_ast", + "ruff_python_parser", + "ruff_source_file", "serde", ] +[[package]] +name = "rustpython-compiler-source" +version = "0.4.0" +dependencies = [ + "ruff_source_file", + "ruff_text_size", +] + [[package]] name = "rustpython-derive" version = "0.4.0" @@ -2113,7 +2208,6 @@ dependencies = [ "quote", "rustpython-compiler-core", "rustpython-doc", - "rustpython-parser-core", "syn 2.0.98", "syn-ext", "textwrap 0.16.1", @@ -2166,48 +2260,6 @@ dependencies = [ "unic-ucd-category", ] -[[package]] -name = "rustpython-parser" -version = "0.4.0" -source = "git+https://github.com/RustPython/Parser.git?rev=d2f137b372ec08ce4a243564a80f8f9153c45a23#d2f137b372ec08ce4a243564a80f8f9153c45a23" -dependencies = [ - "anyhow", - "is-macro", - "itertools 0.11.0", - "lalrpop-util", - "log", - "malachite-bigint", - "num-traits", - "phf", - "phf_codegen", - "rustc-hash 1.1.0", - "rustpython-ast", - "rustpython-parser-core", - "tiny-keccak", - "unic-emoji-char", - "unic-ucd-ident", - "unicode_names2", -] - -[[package]] -name = "rustpython-parser-core" -version = "0.4.0" -source = "git+https://github.com/RustPython/Parser.git?rev=d2f137b372ec08ce4a243564a80f8f9153c45a23#d2f137b372ec08ce4a243564a80f8f9153c45a23" -dependencies = [ - "is-macro", - "memchr", - "rustpython-parser-vendored", -] - -[[package]] -name = "rustpython-parser-vendored" -version = "0.4.0" -source = "git+https://github.com/RustPython/Parser.git?rev=d2f137b372ec08ce4a243564a80f8f9153c45a23#d2f137b372ec08ce4a243564a80f8f9153c45a23" -dependencies = [ - "memchr", - "once_cell", -] - [[package]] name = "rustpython-pylib" version = "0.4.0" @@ -2344,19 +2396,21 @@ dependencies = [ "paste", "rand 0.9.0", "result-like", + "ruff_python_ast", + "ruff_python_parser", + "ruff_source_file", + "ruff_text_size", "rustc_version", "rustix", - "rustpython-ast", "rustpython-codegen", "rustpython-common", "rustpython-compiler", "rustpython-compiler-core", + "rustpython-compiler-source", "rustpython-derive", "rustpython-format", "rustpython-jit", "rustpython-literal", - "rustpython-parser", - "rustpython-parser-core", "rustpython-sre_engine", "rustyline", "schannel", @@ -2388,8 +2442,8 @@ dependencies = [ "console_error_panic_hook", "getrandom 0.2.15", "js-sys", + "ruff_python_parser", "rustpython-common", - "rustpython-parser", "rustpython-pylib", "rustpython-stdlib", "rustpython-vm", @@ -2793,15 +2847,6 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "639ce8ef6d2ba56be0383a94dd13b92138d58de44c62618303bb798fa92bdc00" -[[package]] -name = "tiny-keccak" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" -dependencies = [ - "crunchy", -] - [[package]] name = "tinytemplate" version = "1.2.1" @@ -2879,17 +2924,6 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80d7ff825a6a654ee85a63e80f92f054f904f21e7d12da4e22f9834a4aaa35bc" -[[package]] -name = "unic-emoji-char" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b07221e68897210270a38bde4babb655869637af0f69407f96053a34f76494d" -dependencies = [ - "unic-char-property", - "unic-char-range", - "unic-ucd-version", -] - [[package]] name = "unic-normal" version = "0.9.0" diff --git a/Cargo.toml b/Cargo.toml index 2d66f9775a..ab5e56fa0c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,7 +28,7 @@ rustpython-compiler = { workspace = true } rustpython-pylib = { workspace = true, optional = true } rustpython-stdlib = { workspace = true, optional = true, features = ["compiler"] } rustpython-vm = { workspace = true, features = ["compiler"] } -rustpython-parser = { workspace = true } +ruff_python_parser = { workspace = true } cfg-if = { workspace = true } log = { workspace = true } @@ -94,7 +94,7 @@ x86_64-pc-windows-msvc = { triplet = "x64-windows-static-md", dev-dependencies = [workspace] resolver = "2" members = [ - "compiler", "compiler/core", "compiler/codegen", + "compiler", "compiler/core", "compiler/codegen", "compiler/source", ".", "common", "derive", "jit", "vm", "vm/sre_engine", "pylib", "stdlib", "derive-impl", "wasm/lib", ] @@ -108,6 +108,7 @@ repository = "https://github.com/RustPython/RustPython" license = "MIT" [workspace.dependencies] +rustpython-compiler-source = { path = "compiler/source" } rustpython-compiler-core = { path = "compiler/core", version = "0.4.0" } rustpython-compiler = { path = "compiler", version = "0.4.0" } rustpython-codegen = { path = "compiler/codegen", version = "0.4.0" } @@ -121,15 +122,20 @@ rustpython-stdlib = { path = "stdlib", default-features = false, version = "0.4. rustpython-sre_engine = { path = "vm/sre_engine", version = "0.4.0" } rustpython-doc = { git = "https://github.com/RustPython/__doc__", tag = "0.3.0", version = "0.3.0" } +ruff_python_parser = { git = "https://github.com/astral-sh/ruff.git", tag = "v0.4.10" } +ruff_python_ast = { git = "https://github.com/astral-sh/ruff.git", tag = "v0.4.10" } +ruff_text_size = { git = "https://github.com/astral-sh/ruff.git", tag = "v0.4.10" } +ruff_source_file = { git = "https://github.com/astral-sh/ruff.git", tag = "v0.4.10" } +ruff_python_codegen = { git = "https://github.com/astral-sh/ruff.git", tag = "v0.4.10" } # rustpython-literal = { version = "0.4.0" } # rustpython-parser-core = { version = "0.4.0" } # rustpython-parser = { version = "0.4.0" } # rustpython-ast = { version = "0.4.0" } # rustpython-format= { version = "0.4.0" } rustpython-literal = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "d2f137b372ec08ce4a243564a80f8f9153c45a23" } -rustpython-parser-core = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "d2f137b372ec08ce4a243564a80f8f9153c45a23" } -rustpython-parser = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "d2f137b372ec08ce4a243564a80f8f9153c45a23" } -rustpython-ast = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "d2f137b372ec08ce4a243564a80f8f9153c45a23" } +# rustpython-parser-core = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "d2f137b372ec08ce4a243564a80f8f9153c45a23" } +# rustpython-parser = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "d2f137b372ec08ce4a243564a80f8f9153c45a23" } +# rustpython-ast = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "d2f137b372ec08ce4a243564a80f8f9153c45a23" } rustpython-format = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "d2f137b372ec08ce4a243564a80f8f9153c45a23" } # rustpython-literal = { path = "../RustPython-parser/literal" } # rustpython-parser-core = { path = "../RustPython-parser/core" } diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index af3e2bb5eb..8b28686fd6 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -388,6 +388,8 @@ def test_invalid_position_information(self): with self.assertRaises(ValueError): compile(tree, '', 'exec') + # XXX RUSTPYTHON: we always require that end ranges be present + @unittest.expectedFailure def test_compilation_of_ast_nodes_with_default_end_position_values(self): tree = ast.Module(body=[ ast.Import(names=[ast.alias(name='builtins', lineno=1, col_offset=0)], lineno=1, col_offset=0), @@ -1531,6 +1533,8 @@ def test_literal_eval_malformed_dict_nodes(self): malformed = ast.Dict(keys=[ast.Constant(1)], values=[ast.Constant(2), ast.Constant(3)]) self.assertRaises(ValueError, ast.literal_eval, malformed) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_literal_eval_trailing_ws(self): self.assertEqual(ast.literal_eval(" -1"), -1) self.assertEqual(ast.literal_eval("\t\t-1"), -1) @@ -1549,6 +1553,8 @@ def test_literal_eval_malformed_lineno(self): with self.assertRaisesRegex(ValueError, msg): ast.literal_eval(node) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_literal_eval_syntax_errors(self): with self.assertRaisesRegex(SyntaxError, "unexpected indent"): ast.literal_eval(r''' @@ -1569,6 +1575,8 @@ def test_bad_integer(self): compile(mod, 'test', 'exec') self.assertIn("invalid integer value: None", str(cm.exception)) + # XXX RUSTPYTHON: we always require that end ranges be present + @unittest.expectedFailure def test_level_as_none(self): body = [ast.ImportFrom(module='time', names=[ast.alias(name='sleep', @@ -2034,8 +2042,6 @@ def test_call(self): call = ast.Call(func, args, bad_keywords) self.expr(call, "must have Load context") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_num(self): with warnings.catch_warnings(record=True) as wlog: warnings.filterwarnings('ignore', '', DeprecationWarning) @@ -2733,8 +2739,6 @@ def test_source_segment_multi(self): binop = self._parse_value(s_orig) self.assertEqual(ast.get_source_segment(s_orig, binop.left), s_tuple) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_source_segment_padded(self): s_orig = dedent(''' class C: @@ -2756,8 +2760,6 @@ def test_source_segment_endings(self): self._check_content(s, y, 'y = 1') self._check_content(s, z, 'z = 1') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_source_segment_tabs(self): s = dedent(''' class C: diff --git a/Lib/test/test_codeop.py b/Lib/test/test_codeop.py index 19117fa409..1036b970cd 100644 --- a/Lib/test/test_codeop.py +++ b/Lib/test/test_codeop.py @@ -323,6 +323,8 @@ def assertSyntaxErrorMatches(self, code, message): with self.assertRaisesRegex(SyntaxError, message): compile_command(code, symbol='exec') + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_syntax_errors(self): self.assertSyntaxErrorMatches( dedent("""\ diff --git a/Lib/test/test_eof.py b/Lib/test/test_eof.py index cf12dd3d5d..082103b338 100644 --- a/Lib/test/test_eof.py +++ b/Lib/test/test_eof.py @@ -7,9 +7,9 @@ from test.support import warnings_helper import unittest -# TODO: RUSTPYTHON -@unittest.expectedFailure class EOFTestCase(unittest.TestCase): + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_EOF_single_quote(self): expect = "unterminated string literal (detected at line 1) (, line 1)" for quote in ("'", "\""): @@ -22,6 +22,8 @@ def test_EOF_single_quote(self): else: raise support.TestFailed + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_EOFS(self): expect = ("unterminated triple-quoted string literal (detected at line 1) (, line 1)") try: @@ -32,6 +34,8 @@ def test_EOFS(self): else: raise support.TestFailed + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_EOFS_with_file(self): expect = ("(, line 1)") with os_helper.temp_dir() as temp_dir: @@ -39,6 +43,8 @@ def test_EOFS_with_file(self): rc, out, err = script_helper.assert_python_failure(file_name) self.assertIn(b'unterminated triple-quoted string literal (detected at line 3)', err) + # TODO: RUSTPYTHON + @unittest.expectedFailure @warnings_helper.ignore_warnings(category=SyntaxWarning) def test_eof_with_line_continuation(self): expect = "unexpected EOF while parsing (, line 1)" @@ -49,6 +55,8 @@ def test_eof_with_line_continuation(self): else: raise support.TestFailed + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_line_continuation_EOF(self): """A continuation at the end of input must be an error; bpo2180.""" expect = 'unexpected EOF while parsing (, line 1)' diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 47e7894960..8be8122507 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -50,6 +50,8 @@ def raise_catch(self, exc, excname): self.assertEqual(buf1, buf2) self.assertEqual(exc.__name__, excname) + # TODO: RUSTPYTHON + @unittest.expectedFailure def testRaising(self): self.raise_catch(AttributeError, "AttributeError") self.assertRaises(AttributeError, getattr, sys, "undefined_attribute") diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index b3a38dd652..c0c987ca2f 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -305,8 +305,6 @@ def test_ast_numbers_fstring_with_formatting(self): self.assertEqual(name.col_offset, 22) self.assertEqual(name.end_col_offset, 25) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_ast_line_numbers_multiline_fstring(self): # See bpo-30465 for details. expr = """ @@ -509,8 +507,6 @@ def test_ast_fstring_empty_format_spec(self): self.assertEqual(type(format_spec), ast.JoinedStr) self.assertEqual(len(format_spec.values), 0) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_docstring(self): def f(): f"""Not a docstring""" @@ -535,6 +531,8 @@ def test_ast_compile_time_concat(self): exec(c) self.assertEqual(x[0], "foo3") + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_compile_time_concat_errors(self): self.assertAllRaise( SyntaxError, @@ -550,6 +548,8 @@ def test_literal(self): self.assertEqual(f"a", "a") self.assertEqual(f" ", " ") + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_unterminated_string(self): self.assertAllRaise( SyntaxError, @@ -562,6 +562,8 @@ def test_unterminated_string(self): ], ) + # TODO: RUSTPYTHON + @unittest.expectedFailure @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") def test_mismatched_parens(self): self.assertAllRaise( @@ -666,6 +668,8 @@ def test_double_braces(self): ], ) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_compile_time_concat(self): x = "def" self.assertEqual("abc" f"## {x}ghi", "abc## defghi") @@ -1516,6 +1520,8 @@ def test_assignment(self): ], ) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_del(self): self.assertAllRaise( SyntaxError, diff --git a/Lib/test/test_future_stmt/test_future.py b/Lib/test/test_future_stmt/test_future.py index 4a978cef4a..ad9b5d5a87 100644 --- a/Lib/test/test_future_stmt/test_future.py +++ b/Lib/test/test_future_stmt/test_future.py @@ -67,6 +67,8 @@ def test_badfuture5(self): from test.test_future_stmt import badsyntax_future5 self.check_syntax_error(cm.exception, "badsyntax_future5", 4) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_badfuture6(self): with self.assertRaises(SyntaxError) as cm: from test.test_future_stmt import badsyntax_future6 @@ -132,8 +134,6 @@ def test_unicode_literals_exec(self): exec("from __future__ import unicode_literals; x = ''", {}, scope) self.assertIsInstance(scope["x"], str) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_syntactical_future_repl(self): p = spawn_python('-i') p.stdin.write(b"from __future__ import barry_as_FLUFL\n") @@ -198,6 +198,8 @@ def _exec_future(self, code): ) return scope + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_annotations(self): eq = self.assertAnnotationEqual eq('...') @@ -362,6 +364,8 @@ def test_annotations(self): eq('(((a, b)))', '(a, b)') eq("1 + 2 + 3") + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_fstring_debug_annotations(self): # f-strings with '=' don't round trip very well, so set the expected # result explicitly. @@ -372,6 +376,8 @@ def test_fstring_debug_annotations(self): self.assertAnnotationEqual("f'{x=!a}'", expected="f'x={x!a}'") self.assertAnnotationEqual("f'{x=!s:*^20}'", expected="f'x={x!s:*^20}'") + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_infinity_numbers(self): inf = "1e" + repr(sys.float_info.max_10_exp + 1) infj = f"{inf}j" diff --git a/Lib/test/test_global.py b/Lib/test/test_global.py index f5b38c25ea..4c1bb6c0cf 100644 --- a/Lib/test/test_global.py +++ b/Lib/test/test_global.py @@ -12,6 +12,8 @@ def setUp(self): self.enterContext(check_warnings()) warnings.filterwarnings("error", module="") + # TODO: RUSTPYTHON + @unittest.expectedFailure def test1(self): prog_text_1 = """\ def wrong1(): @@ -22,6 +24,8 @@ def wrong1(): """ check_syntax_error(self, prog_text_1, lineno=4, offset=5) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test2(self): prog_text_2 = """\ def wrong2(): @@ -30,6 +34,8 @@ def wrong2(): """ check_syntax_error(self, prog_text_2, lineno=3, offset=5) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test3(self): prog_text_3 = """\ def wrong3(): diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index a797fd2b22..f94d36fa0b 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -1062,6 +1062,8 @@ def g2(x): self.assertEqual(g2(False), 0) self.assertEqual(g2(True), ('end', 1)) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_yield(self): # Allowed as standalone statement def g(): yield 1 diff --git a/Lib/test/test_keywordonlyarg.py b/Lib/test/test_keywordonlyarg.py index 46dce636df..9e9c6651dd 100644 --- a/Lib/test/test_keywordonlyarg.py +++ b/Lib/test/test_keywordonlyarg.py @@ -40,8 +40,6 @@ def shouldRaiseSyntaxError(s): compile(s, "", "single") self.assertRaises(SyntaxError, shouldRaiseSyntaxError, codestr) - # TODO: RUSTPYTHON - @unittest.expectedFailure def testSyntaxErrorForFunctionDefinition(self): self.assertRaisesSyntaxError("def f(p, *):\n pass\n") self.assertRaisesSyntaxError("def f(p1, *, p1=100):\n pass\n") diff --git a/Lib/test/test_named_expressions.py b/Lib/test/test_named_expressions.py index 797b0f512f..03f5384b63 100644 --- a/Lib/test/test_named_expressions.py +++ b/Lib/test/test_named_expressions.py @@ -4,62 +4,80 @@ class NamedExpressionInvalidTest(unittest.TestCase): + # TODO: RUSTPYTHON: wrong error message + @unittest.expectedFailure def test_named_expression_invalid_01(self): code = """x := 0""" with self.assertRaisesRegex(SyntaxError, "invalid syntax"): exec(code, {}, {}) + # TODO: RUSTPYTHON: wrong error message + @unittest.expectedFailure def test_named_expression_invalid_02(self): code = """x = y := 0""" with self.assertRaisesRegex(SyntaxError, "invalid syntax"): exec(code, {}, {}) + # TODO: RUSTPYTHON: wrong error message + @unittest.expectedFailure def test_named_expression_invalid_03(self): code = """y := f(x)""" with self.assertRaisesRegex(SyntaxError, "invalid syntax"): exec(code, {}, {}) + # TODO: RUSTPYTHON: wrong error message + @unittest.expectedFailure def test_named_expression_invalid_04(self): code = """y0 = y1 := f(x)""" with self.assertRaisesRegex(SyntaxError, "invalid syntax"): exec(code, {}, {}) - # TODO: RUSTPYTHON - @unittest.expectedFailure # wrong error message + # TODO: RUSTPYTHON: wrong error message + @unittest.expectedFailure def test_named_expression_invalid_06(self): code = """((a, b) := (1, 2))""" with self.assertRaisesRegex(SyntaxError, "cannot use assignment expressions with tuple"): exec(code, {}, {}) + # TODO: RUSTPYTHON: wrong error message + @unittest.expectedFailure def test_named_expression_invalid_07(self): code = """def spam(a = b := 42): pass""" with self.assertRaisesRegex(SyntaxError, "invalid syntax"): exec(code, {}, {}) + # TODO: RUSTPYTHON: wrong error message + @unittest.expectedFailure def test_named_expression_invalid_08(self): code = """def spam(a: b := 42 = 5): pass""" with self.assertRaisesRegex(SyntaxError, "invalid syntax"): exec(code, {}, {}) + # TODO: RUSTPYTHON: wrong error message + @unittest.expectedFailure def test_named_expression_invalid_09(self): code = """spam(a=b := 'c')""" with self.assertRaisesRegex(SyntaxError, "invalid syntax"): exec(code, {}, {}) + # TODO: RUSTPYTHON: wrong error message + @unittest.expectedFailure def test_named_expression_invalid_10(self): code = """spam(x = y := f(x))""" with self.assertRaisesRegex(SyntaxError, "invalid syntax"): exec(code, {}, {}) + # TODO: RUSTPYTHON: wrong error message + @unittest.expectedFailure def test_named_expression_invalid_11(self): code = """spam(a=1, b := 2)""" @@ -67,6 +85,8 @@ def test_named_expression_invalid_11(self): "positional argument follows keyword argument"): exec(code, {}, {}) + # TODO: RUSTPYTHON: wrong error message + @unittest.expectedFailure def test_named_expression_invalid_12(self): code = """spam(a=1, (b := 2))""" @@ -74,6 +94,8 @@ def test_named_expression_invalid_12(self): "positional argument follows keyword argument"): exec(code, {}, {}) + # TODO: RUSTPYTHON: wrong error message + @unittest.expectedFailure def test_named_expression_invalid_13(self): code = """spam(a=1, (b := 2))""" @@ -81,14 +103,16 @@ def test_named_expression_invalid_13(self): "positional argument follows keyword argument"): exec(code, {}, {}) + # TODO: RUSTPYTHON: wrong error message + @unittest.expectedFailure def test_named_expression_invalid_14(self): code = """(x := lambda: y := 1)""" with self.assertRaisesRegex(SyntaxError, "invalid syntax"): exec(code, {}, {}) - # TODO: RUSTPYTHON - @unittest.expectedFailure # wrong error message + # TODO: RUSTPYTHON: wrong error message + @unittest.expectedFailure def test_named_expression_invalid_15(self): code = """(lambda: x := 1)""" @@ -96,6 +120,8 @@ def test_named_expression_invalid_15(self): "cannot use assignment expressions with lambda"): exec(code, {}, {}) + # TODO: RUSTPYTHON: wrong error message + @unittest.expectedFailure def test_named_expression_invalid_16(self): code = "[i + 1 for i in i := [1,2]]" diff --git a/Lib/test/test_positional_only_arg.py b/Lib/test/test_positional_only_arg.py index d9d8b03069..e0d784325b 100644 --- a/Lib/test/test_positional_only_arg.py +++ b/Lib/test/test_positional_only_arg.py @@ -22,6 +22,8 @@ def assertRaisesSyntaxError(self, codestr, regex="invalid syntax"): with self.assertRaisesRegex(SyntaxError, regex): compile(codestr + "\n", "", "single") + # TODO: RUSTPYTHON: wrong error message + @unittest.expectedFailure def test_invalid_syntax_errors(self): check_syntax_error(self, "def f(a, b = 5, /, c): pass", "non-default argument follows default argument") check_syntax_error(self, "def f(a = 5, b, /, c): pass", "non-default argument follows default argument") @@ -43,6 +45,8 @@ def test_invalid_syntax_errors(self): check_syntax_error(self, "def f(a, /, c, /, d, *, e): pass") check_syntax_error(self, "def f(a, *, c, /, d, e): pass") + # TODO: RUSTPYTHON: wrong error message + @unittest.expectedFailure def test_invalid_syntax_errors_async(self): check_syntax_error(self, "async def f(a, b = 5, /, c): pass", "non-default argument follows default argument") check_syntax_error(self, "async def f(a = 5, b, /, c): pass", "non-default argument follows default argument") @@ -236,6 +240,8 @@ def test_lambdas(self): x = lambda a, b, /, : a + b self.assertEqual(x(1, 2), 3) + # TODO: RUSTPYTHON: wrong error message + @unittest.expectedFailure def test_invalid_syntax_lambda(self): check_syntax_error(self, "lambda a, b = 5, /, c: None", "non-default argument follows default argument") check_syntax_error(self, "lambda a = 5, b, /, c: None", "non-default argument follows default argument") diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index 9726d3cc1e..7e46773047 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -742,7 +742,8 @@ >>> f(x.y=1) Traceback (most recent call last): SyntaxError: expression cannot contain assignment, perhaps you meant "=="? ->>> f((x)=2) +# TODO: RUSTPYTHON +>>> f((x)=2) # doctest: +SKIP Traceback (most recent call last): SyntaxError: expression cannot contain assignment, perhaps you meant "=="? >>> f(True=1) @@ -1649,7 +1650,8 @@ Invalid pattern matching constructs: - >>> match ...: + # TODO: RUSTPYTHON nothing raised. + >>> match ...: # doctest: +SKIP ... case 42 as _: ... ... Traceback (most recent call last): @@ -1876,12 +1878,16 @@ def test_expression_with_assignment(self): offset=7 ) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_curly_brace_after_primary_raises_immediately(self): self._check_error("f{}", "invalid syntax", mode="single") def test_assign_call(self): self._check_error("f() = 1", "assign") + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_assign_del(self): self._check_error("del (,)", "invalid syntax") self._check_error("del 1", "cannot delete literal") @@ -1994,6 +2000,8 @@ def test_bad_outdent(self): "unindent does not match .* level", subclass=IndentationError) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_kwargs_last(self): self._check_error("int(base=10, '2')", "positional argument follows keyword argument") @@ -2005,6 +2013,8 @@ def test_kwargs_last2(self): "positional argument follows " "keyword argument unpacking") + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_kwargs_last3(self): self._check_error("int(**{'base': 10}, *['2'])", "iterable argument unpacking follows " @@ -2073,8 +2083,6 @@ def test_nested_named_except_blocks(self): code += f"{' '*4*12}pass" self._check_error(code, "too many statically nested blocks") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_barry_as_flufl_with_syntax_errors(self): # The "barry_as_flufl" rule can produce some "bugs-at-a-distance" if # is reading the wrong token in the presence of syntax errors later @@ -2092,6 +2100,8 @@ def func2(): """ self._check_error(code, "expected ':'") + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_invalid_line_continuation_error_position(self): self._check_error(r"a = 3 \ 4", "unexpected character after line continuation character", @@ -2103,6 +2113,8 @@ def test_invalid_line_continuation_error_position(self): "unexpected character after line continuation character", lineno=3, offset=4) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_invalid_line_continuation_left_recursive(self): # Check bpo-42218: SyntaxErrors following left-recursive rules # (t_primary_raw in this case) need to be tested explicitly @@ -2146,6 +2158,8 @@ def case(x): """ compile(code, "", "exec") + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_multiline_compiler_error_points_to_the_end(self): self._check_error( "call(\na=1,\na=1\n)", diff --git a/Lib/test/test_timeit.py b/Lib/test/test_timeit.py index 17e880cb58..72a104fc1a 100644 --- a/Lib/test/test_timeit.py +++ b/Lib/test/test_timeit.py @@ -91,8 +91,6 @@ def test_timer_invalid_setup(self): self.assertRaises(SyntaxError, timeit.Timer, setup='from timeit import *') self.assertRaises(SyntaxError, timeit.Timer, setup=' pass') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_timer_empty_stmt(self): timeit.Timer(stmt='') timeit.Timer(stmt=' \n\t\f') diff --git a/Lib/test/test_unicode_identifiers.py b/Lib/test/test_unicode_identifiers.py index 9e611caf61..d7a0ece253 100644 --- a/Lib/test/test_unicode_identifiers.py +++ b/Lib/test/test_unicode_identifiers.py @@ -2,8 +2,6 @@ class PEP3131Test(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_valid(self): class T: ä = 1 @@ -15,8 +13,6 @@ class T: self.assertEqual(getattr(T, '\u87d2'), 3) self.assertEqual(getattr(T, 'x\U000E0100'), 4) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_non_bmp_normalized(self): 𝔘𝔫𝔦𝔠𝔬𝔡𝔢 = 1 self.assertIn("Unicode", dir()) diff --git a/compiler/Cargo.toml b/compiler/Cargo.toml index cb98093c9e..19ece85a16 100644 --- a/compiler/Cargo.toml +++ b/compiler/Cargo.toml @@ -11,7 +11,13 @@ license.workspace = true [dependencies] rustpython-codegen = { workspace = true } rustpython-compiler-core = { workspace = true } -rustpython-parser = { workspace = true } +rustpython-compiler-source = { workspace = true } +# rustpython-parser = { workspace = true } +ruff_python_parser = { workspace = true } +ruff_python_ast = { workspace = true } +ruff_source_file = { workspace = true } +ruff_text_size = { workspace = true } +thiserror = { workspace = true } [lints] workspace = true \ No newline at end of file diff --git a/compiler/codegen/Cargo.toml b/compiler/codegen/Cargo.toml index 0817a95894..9cf93bc22b 100644 --- a/compiler/codegen/Cargo.toml +++ b/compiler/codegen/Cargo.toml @@ -10,10 +10,16 @@ license.workspace = true [dependencies] -rustpython-ast = { workspace = true, features=["unparse", "constant-optimization"] } +# rustpython-ast = { workspace = true, features=["unparse", "constant-optimization"] } rustpython-common = { workspace = true } -rustpython-parser-core = { workspace = true } +# rustpython-parser-core = { workspace = true } rustpython-compiler-core = { workspace = true } +rustpython-compiler-source = {workspace = true } +ruff_python_parser = { workspace = true } +ruff_python_ast = { workspace = true } +ruff_text_size = { workspace = true } +ruff_source_file = { workspace = true } +ruff_python_codegen = { workspace = true } ahash = { workspace = true } bitflags = { workspace = true } @@ -22,11 +28,13 @@ itertools = { workspace = true } log = { workspace = true } num-complex = { workspace = true } num-traits = { workspace = true } +thiserror = { workspace = true } +malachite-bigint = { workspace = true } [dev-dependencies] -rustpython-parser = { workspace = true } +# rustpython-parser = { workspace = true } insta = { workspace = true } [lints] -workspace = true \ No newline at end of file +workspace = true diff --git a/compiler/codegen/src/compile.rs b/compiler/codegen/src/compile.rs index ce2e5627f0..21087fddd6 100644 --- a/compiler/codegen/src/compile.rs +++ b/compiler/codegen/src/compile.rs @@ -8,23 +8,33 @@ #![deny(clippy::cast_possible_truncation)] use crate::{ - IndexSet, + IndexSet, ToPythonName, error::{CodegenError, CodegenErrorType}, ir, symboltable::{self, SymbolFlags, SymbolScope, SymbolTable}, }; use itertools::Itertools; -use num_complex::Complex64; -use num_traits::ToPrimitive; -use rustpython_ast::located::{self as located_ast, Located}; +use malachite_bigint::BigInt; +use num_complex::Complex; +use num_traits::{Num, ToPrimitive}; +use ruff_python_ast::{ + Alias, Arguments, BoolOp, CmpOp, Comprehension, ConversionFlag, DebugText, Decorator, DictItem, + ExceptHandler, ExceptHandlerExceptHandler, Expr, ExprAttribute, ExprBoolOp, ExprFString, + ExprList, ExprName, ExprStarred, ExprSubscript, ExprTuple, ExprUnaryOp, FString, + FStringElement, FStringElements, FStringPart, Int, Keyword, MatchCase, ModExpression, + ModModule, Operator, Parameters, Pattern, PatternMatchAs, PatternMatchValue, Stmt, StmtExpr, + TypeParam, TypeParamParamSpec, TypeParamTypeVar, TypeParamTypeVarTuple, TypeParams, UnaryOp, + WithItem, +}; +use ruff_source_file::OneIndexed; +use ruff_text_size::{Ranged, TextRange}; +// use rustpython_ast::located::{self as located_ast, Located}; use rustpython_compiler_core::{ Mode, - bytecode::{ - self, Arg as OpArgMarker, CodeObject, ComparisonOperator, ConstantData, Instruction, OpArg, - OpArgType, - }, + bytecode::{self, Arg as OpArgMarker, CodeObject, ConstantData, Instruction, OpArg, OpArgType}, }; -use rustpython_parser_core::source_code::{LineNumber, SourceLocation}; +use rustpython_compiler_source::SourceCode; +// use rustpython_parser_core::source_code::{LineNumber, SourceLocation}; use std::borrow::Cow; type CompileResult = Result; @@ -50,19 +60,26 @@ fn is_forbidden_name(name: &str) -> bool { } /// Main structure holding the state of compilation. -struct Compiler { +struct Compiler<'src> { code_stack: Vec, symbol_table_stack: Vec, - source_path: String, - current_source_location: SourceLocation, + source_code: SourceCode<'src>, + // current_source_location: SourceLocation, + current_source_range: TextRange, qualified_path: Vec, - done_with_future_stmts: bool, + done_with_future_stmts: DoneWithFuture, future_annotations: bool, ctx: CompileContext, class_name: Option, opts: CompileOpts, } +enum DoneWithFuture { + No, + DoneWithDoc, + Yes, +} + #[derive(Debug, Clone, Default)] pub struct CompileOpts { /// How optimized the bytecode output should be; any optimize > 0 does @@ -98,105 +115,78 @@ enum ComprehensionType { Dict, } -/// Compile an located_ast::Mod produced from rustpython_parser::parse() +/// Compile an Mod produced from ruff parser pub fn compile_top( - ast: &located_ast::Mod, - source_path: String, + ast: ruff_python_ast::Mod, + source_code: SourceCode<'_>, mode: Mode, opts: CompileOpts, ) -> CompileResult { match ast { - located_ast::Mod::Module(located_ast::ModModule { body, .. }) => { - compile_program(body, source_path, opts) - } - located_ast::Mod::Interactive(located_ast::ModInteractive { body, .. }) => match mode { - Mode::Single => compile_program_single(body, source_path, opts), - Mode::BlockExpr => compile_block_expression(body, source_path, opts), - _ => unreachable!("only Single and BlockExpr parsed to Interactive"), + ruff_python_ast::Mod::Module(module) => match mode { + Mode::Exec | Mode::Eval => compile_program(&module, source_code, opts), + Mode::Single => compile_program_single(&module, source_code, opts), + Mode::BlockExpr => compile_block_expression(&module, source_code, opts), }, - located_ast::Mod::Expression(located_ast::ModExpression { body, .. }) => { - compile_expression(body, source_path, opts) - } - located_ast::Mod::FunctionType(_) => panic!("can't compile a FunctionType"), + ruff_python_ast::Mod::Expression(expr) => compile_expression(&expr, source_code, opts), } } -/// A helper function for the shared code of the different compile functions -fn compile_impl( - ast: &Ast, - source_path: String, +/// Compile a standard Python program to bytecode +pub fn compile_program( + ast: &ModModule, + source_code: SourceCode<'_>, opts: CompileOpts, - make_symbol_table: impl FnOnce(&Ast) -> Result, - compile: impl FnOnce(&mut Compiler, &Ast, SymbolTable) -> CompileResult<()>, ) -> CompileResult { - let symbol_table = match make_symbol_table(ast) { - Ok(x) => x, - Err(e) => return Err(e.into_codegen_error(source_path)), - }; - - let mut compiler = Compiler::new(opts, source_path, "".to_owned()); - compile(&mut compiler, ast, symbol_table)?; + let symbol_table = SymbolTable::scan_program(ast, source_code.clone()) + .map_err(|e| e.into_codegen_error(source_code.path.to_owned()))?; + let mut compiler = Compiler::new(opts, source_code, "".to_owned()); + compiler.compile_program(ast, symbol_table)?; let code = compiler.pop_code_object(); trace!("Compilation completed: {:?}", code); Ok(code) } -/// Compile a standard Python program to bytecode -pub fn compile_program( - ast: &[located_ast::Stmt], - source_path: String, - opts: CompileOpts, -) -> CompileResult { - compile_impl( - ast, - source_path, - opts, - SymbolTable::scan_program, - Compiler::compile_program, - ) -} - /// Compile a Python program to bytecode for the context of a REPL pub fn compile_program_single( - ast: &[located_ast::Stmt], - source_path: String, + ast: &ModModule, + source_code: SourceCode<'_>, opts: CompileOpts, ) -> CompileResult { - compile_impl( - ast, - source_path, - opts, - SymbolTable::scan_program, - Compiler::compile_program_single, - ) + let symbol_table = SymbolTable::scan_program(ast, source_code.clone()) + .map_err(|e| e.into_codegen_error(source_code.path.to_owned()))?; + let mut compiler = Compiler::new(opts, source_code, "".to_owned()); + compiler.compile_program_single(&ast.body, symbol_table)?; + let code = compiler.pop_code_object(); + trace!("Compilation completed: {:?}", code); + Ok(code) } pub fn compile_block_expression( - ast: &[located_ast::Stmt], - source_path: String, + ast: &ModModule, + source_code: SourceCode<'_>, opts: CompileOpts, ) -> CompileResult { - compile_impl( - ast, - source_path, - opts, - SymbolTable::scan_program, - Compiler::compile_block_expr, - ) + let symbol_table = SymbolTable::scan_program(ast, source_code.clone()) + .map_err(|e| e.into_codegen_error(source_code.path.to_owned()))?; + let mut compiler = Compiler::new(opts, source_code, "".to_owned()); + compiler.compile_block_expr(&ast.body, symbol_table)?; + let code = compiler.pop_code_object(); + trace!("Compilation completed: {:?}", code); + Ok(code) } pub fn compile_expression( - ast: &located_ast::Expr, - source_path: String, + ast: &ModExpression, + source_code: SourceCode<'_>, opts: CompileOpts, ) -> CompileResult { - compile_impl( - ast, - source_path, - opts, - SymbolTable::scan_expr, - Compiler::compile_eval, - ) + let symbol_table = SymbolTable::scan_expr(ast, source_code.clone()) + .map_err(|e| e.into_codegen_error(source_code.path.to_owned()))?; + let mut compiler = Compiler::new(opts, source_code, "".to_owned()); + compiler.compile_eval(ast, symbol_table)?; + let code = compiler.pop_code_object(); + Ok(code) } macro_rules! emit { @@ -220,15 +210,15 @@ struct PatternContext { allow_irrefutable: bool, } -impl Compiler { - fn new(opts: CompileOpts, source_path: String, code_name: String) -> Self { +impl<'src> Compiler<'src> { + fn new(opts: CompileOpts, source_code: SourceCode<'src>, code_name: String) -> Self { let module_code = ir::CodeInfo { flags: bytecode::CodeFlags::NEW_LOCALS, posonlyarg_count: 0, arg_count: 0, kwonlyarg_count: 0, - source_path: source_path.clone(), - first_line_number: LineNumber::MIN, + source_path: source_code.path.to_owned(), + first_line_number: OneIndexed::MIN, obj_name: code_name, blocks: vec![ir::Block::default()], @@ -242,10 +232,11 @@ impl Compiler { Compiler { code_stack: vec![module_code], symbol_table_stack: Vec::new(), - source_path, - current_source_location: SourceLocation::default(), + source_code, + // current_source_location: SourceLocation::default(), + current_source_range: TextRange::default(), qualified_path: Vec::new(), - done_with_future_stmts: false, + done_with_future_stmts: DoneWithFuture::No, future_annotations: false, ctx: CompileContext { loop_data: None, @@ -256,15 +247,18 @@ impl Compiler { opts, } } +} +impl Compiler<'_> { fn error(&mut self, error: CodegenErrorType) -> CodegenError { - self.error_loc(error, self.current_source_location) + self.error_ranged(error, self.current_source_range) } - fn error_loc(&mut self, error: CodegenErrorType, location: SourceLocation) -> CodegenError { + fn error_ranged(&mut self, error: CodegenErrorType, range: TextRange) -> CodegenError { + let location = self.source_code.source_location(range.start()); CodegenError { error, location: Some(location), - source_path: self.source_path.clone(), + source_path: self.source_code.path.to_owned(), } } @@ -296,7 +290,7 @@ impl Compiler { kwonlyarg_count: u32, obj_name: String, ) { - let source_path = self.source_path.clone(); + let source_path = self.source_code.path.to_owned(); let first_line_number = self.get_source_line_number(); let table = self.push_symbol_table(); @@ -373,13 +367,13 @@ impl Compiler { fn compile_program( &mut self, - body: &[located_ast::Stmt], + body: &ModModule, symbol_table: SymbolTable, ) -> CompileResult<()> { let size_before = self.code_stack.len(); self.symbol_table_stack.push(symbol_table); - let (doc, statements) = split_doc(body, &self.opts); + let (doc, statements) = split_doc(&body.body, &self.opts); if let Some(value) = doc { self.emit_load_const(ConstantData::Str { value }); let doc = self.name("__doc__"); @@ -401,14 +395,14 @@ impl Compiler { fn compile_program_single( &mut self, - body: &[located_ast::Stmt], + body: &[Stmt], symbol_table: SymbolTable, ) -> CompileResult<()> { self.symbol_table_stack.push(symbol_table); if let Some((last, body)) = body.split_last() { for statement in body { - if let located_ast::Stmt::Expr(located_ast::StmtExpr { value, .. }) = &statement { + if let Stmt::Expr(StmtExpr { value, .. }) = &statement { self.compile_expression(value)?; emit!(self, Instruction::PrintExpr); } else { @@ -416,7 +410,7 @@ impl Compiler { } } - if let located_ast::Stmt::Expr(located_ast::StmtExpr { value, .. }) = &last { + if let Stmt::Expr(StmtExpr { value, .. }) = &last { self.compile_expression(value)?; emit!(self, Instruction::Duplicate); emit!(self, Instruction::PrintExpr); @@ -434,7 +428,7 @@ impl Compiler { fn compile_block_expr( &mut self, - body: &[located_ast::Stmt], + body: &[Stmt], symbol_table: SymbolTable, ) -> CompileResult<()> { self.symbol_table_stack.push(symbol_table); @@ -443,12 +437,10 @@ impl Compiler { if let Some(last_statement) = body.last() { match last_statement { - located_ast::Stmt::Expr(_) => { + Stmt::Expr(_) => { self.current_block().instructions.pop(); // pop Instruction::Pop } - located_ast::Stmt::FunctionDef(_) - | located_ast::Stmt::AsyncFunctionDef(_) - | located_ast::Stmt::ClassDef(_) => { + Stmt::FunctionDef(_) | Stmt::ClassDef(_) => { let store_inst = self.current_block().instructions.pop().unwrap(); // pop Instruction::Store emit!(self, Instruction::Duplicate); self.current_block().instructions.push(store_inst); @@ -464,16 +456,16 @@ impl Compiler { // Compile statement in eval mode: fn compile_eval( &mut self, - expression: &located_ast::Expr, + expression: &ModExpression, symbol_table: SymbolTable, ) -> CompileResult<()> { self.symbol_table_stack.push(symbol_table); - self.compile_expression(expression)?; + self.compile_expression(&expression.body)?; self.emit_return_value(); Ok(()) } - fn compile_statements(&mut self, statements: &[located_ast::Stmt]) -> CompileResult<()> { + fn compile_statements(&mut self, statements: &[Stmt]) -> CompileResult<()> { for statement in statements { self.compile_statement(statement)? } @@ -508,7 +500,7 @@ impl Compiler { let symbol_table = self.symbol_table_stack.last().unwrap(); let symbol = symbol_table.lookup(name.as_ref()).unwrap_or_else(|| - panic!("The symbol '{name}' must be present in the symbol table, even when it is undefined in python."), + unreachable!("the symbol '{name}' should be present in the symbol table, even when it is undefined in python."), ); let info = self.code_stack.last_mut().unwrap(); let mut cache = &mut info.name_cache; @@ -583,22 +575,28 @@ impl Compiler { Ok(()) } - fn compile_statement(&mut self, statement: &located_ast::Stmt) -> CompileResult<()> { - use located_ast::*; - + fn compile_statement(&mut self, statement: &Stmt) -> CompileResult<()> { + use ruff_python_ast::*; trace!("Compiling {:?}", statement); - self.set_source_location(statement.location()); + self.set_source_range(statement.range()); match &statement { // we do this here because `from __future__` still executes that `from` statement at runtime, // we still need to compile the ImportFrom down below - Stmt::ImportFrom(located_ast::StmtImportFrom { module, names, .. }) + Stmt::ImportFrom(StmtImportFrom { module, names, .. }) if module.as_ref().map(|id| id.as_str()) == Some("__future__") => { self.compile_future_features(names)? } + // ignore module-level doc comments + Stmt::Expr(StmtExpr { value, .. }) + if matches!(&**value, Expr::StringLiteral(..)) + && matches!(self.done_with_future_stmts, DoneWithFuture::No) => + { + self.done_with_future_stmts = DoneWithFuture::DoneWithDoc + } // if we find any other statement, stop accepting future statements - _ => self.done_with_future_stmts = true, + _ => self.done_with_future_stmts = DoneWithFuture::Yes, } match &statement { @@ -633,9 +631,9 @@ impl Compiler { let from_list = if import_star { if self.ctx.in_func() { - return Err(self.error_loc( + return Err(self.error_ranged( CodegenErrorType::FunctionImportStar, - statement.location(), + statement.range(), )); } vec![ConstantData::Str { @@ -654,7 +652,7 @@ impl Compiler { // from .... import (*fromlist) self.emit_load_const(ConstantData::Integer { - value: level.as_ref().map_or(0, |level| level.to_u32()).into(), + value: (*level).into(), }); self.emit_load_const(ConstantData::Tuple { elements: from_list, @@ -699,52 +697,76 @@ impl Compiler { // Handled during symbol table construction. } Stmt::If(StmtIf { - test, body, orelse, .. + test, + body, + elif_else_clauses, + .. }) => { - let after_block = self.new_block(); - if orelse.is_empty() { - // Only if: - self.compile_jump_if(test, false, after_block)?; - self.compile_statements(body)?; - } else { - // if - else: - let else_block = self.new_block(); - self.compile_jump_if(test, false, else_block)?; - self.compile_statements(body)?; - emit!( - self, - Instruction::Jump { - target: after_block, + match elif_else_clauses.as_slice() { + // Only if + [] => { + let after_block = self.new_block(); + self.compile_jump_if(test, false, after_block)?; + self.compile_statements(body)?; + self.switch_to_block(after_block); + } + // If, elif*, elif/else + [rest @ .., tail] => { + let after_block = self.new_block(); + let mut next_block = self.new_block(); + + self.compile_jump_if(test, false, next_block)?; + self.compile_statements(body)?; + emit!( + self, + Instruction::Jump { + target: after_block + } + ); + + for clause in rest { + self.switch_to_block(next_block); + next_block = self.new_block(); + if let Some(test) = &clause.test { + self.compile_jump_if(test, false, next_block)?; + } else { + unreachable!() // must be elif + } + self.compile_statements(&clause.body)?; + emit!( + self, + Instruction::Jump { + target: after_block + } + ); } - ); - // else: - self.switch_to_block(else_block); - self.compile_statements(orelse)?; + self.switch_to_block(next_block); + if let Some(test) = &tail.test { + self.compile_jump_if(test, false, after_block)?; + } + self.compile_statements(&tail.body)?; + self.switch_to_block(after_block); + } } - self.switch_to_block(after_block); } Stmt::While(StmtWhile { test, body, orelse, .. }) => self.compile_while(test, body, orelse)?, - Stmt::With(StmtWith { items, body, .. }) => self.compile_with(items, body, false)?, - Stmt::AsyncWith(StmtAsyncWith { items, body, .. }) => { - self.compile_with(items, body, true)? - } - Stmt::For(StmtFor { - target, - iter, + Stmt::With(StmtWith { + items, body, - orelse, + is_async, .. - }) => self.compile_for(target, iter, body, orelse, false)?, - Stmt::AsyncFor(StmtAsyncFor { + }) => self.compile_with(items, body, *is_async)?, + Stmt::For(StmtFor { target, iter, body, orelse, + is_async, .. - }) => self.compile_for(target, iter, body, orelse, true)?, + }) => self.compile_for(target, iter, body, orelse, *is_async)?, Stmt::Match(StmtMatch { subject, cases, .. }) => self.compile_match(subject, cases)?, Stmt::Raise(StmtRaise { exc, cause, .. }) => { let kind = match exc { @@ -767,64 +789,46 @@ impl Compiler { handlers, orelse, finalbody, + is_star, .. - }) => self.compile_try_statement(body, handlers, orelse, finalbody)?, - Stmt::TryStar(StmtTryStar { - body, - handlers, - orelse, - finalbody, - .. - }) => self.compile_try_star_statement(body, handlers, orelse, finalbody)?, + }) => { + if *is_star { + self.compile_try_star_statement(body, handlers, orelse, finalbody)? + } else { + self.compile_try_statement(body, handlers, orelse, finalbody)? + } + } Stmt::FunctionDef(StmtFunctionDef { name, - args, + parameters, body, decorator_list, returns, type_params, + is_async, .. }) => self.compile_function_def( name.as_str(), - args, + parameters, body, decorator_list, returns.as_deref(), - false, - type_params, - )?, - Stmt::AsyncFunctionDef(StmtAsyncFunctionDef { - name, - args, - body, - decorator_list, - returns, - type_params, - .. - }) => self.compile_function_def( - name.as_str(), - args, - body, - decorator_list, - returns.as_deref(), - true, - type_params, + *is_async, + type_params.as_deref(), )?, Stmt::ClassDef(StmtClassDef { name, body, - bases, - keywords, decorator_list, type_params, + arguments, .. }) => self.compile_class_def( name.as_str(), body, - bases, - keywords, decorator_list, - type_params, + type_params.as_deref(), + arguments.as_deref(), )?, Stmt::Assert(StmtAssert { test, msg, .. }) => { // if some flag, ignore all assert statements! @@ -859,7 +863,7 @@ impl Compiler { } None => { return Err( - self.error_loc(CodegenErrorType::InvalidBreak, statement.location()) + self.error_ranged(CodegenErrorType::InvalidBreak, statement.range()) ); } }, @@ -869,14 +873,14 @@ impl Compiler { } None => { return Err( - self.error_loc(CodegenErrorType::InvalidContinue, statement.location()) + self.error_ranged(CodegenErrorType::InvalidContinue, statement.range()) ); } }, Stmt::Return(StmtReturn { value, .. }) => { if !self.ctx.in_func() { return Err( - self.error_loc(CodegenErrorType::InvalidReturn, statement.location()) + self.error_ranged(CodegenErrorType::InvalidReturn, statement.range()) ); } match value { @@ -887,9 +891,9 @@ impl Compiler { .flags .contains(bytecode::CodeFlags::IS_GENERATOR) { - return Err(self.error_loc( + return Err(self.error_ranged( CodegenErrorType::AsyncReturnValue, - statement.location(), + statement.range(), )); } self.compile_expression(v)?; @@ -933,13 +937,20 @@ impl Compiler { value, .. }) => { - let name_string = name.to_string(); - if !type_params.is_empty() { + // let name_string = name.to_string(); + let Some(name) = name.as_name_expr() else { + // FIXME: is error here? + return Err(self.error(CodegenErrorType::SyntaxError( + "type alias expect name".to_owned(), + ))); + }; + let name_string = name.id.to_string(); + if type_params.is_some() { self.push_symbol_table(); } self.compile_expression(value)?; - self.compile_type_params(type_params)?; - if !type_params.is_empty() { + if let Some(type_params) = type_params { + self.compile_type_params(type_params)?; self.pop_symbol_table(); } self.emit_load_const(ConstantData::Str { @@ -948,33 +959,32 @@ impl Compiler { emit!(self, Instruction::TypeAlias); self.store_name(&name_string)?; } + Stmt::IpyEscapeCommand(_) => todo!(), } Ok(()) } - fn compile_delete(&mut self, expression: &located_ast::Expr) -> CompileResult<()> { + fn compile_delete(&mut self, expression: &Expr) -> CompileResult<()> { + use ruff_python_ast::*; match &expression { - located_ast::Expr::Name(located_ast::ExprName { id, .. }) => { - self.compile_name(id.as_str(), NameUsage::Delete)? - } - located_ast::Expr::Attribute(located_ast::ExprAttribute { value, attr, .. }) => { + Expr::Name(ExprName { id, .. }) => self.compile_name(id.as_str(), NameUsage::Delete)?, + Expr::Attribute(ExprAttribute { value, attr, .. }) => { self.check_forbidden_name(attr.as_str(), NameUsage::Delete)?; self.compile_expression(value)?; let idx = self.name(attr.as_str()); emit!(self, Instruction::DeleteAttr { idx }); } - located_ast::Expr::Subscript(located_ast::ExprSubscript { value, slice, .. }) => { + Expr::Subscript(ExprSubscript { value, slice, .. }) => { self.compile_expression(value)?; self.compile_expression(slice)?; emit!(self, Instruction::DeleteSubscript); } - located_ast::Expr::Tuple(located_ast::ExprTuple { elts, .. }) - | located_ast::Expr::List(located_ast::ExprList { elts, .. }) => { + Expr::Tuple(ExprTuple { elts, .. }) | Expr::List(ExprList { elts, .. }) => { for element in elts { self.compile_delete(element)?; } } - located_ast::Expr::BinOp(_) | located_ast::Expr::UnaryOp(_) => { + Expr::BinOp(_) | Expr::UnaryOp(_) => { return Err(self.error(CodegenErrorType::Delete("expression"))); } _ => return Err(self.error(CodegenErrorType::Delete(expression.python_name()))), @@ -985,9 +995,13 @@ impl Compiler { fn enter_function( &mut self, name: &str, - args: &located_ast::Arguments, + parameters: &Parameters, ) -> CompileResult { - let defaults: Vec<_> = args.defaults().collect(); + let defaults: Vec<_> = std::iter::empty() + .chain(¶meters.posonlyargs) + .chain(¶meters.args) + .filter_map(|x| x.default.as_deref()) + .collect(); let have_defaults = !defaults.is_empty(); if have_defaults { // Construct a tuple: @@ -998,12 +1012,23 @@ impl Compiler { emit!(self, Instruction::BuildTuple { size }); } - let (kw_without_defaults, kw_with_defaults) = args.split_kwonlyargs(); + // TODO: partition_in_place + let mut kw_without_defaults = vec![]; + let mut kw_with_defaults = vec![]; + for kwonlyarg in ¶meters.kwonlyargs { + if let Some(default) = &kwonlyarg.default { + kw_with_defaults.push((&kwonlyarg.parameter, default)); + } else { + kw_without_defaults.push(&kwonlyarg.parameter); + } + } + + // let (kw_without_defaults, kw_with_defaults) = args.split_kwonlyargs(); if !kw_with_defaults.is_empty() { let default_kw_count = kw_with_defaults.len(); for (arg, default) in kw_with_defaults.iter() { self.emit_load_const(ConstantData::Str { - value: arg.arg.to_string(), + value: arg.name.to_string(), }); self.compile_expression(default)?; } @@ -1025,42 +1050,42 @@ impl Compiler { self.push_output( bytecode::CodeFlags::NEW_LOCALS | bytecode::CodeFlags::IS_OPTIMIZED, - args.posonlyargs.len().to_u32(), - (args.posonlyargs.len() + args.args.len()).to_u32(), - args.kwonlyargs.len().to_u32(), + parameters.posonlyargs.len().to_u32(), + (parameters.posonlyargs.len() + parameters.args.len()).to_u32(), + parameters.kwonlyargs.len().to_u32(), name.to_owned(), ); let args_iter = std::iter::empty() - .chain(&args.posonlyargs) - .chain(&args.args) - .map(|arg| arg.as_arg()) + .chain(¶meters.posonlyargs) + .chain(¶meters.args) + .map(|arg| &arg.parameter) .chain(kw_without_defaults) .chain(kw_with_defaults.into_iter().map(|(arg, _)| arg)); for name in args_iter { - self.varname(name.arg.as_str())?; + self.varname(name.name.as_str())?; } - if let Some(name) = args.vararg.as_deref() { + if let Some(name) = parameters.vararg.as_deref() { self.current_code_info().flags |= bytecode::CodeFlags::HAS_VARARGS; - self.varname(name.arg.as_str())?; + self.varname(name.name.as_str())?; } - if let Some(name) = args.kwarg.as_deref() { + if let Some(name) = parameters.kwarg.as_deref() { self.current_code_info().flags |= bytecode::CodeFlags::HAS_VARKEYWORDS; - self.varname(name.arg.as_str())?; + self.varname(name.name.as_str())?; } Ok(func_flags) } - fn prepare_decorators(&mut self, decorator_list: &[located_ast::Expr]) -> CompileResult<()> { + fn prepare_decorators(&mut self, decorator_list: &[Decorator]) -> CompileResult<()> { for decorator in decorator_list { - self.compile_expression(decorator)?; + self.compile_expression(&decorator.expression)?; } Ok(()) } - fn apply_decorators(&mut self, decorator_list: &[located_ast::Expr]) { + fn apply_decorators(&mut self, decorator_list: &[Decorator]) { // Apply decorators: for _ in decorator_list { emit!(self, Instruction::CallFunctionPositional { nargs: 1 }); @@ -1069,14 +1094,10 @@ impl Compiler { /// Store each type parameter so it is accessible to the current scope, and leave a tuple of /// all the type parameters on the stack. - fn compile_type_params(&mut self, type_params: &[located_ast::TypeParam]) -> CompileResult<()> { - for type_param in type_params { + fn compile_type_params(&mut self, type_params: &TypeParams) -> CompileResult<()> { + for type_param in &type_params.type_params { match type_param { - located_ast::TypeParam::TypeVar(located_ast::TypeParamTypeVar { - name, - bound, - .. - }) => { + TypeParam::TypeVar(TypeParamTypeVar { name, bound, .. }) => { if let Some(expr) = &bound { self.compile_expression(expr)?; self.emit_load_const(ConstantData::Str { @@ -1095,9 +1116,7 @@ impl Compiler { self.store_name(name.as_ref())?; } } - located_ast::TypeParam::ParamSpec(located_ast::TypeParamParamSpec { - name, .. - }) => { + TypeParam::ParamSpec(TypeParamParamSpec { name, .. }) => { self.emit_load_const(ConstantData::Str { value: name.to_string(), }); @@ -1105,10 +1124,7 @@ impl Compiler { emit!(self, Instruction::Duplicate); self.store_name(name.as_ref())?; } - located_ast::TypeParam::TypeVarTuple(located_ast::TypeParamTypeVarTuple { - name, - .. - }) => { + TypeParam::TypeVarTuple(TypeParamTypeVarTuple { name, .. }) => { self.emit_load_const(ConstantData::Str { value: name.to_string(), }); @@ -1129,10 +1145,10 @@ impl Compiler { fn compile_try_statement( &mut self, - body: &[located_ast::Stmt], - handlers: &[located_ast::ExceptHandler], - orelse: &[located_ast::Stmt], - finalbody: &[located_ast::Stmt], + body: &[Stmt], + handlers: &[ExceptHandler], + orelse: &[Stmt], + finalbody: &[Stmt], ) -> CompileResult<()> { let handler_block = self.new_block(); let finally_block = self.new_block(); @@ -1164,11 +1180,9 @@ impl Compiler { self.switch_to_block(handler_block); // Exception is on top of stack now for handler in handlers { - let located_ast::ExceptHandler::ExceptHandler( - located_ast::ExceptHandlerExceptHandler { - type_, name, body, .. - }, - ) = &handler; + let ExceptHandler::ExceptHandler(ExceptHandlerExceptHandler { + type_, name, body, .. + }) = &handler; let next_handler = self.new_block(); // If we gave a typ, @@ -1261,10 +1275,10 @@ impl Compiler { fn compile_try_star_statement( &mut self, - _body: &[located_ast::Stmt], - _handlers: &[located_ast::ExceptHandler], - _orelse: &[located_ast::Stmt], - _finalbody: &[located_ast::Stmt], + _body: &[Stmt], + _handlers: &[ExceptHandler], + _orelse: &[Stmt], + _finalbody: &[Stmt], ) -> CompileResult<()> { Err(self.error(CodegenErrorType::NotImplementedYet)) } @@ -1277,21 +1291,21 @@ impl Compiler { fn compile_function_def( &mut self, name: &str, - args: &located_ast::Arguments, - body: &[located_ast::Stmt], - decorator_list: &[located_ast::Expr], - returns: Option<&located_ast::Expr>, // TODO: use type hint somehow.. + parameters: &Parameters, + body: &[Stmt], + decorator_list: &[Decorator], + returns: Option<&Expr>, // TODO: use type hint somehow.. is_async: bool, - type_params: &[located_ast::TypeParam], + type_params: Option<&TypeParams>, ) -> CompileResult<()> { self.prepare_decorators(decorator_list)?; // If there are type params, we need to push a special symbol table just for them - if !type_params.is_empty() { + if type_params.is_some() { self.push_symbol_table(); } - let mut func_flags = self.enter_function(name, args)?; + let mut func_flags = self.enter_function(name, parameters)?; self.current_code_info() .flags .set(bytecode::CodeFlags::IS_COROUTINE, is_async); @@ -1323,7 +1337,7 @@ impl Compiler { // Emit None at end: match body.last() { - Some(located_ast::Stmt::Return(_)) => { + Some(Stmt::Return(_)) => { // the last instruction is a ReturnValue already, we don't need to emit it } _ => { @@ -1337,7 +1351,7 @@ impl Compiler { self.ctx = prev_ctx; // Prepare generic type parameters: - if !type_params.is_empty() { + if let Some(type_params) = type_params { self.compile_type_params(type_params)?; func_flags |= bytecode::MakeFunctionFlags::TYPE_PARAMS; } @@ -1356,17 +1370,17 @@ impl Compiler { num_annotations += 1; } - let args_iter = std::iter::empty() - .chain(&args.posonlyargs) - .chain(&args.args) - .chain(&args.kwonlyargs) - .map(|arg| arg.as_arg()) - .chain(args.vararg.as_deref()) - .chain(args.kwarg.as_deref()); - for arg in args_iter { - if let Some(annotation) = &arg.annotation { + let parameters_iter = std::iter::empty() + .chain(¶meters.posonlyargs) + .chain(¶meters.args) + .chain(¶meters.kwonlyargs) + .map(|x| &x.parameter) + .chain(parameters.vararg.as_deref()) + .chain(parameters.kwarg.as_deref()); + for param in parameters_iter { + if let Some(annotation) = ¶m.annotation { self.emit_load_const(ConstantData::Str { - value: self.mangle(arg.arg.as_str()).into_owned(), + value: self.mangle(param.name.as_str()).into_owned(), }); self.compile_annotation(annotation)?; num_annotations += 1; @@ -1388,7 +1402,7 @@ impl Compiler { } // Pop the special type params symbol table - if !type_params.is_empty() { + if type_params.is_some() { self.pop_symbol_table(); } @@ -1424,7 +1438,7 @@ impl Compiler { let symbol = table.lookup(var).unwrap_or_else(|| { panic!( "couldn't look up var {} in {} in {}", - var, code.obj_name, self.source_path + var, code.obj_name, self.source_code.path ) }); let parent_code = self.code_stack.last().unwrap(); @@ -1453,17 +1467,21 @@ impl Compiler { } // Python/compile.c find_ann - fn find_ann(body: &[located_ast::Stmt]) -> bool { - use located_ast::*; - + fn find_ann(body: &[Stmt]) -> bool { + use ruff_python_ast::*; for statement in body { let res = match &statement { Stmt::AnnAssign(_) => true, Stmt::For(StmtFor { body, orelse, .. }) => { Self::find_ann(body) || Self::find_ann(orelse) } - Stmt::If(StmtIf { body, orelse, .. }) => { - Self::find_ann(body) || Self::find_ann(orelse) + Stmt::If(StmtIf { + body, + elif_else_clauses, + .. + }) => { + Self::find_ann(body) + || elif_else_clauses.iter().any(|x| Self::find_ann(&x.body)) } Stmt::While(StmtWhile { body, orelse, .. }) => { Self::find_ann(body) || Self::find_ann(orelse) @@ -1487,16 +1505,13 @@ impl Compiler { fn compile_class_def( &mut self, name: &str, - body: &[located_ast::Stmt], - bases: &[located_ast::Expr], - keywords: &[located_ast::Keyword], - decorator_list: &[located_ast::Expr], - type_params: &[located_ast::TypeParam], + body: &[Stmt], + decorator_list: &[Decorator], + type_params: Option<&TypeParams>, + arguments: Option<&Arguments>, ) -> CompileResult<()> { self.prepare_decorators(decorator_list)?; - emit!(self, Instruction::LoadBuildClass); - let prev_ctx = self.ctx; self.ctx = CompileContext { func: FunctionContext::NoFunction, @@ -1519,7 +1534,7 @@ impl Compiler { let qualified_name = self.qualified_path.join("."); // If there are type params, we need to push a special symbol table just for them - if !type_params.is_empty() { + if type_params.is_some() { self.push_symbol_table(); } @@ -1571,10 +1586,12 @@ impl Compiler { self.qualified_path.append(global_path_prefix.as_mut()); self.ctx = prev_ctx; + emit!(self, Instruction::LoadBuildClass); + let mut func_flags = bytecode::MakeFunctionFlags::empty(); // Prepare generic type parameters: - if !type_params.is_empty() { + if let Some(type_params) = type_params { self.compile_type_params(type_params)?; func_flags |= bytecode::MakeFunctionFlags::TYPE_PARAMS; } @@ -1584,7 +1601,7 @@ impl Compiler { } // Pop the special type params symbol table - if !type_params.is_empty() { + if type_params.is_some() { self.pop_symbol_table(); } @@ -1602,7 +1619,12 @@ impl Compiler { value: name.to_owned(), }); - let call = self.compile_call_inner(2, bases, keywords)?; + // Call the __build_class__ builtin + let call = if let Some(arguments) = arguments { + self.compile_call_inner(2, arguments)? + } else { + CallType::Positional { nargs: 2 } + }; self.compile_normal_call(call); self.apply_decorators(decorator_list); @@ -1621,12 +1643,7 @@ impl Compiler { }); } - fn compile_while( - &mut self, - test: &located_ast::Expr, - body: &[located_ast::Stmt], - orelse: &[located_ast::Stmt], - ) -> CompileResult<()> { + fn compile_while(&mut self, test: &Expr, body: &[Stmt], orelse: &[Stmt]) -> CompileResult<()> { let while_block = self.new_block(); let else_block = self.new_block(); let after_block = self.new_block(); @@ -1654,11 +1671,11 @@ impl Compiler { fn compile_with( &mut self, - items: &[located_ast::WithItem], - body: &[located_ast::Stmt], + items: &[WithItem], + body: &[Stmt], is_async: bool, ) -> CompileResult<()> { - let with_location = self.current_source_location; + let with_range = self.current_source_range; let Some((item, items)) = items.split_first() else { return Err(self.error(CodegenErrorType::EmptyWithItems)); @@ -1668,7 +1685,7 @@ impl Compiler { let final_block = self.new_block(); self.compile_expression(&item.context_expr)?; - self.set_source_location(with_location); + self.set_source_range(with_range); if is_async { emit!(self, Instruction::BeforeAsyncWith); emit!(self, Instruction::GetAwaitable); @@ -1681,7 +1698,7 @@ impl Compiler { match &item.optional_vars { Some(var) => { - self.set_source_location(var.location()); + self.set_source_range(var.range()); self.compile_store(var)?; } None => { @@ -1697,13 +1714,13 @@ impl Compiler { } self.compile_statements(body)?; } else { - self.set_source_location(with_location); + self.set_source_range(with_range); self.compile_with(items, body, is_async)?; } // sort of "stack up" the layers of with blocks: // with a, b: body -> start_with(a) start_with(b) body() end_with(b) end_with(a) - self.set_source_location(with_location); + self.set_source_range(with_range); emit!(self, Instruction::PopBlock); emit!(self, Instruction::EnterFinally); @@ -1724,10 +1741,10 @@ impl Compiler { fn compile_for( &mut self, - target: &located_ast::Expr, - iter: &located_ast::Expr, - body: &[located_ast::Stmt], - orelse: &[located_ast::Stmt], + target: &Expr, + iter: &Expr, + body: &[Stmt], + orelse: &[Stmt], is_async: bool, ) -> CompileResult<()> { // Start loop @@ -1785,32 +1802,27 @@ impl Compiler { fn compile_pattern_value( &mut self, - value: &located_ast::PatternMatchValue, + value: &PatternMatchValue, _pattern_context: &mut PatternContext, ) -> CompileResult<()> { + use crate::compile::bytecode::ComparisonOperator::*; + self.compile_expression(&value.value)?; - emit!( - self, - Instruction::CompareOperation { - op: ComparisonOperator::Equal - } - ); + emit!(self, Instruction::CompareOperation { op: Equal }); Ok(()) } fn compile_pattern_as( &mut self, - as_pattern: &located_ast::PatternMatchAs, + as_pattern: &PatternMatchAs, pattern_context: &mut PatternContext, ) -> CompileResult<()> { if as_pattern.pattern.is_none() && !pattern_context.allow_irrefutable { // TODO: better error message if let Some(_name) = as_pattern.name.as_ref() { - return Err( - self.error_loc(CodegenErrorType::InvalidMatchCase, as_pattern.location()) - ); + return Err(self.error_ranged(CodegenErrorType::InvalidMatchCase, as_pattern.range)); } - return Err(self.error_loc(CodegenErrorType::InvalidMatchCase, as_pattern.location())); + return Err(self.error_ranged(CodegenErrorType::InvalidMatchCase, as_pattern.range)); } // Need to make a copy for (possibly) storing later: emit!(self, Instruction::Duplicate); @@ -1827,16 +1839,12 @@ impl Compiler { fn compile_pattern_inner( &mut self, - pattern_type: &located_ast::Pattern, + pattern_type: &Pattern, pattern_context: &mut PatternContext, ) -> CompileResult<()> { match &pattern_type { - located_ast::Pattern::MatchValue(value) => { - self.compile_pattern_value(value, pattern_context) - } - located_ast::Pattern::MatchAs(as_pattern) => { - self.compile_pattern_as(as_pattern, pattern_context) - } + Pattern::MatchValue(value) => self.compile_pattern_value(value, pattern_context), + Pattern::MatchAs(as_pattern) => self.compile_pattern_as(as_pattern, pattern_context), _ => { eprintln!("not implemented pattern type: {pattern_type:?}"); Err(self.error(CodegenErrorType::NotImplementedYet)) @@ -1846,7 +1854,7 @@ impl Compiler { fn compile_pattern( &mut self, - pattern_type: &located_ast::Pattern, + pattern_type: &Pattern, pattern_context: &mut PatternContext, ) -> CompileResult<()> { self.compile_pattern_inner(pattern_type, pattern_context)?; @@ -1861,8 +1869,8 @@ impl Compiler { fn compile_match_inner( &mut self, - subject: &located_ast::Expr, - cases: &[located_ast::MatchCase], + subject: &Expr, + cases: &[MatchCase], pattern_context: &mut PatternContext, ) -> CompileResult<()> { self.compile_expression(subject)?; @@ -1917,11 +1925,7 @@ impl Compiler { Ok(()) } - fn compile_match( - &mut self, - subject: &located_ast::Expr, - cases: &[located_ast::MatchCase], - ) -> CompileResult<()> { + fn compile_match(&mut self, subject: &Expr, cases: &[MatchCase]) -> CompileResult<()> { let mut pattern_context = PatternContext { current_block: usize::MAX, blocks: Vec::new(), @@ -1933,9 +1937,9 @@ impl Compiler { fn compile_chained_comparison( &mut self, - left: &located_ast::Expr, - ops: &[located_ast::CmpOp], - exprs: &[located_ast::Expr], + left: &Expr, + ops: &[CmpOp], + exprs: &[Expr], ) -> CompileResult<()> { assert!(!ops.is_empty()); assert_eq!(exprs.len(), ops.len()); @@ -1944,19 +1948,19 @@ impl Compiler { use bytecode::ComparisonOperator::*; use bytecode::TestOperator::*; - let compile_cmpop = |c: &mut Self, op: &located_ast::CmpOp| match op { - located_ast::CmpOp::Eq => emit!(c, Instruction::CompareOperation { op: Equal }), - located_ast::CmpOp::NotEq => emit!(c, Instruction::CompareOperation { op: NotEqual }), - located_ast::CmpOp::Lt => emit!(c, Instruction::CompareOperation { op: Less }), - located_ast::CmpOp::LtE => emit!(c, Instruction::CompareOperation { op: LessOrEqual }), - located_ast::CmpOp::Gt => emit!(c, Instruction::CompareOperation { op: Greater }), - located_ast::CmpOp::GtE => { + let compile_cmpop = |c: &mut Self, op: &CmpOp| match op { + CmpOp::Eq => emit!(c, Instruction::CompareOperation { op: Equal }), + CmpOp::NotEq => emit!(c, Instruction::CompareOperation { op: NotEqual }), + CmpOp::Lt => emit!(c, Instruction::CompareOperation { op: Less }), + CmpOp::LtE => emit!(c, Instruction::CompareOperation { op: LessOrEqual }), + CmpOp::Gt => emit!(c, Instruction::CompareOperation { op: Greater }), + CmpOp::GtE => { emit!(c, Instruction::CompareOperation { op: GreaterOrEqual }) } - located_ast::CmpOp::In => emit!(c, Instruction::TestOperation { op: In }), - located_ast::CmpOp::NotIn => emit!(c, Instruction::TestOperation { op: NotIn }), - located_ast::CmpOp::Is => emit!(c, Instruction::TestOperation { op: Is }), - located_ast::CmpOp::IsNot => emit!(c, Instruction::TestOperation { op: IsNot }), + CmpOp::In => emit!(c, Instruction::TestOperation { op: In }), + CmpOp::NotIn => emit!(c, Instruction::TestOperation { op: NotIn }), + CmpOp::Is => emit!(c, Instruction::TestOperation { op: Is }), + CmpOp::IsNot => emit!(c, Instruction::TestOperation { op: IsNot }), }; // a == b == c == d @@ -2021,10 +2025,14 @@ impl Compiler { Ok(()) } - fn compile_annotation(&mut self, annotation: &located_ast::Expr) -> CompileResult<()> { + fn compile_annotation(&mut self, annotation: &Expr) -> CompileResult<()> { if self.future_annotations { + // FIXME: codegen? + let ident = Default::default(); + let codegen = + ruff_python_codegen::Generator::new(&ident, Default::default(), Default::default()); self.emit_load_const(ConstantData::Str { - value: annotation.to_string(), + value: codegen.expr(annotation), }); } else { self.compile_expression(annotation)?; @@ -2034,9 +2042,9 @@ impl Compiler { fn compile_annotated_assign( &mut self, - target: &located_ast::Expr, - annotation: &located_ast::Expr, - value: Option<&located_ast::Expr>, + target: &Expr, + annotation: &Expr, + value: Option<&Expr>, ) -> CompileResult<()> { if let Some(value) = value { self.compile_expression(value)?; @@ -2051,7 +2059,7 @@ impl Compiler { // Compile annotation: self.compile_annotation(annotation)?; - if let located_ast::Expr::Name(located_ast::ExprName { id, .. }) = &target { + if let Expr::Name(ExprName { id, .. }) = &target { // Store as dict entry in __annotations__ dict: let annotations = self.name("__annotations__"); emit!(self, Instruction::LoadNameAny(annotations)); @@ -2067,29 +2075,26 @@ impl Compiler { Ok(()) } - fn compile_store(&mut self, target: &located_ast::Expr) -> CompileResult<()> { + fn compile_store(&mut self, target: &Expr) -> CompileResult<()> { match &target { - located_ast::Expr::Name(located_ast::ExprName { id, .. }) => { - self.store_name(id.as_str())? - } - located_ast::Expr::Subscript(located_ast::ExprSubscript { value, slice, .. }) => { + Expr::Name(ExprName { id, .. }) => self.store_name(id.as_str())?, + Expr::Subscript(ExprSubscript { value, slice, .. }) => { self.compile_expression(value)?; self.compile_expression(slice)?; emit!(self, Instruction::StoreSubscript); } - located_ast::Expr::Attribute(located_ast::ExprAttribute { value, attr, .. }) => { + Expr::Attribute(ExprAttribute { value, attr, .. }) => { self.check_forbidden_name(attr.as_str(), NameUsage::Store)?; self.compile_expression(value)?; let idx = self.name(attr.as_str()); emit!(self, Instruction::StoreAttr { idx }); } - located_ast::Expr::List(located_ast::ExprList { elts, .. }) - | located_ast::Expr::Tuple(located_ast::ExprTuple { elts, .. }) => { + Expr::List(ExprList { elts, .. }) | Expr::Tuple(ExprTuple { elts, .. }) => { let mut seen_star = false; // Scan for star args: for (i, element) in elts.iter().enumerate() { - if let located_ast::Expr::Starred(_) = &element { + if let Expr::Starred(_) = &element { if seen_star { return Err(self.error(CodegenErrorType::MultipleStarArgs)); } else { @@ -2098,9 +2103,9 @@ impl Compiler { let after = elts.len() - i - 1; let (before, after) = (|| Some((before.to_u8()?, after.to_u8()?)))() .ok_or_else(|| { - self.error_loc( + self.error_ranged( CodegenErrorType::TooManyStarUnpack, - target.location(), + target.range(), ) })?; let args = bytecode::UnpackExArgs { before, after }; @@ -2119,9 +2124,7 @@ impl Compiler { } for element in elts { - if let located_ast::Expr::Starred(located_ast::ExprStarred { value, .. }) = - &element - { + if let Expr::Starred(ExprStarred { value, .. }) = &element { self.compile_store(value)?; } else { self.compile_store(element)?; @@ -2130,7 +2133,7 @@ impl Compiler { } _ => { return Err(self.error(match target { - located_ast::Expr::Starred(_) => CodegenErrorType::SyntaxError( + Expr::Starred(_) => CodegenErrorType::SyntaxError( "starred assignment target must be in a list or tuple".to_owned(), ), _ => CodegenErrorType::Assign(target.python_name()), @@ -2143,9 +2146,9 @@ impl Compiler { fn compile_augassign( &mut self, - target: &located_ast::Expr, - op: &located_ast::Operator, - value: &located_ast::Expr, + target: &Expr, + op: &Operator, + value: &Expr, ) -> CompileResult<()> { enum AugAssignKind<'a> { Name { id: &'a str }, @@ -2154,19 +2157,19 @@ impl Compiler { } let kind = match &target { - located_ast::Expr::Name(located_ast::ExprName { id, .. }) => { + Expr::Name(ExprName { id, .. }) => { let id = id.as_str(); self.compile_name(id, NameUsage::Load)?; AugAssignKind::Name { id } } - located_ast::Expr::Subscript(located_ast::ExprSubscript { value, slice, .. }) => { + Expr::Subscript(ExprSubscript { value, slice, .. }) => { self.compile_expression(value)?; self.compile_expression(slice)?; emit!(self, Instruction::Duplicate2); emit!(self, Instruction::Subscript); AugAssignKind::Subscript } - located_ast::Expr::Attribute(located_ast::ExprAttribute { value, attr, .. }) => { + Expr::Attribute(ExprAttribute { value, attr, .. }) => { let attr = attr.as_str(); self.check_forbidden_name(attr, NameUsage::Store)?; self.compile_expression(value)?; @@ -2203,21 +2206,21 @@ impl Compiler { Ok(()) } - fn compile_op(&mut self, op: &located_ast::Operator, inplace: bool) { + fn compile_op(&mut self, op: &Operator, inplace: bool) { let op = match op { - located_ast::Operator::Add => bytecode::BinaryOperator::Add, - located_ast::Operator::Sub => bytecode::BinaryOperator::Subtract, - located_ast::Operator::Mult => bytecode::BinaryOperator::Multiply, - located_ast::Operator::MatMult => bytecode::BinaryOperator::MatrixMultiply, - located_ast::Operator::Div => bytecode::BinaryOperator::Divide, - located_ast::Operator::FloorDiv => bytecode::BinaryOperator::FloorDivide, - located_ast::Operator::Mod => bytecode::BinaryOperator::Modulo, - located_ast::Operator::Pow => bytecode::BinaryOperator::Power, - located_ast::Operator::LShift => bytecode::BinaryOperator::Lshift, - located_ast::Operator::RShift => bytecode::BinaryOperator::Rshift, - located_ast::Operator::BitOr => bytecode::BinaryOperator::Or, - located_ast::Operator::BitXor => bytecode::BinaryOperator::Xor, - located_ast::Operator::BitAnd => bytecode::BinaryOperator::And, + Operator::Add => bytecode::BinaryOperator::Add, + Operator::Sub => bytecode::BinaryOperator::Subtract, + Operator::Mult => bytecode::BinaryOperator::Multiply, + Operator::MatMult => bytecode::BinaryOperator::MatrixMultiply, + Operator::Div => bytecode::BinaryOperator::Divide, + Operator::FloorDiv => bytecode::BinaryOperator::FloorDivide, + Operator::Mod => bytecode::BinaryOperator::Modulo, + Operator::Pow => bytecode::BinaryOperator::Power, + Operator::LShift => bytecode::BinaryOperator::Lshift, + Operator::RShift => bytecode::BinaryOperator::Rshift, + Operator::BitOr => bytecode::BinaryOperator::Or, + Operator::BitXor => bytecode::BinaryOperator::Xor, + Operator::BitAnd => bytecode::BinaryOperator::And, }; if inplace { emit!(self, Instruction::BinaryOperationInplace { op }) @@ -2236,15 +2239,15 @@ impl Compiler { /// (indicated by the condition parameter). fn compile_jump_if( &mut self, - expression: &located_ast::Expr, + expression: &Expr, condition: bool, target_block: ir::BlockIdx, ) -> CompileResult<()> { // Compile expression for test, and jump to label if false match &expression { - located_ast::Expr::BoolOp(located_ast::ExprBoolOp { op, values, .. }) => { + Expr::BoolOp(ExprBoolOp { op, values, .. }) => { match op { - located_ast::BoolOp::And => { + BoolOp::And => { if condition { // If all values are true. let end_block = self.new_block(); @@ -2265,7 +2268,7 @@ impl Compiler { } } } - located_ast::BoolOp::Or => { + BoolOp::Or => { if condition { // If any of the values is true. for value in values { @@ -2288,8 +2291,8 @@ impl Compiler { } } } - located_ast::Expr::UnaryOp(located_ast::ExprUnaryOp { - op: located_ast::UnaryOp::Not, + Expr::UnaryOp(ExprUnaryOp { + op: UnaryOp::Not, operand, .. }) => { @@ -2320,11 +2323,7 @@ impl Compiler { /// Compile a boolean operation as an expression. /// This means, that the last value remains on the stack. - fn compile_bool_op( - &mut self, - op: &located_ast::BoolOp, - values: &[located_ast::Expr], - ) -> CompileResult<()> { + fn compile_bool_op(&mut self, op: &BoolOp, values: &[Expr]) -> CompileResult<()> { let after_block = self.new_block(); let (last_value, values) = values.split_last().unwrap(); @@ -2332,7 +2331,7 @@ impl Compiler { self.compile_expression(value)?; match op { - located_ast::BoolOp::And => { + BoolOp::And => { emit!( self, Instruction::JumpIfFalseOrPop { @@ -2340,7 +2339,7 @@ impl Compiler { } ); } - located_ast::BoolOp::Or => { + BoolOp::Or => { emit!( self, Instruction::JumpIfTrueOrPop { @@ -2357,44 +2356,36 @@ impl Compiler { Ok(()) } - fn compile_dict( - &mut self, - keys: &[Option], - values: &[located_ast::Expr], - ) -> CompileResult<()> { + fn compile_dict(&mut self, items: &[DictItem]) -> CompileResult<()> { + // FIXME: correct order to build map, etc d = {**a, 'key': 2} should override + // 'key' in dict a let mut size = 0; - let (packed, unpacked): (Vec<_>, Vec<_>) = keys - .iter() - .zip(values.iter()) - .partition(|(k, _)| k.is_some()); - for (key, value) in packed { - self.compile_expression(key.as_ref().unwrap())?; - self.compile_expression(value)?; + let (packed, unpacked): (Vec<_>, Vec<_>) = items.iter().partition(|x| x.key.is_some()); + for item in packed { + self.compile_expression(item.key.as_ref().unwrap())?; + self.compile_expression(&item.value)?; size += 1; } emit!(self, Instruction::BuildMap { size }); - for (_, value) in unpacked { - self.compile_expression(value)?; + for item in unpacked { + self.compile_expression(&item.value)?; emit!(self, Instruction::DictUpdate); } Ok(()) } - fn compile_expression(&mut self, expression: &located_ast::Expr) -> CompileResult<()> { - use located_ast::*; + fn compile_expression(&mut self, expression: &Expr) -> CompileResult<()> { + use ruff_python_ast::*; trace!("Compiling {:?}", expression); - let location = expression.location(); - self.set_source_location(location); + let range = expression.range(); + self.set_source_range(range); match &expression { Expr::Call(ExprCall { - func, - args, - keywords, - .. - }) => self.compile_call(func, args, keywords)?, + func, arguments, .. + }) => self.compile_call(func, arguments)?, Expr::BoolOp(ExprBoolOp { op, values, .. }) => self.compile_bool_op(op, values)?, Expr::BinOp(ExprBinOp { left, op, right, .. @@ -2435,9 +2426,9 @@ impl Compiler { }) => { self.compile_chained_comparison(left, ops, comparators)?; } - Expr::Constant(ExprConstant { value, .. }) => { - self.emit_load_const(compile_constant(value)); - } + // Expr::Constant(ExprConstant { value, .. }) => { + // self.emit_load_const(compile_constant(value)); + // } Expr::List(ExprList { elts, .. }) => { let (size, unpack) = self.gather_elements(0, elts)?; if unpack { @@ -2462,13 +2453,13 @@ impl Compiler { emit!(self, Instruction::BuildSet { size }); } } - Expr::Dict(ExprDict { keys, values, .. }) => { - self.compile_dict(keys, values)?; + Expr::Dict(ExprDict { items, .. }) => { + self.compile_dict(items)?; } Expr::Slice(ExprSlice { lower, upper, step, .. }) => { - let mut compile_bound = |bound: Option<&located_ast::Expr>| match bound { + let mut compile_bound = |bound: Option<&Expr>| match bound { Some(exp) => self.compile_expression(exp), None => { self.emit_load_const(ConstantData::None); @@ -2519,47 +2510,15 @@ impl Compiler { self.emit_load_const(ConstantData::None); emit!(self, Instruction::YieldFrom); } - Expr::JoinedStr(ExprJoinedStr { values, .. }) => { - if let Some(value) = try_get_constant_string(values) { - self.emit_load_const(ConstantData::Str { value }) - } else { - for value in values { - self.compile_expression(value)?; - } - emit!( - self, - Instruction::BuildString { - size: values.len().to_u32(), - } - ) - } - } - Expr::FormattedValue(ExprFormattedValue { - value, - conversion, - format_spec, - .. + Expr::Name(ExprName { id, .. }) => self.load_name(id.as_str())?, + Expr::Lambda(ExprLambda { + parameters, body, .. }) => { - match format_spec { - Some(spec) => self.compile_expression(spec)?, - None => self.emit_load_const(ConstantData::Str { - value: String::new(), - }), - }; - self.compile_expression(value)?; - emit!( - self, - Instruction::FormatValue { - conversion: *conversion, - }, - ); - } - Expr::Name(located_ast::ExprName { id, .. }) => self.load_name(id.as_str())?, - Expr::Lambda(located_ast::ExprLambda { args, body, .. }) => { let prev_ctx = self.ctx; let name = "".to_owned(); - let mut func_flags = self.enter_function(&name, args)?; + let mut func_flags = self + .enter_function(&name, parameters.as_deref().unwrap_or(&Default::default()))?; self.ctx = CompileContext { loop_data: Option::None, @@ -2586,7 +2545,7 @@ impl Compiler { self.ctx = prev_ctx; } - Expr::ListComp(located_ast::ExprListComp { + Expr::ListComp(ExprListComp { elt, generators, .. }) => { self.compile_comprehension( @@ -2609,7 +2568,7 @@ impl Compiler { Self::contains_await(elt), )?; } - Expr::SetComp(located_ast::ExprSetComp { + Expr::SetComp(ExprSetComp { elt, generators, .. }) => { self.compile_comprehension( @@ -2632,7 +2591,7 @@ impl Compiler { Self::contains_await(elt), )?; } - Expr::DictComp(located_ast::ExprDictComp { + Expr::DictComp(ExprDictComp { key, value, generators, @@ -2662,7 +2621,7 @@ impl Compiler { Self::contains_await(key) || Self::contains_await(value), )?; } - Expr::GeneratorExp(located_ast::ExprGeneratorExp { + Expr::Generator(ExprGenerator { elt, generators, .. }) => { self.compile_comprehension( @@ -2684,7 +2643,7 @@ impl Compiler { Expr::Starred(_) => { return Err(self.error(CodegenErrorType::InvalidStarExpr)); } - Expr::IfExp(located_ast::ExprIfExp { + Expr::If(ExprIf { test, body, orelse, .. }) => { let else_block = self.new_block(); @@ -2708,7 +2667,7 @@ impl Compiler { self.switch_to_block(after_block); } - Expr::NamedExpr(located_ast::ExprNamedExpr { + Expr::Named(ExprNamed { target, value, range: _, @@ -2717,11 +2676,50 @@ impl Compiler { emit!(self, Instruction::Duplicate); self.compile_store(target)?; } + Expr::FString(fstring) => { + self.compile_expr_fstring(fstring)?; + } + Expr::StringLiteral(string) => { + self.emit_load_const(ConstantData::Str { + value: string.value.to_str().to_owned(), + }); + } + Expr::BytesLiteral(bytes) => { + let iter = bytes.value.iter().flat_map(|x| x.iter().copied()); + let v: Vec = iter.collect(); + self.emit_load_const(ConstantData::Bytes { value: v }); + } + Expr::NumberLiteral(number) => match &number.value { + Number::Int(int) => { + let value = ruff_int_to_bigint(int).map_err(|e| self.error(e))?; + self.emit_load_const(ConstantData::Integer { value }); + } + Number::Float(float) => { + self.emit_load_const(ConstantData::Float { value: *float }); + } + Number::Complex { real, imag } => { + self.emit_load_const(ConstantData::Complex { + value: Complex::new(*real, *imag), + }); + } + }, + Expr::BooleanLiteral(b) => { + self.emit_load_const(ConstantData::Boolean { value: b.value }); + } + Expr::NoneLiteral(_) => { + self.emit_load_const(ConstantData::None); + } + Expr::EllipsisLiteral(_) => { + self.emit_load_const(ConstantData::Ellipsis); + } + Expr::IpyEscapeCommand(_) => { + panic!("unexpected ipy escape command"); + } } Ok(()) } - fn compile_keywords(&mut self, keywords: &[located_ast::Keyword]) -> CompileResult<()> { + fn compile_keywords(&mut self, keywords: &[Keyword]) -> CompileResult<()> { let mut size = 0; let groupby = keywords.iter().chunk_by(|e| e.arg.is_none()); for (is_unpacking, sub_keywords) in &groupby { @@ -2751,26 +2749,17 @@ impl Compiler { Ok(()) } - fn compile_call( - &mut self, - func: &located_ast::Expr, - args: &[located_ast::Expr], - keywords: &[located_ast::Keyword], - ) -> CompileResult<()> { - let method = - if let located_ast::Expr::Attribute(located_ast::ExprAttribute { - value, attr, .. - }) = &func - { - self.compile_expression(value)?; - let idx = self.name(attr.as_str()); - emit!(self, Instruction::LoadMethod { idx }); - true - } else { - self.compile_expression(func)?; - false - }; - let call = self.compile_call_inner(0, args, keywords)?; + fn compile_call(&mut self, func: &Expr, args: &Arguments) -> CompileResult<()> { + let method = if let Expr::Attribute(ExprAttribute { value, attr, .. }) = &func { + self.compile_expression(value)?; + let idx = self.name(attr.as_str()); + emit!(self, Instruction::LoadMethod { idx }); + true + } else { + self.compile_expression(func)?; + false + }; + let call = self.compile_call_inner(0, args)?; if method { self.compile_method_call(call) } else { @@ -2801,16 +2790,15 @@ impl Compiler { fn compile_call_inner( &mut self, additional_positional: u32, - args: &[located_ast::Expr], - keywords: &[located_ast::Keyword], + arguments: &Arguments, ) -> CompileResult { - let count = (args.len() + keywords.len()).to_u32() + additional_positional; + let count = u32::try_from(arguments.len()).unwrap() + additional_positional; // Normal arguments: - let (size, unpack) = self.gather_elements(additional_positional, args)?; - let has_double_star = keywords.iter().any(|k| k.arg.is_none()); + let (size, unpack) = self.gather_elements(additional_positional, &arguments.args)?; + let has_double_star = arguments.keywords.iter().any(|k| k.arg.is_none()); - for keyword in keywords { + for keyword in &arguments.keywords { if let Some(name) = &keyword.arg { self.check_forbidden_name(name.as_str(), NameUsage::Store)?; } @@ -2825,14 +2813,14 @@ impl Compiler { } // Create an optional map with kw-args: - let has_kwargs = !keywords.is_empty(); + let has_kwargs = !arguments.keywords.is_empty(); if has_kwargs { - self.compile_keywords(keywords)?; + self.compile_keywords(&arguments.keywords)?; } CallType::Ex { has_kwargs } - } else if !keywords.is_empty() { + } else if !arguments.keywords.is_empty() { let mut kwarg_names = vec![]; - for keyword in keywords { + for keyword in &arguments.keywords { if let Some(name) = &keyword.arg { kwarg_names.push(ConstantData::Str { value: name.to_string(), @@ -2857,15 +2845,9 @@ impl Compiler { // Given a vector of expr / star expr generate code which gives either // a list of expressions on the stack, or a list of tuples. - fn gather_elements( - &mut self, - before: u32, - elements: &[located_ast::Expr], - ) -> CompileResult<(u32, bool)> { + fn gather_elements(&mut self, before: u32, elements: &[Expr]) -> CompileResult<(u32, bool)> { // First determine if we have starred elements: - let has_stars = elements - .iter() - .any(|e| matches!(e, located_ast::Expr::Starred(_))); + let has_stars = elements.iter().any(|e| matches!(e, Expr::Starred(_))); let size = if has_stars { let mut size = 0; @@ -2878,9 +2860,7 @@ impl Compiler { let groups = elements .iter() .map(|element| { - if let located_ast::Expr::Starred(located_ast::ExprStarred { value, .. }) = - &element - { + if let Expr::Starred(ExprStarred { value, .. }) = &element { (true, value.as_ref()) } else { (false, element) @@ -2913,7 +2893,7 @@ impl Compiler { Ok((size, has_stars)) } - fn compile_comprehension_element(&mut self, element: &located_ast::Expr) -> CompileResult<()> { + fn compile_comprehension_element(&mut self, element: &Expr) -> CompileResult<()> { self.compile_expression(element).map_err(|e| { if let CodegenErrorType::InvalidStarExpr = e.error { self.error(CodegenErrorType::SyntaxError( @@ -2929,7 +2909,7 @@ impl Compiler { &mut self, name: &str, init_collection: Option, - generators: &[located_ast::Comprehension], + generators: &[Comprehension], compile_element: &dyn Fn(&mut Self) -> CompileResult<()>, comprehension_type: ComprehensionType, element_contains_await: bool, @@ -3110,13 +3090,11 @@ impl Compiler { Ok(()) } - fn compile_future_features( - &mut self, - features: &[located_ast::Alias], - ) -> Result<(), CodegenError> { - if self.done_with_future_stmts { + fn compile_future_features(&mut self, features: &[Alias]) -> Result<(), CodegenError> { + if let DoneWithFuture::Yes = self.done_with_future_stmts { return Err(self.error(CodegenErrorType::InvalidFuturePlacement)); } + self.done_with_future_stmts = DoneWithFuture::DoneWithDoc; for feature in features { match feature.name.as_str() { // Python 3 features; we've already implemented them by default @@ -3135,13 +3113,15 @@ impl Compiler { // Low level helper functions: fn _emit(&mut self, instr: Instruction, arg: OpArg, target: ir::BlockIdx) { - let location = self.current_source_location; + let range = self.current_source_range; + let location = self.source_code.source_location(range.start()); // TODO: insert source filename self.current_block().instructions.push(ir::InstructionInfo { instr, arg, target, location, + // range, }); } @@ -3220,13 +3200,13 @@ impl Compiler { code.current_block = block; } - fn set_source_location(&mut self, location: SourceLocation) { - self.current_source_location = location; + fn set_source_range(&mut self, range: TextRange) { + self.current_source_range = range; } - fn get_source_line_number(&mut self) -> LineNumber { - let location = self.current_source_location; - location.row + fn get_source_line_number(&mut self) -> OneIndexed { + self.source_code + .line_index(self.current_source_range.start()) } fn push_qualified_path(&mut self, name: &str) { @@ -3240,19 +3220,19 @@ impl Compiler { /// Whether the expression contains an await expression and /// thus requires the function to be async. /// Async with and async for are statements, so I won't check for them here - fn contains_await(expression: &located_ast::Expr) -> bool { - use located_ast::*; + fn contains_await(expression: &Expr) -> bool { + use ruff_python_ast::*; match &expression { Expr::Call(ExprCall { - func, - args, - keywords, - .. + func, arguments, .. }) => { Self::contains_await(func) - || args.iter().any(Self::contains_await) - || keywords.iter().any(|kw| Self::contains_await(&kw.value)) + || arguments.args.iter().any(Self::contains_await) + || arguments + .keywords + .iter() + .any(|kw| Self::contains_await(&kw.value)) } Expr::BoolOp(ExprBoolOp { values, .. }) => values.iter().any(Self::contains_await), Expr::BinOp(ExprBinOp { left, right, .. }) => { @@ -3266,14 +3246,13 @@ impl Compiler { Expr::Compare(ExprCompare { left, comparators, .. }) => Self::contains_await(left) || comparators.iter().any(Self::contains_await), - Expr::Constant(ExprConstant { .. }) => false, Expr::List(ExprList { elts, .. }) => elts.iter().any(Self::contains_await), Expr::Tuple(ExprTuple { elts, .. }) => elts.iter().any(Self::contains_await), Expr::Set(ExprSet { elts, .. }) => elts.iter().any(Self::contains_await), - Expr::Dict(ExprDict { keys, values, .. }) => { - keys.iter().flatten().any(Self::contains_await) - || values.iter().any(Self::contains_await) - } + Expr::Dict(ExprDict { items, .. }) => items + .iter() + .flat_map(|item| &item.key) + .any(Self::contains_await), Expr::Slice(ExprSlice { lower, upper, step, .. }) => { @@ -3286,33 +3265,21 @@ impl Compiler { } Expr::Await(ExprAwait { .. }) => true, Expr::YieldFrom(ExprYieldFrom { value, .. }) => Self::contains_await(value), - Expr::JoinedStr(ExprJoinedStr { values, .. }) => { - values.iter().any(Self::contains_await) - } - Expr::FormattedValue(ExprFormattedValue { - value, - conversion: _, - format_spec, - .. - }) => { - Self::contains_await(value) - || format_spec.as_deref().is_some_and(Self::contains_await) - } - Expr::Name(located_ast::ExprName { .. }) => false, - Expr::Lambda(located_ast::ExprLambda { body, .. }) => Self::contains_await(body), - Expr::ListComp(located_ast::ExprListComp { + Expr::Name(ExprName { .. }) => false, + Expr::Lambda(ExprLambda { body, .. }) => Self::contains_await(body), + Expr::ListComp(ExprListComp { elt, generators, .. }) => { Self::contains_await(elt) || generators.iter().any(|jen| Self::contains_await(&jen.iter)) } - Expr::SetComp(located_ast::ExprSetComp { + Expr::SetComp(ExprSetComp { elt, generators, .. }) => { Self::contains_await(elt) || generators.iter().any(|jen| Self::contains_await(&jen.iter)) } - Expr::DictComp(located_ast::ExprDictComp { + Expr::DictComp(ExprDictComp { key, value, generators, @@ -3322,14 +3289,14 @@ impl Compiler { || Self::contains_await(value) || generators.iter().any(|jen| Self::contains_await(&jen.iter)) } - Expr::GeneratorExp(located_ast::ExprGeneratorExp { + Expr::Generator(ExprGenerator { elt, generators, .. }) => { Self::contains_await(elt) || generators.iter().any(|jen| Self::contains_await(&jen.iter)) } Expr::Starred(expr) => Self::contains_await(&expr.value), - Expr::IfExp(located_ast::ExprIfExp { + Expr::If(ExprIf { test, body, orelse, .. }) => { Self::contains_await(test) @@ -3337,13 +3304,165 @@ impl Compiler { || Self::contains_await(orelse) } - Expr::NamedExpr(located_ast::ExprNamedExpr { + Expr::Named(ExprNamed { target, value, range: _, }) => Self::contains_await(target) || Self::contains_await(value), + Expr::FString(ExprFString { value, range: _ }) => { + fn expr_element_contains_await bool>( + expr_element: &FStringExpressionElement, + contains_await: F, + ) -> bool { + contains_await(&expr_element.expression) + || expr_element + .format_spec + .iter() + .flat_map(|spec| spec.elements.expressions()) + .any(|element| expr_element_contains_await(element, contains_await)) + } + + value.elements().any(|element| match element { + FStringElement::Expression(expr_element) => { + expr_element_contains_await(expr_element, Self::contains_await) + } + FStringElement::Literal(_) => false, + }) + } + Expr::StringLiteral(_) + | Expr::BytesLiteral(_) + | Expr::NumberLiteral(_) + | Expr::BooleanLiteral(_) + | Expr::NoneLiteral(_) + | Expr::EllipsisLiteral(_) + | Expr::IpyEscapeCommand(_) => false, } } + + fn compile_expr_fstring(&mut self, fstring: &ExprFString) -> CompileResult<()> { + let fstring = &fstring.value; + for part in fstring { + self.compile_fstring_part(part)?; + } + let part_count: u32 = fstring + .iter() + .len() + .try_into() + .expect("BuildString size overflowed"); + if part_count > 1 { + emit!(self, Instruction::BuildString { size: part_count }); + } + + Ok(()) + } + + fn compile_fstring_part(&mut self, part: &FStringPart) -> CompileResult<()> { + match part { + FStringPart::Literal(string) => { + self.emit_load_const(ConstantData::Str { + value: string.value.to_string(), + }); + Ok(()) + } + FStringPart::FString(fstring) => self.compile_fstring(fstring), + } + } + + fn compile_fstring(&mut self, fstring: &FString) -> CompileResult<()> { + self.compile_fstring_elements(&fstring.elements) + } + + fn compile_fstring_elements( + &mut self, + fstring_elements: &FStringElements, + ) -> CompileResult<()> { + for element in fstring_elements { + match element { + FStringElement::Literal(string) => { + self.emit_load_const(ConstantData::Str { + value: string.value.to_string(), + }); + } + FStringElement::Expression(fstring_expr) => { + let mut conversion = fstring_expr.conversion; + + let debug_text_count = match &fstring_expr.debug_text { + None => 0, + Some(DebugText { leading, trailing }) => { + let range = fstring_expr.expression.range(); + let source = self.source_code.get_range(range); + let source = source.to_string(); + + self.emit_load_const(ConstantData::Str { + value: leading.to_string(), + }); + self.emit_load_const(ConstantData::Str { value: source }); + self.emit_load_const(ConstantData::Str { + value: trailing.to_string(), + }); + + 3 + } + }; + + match &fstring_expr.format_spec { + None => { + self.emit_load_const(ConstantData::Str { + value: String::new(), + }); + // Match CPython behavior: If debug text is present, apply repr conversion. + // See: https://github.com/python/cpython/blob/f61afca262d3a0aa6a8a501db0b1936c60858e35/Parser/action_helpers.c#L1456 + if conversion == ConversionFlag::None && debug_text_count > 0 { + conversion = ConversionFlag::Repr; + } + } + Some(format_spec) => { + self.compile_fstring_elements(&format_spec.elements)?; + } + } + + self.compile_expression(&fstring_expr.expression)?; + + emit!( + self, + Instruction::FormatValue { + conversion: conversion + } + ); + + // concatenate formatted string and debug text (if present) + if debug_text_count > 0 { + emit!( + self, + Instruction::BuildString { + size: debug_text_count + 1 + } + ); + } + } + } + } + + let element_count: u32 = fstring_elements + .len() + .try_into() + .expect("BuildString size overflowed"); + if element_count == 0 { + // ensure to put an empty string on the stack if there aren't any fstring elements + self.emit_load_const(ConstantData::Str { + value: String::new(), + }); + } else if element_count > 1 { + emit!( + self, + Instruction::BuildString { + size: element_count + } + ); + } + + Ok(()) + } } trait EmitArg { @@ -3404,14 +3523,17 @@ fn clean_doc(doc: &str) -> String { } } -fn split_doc<'a>( - body: &'a [located_ast::Stmt], - opts: &CompileOpts, -) -> (Option, &'a [located_ast::Stmt]) { - if let Some((located_ast::Stmt::Expr(expr), body_rest)) = body.split_first() { - if let Some(doc) = try_get_constant_string(std::slice::from_ref(&expr.value)) { +fn split_doc<'a>(body: &'a [Stmt], opts: &CompileOpts) -> (Option, &'a [Stmt]) { + if let Some((Stmt::Expr(expr), body_rest)) = body.split_first() { + let doc_comment = match &*expr.value { + Expr::StringLiteral(value) => Some(&value.value), + // f-strings are not allowed in Python doc comments. + Expr::FString(_) => None, + _ => None, + }; + if let Some(doc) = doc_comment { return if opts.optimize < 2 { - (Some(clean_doc(&doc)), body_rest) + (Some(clean_doc(doc.to_str())), body_rest) } else { (None, body_rest) }; @@ -3420,49 +3542,43 @@ fn split_doc<'a>( (None, body) } -fn try_get_constant_string(values: &[located_ast::Expr]) -> Option { - fn get_constant_string_inner(out_string: &mut String, value: &located_ast::Expr) -> bool { - match value { - located_ast::Expr::Constant(located_ast::ExprConstant { - value: located_ast::Constant::Str(s), - .. - }) => { - out_string.push_str(s); - true - } - located_ast::Expr::JoinedStr(located_ast::ExprJoinedStr { values, .. }) => values - .iter() - .all(|value| get_constant_string_inner(out_string, value)), - _ => false, - } - } - let mut out_string = String::new(); - if values - .iter() - .all(|v| get_constant_string_inner(&mut out_string, v)) - { - Some(out_string) +pub fn ruff_int_to_bigint(int: &Int) -> Result { + if let Some(small) = int.as_u64() { + Ok(BigInt::from(small)) } else { - None + parse_big_integer(int) } } -fn compile_constant(value: &located_ast::Constant) -> ConstantData { - match value { - located_ast::Constant::None => ConstantData::None, - located_ast::Constant::Bool(b) => ConstantData::Boolean { value: *b }, - located_ast::Constant::Str(s) => ConstantData::Str { value: s.clone() }, - located_ast::Constant::Bytes(b) => ConstantData::Bytes { value: b.clone() }, - located_ast::Constant::Int(i) => ConstantData::Integer { value: i.clone() }, - located_ast::Constant::Tuple(t) => ConstantData::Tuple { - elements: t.iter().map(compile_constant).collect(), - }, - located_ast::Constant::Float(f) => ConstantData::Float { value: *f }, - located_ast::Constant::Complex { real, imag } => ConstantData::Complex { - value: Complex64::new(*real, *imag), - }, - located_ast::Constant::Ellipsis => ConstantData::Ellipsis, - } +/// Converts a `ruff` ast integer into a `BigInt`. +/// Unlike small integers, big integers may be stored in one of four possible radix representations. +fn parse_big_integer(int: &Int) -> Result { + // TODO: Improve ruff API + // Can we avoid this copy? + let s = format!("{}", int); + let mut s = s.as_str(); + // See: https://peps.python.org/pep-0515/#literal-grammar + let radix = match s.get(0..2) { + Some("0b" | "0B") => { + s = s.get(2..).unwrap_or(s); + 2 + } + Some("0o" | "0O") => { + s = s.get(2..).unwrap_or(s); + 8 + } + Some("0x" | "0X") => { + s = s.get(2..).unwrap_or(s); + 16 + } + _ => 10, + }; + + BigInt::from_str_radix(s, radix).map_err(|e| { + CodegenErrorType::SyntaxError(format!( + "unparsed integer literal (radix {radix}): {s} ({e})" + )) + }) } // Note: Not a good practice in general. Keep this trait private only for compiler @@ -3476,6 +3592,110 @@ impl ToU32 for usize { } } +#[cfg(test)] +mod tests { + use super::*; + use ruff_python_ast::*; + + /// Test if the compiler can correctly identify fstrings containing an `await` expression. + #[test] + fn test_fstring_contains_await() { + let range = TextRange::default(); + let flags = FStringFlags::default(); + + // f'{x}' + let expr_x = Expr::Name(ExprName { + range, + id: "x".to_owned(), + ctx: ExprContext::Load, + }); + let not_present = &Expr::FString(ExprFString { + range, + value: FStringValue::single(FString { + range, + elements: vec![FStringElement::Expression(FStringExpressionElement { + range, + expression: Box::new(expr_x), + debug_text: None, + conversion: ConversionFlag::None, + format_spec: None, + })] + .into(), + flags, + }), + }); + assert_eq!(Compiler::contains_await(not_present), false); + + // f'{await x}' + let expr_await_x = Expr::Await(ExprAwait { + range, + value: Box::new(Expr::Name(ExprName { + range, + id: "x".to_owned(), + ctx: ExprContext::Load, + })), + }); + let present = &Expr::FString(ExprFString { + range, + value: FStringValue::single(FString { + range, + elements: vec![FStringElement::Expression(FStringExpressionElement { + range, + expression: Box::new(expr_await_x), + debug_text: None, + conversion: ConversionFlag::None, + format_spec: None, + })] + .into(), + flags, + }), + }); + assert_eq!(Compiler::contains_await(present), true); + + // f'{x:{await y}}' + let expr_x = Expr::Name(ExprName { + range, + id: "x".to_owned(), + ctx: ExprContext::Load, + }); + let expr_await_y = Expr::Await(ExprAwait { + range, + value: Box::new(Expr::Name(ExprName { + range, + id: "y".to_owned(), + ctx: ExprContext::Load, + })), + }); + let present = &Expr::FString(ExprFString { + range, + value: FStringValue::single(FString { + range, + elements: vec![FStringElement::Expression(FStringExpressionElement { + range, + expression: Box::new(expr_x), + debug_text: None, + conversion: ConversionFlag::None, + format_spec: Some(Box::new(FStringFormatSpec { + range, + elements: vec![FStringElement::Expression(FStringExpressionElement { + range, + expression: Box::new(expr_await_y), + debug_text: None, + conversion: ConversionFlag::None, + format_spec: None, + })] + .into(), + })), + })] + .into(), + flags, + }), + }); + assert_eq!(Compiler::contains_await(present), true); + } +} + +/* #[cfg(test)] mod tests { use super::*; @@ -3555,3 +3775,4 @@ for stop_exc in (StopIteration('spam'), StopAsyncIteration('ham')): )); } } +*/ diff --git a/compiler/codegen/src/error.rs b/compiler/codegen/src/error.rs index 581f229712..8f38680de0 100644 --- a/compiler/codegen/src/error.rs +++ b/compiler/codegen/src/error.rs @@ -1,6 +1,23 @@ +use ruff_source_file::SourceLocation; use std::fmt; +use thiserror::Error; -pub type CodegenError = rustpython_parser_core::source_code::LocatedError; +// pub type CodegenError = rustpython_parser_core::source_code::LocatedError; + +#[derive(Error, Debug)] +pub struct CodegenError { + pub location: Option, + #[source] + pub error: CodegenErrorType, + pub source_path: String, +} + +impl fmt::Display for CodegenError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // TODO: + self.error.fmt(f) + } +} #[derive(Debug)] #[non_exhaustive] diff --git a/compiler/codegen/src/ir.rs b/compiler/codegen/src/ir.rs index 08e68d283a..2b3e49d036 100644 --- a/compiler/codegen/src/ir.rs +++ b/compiler/codegen/src/ir.rs @@ -1,10 +1,11 @@ use std::ops; use crate::IndexSet; +use ruff_source_file::{OneIndexed, SourceLocation}; use rustpython_compiler_core::bytecode::{ CodeFlags, CodeObject, CodeUnit, ConstantData, InstrDisplayContext, Instruction, Label, OpArg, }; -use rustpython_parser_core::source_code::{LineNumber, SourceLocation}; +// use rustpython_parser_core::source_code::{LineNumber, SourceLocation}; #[derive(Copy, Clone, PartialEq, Eq, Debug)] pub struct BlockIdx(pub u32); @@ -37,11 +38,12 @@ impl ops::IndexMut for Vec { } } -#[derive(Debug, Copy, Clone)] +#[derive(Debug, Clone)] pub struct InstructionInfo { pub instr: Instruction, pub arg: OpArg, pub target: BlockIdx, + // pub range: TextRange, pub location: SourceLocation, } @@ -68,7 +70,7 @@ pub struct CodeInfo { pub arg_count: u32, pub kwonlyarg_count: u32, pub source_path: String, - pub first_line_number: LineNumber, + pub first_line_number: OneIndexed, pub obj_name: String, // Name of the object that created this code object pub blocks: Vec, @@ -134,7 +136,8 @@ impl CodeInfo { *arg = new_arg; } let (extras, lo_arg) = arg.split(); - locations.extend(std::iter::repeat(info.location).take(arg.instr_size())); + locations + .extend(std::iter::repeat(info.location.clone()).take(arg.instr_size())); instructions.extend( extras .map(|byte| CodeUnit::new(Instruction::ExtendedArg, byte)) diff --git a/compiler/codegen/src/lib.rs b/compiler/codegen/src/lib.rs index 910d015a39..ceadb3c364 100644 --- a/compiler/codegen/src/lib.rs +++ b/compiler/codegen/src/lib.rs @@ -14,3 +14,49 @@ pub mod ir; pub mod symboltable; pub use compile::CompileOpts; +use ruff_python_ast::Expr; + +pub trait ToPythonName { + /// Returns a short name for the node suitable for use in error messages. + fn python_name(&self) -> &'static str; +} + +impl ToPythonName for Expr { + fn python_name(&self) -> &'static str { + match self { + Expr::BoolOp { .. } | Expr::BinOp { .. } | Expr::UnaryOp { .. } => "operator", + Expr::Subscript { .. } => "subscript", + Expr::Await { .. } => "await expression", + Expr::Yield { .. } | Expr::YieldFrom { .. } => "yield expression", + Expr::Compare { .. } => "comparison", + Expr::Attribute { .. } => "attribute", + Expr::Call { .. } => "function call", + Expr::BooleanLiteral(b) => { + if b.value { + "True" + } else { + "False" + } + } + Expr::EllipsisLiteral(_) => "ellipsis", + Expr::NoneLiteral(_) => "None", + Expr::NumberLiteral(_) | Expr::BytesLiteral(_) | Expr::StringLiteral(_) => "literal", + Expr::Tuple(_) => "tuple", + Expr::List { .. } => "list", + Expr::Dict { .. } => "dict display", + Expr::Set { .. } => "set display", + Expr::ListComp { .. } => "list comprehension", + Expr::DictComp { .. } => "dict comprehension", + Expr::SetComp { .. } => "set comprehension", + Expr::Generator { .. } => "generator expression", + Expr::Starred { .. } => "starred", + Expr::Slice { .. } => "slice", + Expr::FString { .. } => "f-string expression", + Expr::Name { .. } => "name", + Expr::Lambda { .. } => "lambda", + Expr::If { .. } => "conditional expression", + Expr::Named { .. } => "named expression", + Expr::IpyEscapeCommand(_) => todo!(), + } + } +} diff --git a/compiler/codegen/src/symboltable.rs b/compiler/codegen/src/symboltable.rs index 53d4ff8eb4..4246700cec 100644 --- a/compiler/codegen/src/symboltable.rs +++ b/compiler/codegen/src/symboltable.rs @@ -12,8 +12,16 @@ use crate::{ error::{CodegenError, CodegenErrorType}, }; use bitflags::bitflags; -use rustpython_ast::{self as ast, located::Located}; -use rustpython_parser_core::source_code::{LineNumber, SourceLocation}; +use ruff_python_ast::{ + self as ast, Comprehension, Decorator, Expr, Identifier, ModExpression, ModModule, Parameter, + ParameterWithDefault, Parameters, Pattern, PatternMatchAs, PatternMatchClass, + PatternMatchMapping, PatternMatchOr, PatternMatchSequence, PatternMatchStar, PatternMatchValue, + Stmt, TypeParam, TypeParamParamSpec, TypeParamTypeVar, TypeParamTypeVarTuple, TypeParams, +}; +use ruff_text_size::{Ranged, TextRange}; +use rustpython_compiler_source::{SourceCode, SourceLocation}; +// use rustpython_ast::{self as ast, located::Located}; +// use rustpython_parser_core::source_code::{LineNumber, SourceLocation}; use std::{borrow::Cow, fmt}; /// Captures all symbols in the current scope, and has a list of sub-scopes in this scope. @@ -51,15 +59,18 @@ impl SymbolTable { } } - pub fn scan_program(program: &[ast::located::Stmt]) -> SymbolTableResult { - let mut builder = SymbolTableBuilder::new(); - builder.scan_statements(program)?; + pub fn scan_program( + program: &ModModule, + source_code: SourceCode<'_>, + ) -> SymbolTableResult { + let mut builder = SymbolTableBuilder::new(source_code); + builder.scan_statements(program.body.as_ref())?; builder.finish() } - pub fn scan_expr(expr: &ast::located::Expr) -> SymbolTableResult { - let mut builder = SymbolTableBuilder::new(); - builder.scan_expression(expr, ExpressionContext::Load)?; + pub fn scan_expr(expr: &ModExpression, source_code: SourceCode<'_>) -> SymbolTableResult { + let mut builder = SymbolTableBuilder::new(source_code); + builder.scan_expression(expr.body.as_ref(), ExpressionContext::Load)?; builder.finish() } } @@ -176,11 +187,8 @@ pub struct SymbolTableError { impl SymbolTableError { pub fn into_codegen_error(self, source_path: String) -> CodegenError { CodegenError { + location: self.location, error: CodegenErrorType::SyntaxError(self.error), - location: self.location.map(|l| SourceLocation { - row: l.row, - column: l.column, - }), source_path, } } @@ -551,11 +559,12 @@ enum SymbolUsage { Iter, } -struct SymbolTableBuilder { +struct SymbolTableBuilder<'src> { class_name: Option, // Scope stack. tables: Vec, future_annotations: bool, + source_code: SourceCode<'src>, } /// Enum to indicate in what mode an expression @@ -571,19 +580,20 @@ enum ExpressionContext { IterDefinitionExp, } -impl SymbolTableBuilder { - fn new() -> Self { +impl<'src> SymbolTableBuilder<'src> { + fn new(source_code: SourceCode<'src>) -> Self { let mut this = Self { class_name: None, tables: vec![], future_annotations: false, + source_code, }; this.enter_scope("top", SymbolTableType::Module, 0); this } } -impl SymbolTableBuilder { +impl SymbolTableBuilder<'_> { fn finish(mut self) -> Result { assert_eq!(self.tables.len(), 1); let mut symbol_table = self.tables.pop().unwrap(); @@ -607,38 +617,34 @@ impl SymbolTableBuilder { self.tables.last_mut().unwrap().sub_tables.push(table); } - fn scan_statements(&mut self, statements: &[ast::located::Stmt]) -> SymbolTableResult { + fn line_index_start(&self, range: TextRange) -> u32 { + self.source_code.line_index(range.start()).get() as _ + } + + fn scan_statements(&mut self, statements: &[Stmt]) -> SymbolTableResult { for statement in statements { self.scan_statement(statement)?; } Ok(()) } - fn scan_parameters( - &mut self, - parameters: &[ast::located::ArgWithDefault], - ) -> SymbolTableResult { + fn scan_parameters(&mut self, parameters: &[ParameterWithDefault]) -> SymbolTableResult { for parameter in parameters { - let usage = if parameter.def.annotation.is_some() { - SymbolUsage::AnnotationParameter - } else { - SymbolUsage::Parameter - }; - self.register_name(parameter.def.arg.as_str(), usage, parameter.def.location())?; + self.scan_parameter(¶meter.parameter)?; } Ok(()) } - fn scan_parameter(&mut self, parameter: &ast::located::Arg) -> SymbolTableResult { + fn scan_parameter(&mut self, parameter: &Parameter) -> SymbolTableResult { let usage = if parameter.annotation.is_some() { SymbolUsage::AnnotationParameter } else { SymbolUsage::Parameter }; - self.register_name(parameter.arg.as_str(), usage, parameter.location()) + self.register_ident(¶meter.name, usage) } - fn scan_annotation(&mut self, annotation: &ast::located::Expr) -> SymbolTableResult { + fn scan_annotation(&mut self, annotation: &Expr) -> SymbolTableResult { if self.future_annotations { Ok(()) } else { @@ -646,8 +652,8 @@ impl SymbolTableBuilder { } } - fn scan_statement(&mut self, statement: &ast::located::Stmt) -> SymbolTableResult { - use ast::located::*; + fn scan_statement(&mut self, statement: &Stmt) -> SymbolTableResult { + use ruff_python_ast::*; if let Stmt::ImportFrom(StmtImportFrom { module, names, .. }) = &statement { if module.as_ref().map(|id| id.as_str()) == Some("__future__") { for feature in names { @@ -658,101 +664,109 @@ impl SymbolTableBuilder { } } match &statement { - Stmt::Global(StmtGlobal { names, range }) => { + Stmt::Global(StmtGlobal { names, .. }) => { for name in names { - self.register_name(name.as_str(), SymbolUsage::Global, range.start)?; + self.register_ident(name, SymbolUsage::Global)?; } } - Stmt::Nonlocal(StmtNonlocal { names, range }) => { + Stmt::Nonlocal(StmtNonlocal { names, .. }) => { for name in names { - self.register_name(name.as_str(), SymbolUsage::Nonlocal, range.start)?; + self.register_ident(name, SymbolUsage::Nonlocal)?; } } Stmt::FunctionDef(StmtFunctionDef { name, body, - args, - decorator_list, - type_params, - returns, - range, - .. - }) - | Stmt::AsyncFunctionDef(StmtAsyncFunctionDef { - name, - body, - args, + parameters, decorator_list, type_params, returns, range, .. }) => { - self.scan_expressions(decorator_list, ExpressionContext::Load)?; - self.register_name(name.as_str(), SymbolUsage::Assigned, range.start)?; + self.scan_decorators(decorator_list, ExpressionContext::Load)?; + self.register_ident(name, SymbolUsage::Assigned)?; if let Some(expression) = returns { self.scan_annotation(expression)?; } - if !type_params.is_empty() { + if let Some(type_params) = type_params { self.enter_scope( &format!("", name.as_str()), SymbolTableType::TypeParams, - range.start.row.get(), + // FIXME: line no + self.line_index_start(*range), ); self.scan_type_params(type_params)?; } - self.enter_function(name.as_str(), args, range.start.row)?; + self.enter_scope_with_parameters( + name.as_str(), + parameters, + self.line_index_start(*range), + )?; self.scan_statements(body)?; self.leave_scope(); - if !type_params.is_empty() { + if type_params.is_some() { self.leave_scope(); } } Stmt::ClassDef(StmtClassDef { name, body, - bases, - keywords, + arguments, decorator_list, type_params, range, }) => { - if !type_params.is_empty() { + if let Some(type_params) = type_params { self.enter_scope( &format!("", name.as_str()), SymbolTableType::TypeParams, - range.start.row.get(), + self.line_index_start(type_params.range), ); self.scan_type_params(type_params)?; } - self.enter_scope(name.as_str(), SymbolTableType::Class, range.start.row.get()); + self.enter_scope( + name.as_str(), + SymbolTableType::Class, + self.line_index_start(*range), + ); let prev_class = std::mem::replace(&mut self.class_name, Some(name.to_string())); - self.register_name("__module__", SymbolUsage::Assigned, range.start)?; - self.register_name("__qualname__", SymbolUsage::Assigned, range.start)?; - self.register_name("__doc__", SymbolUsage::Assigned, range.start)?; - self.register_name("__class__", SymbolUsage::Assigned, range.start)?; + self.register_name("__module__", SymbolUsage::Assigned, *range)?; + self.register_name("__qualname__", SymbolUsage::Assigned, *range)?; + self.register_name("__doc__", SymbolUsage::Assigned, *range)?; + self.register_name("__class__", SymbolUsage::Assigned, *range)?; self.scan_statements(body)?; self.leave_scope(); self.class_name = prev_class; - self.scan_expressions(bases, ExpressionContext::Load)?; - for keyword in keywords { - self.scan_expression(&keyword.value, ExpressionContext::Load)?; + if let Some(arguments) = arguments { + self.scan_expressions(&arguments.args, ExpressionContext::Load)?; + for keyword in &arguments.keywords { + self.scan_expression(&keyword.value, ExpressionContext::Load)?; + } } - if !type_params.is_empty() { + if type_params.is_some() { self.leave_scope(); } - self.scan_expressions(decorator_list, ExpressionContext::Load)?; - self.register_name(name.as_str(), SymbolUsage::Assigned, range.start)?; + self.scan_decorators(decorator_list, ExpressionContext::Load)?; + self.register_ident(name, SymbolUsage::Assigned)?; } Stmt::Expr(StmtExpr { value, .. }) => { self.scan_expression(value, ExpressionContext::Load)? } Stmt::If(StmtIf { - test, body, orelse, .. + test, + body, + elif_else_clauses, + .. }) => { self.scan_expression(test, ExpressionContext::Load)?; self.scan_statements(body)?; - self.scan_statements(orelse)?; + for elif in elif_else_clauses { + if let Some(test) = &elif.test { + self.scan_expression(test, ExpressionContext::Load)?; + } + self.scan_statements(&elif.body)?; + } } Stmt::For(StmtFor { target, @@ -760,13 +774,6 @@ impl SymbolTableBuilder { body, orelse, .. - }) - | Stmt::AsyncFor(StmtAsyncFor { - target, - iter, - body, - orelse, - .. }) => { self.scan_expression(target, ExpressionContext::Store)?; self.scan_expression(iter, ExpressionContext::Load)?; @@ -783,18 +790,18 @@ impl SymbolTableBuilder { Stmt::Break(_) | Stmt::Continue(_) | Stmt::Pass(_) => { // No symbols here. } - Stmt::Import(StmtImport { names, range }) - | Stmt::ImportFrom(StmtImportFrom { names, range, .. }) => { + Stmt::Import(StmtImport { names, .. }) + | Stmt::ImportFrom(StmtImportFrom { names, .. }) => { for name in names { if let Some(alias) = &name.asname { // `import my_module as my_alias` - self.register_name(alias.as_str(), SymbolUsage::Imported, range.start)?; + self.register_ident(alias, SymbolUsage::Imported)?; } else { // `import module` self.register_name( name.name.split('.').next().unwrap(), SymbolUsage::Imported, - range.start, + name.name.range, )?; } } @@ -831,11 +838,7 @@ impl SymbolTableBuilder { // https://github.com/python/cpython/blob/main/Python/symtable.c#L1233 match &**target { Expr::Name(ast::ExprName { id, .. }) if *simple => { - self.register_name( - id.as_str(), - SymbolUsage::AnnotationAssigned, - range.start, - )?; + self.register_name(id.as_str(), SymbolUsage::AnnotationAssigned, *range)?; } _ => { self.scan_expression(target, ExpressionContext::Store)?; @@ -846,8 +849,7 @@ impl SymbolTableBuilder { self.scan_expression(value, ExpressionContext::Load)?; } } - Stmt::With(StmtWith { items, body, .. }) - | Stmt::AsyncWith(StmtAsyncWith { items, body, .. }) => { + Stmt::With(StmtWith { items, body, .. }) => { for item in items { self.scan_expression(&item.context_expr, ExpressionContext::Load)?; if let Some(expression) = &item.optional_vars { @@ -861,14 +863,7 @@ impl SymbolTableBuilder { handlers, orelse, finalbody, - range, - }) - | Stmt::TryStar(StmtTryStar { - body, - handlers, - orelse, - finalbody, - range, + .. }) => { self.scan_statements(body)?; for handler in handlers { @@ -882,7 +877,7 @@ impl SymbolTableBuilder { self.scan_expression(expression, ExpressionContext::Load)?; } if let Some(name) = name { - self.register_name(name.as_str(), SymbolUsage::Assigned, range.start)?; + self.register_ident(name, SymbolUsage::Assigned)?; } self.scan_statements(body)?; } @@ -892,8 +887,10 @@ impl SymbolTableBuilder { Stmt::Match(StmtMatch { subject, cases, .. }) => { self.scan_expression(subject, ExpressionContext::Load)?; for case in cases { - // TODO: below - // self.scan_pattern(&case.pattern, ExpressionContext::Load)?; + self.scan_pattern(&case.pattern)?; + if let Some(guard) = &case.guard { + self.scan_expression(guard, ExpressionContext::Load)?; + } self.scan_statements(&case.body)?; } } @@ -909,13 +906,14 @@ impl SymbolTableBuilder { name, value, type_params, - range, + .. }) => { - if !type_params.is_empty() { + if let Some(type_params) = type_params { self.enter_scope( - &name.to_string(), + // &name.to_string(), + "TypeAlias", SymbolTableType::TypeParams, - range.start.row.get(), + self.line_index_start(type_params.range), ); self.scan_type_params(type_params)?; self.scan_expression(value, ExpressionContext::Load)?; @@ -925,13 +923,25 @@ impl SymbolTableBuilder { } self.scan_expression(name, ExpressionContext::Store)?; } + Stmt::IpyEscapeCommand(_) => todo!(), + } + Ok(()) + } + + fn scan_decorators( + &mut self, + decorators: &[Decorator], + context: ExpressionContext, + ) -> SymbolTableResult { + for decorator in decorators { + self.scan_expression(&decorator.expression, context)?; } Ok(()) } fn scan_expressions( &mut self, - expressions: &[ast::located::Expr], + expressions: &[Expr], context: ExpressionContext, ) -> SymbolTableResult { for expression in expressions { @@ -942,10 +952,10 @@ impl SymbolTableBuilder { fn scan_expression( &mut self, - expression: &ast::located::Expr, + expression: &Expr, context: ExpressionContext, ) -> SymbolTableResult { - use ast::located::*; + use ruff_python_ast::*; match expression { Expr::BinOp(ExprBinOp { left, @@ -984,16 +994,12 @@ impl SymbolTableBuilder { }) => { self.scan_expression(value, ExpressionContext::Load)?; } - Expr::Dict(ExprDict { - keys, - values, - range: _, - }) => { - for (key, value) in keys.iter().zip(values.iter()) { - if let Some(key) = key { + Expr::Dict(ExprDict { items, range: _ }) => { + for item in items { + if let Some(key) = &item.key { self.scan_expression(key, context)?; } - self.scan_expression(value, context)?; + self.scan_expression(&item.value, context)?; } } Expr::Await(ExprAwait { value, range: _ }) => { @@ -1012,7 +1018,6 @@ impl SymbolTableBuilder { }) => { self.scan_expression(operand, context)?; } - Expr::Constant(ExprConstant { range: _, .. }) => {} Expr::Starred(ExprStarred { value, range: _, .. }) => { @@ -1039,26 +1044,27 @@ impl SymbolTableBuilder { self.scan_expression(step, context)?; } } - Expr::GeneratorExp(ExprGeneratorExp { + Expr::Generator(ExprGenerator { elt, generators, range, + .. }) => { - self.scan_comprehension("genexpr", elt, None, generators, range.start)?; + self.scan_comprehension("genexpr", elt, None, generators, *range)?; } Expr::ListComp(ExprListComp { elt, generators, range, }) => { - self.scan_comprehension("genexpr", elt, None, generators, range.start)?; + self.scan_comprehension("genexpr", elt, None, generators, *range)?; } Expr::SetComp(ExprSetComp { elt, generators, range, }) => { - self.scan_comprehension("genexpr", elt, None, generators, range.start)?; + self.scan_comprehension("genexpr", elt, None, generators, *range)?; } Expr::DictComp(ExprDictComp { key, @@ -1066,12 +1072,11 @@ impl SymbolTableBuilder { generators, range, }) => { - self.scan_comprehension("genexpr", key, Some(value), generators, range.start)?; + self.scan_comprehension("genexpr", key, Some(value), generators, *range)?; } Expr::Call(ExprCall { func, - args, - keywords, + arguments, range: _, }) => { match context { @@ -1083,43 +1088,27 @@ impl SymbolTableBuilder { } } - self.scan_expressions(args, ExpressionContext::Load)?; - for keyword in keywords { + self.scan_expressions(&arguments.args, ExpressionContext::Load)?; + for keyword in &arguments.keywords { self.scan_expression(&keyword.value, ExpressionContext::Load)?; } } - Expr::FormattedValue(ExprFormattedValue { - value, - format_spec, - range: _, - .. - }) => { - self.scan_expression(value, ExpressionContext::Load)?; - if let Some(spec) = format_spec { - self.scan_expression(spec, ExpressionContext::Load)?; - } - } - Expr::JoinedStr(ExprJoinedStr { values, range: _ }) => { - for value in values { - self.scan_expression(value, ExpressionContext::Load)?; - } - } Expr::Name(ExprName { id, range, .. }) => { let id = id.as_str(); // Determine the contextual usage of this symbol: match context { ExpressionContext::Delete => { - self.register_name(id, SymbolUsage::Assigned, range.start)?; - self.register_name(id, SymbolUsage::Used, range.start)?; + self.register_name(id, SymbolUsage::Assigned, *range)?; + self.register_name(id, SymbolUsage::Used, *range)?; } ExpressionContext::Load | ExpressionContext::IterDefinitionExp => { - self.register_name(id, SymbolUsage::Used, range.start)?; + self.register_name(id, SymbolUsage::Used, *range)?; } ExpressionContext::Store => { - self.register_name(id, SymbolUsage::Assigned, range.start)?; + self.register_name(id, SymbolUsage::Assigned, *range)?; } ExpressionContext::Iter => { - self.register_name(id, SymbolUsage::Iter, range.start)?; + self.register_name(id, SymbolUsage::Iter, *range)?; } } // Interesting stuff about the __class__ variable: @@ -1128,15 +1117,27 @@ impl SymbolTableBuilder { && self.tables.last().unwrap().typ == SymbolTableType::Function && id == "super" { - self.register_name("__class__", SymbolUsage::Used, range.start)?; + self.register_name("__class__", SymbolUsage::Used, *range)?; } } Expr::Lambda(ExprLambda { - args, body, + parameters, range: _, }) => { - self.enter_function("lambda", args, expression.location().row)?; + if let Some(parameters) = parameters { + self.enter_scope_with_parameters( + "lambda", + parameters, + self.line_index_start(expression.range()), + )?; + } else { + self.enter_scope( + "lambda", + SymbolTableType::Function, + self.line_index_start(expression.range()), + ); + } match context { ExpressionContext::IterDefinitionExp => { self.scan_expression(body, ExpressionContext::IterDefinitionExp)?; @@ -1147,7 +1148,25 @@ impl SymbolTableBuilder { } self.leave_scope(); } - Expr::IfExp(ExprIfExp { + Expr::FString(ExprFString { value, .. }) => { + for expr in value.elements().filter_map(|x| x.as_expression()) { + self.scan_expression(&expr.expression, ExpressionContext::Load)?; + if let Some(format_spec) = &expr.format_spec { + for element in format_spec.elements.expressions() { + self.scan_expression(&element.expression, ExpressionContext::Load)? + } + } + } + } + // Constants + Expr::StringLiteral(_) + | Expr::BytesLiteral(_) + | Expr::NumberLiteral(_) + | Expr::BooleanLiteral(_) + | Expr::NoneLiteral(_) + | Expr::EllipsisLiteral(_) => {} + Expr::IpyEscapeCommand(_) => todo!(), + Expr::If(ExprIf { test, body, orelse, @@ -1158,7 +1177,7 @@ impl SymbolTableBuilder { self.scan_expression(orelse, ExpressionContext::Load)?; } - Expr::NamedExpr(ExprNamedExpr { + Expr::Named(ExprNamed { target, value, range, @@ -1167,9 +1186,9 @@ impl SymbolTableBuilder { // comprehension iterator definitions if let ExpressionContext::IterDefinitionExp = context { return Err(SymbolTableError { - error: "assignment expression cannot be used in a comprehension iterable expression".to_string(), - location: Some(target.location()), - }); + error: "assignment expression cannot be used in a comprehension iterable expression".to_string(), + location: Some(self.source_code.source_location(target.range().start())), + }); } self.scan_expression(value, ExpressionContext::Load)?; @@ -1185,13 +1204,13 @@ impl SymbolTableBuilder { self.register_name( id, SymbolUsage::AssignedNamedExprInComprehension, - range.start, + *range, )?; } else { // omit one recursion. When the handling of an store changes for // Identifiers this needs adapted - more forward safe would be // calling scan_expression directly. - self.register_name(id, SymbolUsage::Assigned, range.start)?; + self.register_name(id, SymbolUsage::Assigned, *range)?; } } else { self.scan_expression(target, ExpressionContext::Store)?; @@ -1204,20 +1223,20 @@ impl SymbolTableBuilder { fn scan_comprehension( &mut self, scope_name: &str, - elt1: &ast::located::Expr, - elt2: Option<&ast::located::Expr>, - generators: &[ast::located::Comprehension], - location: SourceLocation, + elt1: &Expr, + elt2: Option<&Expr>, + generators: &[Comprehension], + range: TextRange, ) -> SymbolTableResult { // Comprehensions are compiled as functions, so create a scope for them: self.enter_scope( scope_name, SymbolTableType::Comprehension, - location.row.get(), + self.line_index_start(range), ); // Register the passed argument to the generator function as the name ".0" - self.register_name(".0", SymbolUsage::Parameter, location)?; + self.register_name(".0", SymbolUsage::Parameter, range)?; self.scan_expression(elt1, ExpressionContext::Load)?; if let Some(elt2) = elt2 { @@ -1247,91 +1266,159 @@ impl SymbolTableBuilder { Ok(()) } - fn scan_type_params(&mut self, type_params: &[ast::located::TypeParam]) -> SymbolTableResult { - for type_param in type_params { + fn scan_type_params(&mut self, type_params: &TypeParams) -> SymbolTableResult { + for type_param in &type_params.type_params { match type_param { - ast::located::TypeParam::TypeVar(ast::TypeParamTypeVar { + TypeParam::TypeVar(TypeParamTypeVar { name, bound, range: type_var_range, + .. }) => { - self.register_name(name.as_str(), SymbolUsage::Assigned, type_var_range.start)?; + self.register_name(name.as_str(), SymbolUsage::Assigned, *type_var_range)?; if let Some(binding) = bound { self.scan_expression(binding, ExpressionContext::Load)?; } } - ast::located::TypeParam::ParamSpec(ast::TypeParamParamSpec { + TypeParam::ParamSpec(TypeParamParamSpec { name, range: param_spec_range, + .. }) => { - self.register_name(name, SymbolUsage::Assigned, param_spec_range.start)?; + self.register_name(name, SymbolUsage::Assigned, *param_spec_range)?; } - ast::located::TypeParam::TypeVarTuple(ast::TypeParamTypeVarTuple { + TypeParam::TypeVarTuple(TypeParamTypeVarTuple { name, range: type_var_tuple_range, + .. }) => { - self.register_name(name, SymbolUsage::Assigned, type_var_tuple_range.start)?; + self.register_name(name, SymbolUsage::Assigned, *type_var_tuple_range)?; + } + } + } + Ok(()) + } + + fn scan_patterns(&mut self, patterns: &[Pattern]) -> SymbolTableResult { + for pattern in patterns { + self.scan_pattern(pattern)?; + } + Ok(()) + } + + fn scan_pattern(&mut self, pattern: &Pattern) -> SymbolTableResult { + use Pattern::*; + match pattern { + MatchValue(PatternMatchValue { value, .. }) => { + self.scan_expression(value, ExpressionContext::Load)? + } + MatchSingleton(_) => {} + MatchSequence(PatternMatchSequence { patterns, .. }) => self.scan_patterns(patterns)?, + MatchMapping(PatternMatchMapping { + keys, + patterns, + rest, + .. + }) => { + self.scan_expressions(keys, ExpressionContext::Load)?; + self.scan_patterns(patterns)?; + if let Some(rest) = rest { + self.register_ident(rest, SymbolUsage::Assigned)?; + } + } + MatchClass(PatternMatchClass { cls, arguments, .. }) => { + self.scan_expression(cls, ExpressionContext::Load)?; + self.scan_patterns(&arguments.patterns)?; + for kw in &arguments.keywords { + self.scan_pattern(&kw.pattern)?; + } + } + MatchStar(PatternMatchStar { name, .. }) => { + if let Some(name) = name { + self.register_ident(name, SymbolUsage::Assigned)?; + } + } + MatchAs(PatternMatchAs { pattern, name, .. }) => { + if let Some(pattern) = pattern { + self.scan_pattern(pattern)?; + } + if let Some(name) = name { + self.register_ident(name, SymbolUsage::Assigned)?; } } + MatchOr(PatternMatchOr { patterns, .. }) => self.scan_patterns(patterns)?, } Ok(()) } - fn enter_function( + fn enter_scope_with_parameters( &mut self, name: &str, - args: &ast::located::Arguments, - line_number: LineNumber, + parameters: &Parameters, + line_number: u32, ) -> SymbolTableResult { // Evaluate eventual default parameters: - for default in args + for default in parameters .posonlyargs .iter() - .chain(args.args.iter()) - .chain(args.kwonlyargs.iter()) + .chain(parameters.args.iter()) + .chain(parameters.kwonlyargs.iter()) .filter_map(|arg| arg.default.as_ref()) { self.scan_expression(default, ExpressionContext::Load)?; // not ExprContext? } // Annotations are scanned in outer scope: - for annotation in args + for annotation in parameters .posonlyargs .iter() - .chain(args.args.iter()) - .chain(args.kwonlyargs.iter()) - .filter_map(|arg| arg.def.annotation.as_ref()) + .chain(parameters.args.iter()) + .chain(parameters.kwonlyargs.iter()) + .filter_map(|arg| arg.parameter.annotation.as_ref()) { self.scan_annotation(annotation)?; } - if let Some(annotation) = args.vararg.as_ref().and_then(|arg| arg.annotation.as_ref()) { + if let Some(annotation) = parameters + .vararg + .as_ref() + .and_then(|arg| arg.annotation.as_ref()) + { self.scan_annotation(annotation)?; } - if let Some(annotation) = args.kwarg.as_ref().and_then(|arg| arg.annotation.as_ref()) { + if let Some(annotation) = parameters + .kwarg + .as_ref() + .and_then(|arg| arg.annotation.as_ref()) + { self.scan_annotation(annotation)?; } - self.enter_scope(name, SymbolTableType::Function, line_number.get()); + self.enter_scope(name, SymbolTableType::Function, line_number); // Fill scope with parameter names: - self.scan_parameters(&args.posonlyargs)?; - self.scan_parameters(&args.args)?; - self.scan_parameters(&args.kwonlyargs)?; - if let Some(name) = &args.vararg { + self.scan_parameters(¶meters.posonlyargs)?; + self.scan_parameters(¶meters.args)?; + self.scan_parameters(¶meters.kwonlyargs)?; + if let Some(name) = ¶meters.vararg { self.scan_parameter(name)?; } - if let Some(name) = &args.kwarg { + if let Some(name) = ¶meters.kwarg { self.scan_parameter(name)?; } Ok(()) } + fn register_ident(&mut self, ident: &Identifier, role: SymbolUsage) -> SymbolTableResult { + self.register_name(ident.as_str(), role, ident.range) + } + fn register_name( &mut self, name: &str, role: SymbolUsage, - location: SourceLocation, + range: TextRange, ) -> SymbolTableResult { + let location = self.source_code.source_location(range.start()); let location = Some(location); let scope_depth = self.tables.len(); let table = self.tables.last_mut().unwrap(); diff --git a/compiler/core/Cargo.toml b/compiler/core/Cargo.toml index 619ffcf61e..7621c643d5 100644 --- a/compiler/core/Cargo.toml +++ b/compiler/core/Cargo.toml @@ -9,7 +9,10 @@ repository.workspace = true license.workspace = true [dependencies] -rustpython-parser-core = { workspace = true, features=["location"] } +# rustpython-parser-core = { workspace = true, features=["location"] } +ruff_python_ast = { workspace = true } +ruff_python_parser = { workspace = true } +ruff_source_file = { workspace = true } bitflags = { workspace = true } itertools = { workspace = true } diff --git a/compiler/core/src/bytecode.rs b/compiler/core/src/bytecode.rs index 66284efdc9..4cb80020e7 100644 --- a/compiler/core/src/bytecode.rs +++ b/compiler/core/src/bytecode.rs @@ -5,11 +5,13 @@ use bitflags::bitflags; use itertools::Itertools; use malachite_bigint::BigInt; use num_complex::Complex64; -use rustpython_parser_core::source_code::{OneIndexed, SourceLocation}; +pub use ruff_python_ast::ConversionFlag; +// use rustpython_parser_core::source_code::{OneIndexed, SourceLocation}; +use ruff_source_file::{OneIndexed, SourceLocation}; use std::marker::PhantomData; use std::{collections::BTreeSet, fmt, hash, mem}; -pub use rustpython_parser_core::ConversionFlag; +// pub use rustpython_parser_core::ConversionFlag; pub trait Constant: Sized { type Name: AsRef; diff --git a/compiler/core/src/marshal.rs b/compiler/core/src/marshal.rs index cdc8b8e4ef..1e47a6cac5 100644 --- a/compiler/core/src/marshal.rs +++ b/compiler/core/src/marshal.rs @@ -1,7 +1,7 @@ use crate::bytecode::*; use malachite_bigint::{BigInt, Sign}; use num_complex::Complex64; -use rustpython_parser_core::source_code::{OneIndexed, SourceLocation}; +use ruff_source_file::{OneIndexed, SourceLocation}; use std::convert::Infallible; pub const FORMAT_VERSION: u32 = 4; @@ -185,8 +185,8 @@ pub fn deserialize_code( let locations = (0..len) .map(|_| { Ok(SourceLocation { - row: OneIndexed::new(rdr.read_u32()?).ok_or(MarshalError::InvalidLocation)?, - column: OneIndexed::from_zero_indexed(rdr.read_u32()?), + row: OneIndexed::new(rdr.read_u32()? as _).ok_or(MarshalError::InvalidLocation)?, + column: OneIndexed::from_zero_indexed(rdr.read_u32()? as _), }) }) .collect::>>()?; @@ -200,7 +200,7 @@ pub fn deserialize_code( let len = rdr.read_u32()?; let source_path = bag.make_name(rdr.read_str(len)?); - let first_line_number = OneIndexed::new(rdr.read_u32()?); + let first_line_number = OneIndexed::new(rdr.read_u32()? as _); let max_stackdepth = rdr.read_u32()?; let len = rdr.read_u32()?; @@ -589,8 +589,8 @@ pub fn serialize_code(buf: &mut W, code: &CodeObject) write_len(buf, code.locations.len()); for loc in &*code.locations { - buf.write_u32(loc.row.get()); - buf.write_u32(loc.column.to_zero_indexed()); + buf.write_u32(loc.row.get() as _); + buf.write_u32(loc.column.to_zero_indexed() as _); } buf.write_u16(code.flags.bits()); @@ -601,7 +601,7 @@ pub fn serialize_code(buf: &mut W, code: &CodeObject) write_vec(buf, code.source_path.as_ref().as_bytes()); - buf.write_u32(code.first_line_number.map_or(0, |x| x.get())); + buf.write_u32(code.first_line_number.map_or(0, |x| x.get() as _)); buf.write_u32(code.max_stackdepth); write_vec(buf, code.obj_name.as_ref().as_bytes()); diff --git a/compiler/core/src/mode.rs b/compiler/core/src/mode.rs index 6682540c0d..13cea42b13 100644 --- a/compiler/core/src/mode.rs +++ b/compiler/core/src/mode.rs @@ -1,4 +1,4 @@ -pub use rustpython_parser_core::mode::ModeParseError; +pub use ruff_python_parser::ModeParseError; #[derive(Clone, Copy)] pub enum Mode { @@ -22,12 +22,14 @@ impl std::str::FromStr for Mode { } } -impl From for rustpython_parser_core::Mode { +impl From for ruff_python_parser::Mode { fn from(mode: Mode) -> Self { match mode { Mode::Exec => Self::Module, Mode::Eval => Self::Expression, - Mode::Single | Mode::BlockExpr => Self::Interactive, + // TODO: Improve ruff API + // ruff does not have an interactive mode + Mode::Single | Mode::BlockExpr => Self::Ipython, } } } diff --git a/compiler/source/Cargo.toml b/compiler/source/Cargo.toml new file mode 100644 index 0000000000..373af47cf0 --- /dev/null +++ b/compiler/source/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "rustpython-compiler-source" +description = "RustPython Source and Index" +version.workspace = true +authors.workspace = true +edition.workspace = true +rust-version.workspace = true +repository.workspace = true +license.workspace = true + +[dependencies] +ruff_source_file = { workspace = true } +ruff_text_size = { workspace = true } + +[lints] +workspace = true \ No newline at end of file diff --git a/compiler/source/src/lib.rs b/compiler/source/src/lib.rs new file mode 100644 index 0000000000..2d967e218d --- /dev/null +++ b/compiler/source/src/lib.rs @@ -0,0 +1,42 @@ +pub use ruff_source_file::{LineIndex, OneIndexed as LineNumber, SourceLocation}; +use ruff_text_size::TextRange; +pub use ruff_text_size::TextSize; + +#[derive(Clone)] +pub struct SourceCode<'src> { + pub path: &'src str, + pub text: &'src str, + pub index: LineIndex, +} + +impl<'src> SourceCode<'src> { + pub fn new(path: &'src str, text: &'src str) -> Self { + let index = LineIndex::from_source_text(text); + Self { path, text, index } + } + + pub fn line_index(&self, offset: TextSize) -> LineNumber { + self.index.line_index(offset) + } + + pub fn source_location(&self, offset: TextSize) -> SourceLocation { + self.index.source_location(offset, self.text) + } + + pub fn get_range(&'src self, range: TextRange) -> &'src str { + &self.text[range.start().to_usize()..range.end().to_usize()] + } +} + +pub struct SourceCodeOwned { + pub path: String, + pub text: String, + pub index: LineIndex, +} + +impl SourceCodeOwned { + pub fn new(path: String, text: String) -> Self { + let index = LineIndex::from_source_text(&text); + Self { path, text, index } + } +} diff --git a/compiler/src/lib.rs b/compiler/src/lib.rs index 659dc9974d..9c64de2610 100644 --- a/compiler/src/lib.rs +++ b/compiler/src/lib.rs @@ -1,69 +1,128 @@ +use ruff_source_file::SourceLocation; use rustpython_codegen::{compile, symboltable}; -use rustpython_parser::ast::{self as ast, ConstantOptimizer, fold::Fold}; pub use rustpython_codegen::compile::CompileOpts; pub use rustpython_compiler_core::{Mode, bytecode::CodeObject}; -pub use rustpython_parser::{Parse, source_code::LinearLocator}; +use rustpython_compiler_source::SourceCode; // these modules are out of repository. re-exporting them here for convenience. +pub use ruff_python_ast as ast; +pub use ruff_python_parser as parser; pub use rustpython_codegen as codegen; pub use rustpython_compiler_core as core; -pub use rustpython_parser as parser; +pub use rustpython_compiler_source as source; +use thiserror::Error; -#[derive(Debug)] +#[derive(Error, Debug)] pub enum CompileErrorType { - Codegen(rustpython_codegen::error::CodegenErrorType), - Parse(parser::ParseErrorType), + #[error(transparent)] + Codegen(#[from] codegen::error::CodegenErrorType), + #[error(transparent)] + Parse(#[from] parser::ParseErrorType), } -impl std::error::Error for CompileErrorType { - fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { - match self { - CompileErrorType::Codegen(e) => e.source(), - CompileErrorType::Parse(e) => e.source(), - } - } +#[derive(Error, Debug)] +pub struct ParseError { + #[source] + pub error: parser::ParseErrorType, + pub location: SourceLocation, + pub source_path: String, } -impl std::fmt::Display for CompileErrorType { + +impl std::fmt::Display for ParseError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.error.fmt(f) + } +} + +#[derive(Error, Debug)] +pub enum CompileError { + #[error(transparent)] + Codegen(#[from] codegen::error::CodegenError), + #[error(transparent)] + Parse(#[from] ParseError), +} + +impl CompileError { + pub fn from_ruff_parse_error(error: parser::ParseError, source_code: &SourceCode<'_>) -> Self { + let location = source_code.source_location(error.location.start()); + Self::Parse(ParseError { + error: error.error, + location, + source_path: source_code.path.to_owned(), + }) + } + + pub fn location(&self) -> Option { match self { - CompileErrorType::Codegen(e) => e.fmt(f), - CompileErrorType::Parse(e) => e.fmt(f), + CompileError::Codegen(codegen_error) => codegen_error.location.clone(), + CompileError::Parse(parse_error) => Some(parse_error.location.clone()), } } -} -impl From for CompileErrorType { - fn from(source: rustpython_codegen::error::CodegenErrorType) -> Self { - CompileErrorType::Codegen(source) + + pub fn python_location(&self) -> (usize, usize) { + match self { + CompileError::Codegen(codegen_error) => { + if let Some(location) = &codegen_error.location { + (location.row.get(), location.column.get()) + } else { + (0, 0) + } + } + CompileError::Parse(parse_error) => ( + parse_error.location.row.get(), + parse_error.location.column.get(), + ), + } } -} -impl From for CompileErrorType { - fn from(source: parser::ParseErrorType) -> Self { - CompileErrorType::Parse(source) + + pub fn source_path(&self) -> &str { + match self { + CompileError::Codegen(codegen_error) => &codegen_error.source_path, + CompileError::Parse(parse_error) => &parse_error.source_path, + } } } -pub type CompileError = rustpython_parser::source_code::LocatedError; - /// Compile a given source code into a bytecode object. pub fn compile( source: &str, mode: Mode, - source_path: String, + source_path: &str, opts: CompileOpts, ) -> Result { - let mut locator = LinearLocator::new(source); - let mut ast = match parser::parse(source, mode.into(), &source_path) { - Ok(x) => x, - Err(e) => return Err(locator.locate_error(e)), - }; - if opts.optimize > 0 { - ast = ConstantOptimizer::new() - .fold_mod(ast) - .unwrap_or_else(|e| match e {}); - } - let ast = locator.fold_mod(ast).unwrap_or_else(|e| match e {}); - compile::compile_top(&ast, source_path, mode, opts).map_err(|e| e.into()) + // TODO: do this less hackily; ruff's parser should translate a CRLF line + // break in a multiline string into just an LF in the parsed value + #[cfg(windows)] + let source = &source.replace("\r\n", "\n"); + let source_code = SourceCode::new(source_path, source); + _compile(source_code, mode, opts) + // let index = LineIndex::from_source_text(source); + // let source_code = SourceCode::new(source, &index); + // let mut locator = LinearLocator::new(source); + // let mut ast = match parser::parse(source, mode.into(), &source_path) { + // Ok(x) => x, + // Err(e) => return Err(locator.locate_error(e)), + // }; + + // TODO: + // if opts.optimize > 0 { + // ast = ConstantOptimizer::new() + // .fold_mod(ast) + // .unwrap_or_else(|e| match e {}); + // } + // let ast = locator.fold_mod(ast).unwrap_or_else(|e| match e {}); +} + +fn _compile( + source_code: SourceCode<'_>, + mode: Mode, + opts: CompileOpts, +) -> Result { + let parsed = parser::parse(source_code.text, mode.into()) + .map_err(|err| CompileError::from_ruff_parse_error(err, &source_code))?; + let ast = parsed.into_syntax(); + compile::compile_top(ast, source_code, mode, opts).map_err(|e| e.into()) } pub fn compile_symtable( @@ -71,20 +130,170 @@ pub fn compile_symtable( mode: Mode, source_path: &str, ) -> Result { - let mut locator = LinearLocator::new(source); + let source_code = SourceCode::new(source_path, source); + _compile_symtable(source_code, mode) +} + +pub fn _compile_symtable( + source_code: SourceCode<'_>, + mode: Mode, +) -> Result { let res = match mode { Mode::Exec | Mode::Single | Mode::BlockExpr => { - let ast = - ast::Suite::parse(source, source_path).map_err(|e| locator.locate_error(e))?; - let ast = locator.fold(ast).unwrap(); - symboltable::SymbolTable::scan_program(&ast) + let ast = ruff_python_parser::parse_module(source_code.text) + .map_err(|e| CompileError::from_ruff_parse_error(e, &source_code))?; + symboltable::SymbolTable::scan_program(&ast.into_syntax(), source_code.clone()) } Mode::Eval => { - let expr = - ast::Expr::parse(source, source_path).map_err(|e| locator.locate_error(e))?; - let expr = locator.fold(expr).unwrap(); - symboltable::SymbolTable::scan_expr(&expr) + let ast = + ruff_python_parser::parse(source_code.text, ruff_python_parser::Mode::Expression) + .map_err(|e| CompileError::from_ruff_parse_error(e, &source_code))?; + symboltable::SymbolTable::scan_expr( + &ast.into_syntax().expect_expression(), + source_code.clone(), + ) } }; - res.map_err(|e| e.into_codegen_error(source_path.to_owned()).into()) + res.map_err(|e| e.into_codegen_error(source_code.path.to_owned()).into()) +} + +#[test] +fn test_compile() { + let code = "x = 'abc'"; + let compiled = compile(&code, Mode::Single, "<>", CompileOpts::default()); + dbg!(compiled.expect("compile error")); +} + +#[test] +fn test_compile_phello() { + let code = r#" +initialized = True +def main(): + print("Hello world!") +if __name__ == '__main__': + main() +"#; + let compiled = compile(&code, Mode::Exec, "<>", CompileOpts::default()); + dbg!(compiled.expect("compile error")); +} + +#[test] +fn test_compile_if_elif_else() { + let code = r#" +if False: + pass +elif False: + pass +elif False: + pass +else: + pass +"#; + let compiled = compile(&code, Mode::Exec, "<>", CompileOpts::default()); + dbg!(compiled.expect("compile error")); +} + +#[test] +fn test_compile_lambda() { + let code = r#" +lambda: 'a' +"#; + let compiled = compile(&code, Mode::Exec, "<>", CompileOpts::default()); + dbg!(compiled.expect("compile error")); +} + +#[test] +fn test_compile_lambda2() { + let code = r#" +(lambda x: f'hello, {x}')('world}') +"#; + let compiled = compile(&code, Mode::Exec, "<>", CompileOpts::default()); + dbg!(compiled.expect("compile error")); +} + +#[test] +fn test_compile_lambda3() { + let code = r#" +def g(): + pass +def f(): + if False: + return lambda x: g(x) + elif False: + return g + else: + return g +"#; + let compiled = compile(&code, Mode::Exec, "<>", CompileOpts::default()); + dbg!(compiled.expect("compile error")); +} + +#[test] +fn test_compile_int() { + let code = r#" +a = 0xFF +"#; + let compiled = compile(&code, Mode::Exec, "<>", CompileOpts::default()); + dbg!(compiled.expect("compile error")); +} + +#[test] +fn test_compile_bigint() { + let code = r#" +a = 0xFFFFFFFFFFFFFFFFFFFFFFFF +"#; + let compiled = compile(&code, Mode::Exec, "<>", CompileOpts::default()); + dbg!(compiled.expect("compile error")); +} + +#[test] +fn test_compile_fstring() { + let code1 = r#" +assert f"1" == '1' + "#; + let compiled = compile(&code1, Mode::Exec, "<>", CompileOpts::default()); + dbg!(compiled.expect("compile error")); + + let code2 = r#" +assert f"{1}" == '1' + "#; + let compiled = compile(&code2, Mode::Exec, "<>", CompileOpts::default()); + dbg!(compiled.expect("compile error")); + let code3 = r#" +assert f"{1+1}" == '2' + "#; + let compiled = compile(&code3, Mode::Exec, "<>", CompileOpts::default()); + dbg!(compiled.expect("compile error")); + + let code4 = r#" +assert f"{{{(lambda: f'{1}')}" == '{1' + "#; + let compiled = compile(&code4, Mode::Exec, "<>", CompileOpts::default()); + dbg!(compiled.expect("compile error")); + + let code5 = r#" +assert f"a{1}" == 'a1' + "#; + let compiled = compile(&code5, Mode::Exec, "<>", CompileOpts::default()); + dbg!(compiled.expect("compile error")); + + let code6 = r#" +assert f"{{{(lambda x: f'hello, {x}')('world}')}" == '{hello, world}' + "#; + let compiled = compile(&code6, Mode::Exec, "<>", CompileOpts::default()); + dbg!(compiled.expect("compile error")); +} + +#[test] +fn test_simple_enum() { + let code = r#" +import enum +@enum._simple_enum(enum.IntFlag, boundary=enum.KEEP) +class RegexFlag: + NOFLAG = 0 + DEBUG = 1 +print(RegexFlag.NOFLAG & RegexFlag.DEBUG) +"#; + let compiled = compile(&code, Mode::Exec, "", CompileOpts::default()); + dbg!(compiled.expect("compile error")); } diff --git a/derive-impl/Cargo.toml b/derive-impl/Cargo.toml index debe58106b..ca415dc543 100644 --- a/derive-impl/Cargo.toml +++ b/derive-impl/Cargo.toml @@ -10,7 +10,7 @@ license.workspace = true [dependencies] rustpython-compiler-core = { workspace = true } -rustpython-parser-core = { workspace = true } +# rustpython-parser-core = { workspace = true } rustpython-doc = { workspace = true } itertools = { workspace = true } diff --git a/derive/src/lib.rs b/derive/src/lib.rs index 97ca026e76..a9a3123a5a 100644 --- a/derive/src/lib.rs +++ b/derive/src/lib.rs @@ -246,7 +246,7 @@ impl derive_impl::Compiler for Compiler { module_name: String, ) -> Result> { use rustpython_compiler::{CompileOpts, compile}; - Ok(compile(source, mode, module_name, CompileOpts::default())?) + Ok(compile(source, mode, &module_name, CompileOpts::default())?) } } diff --git a/examples/dis.rs b/examples/dis.rs index ee424f7b34..5b8826fdf1 100644 --- a/examples/dis.rs +++ b/examples/dis.rs @@ -79,7 +79,7 @@ fn display_script( expand_code_objects: bool, ) -> Result<(), Box> { let source = fs::read_to_string(path)?; - let code = compiler::compile(&source, mode, path.to_string_lossy().into_owned(), opts)?; + let code = compiler::compile(&source, mode, &path.to_string_lossy(), opts)?; println!("{}:", path.display()); if expand_code_objects { println!("{}", code.display_expand_code_objects()); diff --git a/examples/parse_folder.rs b/examples/parse_folder.rs index 97a1eaf036..acad76ad76 100644 --- a/examples/parse_folder.rs +++ b/examples/parse_folder.rs @@ -8,7 +8,8 @@ extern crate env_logger; #[macro_use] extern crate log; -use rustpython_parser::{Parse, ast}; +use ruff_python_parser::parse_module; +use rustpython_compiler::ast; use std::{ path::{Path, PathBuf}, time::{Duration, Instant}, @@ -68,18 +69,19 @@ fn parse_python_file(filename: &Path) -> ParsedFile { info!("Parsing file {:?}", filename); match std::fs::read_to_string(filename) { Err(e) => ParsedFile { - // filename: Box::new(filename.to_path_buf()), - // code: "".to_owned(), + filename: Box::new(filename.to_path_buf()), + code: "".to_owned(), num_lines: 0, result: Err(e.to_string()), }, Ok(source) => { let num_lines = source.lines().count(); - let result = - ast::Suite::parse(&source, &filename.to_string_lossy()).map_err(|e| e.to_string()); + let result = parse_module(&source) + .map(|x| x.into_suite()) + .map_err(|e| e.to_string()); ParsedFile { - // filename: Box::new(filename.to_path_buf()), - // code: source.to_string(), + filename: Box::new(filename.to_path_buf()), + code: source.to_string(), num_lines, result, } @@ -132,8 +134,8 @@ struct ScanResult { } struct ParsedFile { - // filename: Box, - // code: String, + filename: Box, + code: String, num_lines: usize, result: ParseResult, } diff --git a/src/shell.rs b/src/shell.rs index 00b6710061..f920b4d011 100644 --- a/src/shell.rs +++ b/src/shell.rs @@ -1,10 +1,12 @@ mod helper; -use rustpython_parser::{ParseErrorType, Tok, lexer::LexicalErrorType}; +use rustpython_compiler::{ + CompileError, ParseError, parser::ParseErrorType, parser::lexer::LexicalErrorType, +}; use rustpython_vm::{ AsObject, PyResult, VirtualMachine, builtins::PyBaseExceptionRef, - compiler::{self, CompileError, CompileErrorType}, + compiler::{self}, readline::{Readline, ReadlineResult}, scope::Scope, }; @@ -35,29 +37,25 @@ fn shell_exec( ShellExecResult::Ok } } - Err(CompileError { - error: CompileErrorType::Parse(ParseErrorType::Lexical(LexicalErrorType::Eof)), - .. - }) - | Err(CompileError { - error: CompileErrorType::Parse(ParseErrorType::Eof), + Err(CompileError::Parse(ParseError { + error: ParseErrorType::Lexical(LexicalErrorType::Eof), .. - }) => ShellExecResult::Continue, + })) => ShellExecResult::Continue, Err(err) => { // bad_error == true if we are handling an error that should be thrown even if we are continuing // if its an indentation error, set to true if we are continuing and the error is on column 0, // since indentations errors on columns other than 0 should be ignored. // if its an unrecognized token for dedent, set to false - let bad_error = match err.error { - CompileErrorType::Parse(ref p) => { + let bad_error = match err { + CompileError::Parse(ref p) => { if matches!( - p, + p.error, ParseErrorType::Lexical(LexicalErrorType::IndentationError) ) { - continuing && err.location.is_some() + continuing // && p.location.is_some() } else { - !matches!(p, ParseErrorType::UnrecognizedToken(Tok::Dedent, _)) + true // !matches!(p, ParseErrorType::UnrecognizedToken(Tok::Dedent, _)) } } _ => true, // It is a bad error for everything else diff --git a/stdlib/src/faulthandler.rs b/stdlib/src/faulthandler.rs index 9ffd931291..e3cd434731 100644 --- a/stdlib/src/faulthandler.rs +++ b/stdlib/src/faulthandler.rs @@ -10,7 +10,7 @@ mod decl { stderr, " File \"{}\", line {} in {}", frame.code.source_path, - frame.current_location().row.to_usize(), + frame.current_location().row, frame.code.obj_name ) } diff --git a/vm/Cargo.toml b/vm/Cargo.toml index a0e357baa9..49b0883bbe 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -19,9 +19,9 @@ freeze-stdlib = ["encodings"] jit = ["rustpython-jit"] threading = ["rustpython-common/threading"] compiler = ["parser", "codegen", "rustpython-compiler"] -ast = ["rustpython-ast"] +ast = ["ruff_python_ast", "ruff_text_size", "ruff_source_file"] codegen = ["rustpython-codegen", "ast"] -parser = ["rustpython-parser", "ast"] +parser = ["ast"] serde = ["dep:serde"] wasmbind = ["chrono/wasmbind", "getrandom/wasm_js", "wasm-bindgen"] @@ -32,10 +32,15 @@ rustpython-common = { workspace = true } rustpython-derive = { workspace = true } rustpython-jit = { workspace = true, optional = true } -rustpython-ast = { workspace = true, optional = true } -rustpython-parser = { workspace = true, optional = true } +ruff_python_ast = { workspace = true, optional = true } +ruff_python_parser = { workspace = true } +ruff_text_size = { workspace = true, optional = true } +ruff_source_file = { workspace = true, optional = true } +# rustpython-ast = { workspace = true, optional = true } +# rustpython-parser = { workspace = true, optional = true } rustpython-compiler-core = { workspace = true } -rustpython-parser-core = { workspace = true } +rustpython-compiler-source = { workspace = true } +# rustpython-parser-core = { workspace = true } rustpython-literal = { workspace = true } rustpython-format = { workspace = true } rustpython-sre_engine = { workspace = true } @@ -87,9 +92,9 @@ unicode_names2 = { workspace = true } # https://github.com/RustPython/RustPython/pull/832#discussion_r275428939 unicode-casing = "0.1.0" # update version all at the same time -unic-ucd-bidi = "0.9.0" -unic-ucd-category = "0.9.0" -unic-ucd-ident = "0.9.0" +unic-ucd-bidi = "0.9.0" +unic-ucd-category = "0.9.0" +unic-ucd-ident = "0.9.0" [target.'cfg(unix)'.dependencies] rustix = { workspace = true } diff --git a/vm/src/builtins/code.rs b/vm/src/builtins/code.rs index 3fa332b2f3..ba2d2dd5c3 100644 --- a/vm/src/builtins/code.rs +++ b/vm/src/builtins/code.rs @@ -11,7 +11,7 @@ use crate::{ convert::ToPyObject, frozen, function::{FuncArgs, OptionalArg}, - source_code::OneIndexed, + source::LineNumber, types::Representable, }; use malachite_bigint::BigInt; @@ -279,7 +279,7 @@ impl PyCode { #[pygetset] fn co_firstlineno(&self) -> u32 { - self.code.first_line_number.map_or(0, |n| n.get()) + self.code.first_line_number.map_or(0, |n| n.get() as _) } #[pygetset] @@ -351,7 +351,7 @@ impl PyCode { }; let first_line_number = match args.co_firstlineno { - OptionalArg::Present(first_line_number) => OneIndexed::new(first_line_number), + OptionalArg::Present(first_line_number) => LineNumber::new(first_line_number as _), OptionalArg::Missing => self.code.first_line_number, }; diff --git a/vm/src/builtins/frame.rs b/vm/src/builtins/frame.rs index 1fd031984a..1b73850190 100644 --- a/vm/src/builtins/frame.rs +++ b/vm/src/builtins/frame.rs @@ -60,7 +60,7 @@ impl Frame { #[pygetset] pub fn f_lineno(&self) -> usize { - self.current_location().row.to_usize() + self.current_location().row.get() } #[pygetset] diff --git a/vm/src/builtins/traceback.rs b/vm/src/builtins/traceback.rs index b0abbd006a..6d88821ae7 100644 --- a/vm/src/builtins/traceback.rs +++ b/vm/src/builtins/traceback.rs @@ -2,7 +2,7 @@ use rustpython_common::lock::PyMutex; use super::PyType; use crate::{ - Context, Py, PyPayload, PyRef, class::PyClassImpl, frame::FrameRef, source_code::LineNumber, + Context, Py, PyPayload, PyRef, class::PyClassImpl, frame::FrameRef, source::LineNumber, }; #[pyclass(module = false, name = "traceback", traverse)] @@ -47,7 +47,7 @@ impl PyTraceback { #[pygetset] fn tb_lineno(&self) -> usize { - self.lineno.to_usize() + self.lineno.get() } #[pygetset] diff --git a/vm/src/compiler.rs b/vm/src/compiler.rs index c81475b6d1..b819fd9a42 100644 --- a/vm/src/compiler.rs +++ b/vm/src/compiler.rs @@ -1,37 +1,49 @@ -#[cfg(feature = "rustpython-codegen")] +#[cfg(feature = "codegen")] pub use rustpython_codegen::CompileOpts; -#[cfg(feature = "rustpython-compiler")] +#[cfg(feature = "compiler")] pub use rustpython_compiler::*; -#[cfg(not(feature = "rustpython-compiler"))] + +#[cfg(not(feature = "compiler"))] +pub use rustpython_compiler_source as source; + +#[cfg(not(feature = "compiler"))] pub use rustpython_compiler_core::Mode; -#[cfg(not(feature = "rustpython-compiler"))] +#[cfg(not(feature = "compiler"))] pub use rustpython_compiler_core as core; -#[cfg(not(feature = "rustpython-compiler"))] -pub use rustpython_parser_core as parser; +#[cfg(not(feature = "compiler"))] +pub use ruff_python_parser as parser; -#[cfg(not(feature = "rustpython-compiler"))] +#[cfg(not(feature = "compiler"))] mod error { - #[cfg(all(feature = "rustpython-parser", feature = "rustpython-codegen"))] + #[cfg(all(feature = "parser", feature = "codegen"))] panic!("Use --features=compiler to enable both parser and codegen"); #[derive(Debug, thiserror::Error)] pub enum CompileErrorType { - #[cfg(feature = "rustpython-codegen")] + #[cfg(feature = "codegen")] #[error(transparent)] - Codegen(#[from] rustpython_codegen::error::CodegenErrorType), - #[cfg(feature = "rustpython-parser")] + Codegen(#[from] super::codegen::error::CodegenErrorType), + #[cfg(feature = "parser")] #[error(transparent)] - Parse(#[from] rustpython_parser::error::ParseErrorType), + Parse(#[from] super::parser::ParseErrorType), } - pub type CompileError = rustpython_parser_core::source_code::LocatedError; + #[derive(Debug, thiserror::Error)] + pub enum CompileError { + #[cfg(feature = "codegen")] + #[error(transparent)] + Codegen(#[from] super::codegen::error::CodegenError), + #[cfg(feature = "parser")] + #[error(transparent)] + Parse(#[from] super::parser::ParseError), + } } -#[cfg(not(feature = "rustpython-compiler"))] +#[cfg(not(feature = "compiler"))] pub use error::{CompileError, CompileErrorType}; -#[cfg(any(feature = "rustpython-parser", feature = "rustpython-codegen"))] +#[cfg(any(feature = "parser", feature = "codegen"))] impl crate::convert::ToPyException for (CompileError, Option<&str>) { fn to_pyexception(&self, vm: &crate::VirtualMachine) -> crate::builtins::PyBaseExceptionRef { vm.new_syntax_error(&self.0, self.1) diff --git a/vm/src/exceptions.rs b/vm/src/exceptions.rs index d2f410b307..9bf7372794 100644 --- a/vm/src/exceptions.rs +++ b/vm/src/exceptions.rs @@ -378,7 +378,7 @@ fn write_traceback_entry( tb_entry.lineno, tb_entry.frame.code.obj_name )?; - print_source_line(output, filename, tb_entry.lineno.to_usize())?; + print_source_line(output, filename, tb_entry.lineno.get())?; Ok(()) } diff --git a/vm/src/frame.rs b/vm/src/frame.rs index ec5dad10a2..cf695cd87b 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -15,7 +15,7 @@ use crate::{ function::{ArgMapping, Either, FuncArgs}, protocol::{PyIter, PyIterReturn}, scope::Scope, - source_code::SourceLocation, + source::SourceLocation, stdlib::{builtins, typing::_typing}, vm::{Context, PyMethod}, }; @@ -166,7 +166,7 @@ impl Frame { } pub fn current_location(&self) -> SourceLocation { - self.code.locations[self.lasti() as usize - 1] + self.code.locations[self.lasti() as usize - 1].clone() } pub fn lasti(&self) -> u32 { @@ -380,7 +380,7 @@ impl ExecutingFrame<'_> { // 2. Add new entry with current execution position (filename, lineno, code_object) to traceback. // 3. Unwind block stack till appropriate handler is found. - let loc = frame.code.locations[idx]; + let loc = frame.code.locations[idx].clone(); let next = exception.traceback(); let new_traceback = PyTraceback::new(next, frame.object.to_owned(), frame.lasti(), loc.row); diff --git a/vm/src/lib.rs b/vm/src/lib.rs index 93adaa13f8..2e4afa3ea1 100644 --- a/vm/src/lib.rs +++ b/vm/src/lib.rs @@ -84,7 +84,7 @@ pub mod warn; #[cfg(windows)] pub mod windows; -pub use self::compiler::parser::source_code; +pub use self::compiler::source; pub use self::convert::{TryFromBorrowedObject, TryFromObject}; pub use self::object::{ AsObject, Py, PyAtomicRef, PyExact, PyObject, PyObjectRef, PyPayload, PyRef, PyRefExact, diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs index 7dd893646e..b5445c65f5 100644 --- a/vm/src/stdlib/ast.rs +++ b/vm/src/stdlib/ast.rs @@ -3,81 +3,55 @@ //! This module makes use of the parser logic, and translates all ast nodes //! into python ast.AST objects. -mod r#gen; +mod pyast; +use crate::builtins::{PyInt, PyStr}; +use crate::stdlib::ast::module::{Mod, ModInteractive}; +use crate::stdlib::ast::node::BoxedSlice; +use crate::stdlib::ast::python::_ast; use crate::{ - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, - VirtualMachine, - builtins::{self, PyDict, PyModule, PyStrRef, PyType}, + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyRefExact, PyResult, + TryFromObject, VirtualMachine, + builtins::PyIntRef, + builtins::{PyDict, PyModule, PyStrRef, PyType}, class::{PyClassImpl, StaticType}, - compiler::CompileError, compiler::core::bytecode::OpArgType, - convert::ToPyException, - source_code::{LinearLocator, OneIndexed, SourceLocation, SourceRange}, + compiler::{CompileError, ParseError}, + convert::ToPyObject, + source::SourceCode, + source::SourceLocation, }; -use num_complex::Complex64; -use num_traits::{ToPrimitive, Zero}; -use rustpython_ast::{self as ast, fold::Fold}; -#[cfg(feature = "rustpython-codegen")] +use node::Node; +use ruff_python_ast as ruff; +use ruff_source_file::OneIndexed; +use ruff_text_size::{Ranged, TextRange, TextSize}; +use rustpython_compiler_source::SourceCodeOwned; + +#[cfg(feature = "parser")] +use ruff_python_parser as parser; +#[cfg(feature = "codegen")] use rustpython_codegen as codegen; -#[cfg(feature = "rustpython-parser")] -use rustpython_parser as parser; - -#[pymodule] -mod _ast { - use crate::{ - AsObject, Context, PyObjectRef, PyPayload, PyResult, VirtualMachine, - builtins::{PyStrRef, PyTupleRef}, - function::FuncArgs, - }; - #[pyattr] - #[pyclass(module = "_ast", name = "AST")] - #[derive(Debug, PyPayload)] - pub(crate) struct NodeAst; - - #[pyclass(flags(BASETYPE, HAS_DICT))] - impl NodeAst { - #[pyslot] - #[pymethod(magic)] - fn init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { - let fields = zelf.get_attr("_fields", vm)?; - let fields: Vec = fields.try_to_value(vm)?; - let numargs = args.args.len(); - if numargs > fields.len() { - return Err(vm.new_type_error(format!( - "{} constructor takes at most {} positional argument{}", - zelf.class().name(), - fields.len(), - if fields.len() == 1 { "" } else { "s" }, - ))); - } - for (name, arg) in fields.iter().zip(args.args) { - zelf.set_attr(name, arg, vm)?; - } - for (key, value) in args.kwargs { - if let Some(pos) = fields.iter().position(|f| f.as_str() == key) { - if pos < numargs { - return Err(vm.new_type_error(format!( - "{} got multiple values for argument '{}'", - zelf.class().name(), - key - ))); - } - } - zelf.set_attr(vm.ctx.intern_str(key), value, vm)?; - } - Ok(()) - } - #[pyattr(name = "_fields")] - fn fields(ctx: &Context) -> PyTupleRef { - ctx.empty_tuple.clone() - } - } - - #[pyattr(name = "PyCF_ONLY_AST")] - use super::PY_COMPILE_FLAG_AST_ONLY; -} +pub(crate) use python::_ast::NodeAst; + +mod python; + +mod argument; +mod basic; +mod constant; +mod elif_else_clause; +mod exception; +mod expression; +mod module; +mod node; +mod operator; +mod other; +mod parameter; +mod pattern; +mod statement; +mod string; +mod type_ignore; +mod type_parameters; fn get_node_field(vm: &VirtualMachine, obj: &PyObject, field: &'static str, typ: &str) -> PyResult { vm.get_attribute_opt(obj.to_owned(), field)? @@ -94,248 +68,197 @@ fn get_node_field_opt( .filter(|obj| !vm.is_none(obj))) } -trait Node: Sized { - fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef; - fn ast_from_object(vm: &VirtualMachine, object: PyObjectRef) -> PyResult; +fn get_int_field( + vm: &VirtualMachine, + obj: &PyObject, + field: &'static str, + typ: &str, +) -> PyResult> { + get_node_field(vm, obj, field, typ)? + .downcast_exact(vm) + .map_err(|_| vm.new_type_error(format!("field \"{field}\" must have integer type"))) } -impl Node for Vec { - fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef { - vm.ctx - .new_list( - self.into_iter() - .map(|node| node.ast_to_object(vm)) - .collect(), - ) - .into() - } - - fn ast_from_object(vm: &VirtualMachine, object: PyObjectRef) -> PyResult { - vm.extract_elements_with(&object, |obj| Node::ast_from_object(vm, obj)) - } +struct PySourceRange { + start: PySourceLocation, + end: PySourceLocation, } -impl Node for Box { - fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef { - (*self).ast_to_object(vm) - } - - fn ast_from_object(vm: &VirtualMachine, object: PyObjectRef) -> PyResult { - T::ast_from_object(vm, object).map(Box::new) - } +pub struct PySourceLocation { + row: Row, + column: Column, } -impl Node for Option { - fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef { - match self { - Some(node) => node.ast_to_object(vm), - None => vm.ctx.none(), - } - } - - fn ast_from_object(vm: &VirtualMachine, object: PyObjectRef) -> PyResult { - if vm.is_none(&object) { - Ok(None) - } else { - Ok(Some(T::ast_from_object(vm, object)?)) +impl PySourceLocation { + fn to_source_location(&self) -> SourceLocation { + SourceLocation { + row: self.row.get_one_indexed(), + column: self.column.get_one_indexed(), } } } -fn range_from_object( - vm: &VirtualMachine, - object: PyObjectRef, - name: &str, -) -> PyResult { - fn make_location(row: u32, column: u32) -> Option { - Some(SourceLocation { - row: OneIndexed::new(row)?, - column: OneIndexed::from_zero_indexed(column), - }) - } - let row = ast::Int::ast_from_object(vm, get_node_field(vm, &object, "lineno", name)?)?; - let column = ast::Int::ast_from_object(vm, get_node_field(vm, &object, "col_offset", name)?)?; - let location = make_location(row.to_u32(), column.to_u32()); - let end_row = get_node_field_opt(vm, &object, "end_lineno")? - .map(|obj| ast::Int::ast_from_object(vm, obj)) - .transpose()?; - let end_column = get_node_field_opt(vm, &object, "end_col_offset")? - .map(|obj| ast::Int::ast_from_object(vm, obj)) - .transpose()?; - let end_location = if let (Some(row), Some(column)) = (end_row, end_column) { - make_location(row.to_u32(), column.to_u32()) - } else { - None - }; - let range = SourceRange { - start: location.unwrap_or_default(), - end: end_location, - }; - Ok(range) -} - -fn node_add_location(dict: &Py, range: SourceRange, vm: &VirtualMachine) { - dict.set_item("lineno", vm.ctx.new_int(range.start.row.get()).into(), vm) - .unwrap(); - dict.set_item( - "col_offset", - vm.ctx.new_int(range.start.column.to_zero_indexed()).into(), - vm, - ) - .unwrap(); - if let Some(end_location) = range.end { - dict.set_item( - "end_lineno", - vm.ctx.new_int(end_location.row.get()).into(), - vm, - ) - .unwrap(); - dict.set_item( - "end_col_offset", - vm.ctx.new_int(end_location.column.to_zero_indexed()).into(), - vm, - ) - .unwrap(); - }; -} +/// A one-based index into the lines. +#[derive(Clone, Copy)] +struct Row(OneIndexed); -impl Node for ast::String { - fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef { - vm.ctx.new_str(self).into() +impl Row { + fn get(self) -> usize { + self.0.get() } - fn ast_from_object(vm: &VirtualMachine, object: PyObjectRef) -> PyResult { - let py_str = PyStrRef::try_from_object(vm, object)?; - Ok(py_str.as_str().to_owned()) + fn get_one_indexed(self) -> OneIndexed { + self.0 } } -impl Node for ast::Identifier { - fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef { - let id: String = self.into(); - vm.ctx.new_str(id).into() - } +/// An UTF-8 index into the line. +#[derive(Clone, Copy)] +struct Column(TextSize); - fn ast_from_object(vm: &VirtualMachine, object: PyObjectRef) -> PyResult { - let py_str = PyStrRef::try_from_object(vm, object)?; - Ok(ast::Identifier::new(py_str.as_str())) +impl Column { + fn get(self) -> usize { + self.0.to_usize() } -} -impl Node for ast::Int { - fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef { - vm.ctx.new_int(self.to_u32()).into() - } - - fn ast_from_object(vm: &VirtualMachine, object: PyObjectRef) -> PyResult { - let value = object.try_into_value(vm)?; - Ok(ast::Int::new(value)) + fn get_one_indexed(self) -> OneIndexed { + OneIndexed::from_zero_indexed(self.get()) } } -impl Node for bool { - fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef { - vm.ctx.new_int(self as u8).into() +fn text_range_to_source_range( + source_code: &SourceCodeOwned, + text_range: TextRange, +) -> PySourceRange { + let index = &source_code.index; + let source = &source_code.text; + + if source.is_empty() { + return PySourceRange { + start: PySourceLocation { + row: Row(OneIndexed::from_zero_indexed(0)), + column: Column(TextSize::new(0)), + }, + end: PySourceLocation { + row: Row(OneIndexed::from_zero_indexed(0)), + column: Column(TextSize::new(0)), + }, + }; } - fn ast_from_object(vm: &VirtualMachine, object: PyObjectRef) -> PyResult { - i32::try_from_object(vm, object).map(|i| i != 0) + let start_row = index.line_index(text_range.start()); + let end_row = index.line_index(text_range.end()); + let start_col = text_range.start() - index.line_start(start_row, source); + let end_col = text_range.end() - index.line_start(end_row, source); + + PySourceRange { + start: PySourceLocation { + row: Row(start_row), + column: Column(start_col), + }, + end: PySourceLocation { + row: Row(end_row), + column: Column(end_col), + }, } } -impl Node for ast::Constant { - fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef { - match self { - ast::Constant::None => vm.ctx.none(), - ast::Constant::Bool(b) => vm.ctx.new_bool(b).into(), - ast::Constant::Str(s) => vm.ctx.new_str(s).into(), - ast::Constant::Bytes(b) => vm.ctx.new_bytes(b).into(), - ast::Constant::Int(i) => vm.ctx.new_int(i).into(), - ast::Constant::Tuple(t) => vm - .ctx - .new_tuple(t.into_iter().map(|c| c.ast_to_object(vm)).collect()) - .into(), - ast::Constant::Float(f) => vm.ctx.new_float(f).into(), - ast::Constant::Complex { real, imag } => vm.new_pyobj(Complex64::new(real, imag)), - ast::Constant::Ellipsis => vm.ctx.ellipsis(), - } - } +fn range_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + name: &str, +) -> PyResult { + let start_row = get_int_field(vm, &object, "lineno", name)?; + let start_column = get_int_field(vm, &object, "col_offset", name)?; + let end_row = get_int_field(vm, &object, "end_lineno", name)?; + let end_column = get_int_field(vm, &object, "end_col_offset", name)?; + + let location = PySourceRange { + start: PySourceLocation { + row: Row(OneIndexed::new(start_row.try_to_primitive(vm)?).unwrap()), + column: Column(TextSize::new(start_column.try_to_primitive(vm)?)), + }, + end: PySourceLocation { + row: Row(OneIndexed::new(end_row.try_to_primitive(vm)?).unwrap()), + column: Column(TextSize::new(end_column.try_to_primitive(vm)?)), + }, + }; - fn ast_from_object(vm: &VirtualMachine, object: PyObjectRef) -> PyResult { - let constant = match_class!(match object { - ref i @ builtins::int::PyInt => { - let value = i.as_bigint(); - if object.class().is(vm.ctx.types.bool_type) { - ast::Constant::Bool(!value.is_zero()) - } else { - ast::Constant::Int(value.clone()) - } - } - ref f @ builtins::float::PyFloat => ast::Constant::Float(f.to_f64()), - ref c @ builtins::complex::PyComplex => { - let c = c.to_complex(); - ast::Constant::Complex { - real: c.re, - imag: c.im, - } - } - ref s @ builtins::pystr::PyStr => ast::Constant::Str(s.as_str().to_owned()), - ref b @ builtins::bytes::PyBytes => ast::Constant::Bytes(b.as_bytes().to_owned()), - ref t @ builtins::tuple::PyTuple => { - ast::Constant::Tuple( - t.iter() - .map(|elt| Self::ast_from_object(vm, elt.clone())) - .collect::>()?, - ) - } - builtins::singletons::PyNone => ast::Constant::None, - builtins::slice::PyEllipsis => ast::Constant::Ellipsis, - obj => - return Err(vm.new_type_error(format!( - "invalid type in Constant: type '{}'", - obj.class().name() - ))), - }); - Ok(constant) - } + Ok(source_range_to_text_range(source_code, location)) } -impl Node for ast::ConversionFlag { - fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef { - vm.ctx.new_int(self as u8).into() - } +fn source_range_to_text_range(source_code: &SourceCodeOwned, location: PySourceRange) -> TextRange { + let source = &source_code.text; + let index = &source_code.index; - fn ast_from_object(vm: &VirtualMachine, object: PyObjectRef) -> PyResult { - i32::try_from_object(vm, object)? - .to_u32() - .and_then(ast::ConversionFlag::from_op_arg) - .ok_or_else(|| vm.new_value_error("invalid conversion flag".to_owned())) + if source.is_empty() { + return TextRange::new(TextSize::new(0), TextSize::new(0)); } + + let start = index.offset( + location.start.row.get_one_indexed(), + location.start.column.get_one_indexed(), + source, + ); + let end = index.offset( + location.end.row.get_one_indexed(), + location.end.column.get_one_indexed(), + source, + ); + + TextRange::new(start, end) } -impl Node for ast::located::Arguments { - fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef { - self.into_python_arguments().ast_to_object(vm) - } - fn ast_from_object(vm: &VirtualMachine, object: PyObjectRef) -> PyResult { - ast::located::PythonArguments::ast_from_object(vm, object) - .map(ast::located::PythonArguments::into_arguments) - } +fn node_add_location( + dict: &Py, + range: TextRange, + vm: &VirtualMachine, + source_code: &SourceCodeOwned, +) { + let range = text_range_to_source_range(source_code, range); + dict.set_item("lineno", vm.ctx.new_int(range.start.row.get()).into(), vm) + .unwrap(); + dict.set_item( + "col_offset", + vm.ctx.new_int(range.start.column.get()).into(), + vm, + ) + .unwrap(); + dict.set_item("end_lineno", vm.ctx.new_int(range.end.row.get()).into(), vm) + .unwrap(); + dict.set_item( + "end_col_offset", + vm.ctx.new_int(range.end.column.get()).into(), + vm, + ) + .unwrap(); } -#[cfg(feature = "rustpython-parser")] +#[cfg(feature = "parser")] pub(crate) fn parse( vm: &VirtualMachine, source: &str, mode: parser::Mode, ) -> Result { - let mut locator = LinearLocator::new(source); - let top = parser::parse(source, mode, "").map_err(|e| locator.locate_error(e))?; - let top = locator.fold_mod(top).unwrap(); - Ok(top.ast_to_object(vm)) + let source_code = SourceCodeOwned::new("".to_owned(), source.to_owned()); + let top = parser::parse(source, mode) + .map_err(|parse_error| ParseError { + error: parse_error.error, + location: text_range_to_source_range(&source_code, parse_error.location) + .start + .to_source_location(), + source_path: "".to_string(), + })? + .into_syntax(); + let top = match top { + ruff::Mod::Module(m) => Mod::Module(m), + ruff::Mod::Expression(e) => Mod::Expression(e), + }; + Ok(top.ast_to_object(vm, &source_code)) } -#[cfg(feature = "rustpython-codegen")] +#[cfg(feature = "codegen")] pub(crate) fn compile( vm: &VirtualMachine, object: PyObjectRef, @@ -348,14 +271,24 @@ pub(crate) fn compile( opts.optimize = optimize; } - let ast = Node::ast_from_object(vm, object)?; - let code = codegen::compile::compile_top(&ast, filename.to_owned(), mode, opts) - .map_err(|err| (CompileError::from(err), None).to_pyexception(vm))?; // FIXME source + let source_code = SourceCodeOwned::new(filename.to_owned(), "".to_owned()); + let ast: Mod = Node::ast_from_object(vm, &source_code, object)?; + let ast = match ast { + Mod::Module(m) => ruff::Mod::Module(m), + Mod::Interactive(ModInteractive { range, body }) => { + ruff::Mod::Module(ruff::ModModule { range, body }) + } + Mod::Expression(e) => ruff::Mod::Expression(e), + Mod::FunctionType(_) => todo!(), + }; + // TODO: create a textual representation of the ast + let text = ""; + let source_code = SourceCode::new(filename, text); + let code = codegen::compile::compile_top(ast, source_code, mode, opts) + .map_err(|err| vm.new_syntax_error(&err.into(), None))?; // FIXME source Ok(vm.ctx.new_code(code).into()) } -// Required crate visibility for inclusion by gen.rs -pub(crate) use _ast::NodeAst; // Used by builtins::compile() pub const PY_COMPILE_FLAG_AST_ONLY: i32 = 0x0400; @@ -398,6 +331,6 @@ pub const PY_COMPILE_FLAGS_MASK: i32 = PY_COMPILE_FLAG_AST_ONLY pub fn make_module(vm: &VirtualMachine) -> PyRef { let module = _ast::make_module(vm); - r#gen::extend_module_nodes(vm, &module); + pyast::extend_module_nodes(vm, &module); module } diff --git a/vm/src/stdlib/ast/argument.rs b/vm/src/stdlib/ast/argument.rs new file mode 100644 index 0000000000..dae2345f0a --- /dev/null +++ b/vm/src/stdlib/ast/argument.rs @@ -0,0 +1,161 @@ +use super::*; + +pub(super) struct PositionalArguments { + pub range: TextRange, + pub args: Box<[ruff::Expr]>, +} + +impl Node for PositionalArguments { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { args, range: _ } = self; + BoxedSlice(args).ast_to_object(vm, source_code) + } + + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + let args: BoxedSlice<_> = Node::ast_from_object(vm, source_code, object)?; + Ok(Self { + args: args.0, + range: TextRange::default(), // TODO + }) + } +} + +pub(super) struct KeywordArguments { + pub range: TextRange, + pub keywords: Box<[ruff::Keyword]>, +} + +impl Node for KeywordArguments { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { keywords, range: _ } = self; + // TODO: use range + BoxedSlice(keywords).ast_to_object(vm, source_code) + } + + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + let keywords: BoxedSlice<_> = Node::ast_from_object(vm, source_code, object)?; + Ok(Self { + keywords: keywords.0, + range: TextRange::default(), // TODO + }) + } +} + +pub(super) fn merge_function_call_arguments( + pos_args: PositionalArguments, + key_args: KeywordArguments, +) -> ruff::Arguments { + let range = pos_args.range.cover(key_args.range); + + ruff::Arguments { + range, + args: pos_args.args, + keywords: key_args.keywords, + } +} + +pub(super) fn split_function_call_arguments( + args: ruff::Arguments, +) -> (PositionalArguments, KeywordArguments) { + let ruff::Arguments { + range: _, + args, + keywords, + } = args; + + let positional_arguments_range = args + .iter() + .map(|item| item.range()) + .reduce(|acc, next| acc.cover(next)) + .unwrap_or_default(); + // debug_assert!(range.contains_range(positional_arguments_range)); + let positional_arguments = PositionalArguments { + range: positional_arguments_range, + args, + }; + + let keyword_arguments_range = keywords + .iter() + .map(|item| item.range()) + .reduce(|acc, next| acc.cover(next)) + .unwrap_or_default(); + // debug_assert!(range.contains_range(keyword_arguments_range)); + let keyword_arguments = KeywordArguments { + range: keyword_arguments_range, + keywords, + }; + + (positional_arguments, keyword_arguments) +} + +pub(super) fn split_class_def_args( + args: Option>, +) -> (Option, Option) { + let args = match args { + None => return (None, None), + Some(args) => *args, + }; + let ruff::Arguments { + range: _, + args, + keywords, + } = args; + + let positional_arguments_range = args + .iter() + .map(|item| item.range()) + .reduce(|acc, next| acc.cover(next)) + .unwrap_or_default(); + // debug_assert!(range.contains_range(positional_arguments_range)); + let positional_arguments = PositionalArguments { + range: positional_arguments_range, + args, + }; + + let keyword_arguments_range = keywords + .iter() + .map(|item| item.range()) + .reduce(|acc, next| acc.cover(next)) + .unwrap_or_default(); + // debug_assert!(range.contains_range(keyword_arguments_range)); + let keyword_arguments = KeywordArguments { + range: keyword_arguments_range, + keywords, + }; + + (Some(positional_arguments), Some(keyword_arguments)) +} + +pub(super) fn merge_class_def_args( + positional_arguments: Option, + keyword_arguments: Option, +) -> Option> { + if positional_arguments.is_none() && keyword_arguments.is_none() { + return None; + } + + let args = if let Some(positional_arguments) = positional_arguments { + positional_arguments.args + } else { + vec![].into_boxed_slice() + }; + let keywords = if let Some(keyword_arguments) = keyword_arguments { + keyword_arguments.keywords + } else { + vec![].into_boxed_slice() + }; + + Some(Box::new(ruff::Arguments { + range: Default::default(), // TODO + args, + keywords, + })) +} diff --git a/vm/src/stdlib/ast/basic.rs b/vm/src/stdlib/ast/basic.rs new file mode 100644 index 0000000000..4ed1e9e03d --- /dev/null +++ b/vm/src/stdlib/ast/basic.rs @@ -0,0 +1,50 @@ +use rustpython_codegen::compile::ruff_int_to_bigint; + +use super::*; + +impl Node for ruff::Identifier { + fn ast_to_object(self, vm: &VirtualMachine, _source_code: &SourceCodeOwned) -> PyObjectRef { + let id = self.as_str(); + vm.ctx.new_str(id).into() + } + + fn ast_from_object( + vm: &VirtualMachine, + _source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + let py_str = PyStrRef::try_from_object(vm, object)?; + Ok(ruff::Identifier::new(py_str.as_str(), TextRange::default())) + } +} + +impl Node for ruff::Int { + fn ast_to_object(self, vm: &VirtualMachine, _source_code: &SourceCodeOwned) -> PyObjectRef { + vm.ctx.new_int(ruff_int_to_bigint(&self).unwrap()).into() + } + + fn ast_from_object( + vm: &VirtualMachine, + _source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + // FIXME: performance + let value: PyIntRef = object.try_into_value(vm)?; + let value = value.as_bigint().to_string(); + Ok(value.parse().unwrap()) + } +} + +impl Node for bool { + fn ast_to_object(self, vm: &VirtualMachine, _source_code: &SourceCodeOwned) -> PyObjectRef { + vm.ctx.new_int(self as u8).into() + } + + fn ast_from_object( + vm: &VirtualMachine, + _source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + i32::try_from_object(vm, object).map(|i| i != 0) + } +} diff --git a/vm/src/stdlib/ast/constant.rs b/vm/src/stdlib/ast/constant.rs new file mode 100644 index 0000000000..29c29b4660 --- /dev/null +++ b/vm/src/stdlib/ast/constant.rs @@ -0,0 +1,396 @@ +use super::*; +use crate::builtins::{PyComplex, PyFrozenSet, PyTuple}; +use ruff::str_prefix::StringLiteralPrefix; + +#[derive(Debug)] +pub(super) struct Constant { + pub(super) range: TextRange, + pub(super) value: ConstantLiteral, +} + +impl Constant { + pub(super) fn new_str( + value: impl Into>, + prefix: StringLiteralPrefix, + range: TextRange, + ) -> Self { + let value = value.into(); + Self { + range, + value: ConstantLiteral::Str { value, prefix }, + } + } + + pub(super) fn new_int(value: ruff::Int, range: TextRange) -> Self { + Self { + range, + value: ConstantLiteral::Int(value), + } + } + + pub(super) fn new_float(value: f64, range: TextRange) -> Self { + Self { + range, + value: ConstantLiteral::Float(value), + } + } + pub(super) fn new_complex(real: f64, imag: f64, range: TextRange) -> Self { + Self { + range, + value: ConstantLiteral::Complex { real, imag }, + } + } + + pub(super) fn new_bytes(value: Box<[u8]>, range: TextRange) -> Self { + Self { + range, + value: ConstantLiteral::Bytes(value), + } + } + + pub(super) fn new_bool(value: bool, range: TextRange) -> Self { + Self { + range, + value: ConstantLiteral::Bool(value), + } + } + + pub(super) fn new_none(range: TextRange) -> Self { + Self { + range, + value: ConstantLiteral::None, + } + } + + pub(super) fn new_ellipsis(range: TextRange) -> Self { + Self { + range, + value: ConstantLiteral::Ellipsis, + } + } + + pub(crate) fn into_expr(self) -> ruff::Expr { + constant_to_ruff_expr(self) + } +} + +#[derive(Debug)] +pub(crate) enum ConstantLiteral { + None, + Bool(bool), + Str { + value: Box, + prefix: StringLiteralPrefix, + }, + Bytes(Box<[u8]>), + Int(ruff::Int), + Tuple(Vec), + FrozenSet(Vec), + Float(f64), + Complex { + real: f64, + imag: f64, + }, + Ellipsis, +} + +// constructor +impl Node for Constant { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { range, value } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprConstant::static_type().to_owned()) + .unwrap(); + let kind = match &value { + ConstantLiteral::Str { + prefix: StringLiteralPrefix::Unicode, + .. + } => vm.ctx.new_str("u").into(), + _ => vm.ctx.none(), + }; + let value = value.ast_to_object(vm, source_code); + let dict = node.as_object().dict().unwrap(); + dict.set_item("value", value, vm).unwrap(); + dict.set_item("kind", kind, vm).unwrap(); + node_add_location(&dict, range, vm, source_code); + node.into() + } + + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + let value_object = get_node_field(vm, &object, "value", "Constant")?; + let value = Node::ast_from_object(vm, source_code, value_object)?; + + Ok(Self { + value, + // kind: get_node_field_opt(_vm, &_object, "kind")? + // .map(|obj| Node::ast_from_object(_vm, obj)) + // .transpose()?, + range: range_from_object(vm, source_code, object, "Constant")?, + }) + } +} + +impl Node for ConstantLiteral { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + match self { + ConstantLiteral::None => vm.ctx.none(), + ConstantLiteral::Bool(value) => vm.ctx.new_bool(value).to_pyobject(vm), + ConstantLiteral::Str { value, .. } => vm.ctx.new_str(value).to_pyobject(vm), + ConstantLiteral::Bytes(value) => vm.ctx.new_bytes(value.into()).to_pyobject(vm), + ConstantLiteral::Int(value) => value.ast_to_object(vm, source_code), + ConstantLiteral::Tuple(value) => { + let value = value + .into_iter() + .map(|c| c.ast_to_object(vm, source_code)) + .collect(); + vm.ctx.new_tuple(value).to_pyobject(vm) + } + ConstantLiteral::FrozenSet(value) => PyFrozenSet::from_iter( + vm, + value.into_iter().map(|c| c.ast_to_object(vm, source_code)), + ) + .unwrap() + .into_pyobject(vm), + ConstantLiteral::Float(value) => vm.ctx.new_float(value).into_pyobject(vm), + ConstantLiteral::Complex { real, imag } => vm + .ctx + .new_complex(num_complex::Complex::new(real, imag)) + .into_pyobject(vm), + ConstantLiteral::Ellipsis => vm.ctx.ellipsis(), + } + } + + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + value_object: PyObjectRef, + ) -> PyResult { + let cls = value_object.class(); + let value = if cls.is(vm.ctx.types.none_type) { + ConstantLiteral::None + } else if cls.is(vm.ctx.types.bool_type) { + ConstantLiteral::Bool(if value_object.is(&vm.ctx.true_value) { + true + } else if value_object.is(&vm.ctx.false_value) { + false + } else { + value_object.try_to_value(vm)? + }) + } else if cls.is(vm.ctx.types.str_type) { + ConstantLiteral::Str { + value: value_object.try_to_value::(vm)?.into(), + prefix: StringLiteralPrefix::Empty, + } + } else if cls.is(vm.ctx.types.bytes_type) { + ConstantLiteral::Bytes(value_object.try_to_value::>(vm)?.into()) + } else if cls.is(vm.ctx.types.int_type) { + ConstantLiteral::Int(Node::ast_from_object(vm, source_code, value_object)?) + } else if cls.is(vm.ctx.types.tuple_type) { + let tuple = value_object.downcast::().map_err(|obj| { + vm.new_type_error(format!( + "Expected type {}, not {}", + PyTuple::static_type().name(), + obj.class().name() + )) + })?; + let tuple = tuple + .into_iter() + .cloned() + .map(|object| Node::ast_from_object(vm, source_code, object)) + .collect::>()?; + ConstantLiteral::Tuple(tuple) + } else if cls.is(vm.ctx.types.frozenset_type) { + let set = value_object.downcast::().unwrap(); + let elements = set + .elements() + .into_iter() + .map(|object| Node::ast_from_object(vm, source_code, object)) + .collect::>()?; + ConstantLiteral::FrozenSet(elements) + } else if cls.is(vm.ctx.types.float_type) { + let float = value_object.try_into_value(vm)?; + ConstantLiteral::Float(float) + } else if cls.is(vm.ctx.types.complex_type) { + let complex = value_object.try_complex(vm)?; + let complex = match complex { + None => { + return Err(vm.new_type_error(format!( + "Expected type {}, not {}", + PyComplex::static_type().name(), + value_object.class().name() + ))); + } + Some((value, _was_coerced)) => value, + }; + ConstantLiteral::Complex { + real: complex.re, + imag: complex.im, + } + } else if cls.is(vm.ctx.types.ellipsis_type) { + ConstantLiteral::Ellipsis + } else { + return Err(vm.new_type_error(format!( + "invalid type in Constant: {}", + value_object.class().name() + ))); + }; + Ok(value) + } +} + +fn constant_to_ruff_expr(value: Constant) -> ruff::Expr { + let Constant { value, range } = value; + match value { + ConstantLiteral::None => ruff::Expr::NoneLiteral(ruff::ExprNoneLiteral { range }), + ConstantLiteral::Bool(value) => { + ruff::Expr::BooleanLiteral(ruff::ExprBooleanLiteral { range, value }) + } + ConstantLiteral::Str { value, prefix } => { + ruff::Expr::StringLiteral(ruff::ExprStringLiteral { + range, + value: ruff::StringLiteralValue::single(ruff::StringLiteral { + range, + value, + flags: ruff::StringLiteralFlags::default().with_prefix(prefix), + }), + }) + } + ConstantLiteral::Bytes(value) => { + ruff::Expr::BytesLiteral(ruff::ExprBytesLiteral { + range, + value: ruff::BytesLiteralValue::single(ruff::BytesLiteral { + range, + value, + flags: Default::default(), // TODO + }), + }) + } + ConstantLiteral::Int(value) => ruff::Expr::NumberLiteral(ruff::ExprNumberLiteral { + range, + value: ruff::Number::Int(value), + }), + ConstantLiteral::Tuple(value) => ruff::Expr::Tuple(ruff::ExprTuple { + range, + elts: value + .into_iter() + .map(|value| { + constant_to_ruff_expr(Constant { + range: TextRange::default(), + value, + }) + }) + .collect(), + ctx: ruff::ExprContext::Load, + // TODO: Does this matter? + parenthesized: true, + }), + ConstantLiteral::FrozenSet(value) => ruff::Expr::Call(ruff::ExprCall { + range, + // idk lol + func: Box::new(ruff::Expr::Name(ruff::ExprName { + range: TextRange::default(), + id: "frozenset".to_owned(), + ctx: ruff::ExprContext::Load, + })), + arguments: ruff::Arguments { + range, + args: value + .into_iter() + .map(|value| { + constant_to_ruff_expr(Constant { + range: TextRange::default(), + value, + }) + }) + .collect(), + keywords: Box::default(), + }, + }), + ConstantLiteral::Float(value) => ruff::Expr::NumberLiteral(ruff::ExprNumberLiteral { + range, + value: ruff::Number::Float(value), + }), + ConstantLiteral::Complex { real, imag } => { + ruff::Expr::NumberLiteral(ruff::ExprNumberLiteral { + range, + value: ruff::Number::Complex { real, imag }, + }) + } + ConstantLiteral::Ellipsis => { + ruff::Expr::EllipsisLiteral(ruff::ExprEllipsisLiteral { range }) + } + } +} + +pub(super) fn number_literal_to_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + constant: ruff::ExprNumberLiteral, +) -> PyObjectRef { + let ruff::ExprNumberLiteral { range, value } = constant; + let c = match value { + ruff::Number::Int(n) => Constant::new_int(n, range), + ruff::Number::Float(n) => Constant::new_float(n, range), + ruff::Number::Complex { real, imag } => Constant::new_complex(real, imag, range), + }; + c.ast_to_object(vm, source_code) +} + +pub(super) fn string_literal_to_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + constant: ruff::ExprStringLiteral, +) -> PyObjectRef { + let ruff::ExprStringLiteral { range, value } = constant; + let prefix = value + .iter() + .next() + .map_or(StringLiteralPrefix::Empty, |part| part.flags.prefix()); + let c = Constant::new_str(value.to_str(), prefix, range); + c.ast_to_object(vm, source_code) +} + +pub(super) fn bytes_literal_to_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + constant: ruff::ExprBytesLiteral, +) -> PyObjectRef { + let ruff::ExprBytesLiteral { range, value } = constant; + let bytes = value.as_slice().iter().flat_map(|b| b.value.iter()); + let c = Constant::new_bytes(bytes.copied().collect(), range); + c.ast_to_object(vm, source_code) +} + +pub(super) fn boolean_literal_to_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + constant: ruff::ExprBooleanLiteral, +) -> PyObjectRef { + let ruff::ExprBooleanLiteral { range, value } = constant; + let c = Constant::new_bool(value, range); + c.ast_to_object(vm, source_code) +} + +pub(super) fn none_literal_to_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + constant: ruff::ExprNoneLiteral, +) -> PyObjectRef { + let ruff::ExprNoneLiteral { range } = constant; + let c = Constant::new_none(range); + c.ast_to_object(vm, source_code) +} + +pub(super) fn ellipsis_literal_to_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + constant: ruff::ExprEllipsisLiteral, +) -> PyObjectRef { + let ruff::ExprEllipsisLiteral { range } = constant; + let c = Constant::new_ellipsis(range); + c.ast_to_object(vm, source_code) +} diff --git a/vm/src/stdlib/ast/elif_else_clause.rs b/vm/src/stdlib/ast/elif_else_clause.rs new file mode 100644 index 0000000000..13f6865eaf --- /dev/null +++ b/vm/src/stdlib/ast/elif_else_clause.rs @@ -0,0 +1,88 @@ +use super::*; + +pub(super) fn ast_to_object( + clause: ruff::ElifElseClause, + mut rest: std::vec::IntoIter, + vm: &VirtualMachine, + source_code: &SourceCodeOwned, +) -> PyObjectRef { + let ruff::ElifElseClause { range, test, body } = clause; + let Some(test) = test else { + assert!(rest.len() == 0); + return body.ast_to_object(vm, source_code); + }; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeStmtIf::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + + dict.set_item("test", test.ast_to_object(vm, source_code), vm) + .unwrap(); + dict.set_item("body", body.ast_to_object(vm, source_code), vm) + .unwrap(); + + let orelse = if let Some(next) = rest.next() { + if next.test.is_some() { + vm.ctx + .new_list(vec![ast_to_object(next, rest, vm, source_code)]) + .into() + } else { + next.body.ast_to_object(vm, source_code) + } + } else { + vm.ctx.new_list(vec![]).into() + }; + dict.set_item("orelse", orelse, vm).unwrap(); + + node_add_location(&dict, range, vm, source_code); + node.into() +} + +pub(super) fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, +) -> PyResult { + let test = Node::ast_from_object(vm, source_code, get_node_field(vm, &object, "test", "If")?)?; + let body = Node::ast_from_object(vm, source_code, get_node_field(vm, &object, "body", "If")?)?; + let orelse: Vec = Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "orelse", "If")?, + )?; + let range = range_from_object(vm, source_code, object, "If")?; + + let elif_else_clauses = if let [ruff::Stmt::If(_)] = &*orelse { + let Some(ruff::Stmt::If(ruff::StmtIf { + range, + test, + body, + mut elif_else_clauses, + })) = orelse.into_iter().next() + else { + unreachable!() + }; + elif_else_clauses.insert( + 0, + ruff::ElifElseClause { + range, + test: Some(*test), + body, + }, + ); + elif_else_clauses + } else { + vec![ruff::ElifElseClause { + range, + test: None, + body: orelse, + }] + }; + + Ok(ruff::StmtIf { + test, + body, + elif_else_clauses, + range, + }) +} diff --git a/vm/src/stdlib/ast/exception.rs b/vm/src/stdlib/ast/exception.rs new file mode 100644 index 0000000000..a76e7b569b --- /dev/null +++ b/vm/src/stdlib/ast/exception.rs @@ -0,0 +1,75 @@ +use super::*; + +// sum +impl Node for ruff::ExceptHandler { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + match self { + ruff::ExceptHandler::ExceptHandler(cons) => cons.ast_to_object(vm, source_code), + } + } + fn ast_from_object( + _vm: &VirtualMachine, + source_code: &SourceCodeOwned, + _object: PyObjectRef, + ) -> PyResult { + let _cls = _object.class(); + Ok( + if _cls.is(pyast::NodeExceptHandlerExceptHandler::static_type()) { + ruff::ExceptHandler::ExceptHandler( + ruff::ExceptHandlerExceptHandler::ast_from_object(_vm, source_code, _object)?, + ) + } else { + return Err(_vm.new_type_error(format!( + "expected some sort of excepthandler, but got {}", + _object.repr(_vm)? + ))); + }, + ) + } +} +// constructor +impl Node for ruff::ExceptHandlerExceptHandler { + fn ast_to_object(self, _vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let ruff::ExceptHandlerExceptHandler { + type_, + name, + body, + range: _range, + } = self; + let node = NodeAst + .into_ref_with_type( + _vm, + pyast::NodeExceptHandlerExceptHandler::static_type().to_owned(), + ) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("type", type_.ast_to_object(_vm, source_code), _vm) + .unwrap(); + dict.set_item("name", name.ast_to_object(_vm, source_code), _vm) + .unwrap(); + dict.set_item("body", body.ast_to_object(_vm, source_code), _vm) + .unwrap(); + node_add_location(&dict, _range, _vm, source_code); + node.into() + } + fn ast_from_object( + _vm: &VirtualMachine, + source_code: &SourceCodeOwned, + _object: PyObjectRef, + ) -> PyResult { + Ok(ruff::ExceptHandlerExceptHandler { + type_: get_node_field_opt(_vm, &_object, "type")? + .map(|obj| Node::ast_from_object(_vm, source_code, obj)) + .transpose()?, + name: get_node_field_opt(_vm, &_object, "name")? + .map(|obj| Node::ast_from_object(_vm, source_code, obj)) + .transpose()?, + body: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "body", "ExceptHandler")?, + )?, + range: range_from_object(_vm, source_code, _object, "ExceptHandler")?, + }) + } +} diff --git a/vm/src/stdlib/ast/expression.rs b/vm/src/stdlib/ast/expression.rs new file mode 100644 index 0000000000..b4c9124ad7 --- /dev/null +++ b/vm/src/stdlib/ast/expression.rs @@ -0,0 +1,1197 @@ +use super::*; +use crate::stdlib::ast::argument::{merge_function_call_arguments, split_function_call_arguments}; +use crate::stdlib::ast::constant::Constant; +use crate::stdlib::ast::string::JoinedStr; + +// sum +impl Node for ruff::Expr { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + match self { + ruff::Expr::BoolOp(cons) => cons.ast_to_object(vm, source_code), + ruff::Expr::Name(cons) => cons.ast_to_object(vm, source_code), + ruff::Expr::BinOp(cons) => cons.ast_to_object(vm, source_code), + ruff::Expr::UnaryOp(cons) => cons.ast_to_object(vm, source_code), + ruff::Expr::Lambda(cons) => cons.ast_to_object(vm, source_code), + ruff::Expr::If(cons) => cons.ast_to_object(vm, source_code), + ruff::Expr::Dict(cons) => cons.ast_to_object(vm, source_code), + ruff::Expr::Set(cons) => cons.ast_to_object(vm, source_code), + ruff::Expr::ListComp(cons) => cons.ast_to_object(vm, source_code), + ruff::Expr::SetComp(cons) => cons.ast_to_object(vm, source_code), + ruff::Expr::DictComp(cons) => cons.ast_to_object(vm, source_code), + ruff::Expr::Generator(cons) => cons.ast_to_object(vm, source_code), + ruff::Expr::Await(cons) => cons.ast_to_object(vm, source_code), + ruff::Expr::Yield(cons) => cons.ast_to_object(vm, source_code), + ruff::Expr::YieldFrom(cons) => cons.ast_to_object(vm, source_code), + ruff::Expr::Compare(cons) => cons.ast_to_object(vm, source_code), + ruff::Expr::Call(cons) => cons.ast_to_object(vm, source_code), + ruff::Expr::Attribute(cons) => cons.ast_to_object(vm, source_code), + ruff::Expr::Subscript(cons) => cons.ast_to_object(vm, source_code), + ruff::Expr::Starred(cons) => cons.ast_to_object(vm, source_code), + ruff::Expr::List(cons) => cons.ast_to_object(vm, source_code), + ruff::Expr::Tuple(cons) => cons.ast_to_object(vm, source_code), + ruff::Expr::Slice(cons) => cons.ast_to_object(vm, source_code), + ruff::Expr::NumberLiteral(cons) => { + constant::number_literal_to_object(vm, source_code, cons) + } + ruff::Expr::StringLiteral(cons) => { + constant::string_literal_to_object(vm, source_code, cons) + } + ruff::Expr::FString(cons) => string::fstring_to_object(vm, source_code, cons), + ruff::Expr::BytesLiteral(cons) => { + constant::bytes_literal_to_object(vm, source_code, cons) + } + ruff::Expr::BooleanLiteral(cons) => { + constant::boolean_literal_to_object(vm, source_code, cons) + } + ruff::Expr::NoneLiteral(cons) => { + constant::none_literal_to_object(vm, source_code, cons) + } + ruff::Expr::EllipsisLiteral(cons) => { + constant::ellipsis_literal_to_object(vm, source_code, cons) + } + ruff::Expr::Named(cons) => cons.ast_to_object(vm, source_code), + ruff::Expr::IpyEscapeCommand(_) => { + unimplemented!("IPython escape command is not allowed in Python AST") + } + } + } + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + let cls = object.class(); + Ok(if cls.is(pyast::NodeExprBoolOp::static_type()) { + ruff::Expr::BoolOp(ruff::ExprBoolOp::ast_from_object(vm, source_code, object)?) + } else if cls.is(pyast::NodeExprNamedExpr::static_type()) { + ruff::Expr::Named(ruff::ExprNamed::ast_from_object(vm, source_code, object)?) + } else if cls.is(pyast::NodeExprBinOp::static_type()) { + ruff::Expr::BinOp(ruff::ExprBinOp::ast_from_object(vm, source_code, object)?) + } else if cls.is(pyast::NodeExprUnaryOp::static_type()) { + ruff::Expr::UnaryOp(ruff::ExprUnaryOp::ast_from_object(vm, source_code, object)?) + } else if cls.is(pyast::NodeExprLambda::static_type()) { + ruff::Expr::Lambda(ruff::ExprLambda::ast_from_object(vm, source_code, object)?) + } else if cls.is(pyast::NodeExprIfExp::static_type()) { + ruff::Expr::If(ruff::ExprIf::ast_from_object(vm, source_code, object)?) + } else if cls.is(pyast::NodeExprDict::static_type()) { + ruff::Expr::Dict(ruff::ExprDict::ast_from_object(vm, source_code, object)?) + } else if cls.is(pyast::NodeExprSet::static_type()) { + ruff::Expr::Set(ruff::ExprSet::ast_from_object(vm, source_code, object)?) + } else if cls.is(pyast::NodeExprListComp::static_type()) { + ruff::Expr::ListComp(ruff::ExprListComp::ast_from_object( + vm, + source_code, + object, + )?) + } else if cls.is(pyast::NodeExprSetComp::static_type()) { + ruff::Expr::SetComp(ruff::ExprSetComp::ast_from_object(vm, source_code, object)?) + } else if cls.is(pyast::NodeExprDictComp::static_type()) { + ruff::Expr::DictComp(ruff::ExprDictComp::ast_from_object( + vm, + source_code, + object, + )?) + } else if cls.is(pyast::NodeExprGeneratorExp::static_type()) { + ruff::Expr::Generator(ruff::ExprGenerator::ast_from_object( + vm, + source_code, + object, + )?) + } else if cls.is(pyast::NodeExprAwait::static_type()) { + ruff::Expr::Await(ruff::ExprAwait::ast_from_object(vm, source_code, object)?) + } else if cls.is(pyast::NodeExprYield::static_type()) { + ruff::Expr::Yield(ruff::ExprYield::ast_from_object(vm, source_code, object)?) + } else if cls.is(pyast::NodeExprYieldFrom::static_type()) { + ruff::Expr::YieldFrom(ruff::ExprYieldFrom::ast_from_object( + vm, + source_code, + object, + )?) + } else if cls.is(pyast::NodeExprCompare::static_type()) { + ruff::Expr::Compare(ruff::ExprCompare::ast_from_object(vm, source_code, object)?) + } else if cls.is(pyast::NodeExprCall::static_type()) { + ruff::Expr::Call(ruff::ExprCall::ast_from_object(vm, source_code, object)?) + } else if cls.is(pyast::NodeExprAttribute::static_type()) { + ruff::Expr::Attribute(ruff::ExprAttribute::ast_from_object( + vm, + source_code, + object, + )?) + } else if cls.is(pyast::NodeExprSubscript::static_type()) { + ruff::Expr::Subscript(ruff::ExprSubscript::ast_from_object( + vm, + source_code, + object, + )?) + } else if cls.is(pyast::NodeExprStarred::static_type()) { + ruff::Expr::Starred(ruff::ExprStarred::ast_from_object(vm, source_code, object)?) + } else if cls.is(pyast::NodeExprName::static_type()) { + ruff::Expr::Name(ruff::ExprName::ast_from_object(vm, source_code, object)?) + } else if cls.is(pyast::NodeExprList::static_type()) { + ruff::Expr::List(ruff::ExprList::ast_from_object(vm, source_code, object)?) + } else if cls.is(pyast::NodeExprTuple::static_type()) { + ruff::Expr::Tuple(ruff::ExprTuple::ast_from_object(vm, source_code, object)?) + } else if cls.is(pyast::NodeExprSlice::static_type()) { + ruff::Expr::Slice(ruff::ExprSlice::ast_from_object(vm, source_code, object)?) + } else if cls.is(pyast::NodeExprConstant::static_type()) { + Constant::ast_from_object(vm, source_code, object)?.into_expr() + } else if cls.is(pyast::NodeExprJoinedStr::static_type()) { + JoinedStr::ast_from_object(vm, source_code, object)?.into_expr() + } else { + return Err(vm.new_type_error(format!( + "expected some sort of expr, but got {}", + object.repr(vm)? + ))); + }) + } +} +// constructor +impl Node for ruff::ExprBoolOp { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { op, values, range } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprBoolOp::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("op", op.ast_to_object(vm, source_code), vm) + .unwrap(); + dict.set_item("values", values.ast_to_object(vm, source_code), vm) + .unwrap(); + node_add_location(&dict, range, vm, source_code); + node.into() + } + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + Ok(Self { + op: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "op", "BoolOp")?, + )?, + values: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "values", "BoolOp")?, + )?, + range: range_from_object(vm, source_code, object, "BoolOp")?, + }) + } +} +// constructor +impl Node for ruff::ExprNamed { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { + target, + value, + range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprNamedExpr::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("target", target.ast_to_object(vm, source_code), vm) + .unwrap(); + dict.set_item("value", value.ast_to_object(vm, source_code), vm) + .unwrap(); + node_add_location(&dict, range, vm, source_code); + node.into() + } + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + Ok(Self { + target: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "target", "NamedExpr")?, + )?, + value: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "value", "NamedExpr")?, + )?, + range: range_from_object(vm, source_code, object, "NamedExpr")?, + }) + } +} +// constructor +impl Node for ruff::ExprBinOp { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { + left, + op, + right, + range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprBinOp::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("left", left.ast_to_object(vm, source_code), vm) + .unwrap(); + dict.set_item("op", op.ast_to_object(vm, source_code), vm) + .unwrap(); + dict.set_item("right", right.ast_to_object(vm, source_code), vm) + .unwrap(); + node_add_location(&dict, range, vm, source_code); + node.into() + } + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + Ok(Self { + left: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "left", "BinOp")?, + )?, + op: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "op", "BinOp")?, + )?, + right: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "right", "BinOp")?, + )?, + range: range_from_object(vm, source_code, object, "BinOp")?, + }) + } +} +// constructor +impl Node for ruff::ExprUnaryOp { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { op, operand, range } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprUnaryOp::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("op", op.ast_to_object(vm, source_code), vm) + .unwrap(); + dict.set_item("operand", operand.ast_to_object(vm, source_code), vm) + .unwrap(); + node_add_location(&dict, range, vm, source_code); + node.into() + } + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + Ok(Self { + op: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "op", "UnaryOp")?, + )?, + operand: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "operand", "UnaryOp")?, + )?, + range: range_from_object(vm, source_code, object, "UnaryOp")?, + }) + } +} +// constructor +impl Node for ruff::ExprLambda { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { + parameters, + body, + range: _range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprLambda::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("args", parameters.ast_to_object(vm, source_code), vm) + .unwrap(); + dict.set_item("body", body.ast_to_object(vm, source_code), vm) + .unwrap(); + node_add_location(&dict, _range, vm, source_code); + node.into() + } + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + Ok(Self { + parameters: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "args", "Lambda")?, + )?, + body: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "body", "Lambda")?, + )?, + range: range_from_object(vm, source_code, object, "Lambda")?, + }) + } +} +// constructor +impl Node for ruff::ExprIf { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { + test, + body, + orelse, + range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprIfExp::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("test", test.ast_to_object(vm, source_code), vm) + .unwrap(); + dict.set_item("body", body.ast_to_object(vm, source_code), vm) + .unwrap(); + dict.set_item("orelse", orelse.ast_to_object(vm, source_code), vm) + .unwrap(); + node_add_location(&dict, range, vm, source_code); + node.into() + } + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + Ok(Self { + test: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "test", "IfExp")?, + )?, + body: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "body", "IfExp")?, + )?, + orelse: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "orelse", "IfExp")?, + )?, + range: range_from_object(vm, source_code, object, "IfExp")?, + }) + } +} +// constructor +impl Node for ruff::ExprDict { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { items, range } = self; + let (keys, values) = + items + .into_iter() + .fold((vec![], vec![]), |(mut keys, mut values), item| { + keys.push(item.key); + values.push(item.value); + (keys, values) + }); + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprDict::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("keys", keys.ast_to_object(vm, source_code), vm) + .unwrap(); + dict.set_item("values", values.ast_to_object(vm, source_code), vm) + .unwrap(); + node_add_location(&dict, range, vm, source_code); + node.into() + } + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + let keys: Vec> = Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "keys", "Dict")?, + )?; + let values: Vec<_> = Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "values", "Dict")?, + )?; + let items = keys + .into_iter() + .zip(values) + .map(|(key, value)| ruff::DictItem { key, value }) + .collect(); + Ok(Self { + items, + range: range_from_object(vm, source_code, object, "Dict")?, + }) + } +} +// constructor +impl Node for ruff::ExprSet { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { elts, range } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprSet::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("elts", elts.ast_to_object(vm, source_code), vm) + .unwrap(); + node_add_location(&dict, range, vm, source_code); + node.into() + } + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + Ok(Self { + elts: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "elts", "Set")?, + )?, + range: range_from_object(vm, source_code, object, "Set")?, + }) + } +} +// constructor +impl Node for ruff::ExprListComp { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { + elt, + generators, + range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprListComp::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("elt", elt.ast_to_object(vm, source_code), vm) + .unwrap(); + dict.set_item("generators", generators.ast_to_object(vm, source_code), vm) + .unwrap(); + node_add_location(&dict, range, vm, source_code); + node.into() + } + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + Ok(Self { + elt: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "elt", "ListComp")?, + )?, + generators: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "generators", "ListComp")?, + )?, + range: range_from_object(vm, source_code, object, "ListComp")?, + }) + } +} +// constructor +impl Node for ruff::ExprSetComp { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { + elt, + generators, + range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprSetComp::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("elt", elt.ast_to_object(vm, source_code), vm) + .unwrap(); + dict.set_item("generators", generators.ast_to_object(vm, source_code), vm) + .unwrap(); + node_add_location(&dict, range, vm, source_code); + node.into() + } + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + Ok(Self { + elt: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "elt", "SetComp")?, + )?, + generators: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "generators", "SetComp")?, + )?, + range: range_from_object(vm, source_code, object, "SetComp")?, + }) + } +} +// constructor +impl Node for ruff::ExprDictComp { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { + key, + value, + generators, + range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprDictComp::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("key", key.ast_to_object(vm, source_code), vm) + .unwrap(); + dict.set_item("value", value.ast_to_object(vm, source_code), vm) + .unwrap(); + dict.set_item("generators", generators.ast_to_object(vm, source_code), vm) + .unwrap(); + node_add_location(&dict, range, vm, source_code); + node.into() + } + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + Ok(Self { + key: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "key", "DictComp")?, + )?, + value: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "value", "DictComp")?, + )?, + generators: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "generators", "DictComp")?, + )?, + range: range_from_object(vm, source_code, object, "DictComp")?, + }) + } +} +// constructor +impl Node for ruff::ExprGenerator { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { + elt, + generators, + range, + parenthesized: _, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprGeneratorExp::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("elt", elt.ast_to_object(vm, source_code), vm) + .unwrap(); + dict.set_item("generators", generators.ast_to_object(vm, source_code), vm) + .unwrap(); + node_add_location(&dict, range, vm, source_code); + node.into() + } + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + Ok(Self { + elt: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "elt", "GeneratorExp")?, + )?, + generators: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "generators", "GeneratorExp")?, + )?, + range: range_from_object(vm, source_code, object, "GeneratorExp")?, + // TODO: Is this correct? + parenthesized: true, + }) + } +} +// constructor +impl Node for ruff::ExprAwait { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { value, range } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprAwait::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("value", value.ast_to_object(vm, source_code), vm) + .unwrap(); + node_add_location(&dict, range, vm, source_code); + node.into() + } + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + Ok(Self { + value: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "value", "Await")?, + )?, + range: range_from_object(vm, source_code, object, "Await")?, + }) + } +} +// constructor +impl Node for ruff::ExprYield { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let ruff::ExprYield { value, range } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprYield::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("value", value.ast_to_object(vm, source_code), vm) + .unwrap(); + node_add_location(&dict, range, vm, source_code); + node.into() + } + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + Ok(ruff::ExprYield { + value: get_node_field_opt(vm, &object, "value")? + .map(|obj| Node::ast_from_object(vm, source_code, obj)) + .transpose()?, + range: range_from_object(vm, source_code, object, "Yield")?, + }) + } +} +// constructor +impl Node for ruff::ExprYieldFrom { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { value, range } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprYieldFrom::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("value", value.ast_to_object(vm, source_code), vm) + .unwrap(); + node_add_location(&dict, range, vm, source_code); + node.into() + } + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + Ok(Self { + value: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "value", "YieldFrom")?, + )?, + range: range_from_object(vm, source_code, object, "YieldFrom")?, + }) + } +} +// constructor +impl Node for ruff::ExprCompare { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { + left, + ops, + comparators, + range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprCompare::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("left", left.ast_to_object(vm, source_code), vm) + .unwrap(); + dict.set_item("ops", BoxedSlice(ops).ast_to_object(vm, source_code), vm) + .unwrap(); + dict.set_item( + "comparators", + BoxedSlice(comparators).ast_to_object(vm, source_code), + vm, + ) + .unwrap(); + node_add_location(&dict, range, vm, source_code); + node.into() + } + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + Ok(Self { + left: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "left", "Compare")?, + )?, + ops: { + let ops: BoxedSlice<_> = Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "ops", "Compare")?, + )?; + ops.0 + }, + comparators: { + let comparators: BoxedSlice<_> = Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "comparators", "Compare")?, + )?; + comparators.0 + }, + range: range_from_object(vm, source_code, object, "Compare")?, + }) + } +} +// constructor +impl Node for ruff::ExprCall { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { + func, + arguments, + range, + } = self; + let (positional_arguments, keyword_arguments) = split_function_call_arguments(arguments); + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprCall::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("func", func.ast_to_object(vm, source_code), vm) + .unwrap(); + dict.set_item( + "args", + positional_arguments.ast_to_object(vm, source_code), + vm, + ) + .unwrap(); + dict.set_item( + "keywords", + keyword_arguments.ast_to_object(vm, source_code), + vm, + ) + .unwrap(); + node_add_location(&dict, range, vm, source_code); + node.into() + } + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + Ok(Self { + func: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "func", "Call")?, + )?, + arguments: merge_function_call_arguments( + Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "args", "Call")?, + )?, + Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "keywords", "Call")?, + )?, + ), + range: range_from_object(vm, source_code, object, "Call")?, + }) + } +} + +// constructor +impl Node for ruff::ExprAttribute { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { + value, + attr, + ctx, + range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprAttribute::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("value", value.ast_to_object(vm, source_code), vm) + .unwrap(); + dict.set_item("attr", attr.ast_to_object(vm, source_code), vm) + .unwrap(); + dict.set_item("ctx", ctx.ast_to_object(vm, source_code), vm) + .unwrap(); + node_add_location(&dict, range, vm, source_code); + node.into() + } + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + Ok(Self { + value: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "value", "Attribute")?, + )?, + attr: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "attr", "Attribute")?, + )?, + ctx: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "ctx", "Attribute")?, + )?, + range: range_from_object(vm, source_code, object, "Attribute")?, + }) + } +} +// constructor +impl Node for ruff::ExprSubscript { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { + value, + slice, + ctx, + range: _range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprSubscript::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("value", value.ast_to_object(vm, source_code), vm) + .unwrap(); + dict.set_item("slice", slice.ast_to_object(vm, source_code), vm) + .unwrap(); + dict.set_item("ctx", ctx.ast_to_object(vm, source_code), vm) + .unwrap(); + node_add_location(&dict, _range, vm, source_code); + node.into() + } + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + Ok(Self { + value: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "value", "Subscript")?, + )?, + slice: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "slice", "Subscript")?, + )?, + ctx: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "ctx", "Subscript")?, + )?, + range: range_from_object(vm, source_code, object, "Subscript")?, + }) + } +} +// constructor +impl Node for ruff::ExprStarred { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { value, ctx, range } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprStarred::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("value", value.ast_to_object(vm, source_code), vm) + .unwrap(); + dict.set_item("ctx", ctx.ast_to_object(vm, source_code), vm) + .unwrap(); + node_add_location(&dict, range, vm, source_code); + node.into() + } + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + Ok(Self { + value: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "value", "Starred")?, + )?, + ctx: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "ctx", "Starred")?, + )?, + range: range_from_object(vm, source_code, object, "Starred")?, + }) + } +} +// constructor +impl Node for ruff::ExprName { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { id, ctx, range } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprName::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("id", id.to_pyobject(vm), vm).unwrap(); + dict.set_item("ctx", ctx.ast_to_object(vm, source_code), vm) + .unwrap(); + node_add_location(&dict, range, vm, source_code); + node.into() + } + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + Ok(Self { + id: get_node_field(vm, &object, "id", "Name")?.try_into_value(vm)?, + ctx: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "ctx", "Name")?, + )?, + range: range_from_object(vm, source_code, object, "Name")?, + }) + } +} +// constructor +impl Node for ruff::ExprList { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let ruff::ExprList { elts, ctx, range } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprList::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("elts", elts.ast_to_object(vm, source_code), vm) + .unwrap(); + dict.set_item("ctx", ctx.ast_to_object(vm, source_code), vm) + .unwrap(); + node_add_location(&dict, range, vm, source_code); + node.into() + } + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + Ok(ruff::ExprList { + elts: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "elts", "List")?, + )?, + ctx: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "ctx", "List")?, + )?, + range: range_from_object(vm, source_code, object, "List")?, + }) + } +} +// constructor +impl Node for ruff::ExprTuple { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { + elts, + ctx, + range: _range, + parenthesized: _, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprTuple::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("elts", elts.ast_to_object(vm, source_code), vm) + .unwrap(); + dict.set_item("ctx", ctx.ast_to_object(vm, source_code), vm) + .unwrap(); + node_add_location(&dict, _range, vm, source_code); + node.into() + } + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + Ok(Self { + elts: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "elts", "Tuple")?, + )?, + ctx: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "ctx", "Tuple")?, + )?, + range: range_from_object(vm, source_code, object, "Tuple")?, + parenthesized: true, // TODO: is this correct? + }) + } +} +// constructor +impl Node for ruff::ExprSlice { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { + lower, + upper, + step, + range: _range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprSlice::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("lower", lower.ast_to_object(vm, source_code), vm) + .unwrap(); + dict.set_item("upper", upper.ast_to_object(vm, source_code), vm) + .unwrap(); + dict.set_item("step", step.ast_to_object(vm, source_code), vm) + .unwrap(); + node_add_location(&dict, _range, vm, source_code); + node.into() + } + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + Ok(Self { + lower: get_node_field_opt(vm, &object, "lower")? + .map(|obj| Node::ast_from_object(vm, source_code, obj)) + .transpose()?, + upper: get_node_field_opt(vm, &object, "upper")? + .map(|obj| Node::ast_from_object(vm, source_code, obj)) + .transpose()?, + step: get_node_field_opt(vm, &object, "step")? + .map(|obj| Node::ast_from_object(vm, source_code, obj)) + .transpose()?, + range: range_from_object(vm, source_code, object, "Slice")?, + }) + } +} +// sum +impl Node for ruff::ExprContext { + fn ast_to_object(self, vm: &VirtualMachine, _source_code: &SourceCodeOwned) -> PyObjectRef { + let node_type = match self { + ruff::ExprContext::Load => pyast::NodeExprContextLoad::static_type(), + ruff::ExprContext::Store => pyast::NodeExprContextStore::static_type(), + ruff::ExprContext::Del => pyast::NodeExprContextDel::static_type(), + ruff::ExprContext::Invalid => { + unimplemented!("Invalid expression context is not allowed in Python AST") + } + }; + NodeAst + .into_ref_with_type(vm, node_type.to_owned()) + .unwrap() + .into() + } + fn ast_from_object( + vm: &VirtualMachine, + _source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + let _cls = object.class(); + Ok(if _cls.is(pyast::NodeExprContextLoad::static_type()) { + ruff::ExprContext::Load + } else if _cls.is(pyast::NodeExprContextStore::static_type()) { + ruff::ExprContext::Store + } else if _cls.is(pyast::NodeExprContextDel::static_type()) { + ruff::ExprContext::Del + } else { + return Err(vm.new_type_error(format!( + "expected some sort of expr_context, but got {}", + object.repr(vm)? + ))); + }) + } +} + +// product +impl Node for ruff::Comprehension { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { + target, + iter, + ifs, + is_async, + range: _range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeComprehension::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("target", target.ast_to_object(vm, source_code), vm) + .unwrap(); + dict.set_item("iter", iter.ast_to_object(vm, source_code), vm) + .unwrap(); + dict.set_item("ifs", ifs.ast_to_object(vm, source_code), vm) + .unwrap(); + dict.set_item("is_async", is_async.ast_to_object(vm, source_code), vm) + .unwrap(); + node.into() + } + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + Ok(Self { + target: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "target", "comprehension")?, + )?, + iter: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "iter", "comprehension")?, + )?, + ifs: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "ifs", "comprehension")?, + )?, + is_async: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "is_async", "comprehension")?, + )?, + range: Default::default(), + }) + } +} diff --git a/vm/src/stdlib/ast/gen.rs b/vm/src/stdlib/ast/gen.rs deleted file mode 100644 index d3969b9024..0000000000 --- a/vm/src/stdlib/ast/gen.rs +++ /dev/null @@ -1,5496 +0,0 @@ -// File automatically generated by ast/asdl_rs.py. - -#![allow(clippy::all)] - -use super::*; -use crate::common::ascii; -#[pyclass(module = "_ast", name = "mod", base = "NodeAst")] -struct NodeMod; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeMod {} -#[pyclass(module = "_ast", name = "Module", base = "NodeMod")] -struct NodeModModule; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeModModule { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("body")).into(), - ctx.new_str(ascii!("type_ignores")).into(), - ]) - .into(), - ); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "Interactive", base = "NodeMod")] -struct NodeModInteractive; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeModInteractive { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ctx.new_str(ascii!("body")).into()]) - .into(), - ); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "Expression", base = "NodeMod")] -struct NodeModExpression; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeModExpression { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ctx.new_str(ascii!("body")).into()]) - .into(), - ); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "FunctionType", base = "NodeMod")] -struct NodeModFunctionType; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeModFunctionType { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("argtypes")).into(), - ctx.new_str(ascii!("returns")).into(), - ]) - .into(), - ); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "stmt", base = "NodeAst")] -struct NodeStmt; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmt {} -#[pyclass(module = "_ast", name = "FunctionDef", base = "NodeStmt")] -struct NodeStmtFunctionDef; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtFunctionDef { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("name")).into(), - ctx.new_str(ascii!("args")).into(), - ctx.new_str(ascii!("body")).into(), - ctx.new_str(ascii!("decorator_list")).into(), - ctx.new_str(ascii!("returns")).into(), - ctx.new_str(ascii!("type_comment")).into(), - ctx.new_str(ascii!("type_params")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "AsyncFunctionDef", base = "NodeStmt")] -struct NodeStmtAsyncFunctionDef; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtAsyncFunctionDef { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("name")).into(), - ctx.new_str(ascii!("args")).into(), - ctx.new_str(ascii!("body")).into(), - ctx.new_str(ascii!("decorator_list")).into(), - ctx.new_str(ascii!("returns")).into(), - ctx.new_str(ascii!("type_comment")).into(), - ctx.new_str(ascii!("type_params")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "ClassDef", base = "NodeStmt")] -struct NodeStmtClassDef; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtClassDef { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("name")).into(), - ctx.new_str(ascii!("bases")).into(), - ctx.new_str(ascii!("keywords")).into(), - ctx.new_str(ascii!("body")).into(), - ctx.new_str(ascii!("decorator_list")).into(), - ctx.new_str(ascii!("type_params")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Return", base = "NodeStmt")] -struct NodeStmtReturn; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtReturn { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ctx.new_str(ascii!("value")).into()]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Delete", base = "NodeStmt")] -struct NodeStmtDelete; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtDelete { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ctx.new_str(ascii!("targets")).into()]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Assign", base = "NodeStmt")] -struct NodeStmtAssign; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtAssign { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("targets")).into(), - ctx.new_str(ascii!("value")).into(), - ctx.new_str(ascii!("type_comment")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "TypeAlias", base = "NodeStmt")] -struct NodeStmtTypeAlias; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtTypeAlias { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("name")).into(), - ctx.new_str(ascii!("type_params")).into(), - ctx.new_str(ascii!("value")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "AugAssign", base = "NodeStmt")] -struct NodeStmtAugAssign; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtAugAssign { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("target")).into(), - ctx.new_str(ascii!("op")).into(), - ctx.new_str(ascii!("value")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "AnnAssign", base = "NodeStmt")] -struct NodeStmtAnnAssign; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtAnnAssign { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("target")).into(), - ctx.new_str(ascii!("annotation")).into(), - ctx.new_str(ascii!("value")).into(), - ctx.new_str(ascii!("simple")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "For", base = "NodeStmt")] -struct NodeStmtFor; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtFor { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("target")).into(), - ctx.new_str(ascii!("iter")).into(), - ctx.new_str(ascii!("body")).into(), - ctx.new_str(ascii!("orelse")).into(), - ctx.new_str(ascii!("type_comment")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "AsyncFor", base = "NodeStmt")] -struct NodeStmtAsyncFor; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtAsyncFor { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("target")).into(), - ctx.new_str(ascii!("iter")).into(), - ctx.new_str(ascii!("body")).into(), - ctx.new_str(ascii!("orelse")).into(), - ctx.new_str(ascii!("type_comment")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "While", base = "NodeStmt")] -struct NodeStmtWhile; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtWhile { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("test")).into(), - ctx.new_str(ascii!("body")).into(), - ctx.new_str(ascii!("orelse")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "If", base = "NodeStmt")] -struct NodeStmtIf; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtIf { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("test")).into(), - ctx.new_str(ascii!("body")).into(), - ctx.new_str(ascii!("orelse")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "With", base = "NodeStmt")] -struct NodeStmtWith; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtWith { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("items")).into(), - ctx.new_str(ascii!("body")).into(), - ctx.new_str(ascii!("type_comment")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "AsyncWith", base = "NodeStmt")] -struct NodeStmtAsyncWith; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtAsyncWith { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("items")).into(), - ctx.new_str(ascii!("body")).into(), - ctx.new_str(ascii!("type_comment")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Match", base = "NodeStmt")] -struct NodeStmtMatch; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtMatch { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("subject")).into(), - ctx.new_str(ascii!("cases")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Raise", base = "NodeStmt")] -struct NodeStmtRaise; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtRaise { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("exc")).into(), - ctx.new_str(ascii!("cause")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Try", base = "NodeStmt")] -struct NodeStmtTry; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtTry { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("body")).into(), - ctx.new_str(ascii!("handlers")).into(), - ctx.new_str(ascii!("orelse")).into(), - ctx.new_str(ascii!("finalbody")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "TryStar", base = "NodeStmt")] -struct NodeStmtTryStar; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtTryStar { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("body")).into(), - ctx.new_str(ascii!("handlers")).into(), - ctx.new_str(ascii!("orelse")).into(), - ctx.new_str(ascii!("finalbody")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Assert", base = "NodeStmt")] -struct NodeStmtAssert; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtAssert { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("test")).into(), - ctx.new_str(ascii!("msg")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Import", base = "NodeStmt")] -struct NodeStmtImport; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtImport { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ctx.new_str(ascii!("names")).into()]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "ImportFrom", base = "NodeStmt")] -struct NodeStmtImportFrom; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtImportFrom { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("module")).into(), - ctx.new_str(ascii!("names")).into(), - ctx.new_str(ascii!("level")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Global", base = "NodeStmt")] -struct NodeStmtGlobal; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtGlobal { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ctx.new_str(ascii!("names")).into()]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Nonlocal", base = "NodeStmt")] -struct NodeStmtNonlocal; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtNonlocal { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ctx.new_str(ascii!("names")).into()]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Expr", base = "NodeStmt")] -struct NodeStmtExpr; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtExpr { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ctx.new_str(ascii!("value")).into()]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Pass", base = "NodeStmt")] -struct NodeStmtPass; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtPass { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Break", base = "NodeStmt")] -struct NodeStmtBreak; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtBreak { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Continue", base = "NodeStmt")] -struct NodeStmtContinue; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtContinue { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "expr", base = "NodeAst")] -struct NodeExpr; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExpr {} -#[pyclass(module = "_ast", name = "BoolOp", base = "NodeExpr")] -struct NodeExprBoolOp; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprBoolOp { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("op")).into(), - ctx.new_str(ascii!("values")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "NamedExpr", base = "NodeExpr")] -struct NodeExprNamedExpr; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprNamedExpr { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("target")).into(), - ctx.new_str(ascii!("value")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "BinOp", base = "NodeExpr")] -struct NodeExprBinOp; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprBinOp { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("left")).into(), - ctx.new_str(ascii!("op")).into(), - ctx.new_str(ascii!("right")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "UnaryOp", base = "NodeExpr")] -struct NodeExprUnaryOp; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprUnaryOp { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("op")).into(), - ctx.new_str(ascii!("operand")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Lambda", base = "NodeExpr")] -struct NodeExprLambda; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprLambda { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("args")).into(), - ctx.new_str(ascii!("body")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "IfExp", base = "NodeExpr")] -struct NodeExprIfExp; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprIfExp { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("test")).into(), - ctx.new_str(ascii!("body")).into(), - ctx.new_str(ascii!("orelse")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Dict", base = "NodeExpr")] -struct NodeExprDict; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprDict { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("keys")).into(), - ctx.new_str(ascii!("values")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Set", base = "NodeExpr")] -struct NodeExprSet; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprSet { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ctx.new_str(ascii!("elts")).into()]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "ListComp", base = "NodeExpr")] -struct NodeExprListComp; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprListComp { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("elt")).into(), - ctx.new_str(ascii!("generators")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "SetComp", base = "NodeExpr")] -struct NodeExprSetComp; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprSetComp { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("elt")).into(), - ctx.new_str(ascii!("generators")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "DictComp", base = "NodeExpr")] -struct NodeExprDictComp; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprDictComp { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("key")).into(), - ctx.new_str(ascii!("value")).into(), - ctx.new_str(ascii!("generators")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "GeneratorExp", base = "NodeExpr")] -struct NodeExprGeneratorExp; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprGeneratorExp { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("elt")).into(), - ctx.new_str(ascii!("generators")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Await", base = "NodeExpr")] -struct NodeExprAwait; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprAwait { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ctx.new_str(ascii!("value")).into()]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Yield", base = "NodeExpr")] -struct NodeExprYield; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprYield { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ctx.new_str(ascii!("value")).into()]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "YieldFrom", base = "NodeExpr")] -struct NodeExprYieldFrom; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprYieldFrom { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ctx.new_str(ascii!("value")).into()]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Compare", base = "NodeExpr")] -struct NodeExprCompare; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprCompare { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("left")).into(), - ctx.new_str(ascii!("ops")).into(), - ctx.new_str(ascii!("comparators")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Call", base = "NodeExpr")] -struct NodeExprCall; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprCall { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("func")).into(), - ctx.new_str(ascii!("args")).into(), - ctx.new_str(ascii!("keywords")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "FormattedValue", base = "NodeExpr")] -struct NodeExprFormattedValue; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprFormattedValue { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("value")).into(), - ctx.new_str(ascii!("conversion")).into(), - ctx.new_str(ascii!("format_spec")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "JoinedStr", base = "NodeExpr")] -struct NodeExprJoinedStr; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprJoinedStr { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ctx.new_str(ascii!("values")).into()]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Constant", base = "NodeExpr")] -struct NodeExprConstant; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprConstant { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("value")).into(), - ctx.new_str(ascii!("kind")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Attribute", base = "NodeExpr")] -struct NodeExprAttribute; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprAttribute { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("value")).into(), - ctx.new_str(ascii!("attr")).into(), - ctx.new_str(ascii!("ctx")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Subscript", base = "NodeExpr")] -struct NodeExprSubscript; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprSubscript { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("value")).into(), - ctx.new_str(ascii!("slice")).into(), - ctx.new_str(ascii!("ctx")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Starred", base = "NodeExpr")] -struct NodeExprStarred; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprStarred { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("value")).into(), - ctx.new_str(ascii!("ctx")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Name", base = "NodeExpr")] -struct NodeExprName; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprName { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("id")).into(), - ctx.new_str(ascii!("ctx")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "List", base = "NodeExpr")] -struct NodeExprList; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprList { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("elts")).into(), - ctx.new_str(ascii!("ctx")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Tuple", base = "NodeExpr")] -struct NodeExprTuple; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprTuple { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("elts")).into(), - ctx.new_str(ascii!("ctx")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Slice", base = "NodeExpr")] -struct NodeExprSlice; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprSlice { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("lower")).into(), - ctx.new_str(ascii!("upper")).into(), - ctx.new_str(ascii!("step")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "expr_context", base = "NodeAst")] -struct NodeExprContext; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprContext {} -#[pyclass(module = "_ast", name = "Load", base = "NodeExprContext")] -struct NodeExprContextLoad; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprContextLoad { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "Store", base = "NodeExprContext")] -struct NodeExprContextStore; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprContextStore { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "Del", base = "NodeExprContext")] -struct NodeExprContextDel; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprContextDel { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "boolop", base = "NodeAst")] -struct NodeBoolOp; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeBoolOp {} -#[pyclass(module = "_ast", name = "And", base = "NodeBoolOp")] -struct NodeBoolOpAnd; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeBoolOpAnd { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "Or", base = "NodeBoolOp")] -struct NodeBoolOpOr; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeBoolOpOr { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "operator", base = "NodeAst")] -struct NodeOperator; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeOperator {} -#[pyclass(module = "_ast", name = "Add", base = "NodeOperator")] -struct NodeOperatorAdd; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeOperatorAdd { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "Sub", base = "NodeOperator")] -struct NodeOperatorSub; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeOperatorSub { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "Mult", base = "NodeOperator")] -struct NodeOperatorMult; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeOperatorMult { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "MatMult", base = "NodeOperator")] -struct NodeOperatorMatMult; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeOperatorMatMult { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "Div", base = "NodeOperator")] -struct NodeOperatorDiv; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeOperatorDiv { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "Mod", base = "NodeOperator")] -struct NodeOperatorMod; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeOperatorMod { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "Pow", base = "NodeOperator")] -struct NodeOperatorPow; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeOperatorPow { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "LShift", base = "NodeOperator")] -struct NodeOperatorLShift; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeOperatorLShift { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "RShift", base = "NodeOperator")] -struct NodeOperatorRShift; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeOperatorRShift { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "BitOr", base = "NodeOperator")] -struct NodeOperatorBitOr; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeOperatorBitOr { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "BitXor", base = "NodeOperator")] -struct NodeOperatorBitXor; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeOperatorBitXor { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "BitAnd", base = "NodeOperator")] -struct NodeOperatorBitAnd; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeOperatorBitAnd { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "FloorDiv", base = "NodeOperator")] -struct NodeOperatorFloorDiv; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeOperatorFloorDiv { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "unaryop", base = "NodeAst")] -struct NodeUnaryOp; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeUnaryOp {} -#[pyclass(module = "_ast", name = "Invert", base = "NodeUnaryOp")] -struct NodeUnaryOpInvert; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeUnaryOpInvert { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "Not", base = "NodeUnaryOp")] -struct NodeUnaryOpNot; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeUnaryOpNot { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "UAdd", base = "NodeUnaryOp")] -struct NodeUnaryOpUAdd; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeUnaryOpUAdd { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "USub", base = "NodeUnaryOp")] -struct NodeUnaryOpUSub; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeUnaryOpUSub { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "cmpop", base = "NodeAst")] -struct NodeCmpOp; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeCmpOp {} -#[pyclass(module = "_ast", name = "Eq", base = "NodeCmpOp")] -struct NodeCmpOpEq; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeCmpOpEq { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "NotEq", base = "NodeCmpOp")] -struct NodeCmpOpNotEq; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeCmpOpNotEq { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "Lt", base = "NodeCmpOp")] -struct NodeCmpOpLt; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeCmpOpLt { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "LtE", base = "NodeCmpOp")] -struct NodeCmpOpLtE; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeCmpOpLtE { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "Gt", base = "NodeCmpOp")] -struct NodeCmpOpGt; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeCmpOpGt { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "GtE", base = "NodeCmpOp")] -struct NodeCmpOpGtE; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeCmpOpGtE { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "Is", base = "NodeCmpOp")] -struct NodeCmpOpIs; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeCmpOpIs { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "IsNot", base = "NodeCmpOp")] -struct NodeCmpOpIsNot; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeCmpOpIsNot { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "In", base = "NodeCmpOp")] -struct NodeCmpOpIn; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeCmpOpIn { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "NotIn", base = "NodeCmpOp")] -struct NodeCmpOpNotIn; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeCmpOpNotIn { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "comprehension", base = "NodeAst")] -struct NodeComprehension; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeComprehension { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("target")).into(), - ctx.new_str(ascii!("iter")).into(), - ctx.new_str(ascii!("ifs")).into(), - ctx.new_str(ascii!("is_async")).into(), - ]) - .into(), - ); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "excepthandler", base = "NodeAst")] -struct NodeExceptHandler; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExceptHandler {} -#[pyclass(module = "_ast", name = "ExceptHandler", base = "NodeExceptHandler")] -struct NodeExceptHandlerExceptHandler; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExceptHandlerExceptHandler { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("type")).into(), - ctx.new_str(ascii!("name")).into(), - ctx.new_str(ascii!("body")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "arguments", base = "NodeAst")] -struct NodeArguments; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeArguments { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("posonlyargs")).into(), - ctx.new_str(ascii!("args")).into(), - ctx.new_str(ascii!("vararg")).into(), - ctx.new_str(ascii!("kwonlyargs")).into(), - ctx.new_str(ascii!("kw_defaults")).into(), - ctx.new_str(ascii!("kwarg")).into(), - ctx.new_str(ascii!("defaults")).into(), - ]) - .into(), - ); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "arg", base = "NodeAst")] -struct NodeArg; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeArg { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("arg")).into(), - ctx.new_str(ascii!("annotation")).into(), - ctx.new_str(ascii!("type_comment")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "keyword", base = "NodeAst")] -struct NodeKeyword; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeKeyword { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("arg")).into(), - ctx.new_str(ascii!("value")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "alias", base = "NodeAst")] -struct NodeAlias; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeAlias { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("name")).into(), - ctx.new_str(ascii!("asname")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "withitem", base = "NodeAst")] -struct NodeWithItem; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeWithItem { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("context_expr")).into(), - ctx.new_str(ascii!("optional_vars")).into(), - ]) - .into(), - ); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "match_case", base = "NodeAst")] -struct NodeMatchCase; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeMatchCase { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("pattern")).into(), - ctx.new_str(ascii!("guard")).into(), - ctx.new_str(ascii!("body")).into(), - ]) - .into(), - ); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "pattern", base = "NodeAst")] -struct NodePattern; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodePattern {} -#[pyclass(module = "_ast", name = "MatchValue", base = "NodePattern")] -struct NodePatternMatchValue; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodePatternMatchValue { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ctx.new_str(ascii!("value")).into()]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "MatchSingleton", base = "NodePattern")] -struct NodePatternMatchSingleton; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodePatternMatchSingleton { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ctx.new_str(ascii!("value")).into()]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "MatchSequence", base = "NodePattern")] -struct NodePatternMatchSequence; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodePatternMatchSequence { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ctx.new_str(ascii!("patterns")).into()]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "MatchMapping", base = "NodePattern")] -struct NodePatternMatchMapping; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodePatternMatchMapping { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("keys")).into(), - ctx.new_str(ascii!("patterns")).into(), - ctx.new_str(ascii!("rest")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "MatchClass", base = "NodePattern")] -struct NodePatternMatchClass; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodePatternMatchClass { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("cls")).into(), - ctx.new_str(ascii!("patterns")).into(), - ctx.new_str(ascii!("kwd_attrs")).into(), - ctx.new_str(ascii!("kwd_patterns")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "MatchStar", base = "NodePattern")] -struct NodePatternMatchStar; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodePatternMatchStar { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ctx.new_str(ascii!("name")).into()]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "MatchAs", base = "NodePattern")] -struct NodePatternMatchAs; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodePatternMatchAs { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("pattern")).into(), - ctx.new_str(ascii!("name")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "MatchOr", base = "NodePattern")] -struct NodePatternMatchOr; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodePatternMatchOr { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ctx.new_str(ascii!("patterns")).into()]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "type_ignore", base = "NodeAst")] -struct NodeTypeIgnore; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeTypeIgnore {} -#[pyclass(module = "_ast", name = "TypeIgnore", base = "NodeTypeIgnore")] -struct NodeTypeIgnoreTypeIgnore; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeTypeIgnoreTypeIgnore { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("tag")).into(), - ]) - .into(), - ); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "type_param", base = "NodeAst")] -struct NodeTypeParam; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeTypeParam {} -#[pyclass(module = "_ast", name = "TypeVar", base = "NodeTypeParam")] -struct NodeTypeParamTypeVar; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeTypeParamTypeVar { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("name")).into(), - ctx.new_str(ascii!("bound")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "ParamSpec", base = "NodeTypeParam")] -struct NodeTypeParamParamSpec; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeTypeParamParamSpec { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ctx.new_str(ascii!("name")).into()]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "TypeVarTuple", base = "NodeTypeParam")] -struct NodeTypeParamTypeVarTuple; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeTypeParamTypeVarTuple { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ctx.new_str(ascii!("name")).into()]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} - -// sum -impl Node for ast::located::Mod { - fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef { - match self { - ast::located::Mod::Module(cons) => cons.ast_to_object(vm), - ast::located::Mod::Interactive(cons) => cons.ast_to_object(vm), - ast::located::Mod::Expression(cons) => cons.ast_to_object(vm), - ast::located::Mod::FunctionType(cons) => cons.ast_to_object(vm), - } - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - let _cls = _object.class(); - Ok(if _cls.is(NodeModModule::static_type()) { - ast::located::Mod::Module(ast::located::ModModule::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeModInteractive::static_type()) { - ast::located::Mod::Interactive(ast::located::ModInteractive::ast_from_object( - _vm, _object, - )?) - } else if _cls.is(NodeModExpression::static_type()) { - ast::located::Mod::Expression(ast::located::ModExpression::ast_from_object( - _vm, _object, - )?) - } else if _cls.is(NodeModFunctionType::static_type()) { - ast::located::Mod::FunctionType(ast::located::ModFunctionType::ast_from_object( - _vm, _object, - )?) - } else { - return Err(_vm.new_type_error(format!( - "expected some sort of mod, but got {}", - _object.repr(_vm)? - ))); - }) - } -} -// constructor -impl Node for ast::located::ModModule { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ModModule { - body, - type_ignores, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeModModule::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("body", body.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("type_ignores", type_ignores.ast_to_object(_vm), _vm) - .unwrap(); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::ModModule { - body: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "body", "Module")?)?, - type_ignores: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "type_ignores", "Module")?, - )?, - range: Default::default(), - }) - } -} -// constructor -impl Node for ast::located::ModInteractive { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ModInteractive { - body, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeModInteractive::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("body", body.ast_to_object(_vm), _vm).unwrap(); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::ModInteractive { - body: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "body", "Interactive")?, - )?, - range: Default::default(), - }) - } -} -// constructor -impl Node for ast::located::ModExpression { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ModExpression { - body, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeModExpression::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("body", body.ast_to_object(_vm), _vm).unwrap(); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::ModExpression { - body: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "body", "Expression")?)?, - range: Default::default(), - }) - } -} -// constructor -impl Node for ast::located::ModFunctionType { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ModFunctionType { - argtypes, - returns, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeModFunctionType::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("argtypes", argtypes.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("returns", returns.ast_to_object(_vm), _vm) - .unwrap(); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::ModFunctionType { - argtypes: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "argtypes", "FunctionType")?, - )?, - returns: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "returns", "FunctionType")?, - )?, - range: Default::default(), - }) - } -} -// sum -impl Node for ast::located::Stmt { - fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef { - match self { - ast::located::Stmt::FunctionDef(cons) => cons.ast_to_object(vm), - ast::located::Stmt::AsyncFunctionDef(cons) => cons.ast_to_object(vm), - ast::located::Stmt::ClassDef(cons) => cons.ast_to_object(vm), - ast::located::Stmt::Return(cons) => cons.ast_to_object(vm), - ast::located::Stmt::Delete(cons) => cons.ast_to_object(vm), - ast::located::Stmt::Assign(cons) => cons.ast_to_object(vm), - ast::located::Stmt::TypeAlias(cons) => cons.ast_to_object(vm), - ast::located::Stmt::AugAssign(cons) => cons.ast_to_object(vm), - ast::located::Stmt::AnnAssign(cons) => cons.ast_to_object(vm), - ast::located::Stmt::For(cons) => cons.ast_to_object(vm), - ast::located::Stmt::AsyncFor(cons) => cons.ast_to_object(vm), - ast::located::Stmt::While(cons) => cons.ast_to_object(vm), - ast::located::Stmt::If(cons) => cons.ast_to_object(vm), - ast::located::Stmt::With(cons) => cons.ast_to_object(vm), - ast::located::Stmt::AsyncWith(cons) => cons.ast_to_object(vm), - ast::located::Stmt::Match(cons) => cons.ast_to_object(vm), - ast::located::Stmt::Raise(cons) => cons.ast_to_object(vm), - ast::located::Stmt::Try(cons) => cons.ast_to_object(vm), - ast::located::Stmt::TryStar(cons) => cons.ast_to_object(vm), - ast::located::Stmt::Assert(cons) => cons.ast_to_object(vm), - ast::located::Stmt::Import(cons) => cons.ast_to_object(vm), - ast::located::Stmt::ImportFrom(cons) => cons.ast_to_object(vm), - ast::located::Stmt::Global(cons) => cons.ast_to_object(vm), - ast::located::Stmt::Nonlocal(cons) => cons.ast_to_object(vm), - ast::located::Stmt::Expr(cons) => cons.ast_to_object(vm), - ast::located::Stmt::Pass(cons) => cons.ast_to_object(vm), - ast::located::Stmt::Break(cons) => cons.ast_to_object(vm), - ast::located::Stmt::Continue(cons) => cons.ast_to_object(vm), - } - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - let _cls = _object.class(); - Ok(if _cls.is(NodeStmtFunctionDef::static_type()) { - ast::located::Stmt::FunctionDef(ast::located::StmtFunctionDef::ast_from_object( - _vm, _object, - )?) - } else if _cls.is(NodeStmtAsyncFunctionDef::static_type()) { - ast::located::Stmt::AsyncFunctionDef( - ast::located::StmtAsyncFunctionDef::ast_from_object(_vm, _object)?, - ) - } else if _cls.is(NodeStmtClassDef::static_type()) { - ast::located::Stmt::ClassDef(ast::located::StmtClassDef::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeStmtReturn::static_type()) { - ast::located::Stmt::Return(ast::located::StmtReturn::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeStmtDelete::static_type()) { - ast::located::Stmt::Delete(ast::located::StmtDelete::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeStmtAssign::static_type()) { - ast::located::Stmt::Assign(ast::located::StmtAssign::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeStmtTypeAlias::static_type()) { - ast::located::Stmt::TypeAlias(ast::located::StmtTypeAlias::ast_from_object( - _vm, _object, - )?) - } else if _cls.is(NodeStmtAugAssign::static_type()) { - ast::located::Stmt::AugAssign(ast::located::StmtAugAssign::ast_from_object( - _vm, _object, - )?) - } else if _cls.is(NodeStmtAnnAssign::static_type()) { - ast::located::Stmt::AnnAssign(ast::located::StmtAnnAssign::ast_from_object( - _vm, _object, - )?) - } else if _cls.is(NodeStmtFor::static_type()) { - ast::located::Stmt::For(ast::located::StmtFor::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeStmtAsyncFor::static_type()) { - ast::located::Stmt::AsyncFor(ast::located::StmtAsyncFor::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeStmtWhile::static_type()) { - ast::located::Stmt::While(ast::located::StmtWhile::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeStmtIf::static_type()) { - ast::located::Stmt::If(ast::located::StmtIf::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeStmtWith::static_type()) { - ast::located::Stmt::With(ast::located::StmtWith::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeStmtAsyncWith::static_type()) { - ast::located::Stmt::AsyncWith(ast::located::StmtAsyncWith::ast_from_object( - _vm, _object, - )?) - } else if _cls.is(NodeStmtMatch::static_type()) { - ast::located::Stmt::Match(ast::located::StmtMatch::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeStmtRaise::static_type()) { - ast::located::Stmt::Raise(ast::located::StmtRaise::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeStmtTry::static_type()) { - ast::located::Stmt::Try(ast::located::StmtTry::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeStmtTryStar::static_type()) { - ast::located::Stmt::TryStar(ast::located::StmtTryStar::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeStmtAssert::static_type()) { - ast::located::Stmt::Assert(ast::located::StmtAssert::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeStmtImport::static_type()) { - ast::located::Stmt::Import(ast::located::StmtImport::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeStmtImportFrom::static_type()) { - ast::located::Stmt::ImportFrom(ast::located::StmtImportFrom::ast_from_object( - _vm, _object, - )?) - } else if _cls.is(NodeStmtGlobal::static_type()) { - ast::located::Stmt::Global(ast::located::StmtGlobal::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeStmtNonlocal::static_type()) { - ast::located::Stmt::Nonlocal(ast::located::StmtNonlocal::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeStmtExpr::static_type()) { - ast::located::Stmt::Expr(ast::located::StmtExpr::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeStmtPass::static_type()) { - ast::located::Stmt::Pass(ast::located::StmtPass::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeStmtBreak::static_type()) { - ast::located::Stmt::Break(ast::located::StmtBreak::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeStmtContinue::static_type()) { - ast::located::Stmt::Continue(ast::located::StmtContinue::ast_from_object(_vm, _object)?) - } else { - return Err(_vm.new_type_error(format!( - "expected some sort of stmt, but got {}", - _object.repr(_vm)? - ))); - }) - } -} -// constructor -impl Node for ast::located::StmtFunctionDef { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtFunctionDef { - name, - args, - body, - decorator_list, - returns, - type_comment, - type_params, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtFunctionDef::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("name", name.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("args", args.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("body", body.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("decorator_list", decorator_list.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("returns", returns.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("type_comment", type_comment.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("type_params", type_params.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::StmtFunctionDef { - name: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "name", "FunctionDef")?, - )?, - args: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "args", "FunctionDef")?, - )?, - body: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "body", "FunctionDef")?, - )?, - decorator_list: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "decorator_list", "FunctionDef")?, - )?, - returns: get_node_field_opt(_vm, &_object, "returns")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - type_comment: get_node_field_opt(_vm, &_object, "type_comment")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - type_params: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "type_params", "FunctionDef")?, - )?, - range: range_from_object(_vm, _object, "FunctionDef")?, - }) - } -} -// constructor -impl Node for ast::located::StmtAsyncFunctionDef { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtAsyncFunctionDef { - name, - args, - body, - decorator_list, - returns, - type_comment, - type_params, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtAsyncFunctionDef::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("name", name.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("args", args.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("body", body.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("decorator_list", decorator_list.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("returns", returns.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("type_comment", type_comment.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("type_params", type_params.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::StmtAsyncFunctionDef { - name: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "name", "AsyncFunctionDef")?, - )?, - args: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "args", "AsyncFunctionDef")?, - )?, - body: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "body", "AsyncFunctionDef")?, - )?, - decorator_list: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "decorator_list", "AsyncFunctionDef")?, - )?, - returns: get_node_field_opt(_vm, &_object, "returns")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - type_comment: get_node_field_opt(_vm, &_object, "type_comment")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - type_params: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "type_params", "AsyncFunctionDef")?, - )?, - range: range_from_object(_vm, _object, "AsyncFunctionDef")?, - }) - } -} -// constructor -impl Node for ast::located::StmtClassDef { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtClassDef { - name, - bases, - keywords, - body, - decorator_list, - type_params, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtClassDef::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("name", name.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("bases", bases.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("keywords", keywords.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("body", body.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("decorator_list", decorator_list.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("type_params", type_params.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::StmtClassDef { - name: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "name", "ClassDef")?)?, - bases: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "bases", "ClassDef")?)?, - keywords: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "keywords", "ClassDef")?, - )?, - body: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "body", "ClassDef")?)?, - decorator_list: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "decorator_list", "ClassDef")?, - )?, - type_params: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "type_params", "ClassDef")?, - )?, - range: range_from_object(_vm, _object, "ClassDef")?, - }) - } -} -// constructor -impl Node for ast::located::StmtReturn { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtReturn { - value, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtReturn::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("value", value.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::StmtReturn { - value: get_node_field_opt(_vm, &_object, "value")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - range: range_from_object(_vm, _object, "Return")?, - }) - } -} -// constructor -impl Node for ast::located::StmtDelete { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtDelete { - targets, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtDelete::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("targets", targets.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::StmtDelete { - targets: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "targets", "Delete")?, - )?, - range: range_from_object(_vm, _object, "Delete")?, - }) - } -} -// constructor -impl Node for ast::located::StmtAssign { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtAssign { - targets, - value, - type_comment, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtAssign::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("targets", targets.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("value", value.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("type_comment", type_comment.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::StmtAssign { - targets: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "targets", "Assign")?, - )?, - value: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "value", "Assign")?)?, - type_comment: get_node_field_opt(_vm, &_object, "type_comment")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - range: range_from_object(_vm, _object, "Assign")?, - }) - } -} -// constructor -impl Node for ast::located::StmtTypeAlias { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtTypeAlias { - name, - type_params, - value, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtTypeAlias::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("name", name.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("type_params", type_params.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("value", value.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::StmtTypeAlias { - name: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "name", "TypeAlias")?)?, - type_params: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "type_params", "TypeAlias")?, - )?, - value: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "value", "TypeAlias")?, - )?, - range: range_from_object(_vm, _object, "TypeAlias")?, - }) - } -} -// constructor -impl Node for ast::located::StmtAugAssign { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtAugAssign { - target, - op, - value, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtAugAssign::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("target", target.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("op", op.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("value", value.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::StmtAugAssign { - target: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "target", "AugAssign")?, - )?, - op: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "op", "AugAssign")?)?, - value: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "value", "AugAssign")?, - )?, - range: range_from_object(_vm, _object, "AugAssign")?, - }) - } -} -// constructor -impl Node for ast::located::StmtAnnAssign { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtAnnAssign { - target, - annotation, - value, - simple, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtAnnAssign::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("target", target.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("annotation", annotation.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("value", value.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("simple", simple.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::StmtAnnAssign { - target: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "target", "AnnAssign")?, - )?, - annotation: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "annotation", "AnnAssign")?, - )?, - value: get_node_field_opt(_vm, &_object, "value")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - simple: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "simple", "AnnAssign")?, - )?, - range: range_from_object(_vm, _object, "AnnAssign")?, - }) - } -} -// constructor -impl Node for ast::located::StmtFor { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtFor { - target, - iter, - body, - orelse, - type_comment, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtFor::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("target", target.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("iter", iter.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("body", body.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("orelse", orelse.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("type_comment", type_comment.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::StmtFor { - target: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "target", "For")?)?, - iter: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "iter", "For")?)?, - body: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "body", "For")?)?, - orelse: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "orelse", "For")?)?, - type_comment: get_node_field_opt(_vm, &_object, "type_comment")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - range: range_from_object(_vm, _object, "For")?, - }) - } -} -// constructor -impl Node for ast::located::StmtAsyncFor { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtAsyncFor { - target, - iter, - body, - orelse, - type_comment, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtAsyncFor::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("target", target.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("iter", iter.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("body", body.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("orelse", orelse.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("type_comment", type_comment.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::StmtAsyncFor { - target: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "target", "AsyncFor")?, - )?, - iter: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "iter", "AsyncFor")?)?, - body: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "body", "AsyncFor")?)?, - orelse: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "orelse", "AsyncFor")?, - )?, - type_comment: get_node_field_opt(_vm, &_object, "type_comment")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - range: range_from_object(_vm, _object, "AsyncFor")?, - }) - } -} -// constructor -impl Node for ast::located::StmtWhile { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtWhile { - test, - body, - orelse, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtWhile::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("test", test.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("body", body.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("orelse", orelse.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::StmtWhile { - test: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "test", "While")?)?, - body: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "body", "While")?)?, - orelse: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "orelse", "While")?)?, - range: range_from_object(_vm, _object, "While")?, - }) - } -} -// constructor -impl Node for ast::located::StmtIf { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtIf { - test, - body, - orelse, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtIf::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("test", test.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("body", body.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("orelse", orelse.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::StmtIf { - test: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "test", "If")?)?, - body: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "body", "If")?)?, - orelse: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "orelse", "If")?)?, - range: range_from_object(_vm, _object, "If")?, - }) - } -} -// constructor -impl Node for ast::located::StmtWith { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtWith { - items, - body, - type_comment, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtWith::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("items", items.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("body", body.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("type_comment", type_comment.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::StmtWith { - items: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "items", "With")?)?, - body: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "body", "With")?)?, - type_comment: get_node_field_opt(_vm, &_object, "type_comment")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - range: range_from_object(_vm, _object, "With")?, - }) - } -} -// constructor -impl Node for ast::located::StmtAsyncWith { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtAsyncWith { - items, - body, - type_comment, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtAsyncWith::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("items", items.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("body", body.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("type_comment", type_comment.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::StmtAsyncWith { - items: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "items", "AsyncWith")?, - )?, - body: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "body", "AsyncWith")?)?, - type_comment: get_node_field_opt(_vm, &_object, "type_comment")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - range: range_from_object(_vm, _object, "AsyncWith")?, - }) - } -} -// constructor -impl Node for ast::located::StmtMatch { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtMatch { - subject, - cases, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtMatch::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("subject", subject.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("cases", cases.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::StmtMatch { - subject: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "subject", "Match")?, - )?, - cases: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "cases", "Match")?)?, - range: range_from_object(_vm, _object, "Match")?, - }) - } -} -// constructor -impl Node for ast::located::StmtRaise { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtRaise { - exc, - cause, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtRaise::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("exc", exc.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("cause", cause.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::StmtRaise { - exc: get_node_field_opt(_vm, &_object, "exc")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - cause: get_node_field_opt(_vm, &_object, "cause")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - range: range_from_object(_vm, _object, "Raise")?, - }) - } -} -// constructor -impl Node for ast::located::StmtTry { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtTry { - body, - handlers, - orelse, - finalbody, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtTry::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("body", body.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("handlers", handlers.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("orelse", orelse.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("finalbody", finalbody.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::StmtTry { - body: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "body", "Try")?)?, - handlers: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "handlers", "Try")?, - )?, - orelse: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "orelse", "Try")?)?, - finalbody: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "finalbody", "Try")?, - )?, - range: range_from_object(_vm, _object, "Try")?, - }) - } -} -// constructor -impl Node for ast::located::StmtTryStar { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtTryStar { - body, - handlers, - orelse, - finalbody, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtTryStar::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("body", body.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("handlers", handlers.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("orelse", orelse.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("finalbody", finalbody.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::StmtTryStar { - body: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "body", "TryStar")?)?, - handlers: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "handlers", "TryStar")?, - )?, - orelse: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "orelse", "TryStar")?, - )?, - finalbody: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "finalbody", "TryStar")?, - )?, - range: range_from_object(_vm, _object, "TryStar")?, - }) - } -} -// constructor -impl Node for ast::located::StmtAssert { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtAssert { - test, - msg, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtAssert::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("test", test.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("msg", msg.ast_to_object(_vm), _vm).unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::StmtAssert { - test: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "test", "Assert")?)?, - msg: get_node_field_opt(_vm, &_object, "msg")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - range: range_from_object(_vm, _object, "Assert")?, - }) - } -} -// constructor -impl Node for ast::located::StmtImport { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtImport { - names, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtImport::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("names", names.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::StmtImport { - names: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "names", "Import")?)?, - range: range_from_object(_vm, _object, "Import")?, - }) - } -} -// constructor -impl Node for ast::located::StmtImportFrom { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtImportFrom { - module, - names, - level, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtImportFrom::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("module", module.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("names", names.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("level", level.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::StmtImportFrom { - module: get_node_field_opt(_vm, &_object, "module")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - names: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "names", "ImportFrom")?, - )?, - level: get_node_field_opt(_vm, &_object, "level")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - range: range_from_object(_vm, _object, "ImportFrom")?, - }) - } -} -// constructor -impl Node for ast::located::StmtGlobal { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtGlobal { - names, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtGlobal::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("names", names.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::StmtGlobal { - names: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "names", "Global")?)?, - range: range_from_object(_vm, _object, "Global")?, - }) - } -} -// constructor -impl Node for ast::located::StmtNonlocal { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtNonlocal { - names, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtNonlocal::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("names", names.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::StmtNonlocal { - names: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "names", "Nonlocal")?)?, - range: range_from_object(_vm, _object, "Nonlocal")?, - }) - } -} -// constructor -impl Node for ast::located::StmtExpr { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtExpr { - value, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtExpr::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("value", value.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::StmtExpr { - value: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "value", "Expr")?)?, - range: range_from_object(_vm, _object, "Expr")?, - }) - } -} -// constructor -impl Node for ast::located::StmtPass { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtPass { range: _range } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtPass::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::StmtPass { - range: range_from_object(_vm, _object, "Pass")?, - }) - } -} -// constructor -impl Node for ast::located::StmtBreak { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtBreak { range: _range } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtBreak::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::StmtBreak { - range: range_from_object(_vm, _object, "Break")?, - }) - } -} -// constructor -impl Node for ast::located::StmtContinue { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtContinue { range: _range } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtContinue::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::StmtContinue { - range: range_from_object(_vm, _object, "Continue")?, - }) - } -} -// sum -impl Node for ast::located::Expr { - fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef { - match self { - ast::located::Expr::BoolOp(cons) => cons.ast_to_object(vm), - ast::located::Expr::NamedExpr(cons) => cons.ast_to_object(vm), - ast::located::Expr::BinOp(cons) => cons.ast_to_object(vm), - ast::located::Expr::UnaryOp(cons) => cons.ast_to_object(vm), - ast::located::Expr::Lambda(cons) => cons.ast_to_object(vm), - ast::located::Expr::IfExp(cons) => cons.ast_to_object(vm), - ast::located::Expr::Dict(cons) => cons.ast_to_object(vm), - ast::located::Expr::Set(cons) => cons.ast_to_object(vm), - ast::located::Expr::ListComp(cons) => cons.ast_to_object(vm), - ast::located::Expr::SetComp(cons) => cons.ast_to_object(vm), - ast::located::Expr::DictComp(cons) => cons.ast_to_object(vm), - ast::located::Expr::GeneratorExp(cons) => cons.ast_to_object(vm), - ast::located::Expr::Await(cons) => cons.ast_to_object(vm), - ast::located::Expr::Yield(cons) => cons.ast_to_object(vm), - ast::located::Expr::YieldFrom(cons) => cons.ast_to_object(vm), - ast::located::Expr::Compare(cons) => cons.ast_to_object(vm), - ast::located::Expr::Call(cons) => cons.ast_to_object(vm), - ast::located::Expr::FormattedValue(cons) => cons.ast_to_object(vm), - ast::located::Expr::JoinedStr(cons) => cons.ast_to_object(vm), - ast::located::Expr::Constant(cons) => cons.ast_to_object(vm), - ast::located::Expr::Attribute(cons) => cons.ast_to_object(vm), - ast::located::Expr::Subscript(cons) => cons.ast_to_object(vm), - ast::located::Expr::Starred(cons) => cons.ast_to_object(vm), - ast::located::Expr::Name(cons) => cons.ast_to_object(vm), - ast::located::Expr::List(cons) => cons.ast_to_object(vm), - ast::located::Expr::Tuple(cons) => cons.ast_to_object(vm), - ast::located::Expr::Slice(cons) => cons.ast_to_object(vm), - } - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - let _cls = _object.class(); - Ok(if _cls.is(NodeExprBoolOp::static_type()) { - ast::located::Expr::BoolOp(ast::located::ExprBoolOp::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeExprNamedExpr::static_type()) { - ast::located::Expr::NamedExpr(ast::located::ExprNamedExpr::ast_from_object( - _vm, _object, - )?) - } else if _cls.is(NodeExprBinOp::static_type()) { - ast::located::Expr::BinOp(ast::located::ExprBinOp::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeExprUnaryOp::static_type()) { - ast::located::Expr::UnaryOp(ast::located::ExprUnaryOp::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeExprLambda::static_type()) { - ast::located::Expr::Lambda(ast::located::ExprLambda::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeExprIfExp::static_type()) { - ast::located::Expr::IfExp(ast::located::ExprIfExp::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeExprDict::static_type()) { - ast::located::Expr::Dict(ast::located::ExprDict::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeExprSet::static_type()) { - ast::located::Expr::Set(ast::located::ExprSet::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeExprListComp::static_type()) { - ast::located::Expr::ListComp(ast::located::ExprListComp::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeExprSetComp::static_type()) { - ast::located::Expr::SetComp(ast::located::ExprSetComp::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeExprDictComp::static_type()) { - ast::located::Expr::DictComp(ast::located::ExprDictComp::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeExprGeneratorExp::static_type()) { - ast::located::Expr::GeneratorExp(ast::located::ExprGeneratorExp::ast_from_object( - _vm, _object, - )?) - } else if _cls.is(NodeExprAwait::static_type()) { - ast::located::Expr::Await(ast::located::ExprAwait::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeExprYield::static_type()) { - ast::located::Expr::Yield(ast::located::ExprYield::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeExprYieldFrom::static_type()) { - ast::located::Expr::YieldFrom(ast::located::ExprYieldFrom::ast_from_object( - _vm, _object, - )?) - } else if _cls.is(NodeExprCompare::static_type()) { - ast::located::Expr::Compare(ast::located::ExprCompare::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeExprCall::static_type()) { - ast::located::Expr::Call(ast::located::ExprCall::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeExprFormattedValue::static_type()) { - ast::located::Expr::FormattedValue(ast::located::ExprFormattedValue::ast_from_object( - _vm, _object, - )?) - } else if _cls.is(NodeExprJoinedStr::static_type()) { - ast::located::Expr::JoinedStr(ast::located::ExprJoinedStr::ast_from_object( - _vm, _object, - )?) - } else if _cls.is(NodeExprConstant::static_type()) { - ast::located::Expr::Constant(ast::located::ExprConstant::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeExprAttribute::static_type()) { - ast::located::Expr::Attribute(ast::located::ExprAttribute::ast_from_object( - _vm, _object, - )?) - } else if _cls.is(NodeExprSubscript::static_type()) { - ast::located::Expr::Subscript(ast::located::ExprSubscript::ast_from_object( - _vm, _object, - )?) - } else if _cls.is(NodeExprStarred::static_type()) { - ast::located::Expr::Starred(ast::located::ExprStarred::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeExprName::static_type()) { - ast::located::Expr::Name(ast::located::ExprName::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeExprList::static_type()) { - ast::located::Expr::List(ast::located::ExprList::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeExprTuple::static_type()) { - ast::located::Expr::Tuple(ast::located::ExprTuple::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeExprSlice::static_type()) { - ast::located::Expr::Slice(ast::located::ExprSlice::ast_from_object(_vm, _object)?) - } else { - return Err(_vm.new_type_error(format!( - "expected some sort of expr, but got {}", - _object.repr(_vm)? - ))); - }) - } -} -// constructor -impl Node for ast::located::ExprBoolOp { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprBoolOp { - op, - values, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprBoolOp::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("op", op.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("values", values.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::ExprBoolOp { - op: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "op", "BoolOp")?)?, - values: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "values", "BoolOp")?)?, - range: range_from_object(_vm, _object, "BoolOp")?, - }) - } -} -// constructor -impl Node for ast::located::ExprNamedExpr { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprNamedExpr { - target, - value, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprNamedExpr::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("target", target.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("value", value.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::ExprNamedExpr { - target: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "target", "NamedExpr")?, - )?, - value: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "value", "NamedExpr")?, - )?, - range: range_from_object(_vm, _object, "NamedExpr")?, - }) - } -} -// constructor -impl Node for ast::located::ExprBinOp { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprBinOp { - left, - op, - right, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprBinOp::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("left", left.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("op", op.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("right", right.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::ExprBinOp { - left: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "left", "BinOp")?)?, - op: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "op", "BinOp")?)?, - right: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "right", "BinOp")?)?, - range: range_from_object(_vm, _object, "BinOp")?, - }) - } -} -// constructor -impl Node for ast::located::ExprUnaryOp { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprUnaryOp { - op, - operand, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprUnaryOp::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("op", op.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("operand", operand.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::ExprUnaryOp { - op: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "op", "UnaryOp")?)?, - operand: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "operand", "UnaryOp")?, - )?, - range: range_from_object(_vm, _object, "UnaryOp")?, - }) - } -} -// constructor -impl Node for ast::located::ExprLambda { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprLambda { - args, - body, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprLambda::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("args", args.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("body", body.ast_to_object(_vm), _vm).unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::ExprLambda { - args: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "args", "Lambda")?)?, - body: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "body", "Lambda")?)?, - range: range_from_object(_vm, _object, "Lambda")?, - }) - } -} -// constructor -impl Node for ast::located::ExprIfExp { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprIfExp { - test, - body, - orelse, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprIfExp::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("test", test.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("body", body.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("orelse", orelse.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::ExprIfExp { - test: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "test", "IfExp")?)?, - body: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "body", "IfExp")?)?, - orelse: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "orelse", "IfExp")?)?, - range: range_from_object(_vm, _object, "IfExp")?, - }) - } -} -// constructor -impl Node for ast::located::ExprDict { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprDict { - keys, - values, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprDict::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("keys", keys.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("values", values.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::ExprDict { - keys: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "keys", "Dict")?)?, - values: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "values", "Dict")?)?, - range: range_from_object(_vm, _object, "Dict")?, - }) - } -} -// constructor -impl Node for ast::located::ExprSet { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprSet { - elts, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprSet::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("elts", elts.ast_to_object(_vm), _vm).unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::ExprSet { - elts: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "elts", "Set")?)?, - range: range_from_object(_vm, _object, "Set")?, - }) - } -} -// constructor -impl Node for ast::located::ExprListComp { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprListComp { - elt, - generators, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprListComp::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("elt", elt.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("generators", generators.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::ExprListComp { - elt: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "elt", "ListComp")?)?, - generators: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "generators", "ListComp")?, - )?, - range: range_from_object(_vm, _object, "ListComp")?, - }) - } -} -// constructor -impl Node for ast::located::ExprSetComp { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprSetComp { - elt, - generators, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprSetComp::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("elt", elt.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("generators", generators.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::ExprSetComp { - elt: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "elt", "SetComp")?)?, - generators: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "generators", "SetComp")?, - )?, - range: range_from_object(_vm, _object, "SetComp")?, - }) - } -} -// constructor -impl Node for ast::located::ExprDictComp { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprDictComp { - key, - value, - generators, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprDictComp::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("key", key.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("value", value.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("generators", generators.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::ExprDictComp { - key: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "key", "DictComp")?)?, - value: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "value", "DictComp")?)?, - generators: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "generators", "DictComp")?, - )?, - range: range_from_object(_vm, _object, "DictComp")?, - }) - } -} -// constructor -impl Node for ast::located::ExprGeneratorExp { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprGeneratorExp { - elt, - generators, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprGeneratorExp::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("elt", elt.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("generators", generators.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::ExprGeneratorExp { - elt: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "elt", "GeneratorExp")?)?, - generators: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "generators", "GeneratorExp")?, - )?, - range: range_from_object(_vm, _object, "GeneratorExp")?, - }) - } -} -// constructor -impl Node for ast::located::ExprAwait { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprAwait { - value, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprAwait::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("value", value.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::ExprAwait { - value: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "value", "Await")?)?, - range: range_from_object(_vm, _object, "Await")?, - }) - } -} -// constructor -impl Node for ast::located::ExprYield { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprYield { - value, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprYield::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("value", value.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::ExprYield { - value: get_node_field_opt(_vm, &_object, "value")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - range: range_from_object(_vm, _object, "Yield")?, - }) - } -} -// constructor -impl Node for ast::located::ExprYieldFrom { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprYieldFrom { - value, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprYieldFrom::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("value", value.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::ExprYieldFrom { - value: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "value", "YieldFrom")?, - )?, - range: range_from_object(_vm, _object, "YieldFrom")?, - }) - } -} -// constructor -impl Node for ast::located::ExprCompare { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprCompare { - left, - ops, - comparators, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprCompare::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("left", left.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("ops", ops.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("comparators", comparators.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::ExprCompare { - left: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "left", "Compare")?)?, - ops: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "ops", "Compare")?)?, - comparators: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "comparators", "Compare")?, - )?, - range: range_from_object(_vm, _object, "Compare")?, - }) - } -} -// constructor -impl Node for ast::located::ExprCall { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprCall { - func, - args, - keywords, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprCall::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("func", func.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("args", args.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("keywords", keywords.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::ExprCall { - func: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "func", "Call")?)?, - args: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "args", "Call")?)?, - keywords: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "keywords", "Call")?, - )?, - range: range_from_object(_vm, _object, "Call")?, - }) - } -} -// constructor -impl Node for ast::located::ExprFormattedValue { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprFormattedValue { - value, - conversion, - format_spec, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprFormattedValue::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("value", value.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("conversion", conversion.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("format_spec", format_spec.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::ExprFormattedValue { - value: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "value", "FormattedValue")?, - )?, - conversion: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "conversion", "FormattedValue")?, - )?, - format_spec: get_node_field_opt(_vm, &_object, "format_spec")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - range: range_from_object(_vm, _object, "FormattedValue")?, - }) - } -} -// constructor -impl Node for ast::located::ExprJoinedStr { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprJoinedStr { - values, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprJoinedStr::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("values", values.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::ExprJoinedStr { - values: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "values", "JoinedStr")?, - )?, - range: range_from_object(_vm, _object, "JoinedStr")?, - }) - } -} -// constructor -impl Node for ast::located::ExprConstant { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprConstant { - value, - kind, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprConstant::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("value", value.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("kind", kind.ast_to_object(_vm), _vm).unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::ExprConstant { - value: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "value", "Constant")?)?, - kind: get_node_field_opt(_vm, &_object, "kind")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - range: range_from_object(_vm, _object, "Constant")?, - }) - } -} -// constructor -impl Node for ast::located::ExprAttribute { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprAttribute { - value, - attr, - ctx, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprAttribute::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("value", value.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("attr", attr.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("ctx", ctx.ast_to_object(_vm), _vm).unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::ExprAttribute { - value: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "value", "Attribute")?, - )?, - attr: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "attr", "Attribute")?)?, - ctx: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "ctx", "Attribute")?)?, - range: range_from_object(_vm, _object, "Attribute")?, - }) - } -} -// constructor -impl Node for ast::located::ExprSubscript { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprSubscript { - value, - slice, - ctx, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprSubscript::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("value", value.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("slice", slice.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("ctx", ctx.ast_to_object(_vm), _vm).unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::ExprSubscript { - value: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "value", "Subscript")?, - )?, - slice: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "slice", "Subscript")?, - )?, - ctx: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "ctx", "Subscript")?)?, - range: range_from_object(_vm, _object, "Subscript")?, - }) - } -} -// constructor -impl Node for ast::located::ExprStarred { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprStarred { - value, - ctx, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprStarred::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("value", value.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("ctx", ctx.ast_to_object(_vm), _vm).unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::ExprStarred { - value: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "value", "Starred")?)?, - ctx: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "ctx", "Starred")?)?, - range: range_from_object(_vm, _object, "Starred")?, - }) - } -} -// constructor -impl Node for ast::located::ExprName { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprName { - id, - ctx, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprName::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("id", id.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("ctx", ctx.ast_to_object(_vm), _vm).unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::ExprName { - id: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "id", "Name")?)?, - ctx: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "ctx", "Name")?)?, - range: range_from_object(_vm, _object, "Name")?, - }) - } -} -// constructor -impl Node for ast::located::ExprList { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprList { - elts, - ctx, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprList::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("elts", elts.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("ctx", ctx.ast_to_object(_vm), _vm).unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::ExprList { - elts: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "elts", "List")?)?, - ctx: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "ctx", "List")?)?, - range: range_from_object(_vm, _object, "List")?, - }) - } -} -// constructor -impl Node for ast::located::ExprTuple { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprTuple { - elts, - ctx, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprTuple::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("elts", elts.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("ctx", ctx.ast_to_object(_vm), _vm).unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::ExprTuple { - elts: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "elts", "Tuple")?)?, - ctx: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "ctx", "Tuple")?)?, - range: range_from_object(_vm, _object, "Tuple")?, - }) - } -} -// constructor -impl Node for ast::located::ExprSlice { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprSlice { - lower, - upper, - step, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprSlice::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("lower", lower.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("upper", upper.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("step", step.ast_to_object(_vm), _vm).unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::ExprSlice { - lower: get_node_field_opt(_vm, &_object, "lower")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - upper: get_node_field_opt(_vm, &_object, "upper")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - step: get_node_field_opt(_vm, &_object, "step")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - range: range_from_object(_vm, _object, "Slice")?, - }) - } -} -// sum -impl Node for ast::located::ExprContext { - fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef { - let node_type = match self { - ast::located::ExprContext::Load => NodeExprContextLoad::static_type(), - ast::located::ExprContext::Store => NodeExprContextStore::static_type(), - ast::located::ExprContext::Del => NodeExprContextDel::static_type(), - }; - NodeAst - .into_ref_with_type(vm, node_type.to_owned()) - .unwrap() - .into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - let _cls = _object.class(); - Ok(if _cls.is(NodeExprContextLoad::static_type()) { - ast::located::ExprContext::Load - } else if _cls.is(NodeExprContextStore::static_type()) { - ast::located::ExprContext::Store - } else if _cls.is(NodeExprContextDel::static_type()) { - ast::located::ExprContext::Del - } else { - return Err(_vm.new_type_error(format!( - "expected some sort of expr_context, but got {}", - _object.repr(_vm)? - ))); - }) - } -} -// sum -impl Node for ast::located::BoolOp { - fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef { - let node_type = match self { - ast::located::BoolOp::And => NodeBoolOpAnd::static_type(), - ast::located::BoolOp::Or => NodeBoolOpOr::static_type(), - }; - NodeAst - .into_ref_with_type(vm, node_type.to_owned()) - .unwrap() - .into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - let _cls = _object.class(); - Ok(if _cls.is(NodeBoolOpAnd::static_type()) { - ast::located::BoolOp::And - } else if _cls.is(NodeBoolOpOr::static_type()) { - ast::located::BoolOp::Or - } else { - return Err(_vm.new_type_error(format!( - "expected some sort of boolop, but got {}", - _object.repr(_vm)? - ))); - }) - } -} -// sum -impl Node for ast::located::Operator { - fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef { - let node_type = match self { - ast::located::Operator::Add => NodeOperatorAdd::static_type(), - ast::located::Operator::Sub => NodeOperatorSub::static_type(), - ast::located::Operator::Mult => NodeOperatorMult::static_type(), - ast::located::Operator::MatMult => NodeOperatorMatMult::static_type(), - ast::located::Operator::Div => NodeOperatorDiv::static_type(), - ast::located::Operator::Mod => NodeOperatorMod::static_type(), - ast::located::Operator::Pow => NodeOperatorPow::static_type(), - ast::located::Operator::LShift => NodeOperatorLShift::static_type(), - ast::located::Operator::RShift => NodeOperatorRShift::static_type(), - ast::located::Operator::BitOr => NodeOperatorBitOr::static_type(), - ast::located::Operator::BitXor => NodeOperatorBitXor::static_type(), - ast::located::Operator::BitAnd => NodeOperatorBitAnd::static_type(), - ast::located::Operator::FloorDiv => NodeOperatorFloorDiv::static_type(), - }; - NodeAst - .into_ref_with_type(vm, node_type.to_owned()) - .unwrap() - .into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - let _cls = _object.class(); - Ok(if _cls.is(NodeOperatorAdd::static_type()) { - ast::located::Operator::Add - } else if _cls.is(NodeOperatorSub::static_type()) { - ast::located::Operator::Sub - } else if _cls.is(NodeOperatorMult::static_type()) { - ast::located::Operator::Mult - } else if _cls.is(NodeOperatorMatMult::static_type()) { - ast::located::Operator::MatMult - } else if _cls.is(NodeOperatorDiv::static_type()) { - ast::located::Operator::Div - } else if _cls.is(NodeOperatorMod::static_type()) { - ast::located::Operator::Mod - } else if _cls.is(NodeOperatorPow::static_type()) { - ast::located::Operator::Pow - } else if _cls.is(NodeOperatorLShift::static_type()) { - ast::located::Operator::LShift - } else if _cls.is(NodeOperatorRShift::static_type()) { - ast::located::Operator::RShift - } else if _cls.is(NodeOperatorBitOr::static_type()) { - ast::located::Operator::BitOr - } else if _cls.is(NodeOperatorBitXor::static_type()) { - ast::located::Operator::BitXor - } else if _cls.is(NodeOperatorBitAnd::static_type()) { - ast::located::Operator::BitAnd - } else if _cls.is(NodeOperatorFloorDiv::static_type()) { - ast::located::Operator::FloorDiv - } else { - return Err(_vm.new_type_error(format!( - "expected some sort of operator, but got {}", - _object.repr(_vm)? - ))); - }) - } -} -// sum -impl Node for ast::located::UnaryOp { - fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef { - let node_type = match self { - ast::located::UnaryOp::Invert => NodeUnaryOpInvert::static_type(), - ast::located::UnaryOp::Not => NodeUnaryOpNot::static_type(), - ast::located::UnaryOp::UAdd => NodeUnaryOpUAdd::static_type(), - ast::located::UnaryOp::USub => NodeUnaryOpUSub::static_type(), - }; - NodeAst - .into_ref_with_type(vm, node_type.to_owned()) - .unwrap() - .into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - let _cls = _object.class(); - Ok(if _cls.is(NodeUnaryOpInvert::static_type()) { - ast::located::UnaryOp::Invert - } else if _cls.is(NodeUnaryOpNot::static_type()) { - ast::located::UnaryOp::Not - } else if _cls.is(NodeUnaryOpUAdd::static_type()) { - ast::located::UnaryOp::UAdd - } else if _cls.is(NodeUnaryOpUSub::static_type()) { - ast::located::UnaryOp::USub - } else { - return Err(_vm.new_type_error(format!( - "expected some sort of unaryop, but got {}", - _object.repr(_vm)? - ))); - }) - } -} -// sum -impl Node for ast::located::CmpOp { - fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef { - let node_type = match self { - ast::located::CmpOp::Eq => NodeCmpOpEq::static_type(), - ast::located::CmpOp::NotEq => NodeCmpOpNotEq::static_type(), - ast::located::CmpOp::Lt => NodeCmpOpLt::static_type(), - ast::located::CmpOp::LtE => NodeCmpOpLtE::static_type(), - ast::located::CmpOp::Gt => NodeCmpOpGt::static_type(), - ast::located::CmpOp::GtE => NodeCmpOpGtE::static_type(), - ast::located::CmpOp::Is => NodeCmpOpIs::static_type(), - ast::located::CmpOp::IsNot => NodeCmpOpIsNot::static_type(), - ast::located::CmpOp::In => NodeCmpOpIn::static_type(), - ast::located::CmpOp::NotIn => NodeCmpOpNotIn::static_type(), - }; - NodeAst - .into_ref_with_type(vm, node_type.to_owned()) - .unwrap() - .into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - let _cls = _object.class(); - Ok(if _cls.is(NodeCmpOpEq::static_type()) { - ast::located::CmpOp::Eq - } else if _cls.is(NodeCmpOpNotEq::static_type()) { - ast::located::CmpOp::NotEq - } else if _cls.is(NodeCmpOpLt::static_type()) { - ast::located::CmpOp::Lt - } else if _cls.is(NodeCmpOpLtE::static_type()) { - ast::located::CmpOp::LtE - } else if _cls.is(NodeCmpOpGt::static_type()) { - ast::located::CmpOp::Gt - } else if _cls.is(NodeCmpOpGtE::static_type()) { - ast::located::CmpOp::GtE - } else if _cls.is(NodeCmpOpIs::static_type()) { - ast::located::CmpOp::Is - } else if _cls.is(NodeCmpOpIsNot::static_type()) { - ast::located::CmpOp::IsNot - } else if _cls.is(NodeCmpOpIn::static_type()) { - ast::located::CmpOp::In - } else if _cls.is(NodeCmpOpNotIn::static_type()) { - ast::located::CmpOp::NotIn - } else { - return Err(_vm.new_type_error(format!( - "expected some sort of cmpop, but got {}", - _object.repr(_vm)? - ))); - }) - } -} -// product -impl Node for ast::located::Comprehension { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::Comprehension { - target, - iter, - ifs, - is_async, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeComprehension::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("target", target.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("iter", iter.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("ifs", ifs.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("is_async", is_async.ast_to_object(_vm), _vm) - .unwrap(); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::Comprehension { - target: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "target", "comprehension")?, - )?, - iter: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "iter", "comprehension")?, - )?, - ifs: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "ifs", "comprehension")?, - )?, - is_async: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "is_async", "comprehension")?, - )?, - range: Default::default(), - }) - } -} -// sum -impl Node for ast::located::ExceptHandler { - fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef { - match self { - ast::located::ExceptHandler::ExceptHandler(cons) => cons.ast_to_object(vm), - } - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - let _cls = _object.class(); - Ok(if _cls.is(NodeExceptHandlerExceptHandler::static_type()) { - ast::located::ExceptHandler::ExceptHandler( - ast::located::ExceptHandlerExceptHandler::ast_from_object(_vm, _object)?, - ) - } else { - return Err(_vm.new_type_error(format!( - "expected some sort of excepthandler, but got {}", - _object.repr(_vm)? - ))); - }) - } -} -// constructor -impl Node for ast::located::ExceptHandlerExceptHandler { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExceptHandlerExceptHandler { - type_, - name, - body, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type( - _vm, - NodeExceptHandlerExceptHandler::static_type().to_owned(), - ) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("type", type_.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("name", name.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("body", body.ast_to_object(_vm), _vm).unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::ExceptHandlerExceptHandler { - type_: get_node_field_opt(_vm, &_object, "type")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - name: get_node_field_opt(_vm, &_object, "name")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - body: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "body", "ExceptHandler")?, - )?, - range: range_from_object(_vm, _object, "ExceptHandler")?, - }) - } -} -// product -impl Node for ast::located::PythonArguments { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::PythonArguments { - posonlyargs, - args, - vararg, - kwonlyargs, - kw_defaults, - kwarg, - defaults, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeArguments::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("posonlyargs", posonlyargs.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("args", args.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("vararg", vararg.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("kwonlyargs", kwonlyargs.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("kw_defaults", kw_defaults.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("kwarg", kwarg.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("defaults", defaults.ast_to_object(_vm), _vm) - .unwrap(); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::PythonArguments { - posonlyargs: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "posonlyargs", "arguments")?, - )?, - args: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "args", "arguments")?)?, - vararg: get_node_field_opt(_vm, &_object, "vararg")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - kwonlyargs: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "kwonlyargs", "arguments")?, - )?, - kw_defaults: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "kw_defaults", "arguments")?, - )?, - kwarg: get_node_field_opt(_vm, &_object, "kwarg")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - defaults: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "defaults", "arguments")?, - )?, - range: Default::default(), - }) - } -} -// product -impl Node for ast::located::Arg { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::Arg { - arg, - annotation, - type_comment, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeArg::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("arg", arg.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("annotation", annotation.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("type_comment", type_comment.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::Arg { - arg: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "arg", "arg")?)?, - annotation: get_node_field_opt(_vm, &_object, "annotation")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - type_comment: get_node_field_opt(_vm, &_object, "type_comment")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - range: range_from_object(_vm, _object, "arg")?, - }) - } -} -// product -impl Node for ast::located::Keyword { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::Keyword { - arg, - value, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeKeyword::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("arg", arg.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("value", value.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::Keyword { - arg: get_node_field_opt(_vm, &_object, "arg")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - value: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "value", "keyword")?)?, - range: range_from_object(_vm, _object, "keyword")?, - }) - } -} -// product -impl Node for ast::located::Alias { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::Alias { - name, - asname, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeAlias::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("name", name.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("asname", asname.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::Alias { - name: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "name", "alias")?)?, - asname: get_node_field_opt(_vm, &_object, "asname")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - range: range_from_object(_vm, _object, "alias")?, - }) - } -} -// product -impl Node for ast::located::WithItem { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::WithItem { - context_expr, - optional_vars, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeWithItem::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("context_expr", context_expr.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("optional_vars", optional_vars.ast_to_object(_vm), _vm) - .unwrap(); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::WithItem { - context_expr: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "context_expr", "withitem")?, - )?, - optional_vars: get_node_field_opt(_vm, &_object, "optional_vars")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - range: Default::default(), - }) - } -} -// product -impl Node for ast::located::MatchCase { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::MatchCase { - pattern, - guard, - body, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeMatchCase::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("pattern", pattern.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("guard", guard.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("body", body.ast_to_object(_vm), _vm).unwrap(); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::MatchCase { - pattern: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "pattern", "match_case")?, - )?, - guard: get_node_field_opt(_vm, &_object, "guard")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - body: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "body", "match_case")?)?, - range: Default::default(), - }) - } -} -// sum -impl Node for ast::located::Pattern { - fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef { - match self { - ast::located::Pattern::MatchValue(cons) => cons.ast_to_object(vm), - ast::located::Pattern::MatchSingleton(cons) => cons.ast_to_object(vm), - ast::located::Pattern::MatchSequence(cons) => cons.ast_to_object(vm), - ast::located::Pattern::MatchMapping(cons) => cons.ast_to_object(vm), - ast::located::Pattern::MatchClass(cons) => cons.ast_to_object(vm), - ast::located::Pattern::MatchStar(cons) => cons.ast_to_object(vm), - ast::located::Pattern::MatchAs(cons) => cons.ast_to_object(vm), - ast::located::Pattern::MatchOr(cons) => cons.ast_to_object(vm), - } - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - let _cls = _object.class(); - Ok(if _cls.is(NodePatternMatchValue::static_type()) { - ast::located::Pattern::MatchValue(ast::located::PatternMatchValue::ast_from_object( - _vm, _object, - )?) - } else if _cls.is(NodePatternMatchSingleton::static_type()) { - ast::located::Pattern::MatchSingleton( - ast::located::PatternMatchSingleton::ast_from_object(_vm, _object)?, - ) - } else if _cls.is(NodePatternMatchSequence::static_type()) { - ast::located::Pattern::MatchSequence( - ast::located::PatternMatchSequence::ast_from_object(_vm, _object)?, - ) - } else if _cls.is(NodePatternMatchMapping::static_type()) { - ast::located::Pattern::MatchMapping(ast::located::PatternMatchMapping::ast_from_object( - _vm, _object, - )?) - } else if _cls.is(NodePatternMatchClass::static_type()) { - ast::located::Pattern::MatchClass(ast::located::PatternMatchClass::ast_from_object( - _vm, _object, - )?) - } else if _cls.is(NodePatternMatchStar::static_type()) { - ast::located::Pattern::MatchStar(ast::located::PatternMatchStar::ast_from_object( - _vm, _object, - )?) - } else if _cls.is(NodePatternMatchAs::static_type()) { - ast::located::Pattern::MatchAs(ast::located::PatternMatchAs::ast_from_object( - _vm, _object, - )?) - } else if _cls.is(NodePatternMatchOr::static_type()) { - ast::located::Pattern::MatchOr(ast::located::PatternMatchOr::ast_from_object( - _vm, _object, - )?) - } else { - return Err(_vm.new_type_error(format!( - "expected some sort of pattern, but got {}", - _object.repr(_vm)? - ))); - }) - } -} -// constructor -impl Node for ast::located::PatternMatchValue { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::PatternMatchValue { - value, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodePatternMatchValue::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("value", value.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::PatternMatchValue { - value: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "value", "MatchValue")?, - )?, - range: range_from_object(_vm, _object, "MatchValue")?, - }) - } -} -// constructor -impl Node for ast::located::PatternMatchSingleton { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::PatternMatchSingleton { - value, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodePatternMatchSingleton::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("value", value.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::PatternMatchSingleton { - value: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "value", "MatchSingleton")?, - )?, - range: range_from_object(_vm, _object, "MatchSingleton")?, - }) - } -} -// constructor -impl Node for ast::located::PatternMatchSequence { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::PatternMatchSequence { - patterns, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodePatternMatchSequence::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("patterns", patterns.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::PatternMatchSequence { - patterns: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "patterns", "MatchSequence")?, - )?, - range: range_from_object(_vm, _object, "MatchSequence")?, - }) - } -} -// constructor -impl Node for ast::located::PatternMatchMapping { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::PatternMatchMapping { - keys, - patterns, - rest, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodePatternMatchMapping::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("keys", keys.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("patterns", patterns.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("rest", rest.ast_to_object(_vm), _vm).unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::PatternMatchMapping { - keys: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "keys", "MatchMapping")?, - )?, - patterns: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "patterns", "MatchMapping")?, - )?, - rest: get_node_field_opt(_vm, &_object, "rest")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - range: range_from_object(_vm, _object, "MatchMapping")?, - }) - } -} -// constructor -impl Node for ast::located::PatternMatchClass { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::PatternMatchClass { - cls, - patterns, - kwd_attrs, - kwd_patterns, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodePatternMatchClass::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("cls", cls.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("patterns", patterns.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("kwd_attrs", kwd_attrs.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("kwd_patterns", kwd_patterns.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::PatternMatchClass { - cls: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "cls", "MatchClass")?)?, - patterns: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "patterns", "MatchClass")?, - )?, - kwd_attrs: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "kwd_attrs", "MatchClass")?, - )?, - kwd_patterns: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "kwd_patterns", "MatchClass")?, - )?, - range: range_from_object(_vm, _object, "MatchClass")?, - }) - } -} -// constructor -impl Node for ast::located::PatternMatchStar { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::PatternMatchStar { - name, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodePatternMatchStar::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("name", name.ast_to_object(_vm), _vm).unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::PatternMatchStar { - name: get_node_field_opt(_vm, &_object, "name")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - range: range_from_object(_vm, _object, "MatchStar")?, - }) - } -} -// constructor -impl Node for ast::located::PatternMatchAs { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::PatternMatchAs { - pattern, - name, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodePatternMatchAs::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("pattern", pattern.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("name", name.ast_to_object(_vm), _vm).unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::PatternMatchAs { - pattern: get_node_field_opt(_vm, &_object, "pattern")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - name: get_node_field_opt(_vm, &_object, "name")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - range: range_from_object(_vm, _object, "MatchAs")?, - }) - } -} -// constructor -impl Node for ast::located::PatternMatchOr { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::PatternMatchOr { - patterns, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodePatternMatchOr::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("patterns", patterns.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::PatternMatchOr { - patterns: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "patterns", "MatchOr")?, - )?, - range: range_from_object(_vm, _object, "MatchOr")?, - }) - } -} -// sum -impl Node for ast::located::TypeIgnore { - fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef { - match self { - ast::located::TypeIgnore::TypeIgnore(cons) => cons.ast_to_object(vm), - } - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - let _cls = _object.class(); - Ok(if _cls.is(NodeTypeIgnoreTypeIgnore::static_type()) { - ast::located::TypeIgnore::TypeIgnore( - ast::located::TypeIgnoreTypeIgnore::ast_from_object(_vm, _object)?, - ) - } else { - return Err(_vm.new_type_error(format!( - "expected some sort of type_ignore, but got {}", - _object.repr(_vm)? - ))); - }) - } -} -// constructor -impl Node for ast::located::TypeIgnoreTypeIgnore { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::TypeIgnoreTypeIgnore { - lineno, - tag, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeTypeIgnoreTypeIgnore::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("lineno", lineno.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("tag", tag.ast_to_object(_vm), _vm).unwrap(); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::TypeIgnoreTypeIgnore { - lineno: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "lineno", "TypeIgnore")?, - )?, - tag: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "tag", "TypeIgnore")?)?, - range: Default::default(), - }) - } -} -// sum -impl Node for ast::located::TypeParam { - fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef { - match self { - ast::located::TypeParam::TypeVar(cons) => cons.ast_to_object(vm), - ast::located::TypeParam::ParamSpec(cons) => cons.ast_to_object(vm), - ast::located::TypeParam::TypeVarTuple(cons) => cons.ast_to_object(vm), - } - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - let _cls = _object.class(); - Ok(if _cls.is(NodeTypeParamTypeVar::static_type()) { - ast::located::TypeParam::TypeVar(ast::located::TypeParamTypeVar::ast_from_object( - _vm, _object, - )?) - } else if _cls.is(NodeTypeParamParamSpec::static_type()) { - ast::located::TypeParam::ParamSpec(ast::located::TypeParamParamSpec::ast_from_object( - _vm, _object, - )?) - } else if _cls.is(NodeTypeParamTypeVarTuple::static_type()) { - ast::located::TypeParam::TypeVarTuple( - ast::located::TypeParamTypeVarTuple::ast_from_object(_vm, _object)?, - ) - } else { - return Err(_vm.new_type_error(format!( - "expected some sort of type_param, but got {}", - _object.repr(_vm)? - ))); - }) - } -} -// constructor -impl Node for ast::located::TypeParamTypeVar { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::TypeParamTypeVar { - name, - bound, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeTypeParamTypeVar::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("name", name.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("bound", bound.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::TypeParamTypeVar { - name: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "name", "TypeVar")?)?, - bound: get_node_field_opt(_vm, &_object, "bound")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - range: range_from_object(_vm, _object, "TypeVar")?, - }) - } -} -// constructor -impl Node for ast::located::TypeParamParamSpec { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::TypeParamParamSpec { - name, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeTypeParamParamSpec::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("name", name.ast_to_object(_vm), _vm).unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::TypeParamParamSpec { - name: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "name", "ParamSpec")?)?, - range: range_from_object(_vm, _object, "ParamSpec")?, - }) - } -} -// constructor -impl Node for ast::located::TypeParamTypeVarTuple { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::TypeParamTypeVarTuple { - name, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeTypeParamTypeVarTuple::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("name", name.ast_to_object(_vm), _vm).unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult { - Ok(ast::located::TypeParamTypeVarTuple { - name: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "name", "TypeVarTuple")?, - )?, - range: range_from_object(_vm, _object, "TypeVarTuple")?, - }) - } -} - -pub fn extend_module_nodes(vm: &VirtualMachine, module: &Py) { - extend_module!(vm, module, { - "mod" => NodeMod::make_class(&vm.ctx), - "Module" => NodeModModule::make_class(&vm.ctx), - "Interactive" => NodeModInteractive::make_class(&vm.ctx), - "Expression" => NodeModExpression::make_class(&vm.ctx), - "FunctionType" => NodeModFunctionType::make_class(&vm.ctx), - "stmt" => NodeStmt::make_class(&vm.ctx), - "FunctionDef" => NodeStmtFunctionDef::make_class(&vm.ctx), - "AsyncFunctionDef" => NodeStmtAsyncFunctionDef::make_class(&vm.ctx), - "ClassDef" => NodeStmtClassDef::make_class(&vm.ctx), - "Return" => NodeStmtReturn::make_class(&vm.ctx), - "Delete" => NodeStmtDelete::make_class(&vm.ctx), - "Assign" => NodeStmtAssign::make_class(&vm.ctx), - "TypeAlias" => NodeStmtTypeAlias::make_class(&vm.ctx), - "AugAssign" => NodeStmtAugAssign::make_class(&vm.ctx), - "AnnAssign" => NodeStmtAnnAssign::make_class(&vm.ctx), - "For" => NodeStmtFor::make_class(&vm.ctx), - "AsyncFor" => NodeStmtAsyncFor::make_class(&vm.ctx), - "While" => NodeStmtWhile::make_class(&vm.ctx), - "If" => NodeStmtIf::make_class(&vm.ctx), - "With" => NodeStmtWith::make_class(&vm.ctx), - "AsyncWith" => NodeStmtAsyncWith::make_class(&vm.ctx), - "Match" => NodeStmtMatch::make_class(&vm.ctx), - "Raise" => NodeStmtRaise::make_class(&vm.ctx), - "Try" => NodeStmtTry::make_class(&vm.ctx), - "TryStar" => NodeStmtTryStar::make_class(&vm.ctx), - "Assert" => NodeStmtAssert::make_class(&vm.ctx), - "Import" => NodeStmtImport::make_class(&vm.ctx), - "ImportFrom" => NodeStmtImportFrom::make_class(&vm.ctx), - "Global" => NodeStmtGlobal::make_class(&vm.ctx), - "Nonlocal" => NodeStmtNonlocal::make_class(&vm.ctx), - "Expr" => NodeStmtExpr::make_class(&vm.ctx), - "Pass" => NodeStmtPass::make_class(&vm.ctx), - "Break" => NodeStmtBreak::make_class(&vm.ctx), - "Continue" => NodeStmtContinue::make_class(&vm.ctx), - "expr" => NodeExpr::make_class(&vm.ctx), - "BoolOp" => NodeExprBoolOp::make_class(&vm.ctx), - "NamedExpr" => NodeExprNamedExpr::make_class(&vm.ctx), - "BinOp" => NodeExprBinOp::make_class(&vm.ctx), - "UnaryOp" => NodeExprUnaryOp::make_class(&vm.ctx), - "Lambda" => NodeExprLambda::make_class(&vm.ctx), - "IfExp" => NodeExprIfExp::make_class(&vm.ctx), - "Dict" => NodeExprDict::make_class(&vm.ctx), - "Set" => NodeExprSet::make_class(&vm.ctx), - "ListComp" => NodeExprListComp::make_class(&vm.ctx), - "SetComp" => NodeExprSetComp::make_class(&vm.ctx), - "DictComp" => NodeExprDictComp::make_class(&vm.ctx), - "GeneratorExp" => NodeExprGeneratorExp::make_class(&vm.ctx), - "Await" => NodeExprAwait::make_class(&vm.ctx), - "Yield" => NodeExprYield::make_class(&vm.ctx), - "YieldFrom" => NodeExprYieldFrom::make_class(&vm.ctx), - "Compare" => NodeExprCompare::make_class(&vm.ctx), - "Call" => NodeExprCall::make_class(&vm.ctx), - "FormattedValue" => NodeExprFormattedValue::make_class(&vm.ctx), - "JoinedStr" => NodeExprJoinedStr::make_class(&vm.ctx), - "Constant" => NodeExprConstant::make_class(&vm.ctx), - "Attribute" => NodeExprAttribute::make_class(&vm.ctx), - "Subscript" => NodeExprSubscript::make_class(&vm.ctx), - "Starred" => NodeExprStarred::make_class(&vm.ctx), - "Name" => NodeExprName::make_class(&vm.ctx), - "List" => NodeExprList::make_class(&vm.ctx), - "Tuple" => NodeExprTuple::make_class(&vm.ctx), - "Slice" => NodeExprSlice::make_class(&vm.ctx), - "expr_context" => NodeExprContext::make_class(&vm.ctx), - "Load" => NodeExprContextLoad::make_class(&vm.ctx), - "Store" => NodeExprContextStore::make_class(&vm.ctx), - "Del" => NodeExprContextDel::make_class(&vm.ctx), - "boolop" => NodeBoolOp::make_class(&vm.ctx), - "And" => NodeBoolOpAnd::make_class(&vm.ctx), - "Or" => NodeBoolOpOr::make_class(&vm.ctx), - "operator" => NodeOperator::make_class(&vm.ctx), - "Add" => NodeOperatorAdd::make_class(&vm.ctx), - "Sub" => NodeOperatorSub::make_class(&vm.ctx), - "Mult" => NodeOperatorMult::make_class(&vm.ctx), - "MatMult" => NodeOperatorMatMult::make_class(&vm.ctx), - "Div" => NodeOperatorDiv::make_class(&vm.ctx), - "Mod" => NodeOperatorMod::make_class(&vm.ctx), - "Pow" => NodeOperatorPow::make_class(&vm.ctx), - "LShift" => NodeOperatorLShift::make_class(&vm.ctx), - "RShift" => NodeOperatorRShift::make_class(&vm.ctx), - "BitOr" => NodeOperatorBitOr::make_class(&vm.ctx), - "BitXor" => NodeOperatorBitXor::make_class(&vm.ctx), - "BitAnd" => NodeOperatorBitAnd::make_class(&vm.ctx), - "FloorDiv" => NodeOperatorFloorDiv::make_class(&vm.ctx), - "unaryop" => NodeUnaryOp::make_class(&vm.ctx), - "Invert" => NodeUnaryOpInvert::make_class(&vm.ctx), - "Not" => NodeUnaryOpNot::make_class(&vm.ctx), - "UAdd" => NodeUnaryOpUAdd::make_class(&vm.ctx), - "USub" => NodeUnaryOpUSub::make_class(&vm.ctx), - "cmpop" => NodeCmpOp::make_class(&vm.ctx), - "Eq" => NodeCmpOpEq::make_class(&vm.ctx), - "NotEq" => NodeCmpOpNotEq::make_class(&vm.ctx), - "Lt" => NodeCmpOpLt::make_class(&vm.ctx), - "LtE" => NodeCmpOpLtE::make_class(&vm.ctx), - "Gt" => NodeCmpOpGt::make_class(&vm.ctx), - "GtE" => NodeCmpOpGtE::make_class(&vm.ctx), - "Is" => NodeCmpOpIs::make_class(&vm.ctx), - "IsNot" => NodeCmpOpIsNot::make_class(&vm.ctx), - "In" => NodeCmpOpIn::make_class(&vm.ctx), - "NotIn" => NodeCmpOpNotIn::make_class(&vm.ctx), - "comprehension" => NodeComprehension::make_class(&vm.ctx), - "excepthandler" => NodeExceptHandler::make_class(&vm.ctx), - "ExceptHandler" => NodeExceptHandlerExceptHandler::make_class(&vm.ctx), - "arguments" => NodeArguments::make_class(&vm.ctx), - "arg" => NodeArg::make_class(&vm.ctx), - "keyword" => NodeKeyword::make_class(&vm.ctx), - "alias" => NodeAlias::make_class(&vm.ctx), - "withitem" => NodeWithItem::make_class(&vm.ctx), - "match_case" => NodeMatchCase::make_class(&vm.ctx), - "pattern" => NodePattern::make_class(&vm.ctx), - "MatchValue" => NodePatternMatchValue::make_class(&vm.ctx), - "MatchSingleton" => NodePatternMatchSingleton::make_class(&vm.ctx), - "MatchSequence" => NodePatternMatchSequence::make_class(&vm.ctx), - "MatchMapping" => NodePatternMatchMapping::make_class(&vm.ctx), - "MatchClass" => NodePatternMatchClass::make_class(&vm.ctx), - "MatchStar" => NodePatternMatchStar::make_class(&vm.ctx), - "MatchAs" => NodePatternMatchAs::make_class(&vm.ctx), - "MatchOr" => NodePatternMatchOr::make_class(&vm.ctx), - "type_ignore" => NodeTypeIgnore::make_class(&vm.ctx), - "TypeIgnore" => NodeTypeIgnoreTypeIgnore::make_class(&vm.ctx), - "type_param" => NodeTypeParam::make_class(&vm.ctx), - "TypeVar" => NodeTypeParamTypeVar::make_class(&vm.ctx), - "ParamSpec" => NodeTypeParamParamSpec::make_class(&vm.ctx), - "TypeVarTuple" => NodeTypeParamTypeVarTuple::make_class(&vm.ctx), - }) -} diff --git a/vm/src/stdlib/ast/module.rs b/vm/src/stdlib/ast/module.rs new file mode 100644 index 0000000000..480ced0b6f --- /dev/null +++ b/vm/src/stdlib/ast/module.rs @@ -0,0 +1,221 @@ +use super::*; +use crate::stdlib::ast::type_ignore::TypeIgnore; + +/// Represents the different types of Python module structures. +/// +/// This enum is used to represent the various possible forms of a Python module +/// in an Abstract Syntax Tree (AST). It can correspond to: +/// +/// - `Module`: A standard Python script, containing a sequence of statements +/// (e.g., assignments, function calls), possibly with type ignores. +/// - `Interactive`: A representation of code executed in an interactive +/// Python session (e.g., the REPL or Jupyter notebooks), where statements +/// are evaluated one at a time. +/// - `Expression`: A single expression without any surrounding statements. +/// This is typically used in scenarios like `eval()` or in expression-only +/// contexts. +/// - `FunctionType`: A function signature with argument and return type +/// annotations, representing the type hints of a function (e.g., `def add(x: int, y: int) -> int`). +pub(super) enum Mod { + Module(ruff::ModModule), + Interactive(ModInteractive), + Expression(ruff::ModExpression), + FunctionType(ModFunctionType), +} + +// sum +impl Node for Mod { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + match self { + Self::Module(cons) => cons.ast_to_object(vm, source_code), + Self::Interactive(cons) => cons.ast_to_object(vm, source_code), + Self::Expression(cons) => cons.ast_to_object(vm, source_code), + Self::FunctionType(cons) => cons.ast_to_object(vm, source_code), + } + } + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + let cls = object.class(); + Ok(if cls.is(pyast::NodeModModule::static_type()) { + Self::Module(ruff::ModModule::ast_from_object(vm, source_code, object)?) + } else if cls.is(pyast::NodeModInteractive::static_type()) { + Self::Interactive(ModInteractive::ast_from_object(vm, source_code, object)?) + } else if cls.is(pyast::NodeModExpression::static_type()) { + Self::Expression(ruff::ModExpression::ast_from_object( + vm, + source_code, + object, + )?) + } else if cls.is(pyast::NodeModFunctionType::static_type()) { + Self::FunctionType(ModFunctionType::ast_from_object(vm, source_code, object)?) + } else { + return Err(vm.new_type_error(format!( + "expected some sort of mod, but got {}", + object.repr(vm)? + ))); + }) + } +} +// constructor +impl Node for ruff::ModModule { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let ruff::ModModule { + body, + // type_ignores, + range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeModModule::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("body", body.ast_to_object(vm, source_code), vm) + .unwrap(); + // TODO: Improve ruff API + // ruff ignores type_ignore comments currently. + let type_ignores: Vec = vec![]; + dict.set_item( + "type_ignores", + type_ignores.ast_to_object(vm, source_code), + vm, + ) + .unwrap(); + node_add_location(&dict, range, vm, source_code); + node.into() + } + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + Ok(ruff::ModModule { + body: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "body", "Module")?, + )?, + // type_ignores: Node::ast_from_object( + // _vm, + // get_node_field(_vm, &_object, "type_ignores", "Module")?, + // )?, + range: Default::default(), + }) + } +} + +pub(super) struct ModInteractive { + pub(crate) range: TextRange, + pub(crate) body: Vec, +} + +// constructor +impl Node for ModInteractive { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { body, range } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeModInteractive::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("body", body.ast_to_object(vm, source_code), vm) + .unwrap(); + node_add_location(&dict, range, vm, source_code); + node.into() + } + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + Ok(Self { + body: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "body", "Interactive")?, + )?, + range: Default::default(), + }) + } +} +// constructor +impl Node for ruff::ModExpression { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { body, range } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeModExpression::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("body", body.ast_to_object(vm, source_code), vm) + .unwrap(); + node_add_location(&dict, range, vm, source_code); + node.into() + } + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + Ok(Self { + body: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "body", "Expression")?, + )?, + range: Default::default(), + }) + } +} + +pub(super) struct ModFunctionType { + pub(crate) argtypes: Box<[ruff::Expr]>, + pub(crate) returns: ruff::Expr, + pub(crate) range: TextRange, +} + +// constructor +impl Node for ModFunctionType { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let ModFunctionType { + argtypes, + returns, + range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeModFunctionType::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item( + "argtypes", + BoxedSlice(argtypes).ast_to_object(vm, source_code), + vm, + ) + .unwrap(); + dict.set_item("returns", returns.ast_to_object(vm, source_code), vm) + .unwrap(); + node_add_location(&dict, range, vm, source_code); + node.into() + } + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + Ok(ModFunctionType { + argtypes: { + let argtypes: BoxedSlice<_> = Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "argtypes", "FunctionType")?, + )?; + argtypes.0 + }, + returns: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "returns", "FunctionType")?, + )?, + range: Default::default(), + }) + } +} diff --git a/vm/src/stdlib/ast/node.rs b/vm/src/stdlib/ast/node.rs new file mode 100644 index 0000000000..03c62d7eb9 --- /dev/null +++ b/vm/src/stdlib/ast/node.rs @@ -0,0 +1,93 @@ +use crate::{PyObjectRef, PyResult, VirtualMachine}; +use rustpython_compiler_source::SourceCodeOwned; + +pub(crate) trait Node: Sized { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef; + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult; + /// Used in `Option::ast_from_object`; if `true`, that impl will return None. + fn is_none(&self) -> bool { + false + } +} + +impl Node for Vec { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + vm.ctx + .new_list( + self.into_iter() + .map(|node| node.ast_to_object(vm, source_code)) + .collect(), + ) + .into() + } + + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + vm.extract_elements_with(&object, |obj| Node::ast_from_object(vm, source_code, obj)) + } +} + +impl Node for Box { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + (*self).ast_to_object(vm, source_code) + } + + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + T::ast_from_object(vm, source_code, object).map(Box::new) + } + + fn is_none(&self) -> bool { + (**self).is_none() + } +} + +impl Node for Option { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + match self { + Some(node) => node.ast_to_object(vm, source_code), + None => vm.ctx.none(), + } + } + + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + if vm.is_none(&object) { + Ok(None) + } else { + let x = T::ast_from_object(vm, source_code, object)?; + Ok((!x.is_none()).then_some(x)) + } + } +} + +pub(super) struct BoxedSlice(pub(super) Box<[T]>); + +impl Node for BoxedSlice { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + self.0.into_vec().ast_to_object(vm, source_code) + } + + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + Ok(Self( + as Node>::ast_from_object(vm, source_code, object)?.into_boxed_slice(), + )) + } +} diff --git a/vm/src/stdlib/ast/operator.rs b/vm/src/stdlib/ast/operator.rs new file mode 100644 index 0000000000..bf11c5e1c0 --- /dev/null +++ b/vm/src/stdlib/ast/operator.rs @@ -0,0 +1,185 @@ +use super::*; + +// sum +impl Node for ruff::BoolOp { + fn ast_to_object(self, vm: &VirtualMachine, _source_code: &SourceCodeOwned) -> PyObjectRef { + let node_type = match self { + ruff::BoolOp::And => pyast::NodeBoolOpAnd::static_type(), + ruff::BoolOp::Or => pyast::NodeBoolOpOr::static_type(), + }; + NodeAst + .into_ref_with_type(vm, node_type.to_owned()) + .unwrap() + .into() + } + fn ast_from_object( + _vm: &VirtualMachine, + _source_code: &SourceCodeOwned, + _object: PyObjectRef, + ) -> PyResult { + let _cls = _object.class(); + Ok(if _cls.is(pyast::NodeBoolOpAnd::static_type()) { + ruff::BoolOp::And + } else if _cls.is(pyast::NodeBoolOpOr::static_type()) { + ruff::BoolOp::Or + } else { + return Err(_vm.new_type_error(format!( + "expected some sort of boolop, but got {}", + _object.repr(_vm)? + ))); + }) + } +} +// sum +impl Node for ruff::Operator { + fn ast_to_object(self, vm: &VirtualMachine, _source_code: &SourceCodeOwned) -> PyObjectRef { + let node_type = match self { + ruff::Operator::Add => pyast::NodeOperatorAdd::static_type(), + ruff::Operator::Sub => pyast::NodeOperatorSub::static_type(), + ruff::Operator::Mult => pyast::NodeOperatorMult::static_type(), + ruff::Operator::MatMult => pyast::NodeOperatorMatMult::static_type(), + ruff::Operator::Div => pyast::NodeOperatorDiv::static_type(), + ruff::Operator::Mod => pyast::NodeOperatorMod::static_type(), + ruff::Operator::Pow => pyast::NodeOperatorPow::static_type(), + ruff::Operator::LShift => pyast::NodeOperatorLShift::static_type(), + ruff::Operator::RShift => pyast::NodeOperatorRShift::static_type(), + ruff::Operator::BitOr => pyast::NodeOperatorBitOr::static_type(), + ruff::Operator::BitXor => pyast::NodeOperatorBitXor::static_type(), + ruff::Operator::BitAnd => pyast::NodeOperatorBitAnd::static_type(), + ruff::Operator::FloorDiv => pyast::NodeOperatorFloorDiv::static_type(), + }; + NodeAst + .into_ref_with_type(vm, node_type.to_owned()) + .unwrap() + .into() + } + fn ast_from_object( + _vm: &VirtualMachine, + _source_code: &SourceCodeOwned, + _object: PyObjectRef, + ) -> PyResult { + let _cls = _object.class(); + Ok(if _cls.is(pyast::NodeOperatorAdd::static_type()) { + ruff::Operator::Add + } else if _cls.is(pyast::NodeOperatorSub::static_type()) { + ruff::Operator::Sub + } else if _cls.is(pyast::NodeOperatorMult::static_type()) { + ruff::Operator::Mult + } else if _cls.is(pyast::NodeOperatorMatMult::static_type()) { + ruff::Operator::MatMult + } else if _cls.is(pyast::NodeOperatorDiv::static_type()) { + ruff::Operator::Div + } else if _cls.is(pyast::NodeOperatorMod::static_type()) { + ruff::Operator::Mod + } else if _cls.is(pyast::NodeOperatorPow::static_type()) { + ruff::Operator::Pow + } else if _cls.is(pyast::NodeOperatorLShift::static_type()) { + ruff::Operator::LShift + } else if _cls.is(pyast::NodeOperatorRShift::static_type()) { + ruff::Operator::RShift + } else if _cls.is(pyast::NodeOperatorBitOr::static_type()) { + ruff::Operator::BitOr + } else if _cls.is(pyast::NodeOperatorBitXor::static_type()) { + ruff::Operator::BitXor + } else if _cls.is(pyast::NodeOperatorBitAnd::static_type()) { + ruff::Operator::BitAnd + } else if _cls.is(pyast::NodeOperatorFloorDiv::static_type()) { + ruff::Operator::FloorDiv + } else { + return Err(_vm.new_type_error(format!( + "expected some sort of operator, but got {}", + _object.repr(_vm)? + ))); + }) + } +} +// sum +impl Node for ruff::UnaryOp { + fn ast_to_object(self, vm: &VirtualMachine, _source_code: &SourceCodeOwned) -> PyObjectRef { + let node_type = match self { + ruff::UnaryOp::Invert => pyast::NodeUnaryOpInvert::static_type(), + ruff::UnaryOp::Not => pyast::NodeUnaryOpNot::static_type(), + ruff::UnaryOp::UAdd => pyast::NodeUnaryOpUAdd::static_type(), + ruff::UnaryOp::USub => pyast::NodeUnaryOpUSub::static_type(), + }; + NodeAst + .into_ref_with_type(vm, node_type.to_owned()) + .unwrap() + .into() + } + fn ast_from_object( + _vm: &VirtualMachine, + _source_code: &SourceCodeOwned, + _object: PyObjectRef, + ) -> PyResult { + let _cls = _object.class(); + Ok(if _cls.is(pyast::NodeUnaryOpInvert::static_type()) { + ruff::UnaryOp::Invert + } else if _cls.is(pyast::NodeUnaryOpNot::static_type()) { + ruff::UnaryOp::Not + } else if _cls.is(pyast::NodeUnaryOpUAdd::static_type()) { + ruff::UnaryOp::UAdd + } else if _cls.is(pyast::NodeUnaryOpUSub::static_type()) { + ruff::UnaryOp::USub + } else { + return Err(_vm.new_type_error(format!( + "expected some sort of unaryop, but got {}", + _object.repr(_vm)? + ))); + }) + } +} +// sum +impl Node for ruff::CmpOp { + fn ast_to_object(self, vm: &VirtualMachine, _source_code: &SourceCodeOwned) -> PyObjectRef { + let node_type = match self { + ruff::CmpOp::Eq => pyast::NodeCmpOpEq::static_type(), + ruff::CmpOp::NotEq => pyast::NodeCmpOpNotEq::static_type(), + ruff::CmpOp::Lt => pyast::NodeCmpOpLt::static_type(), + ruff::CmpOp::LtE => pyast::NodeCmpOpLtE::static_type(), + ruff::CmpOp::Gt => pyast::NodeCmpOpGt::static_type(), + ruff::CmpOp::GtE => pyast::NodeCmpOpGtE::static_type(), + ruff::CmpOp::Is => pyast::NodeCmpOpIs::static_type(), + ruff::CmpOp::IsNot => pyast::NodeCmpOpIsNot::static_type(), + ruff::CmpOp::In => pyast::NodeCmpOpIn::static_type(), + ruff::CmpOp::NotIn => pyast::NodeCmpOpNotIn::static_type(), + }; + NodeAst + .into_ref_with_type(vm, node_type.to_owned()) + .unwrap() + .into() + } + fn ast_from_object( + _vm: &VirtualMachine, + _source_code: &SourceCodeOwned, + _object: PyObjectRef, + ) -> PyResult { + let _cls = _object.class(); + Ok(if _cls.is(pyast::NodeCmpOpEq::static_type()) { + ruff::CmpOp::Eq + } else if _cls.is(pyast::NodeCmpOpNotEq::static_type()) { + ruff::CmpOp::NotEq + } else if _cls.is(pyast::NodeCmpOpLt::static_type()) { + ruff::CmpOp::Lt + } else if _cls.is(pyast::NodeCmpOpLtE::static_type()) { + ruff::CmpOp::LtE + } else if _cls.is(pyast::NodeCmpOpGt::static_type()) { + ruff::CmpOp::Gt + } else if _cls.is(pyast::NodeCmpOpGtE::static_type()) { + ruff::CmpOp::GtE + } else if _cls.is(pyast::NodeCmpOpIs::static_type()) { + ruff::CmpOp::Is + } else if _cls.is(pyast::NodeCmpOpIsNot::static_type()) { + ruff::CmpOp::IsNot + } else if _cls.is(pyast::NodeCmpOpIn::static_type()) { + ruff::CmpOp::In + } else if _cls.is(pyast::NodeCmpOpNotIn::static_type()) { + ruff::CmpOp::NotIn + } else { + return Err(_vm.new_type_error(format!( + "expected some sort of cmpop, but got {}", + _object.repr(_vm)? + ))); + }) + } +} diff --git a/vm/src/stdlib/ast/other.rs b/vm/src/stdlib/ast/other.rs new file mode 100644 index 0000000000..09ffbd4077 --- /dev/null +++ b/vm/src/stdlib/ast/other.rs @@ -0,0 +1,131 @@ +use super::*; +use num_traits::ToPrimitive; + +impl Node for ruff::ConversionFlag { + fn ast_to_object(self, vm: &VirtualMachine, _source_code: &SourceCodeOwned) -> PyObjectRef { + vm.ctx.new_int(self as u8).into() + } + + fn ast_from_object( + vm: &VirtualMachine, + _source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + i32::try_from_object(vm, object)? + .to_u32() + .and_then(ruff::ConversionFlag::from_op_arg) + .ok_or_else(|| vm.new_value_error("invalid conversion flag".to_owned())) + } +} + +// /// This is just a string, not strictly an AST node. But it makes AST conversions easier. +// impl Node for ruff::name::Name { +// fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { +// vm.ctx.new_str(self.as_str()).to_pyobject(vm) +// } + +// fn ast_from_object(vm: &VirtualMachine, source_code: &SourceCodeOwned, object: PyObjectRef) -> PyResult { +// match object.downcast::() { +// Ok(name) => Ok(Self::new(name)), +// Err(_) => Err(vm.new_value_error("expected str for name".to_owned())), +// } +// } +// } + +impl Node for ruff::Decorator { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + ruff::Expr::ast_to_object(self.expression, vm, source_code) + } + + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + let expression = ruff::Expr::ast_from_object(vm, source_code, object)?; + let range = expression.range(); + Ok(ruff::Decorator { expression, range }) + } +} + +// product +impl Node for ruff::Alias { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { + name, + asname, + range: _range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeAlias::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("name", name.ast_to_object(vm, source_code), vm) + .unwrap(); + dict.set_item("asname", asname.ast_to_object(vm, source_code), vm) + .unwrap(); + node_add_location(&dict, _range, vm, source_code); + node.into() + } + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + Ok(Self { + name: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "name", "alias")?, + )?, + asname: get_node_field_opt(vm, &object, "asname")? + .map(|obj| Node::ast_from_object(vm, source_code, obj)) + .transpose()?, + range: range_from_object(vm, source_code, object, "alias")?, + }) + } +} +// product +impl Node for ruff::WithItem { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { + context_expr, + optional_vars, + range: _range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeWithItem::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item( + "context_expr", + context_expr.ast_to_object(vm, source_code), + vm, + ) + .unwrap(); + dict.set_item( + "optional_vars", + optional_vars.ast_to_object(vm, source_code), + vm, + ) + .unwrap(); + node.into() + } + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + Ok(Self { + context_expr: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "context_expr", "withitem")?, + )?, + optional_vars: get_node_field_opt(vm, &object, "optional_vars")? + .map(|obj| Node::ast_from_object(vm, source_code, obj)) + .transpose()?, + range: Default::default(), + }) + } +} diff --git a/vm/src/stdlib/ast/parameter.rs b/vm/src/stdlib/ast/parameter.rs new file mode 100644 index 0000000000..82c22d020c --- /dev/null +++ b/vm/src/stdlib/ast/parameter.rs @@ -0,0 +1,401 @@ +use super::*; + +// product +impl Node for ruff::Parameters { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { + posonlyargs, + args, + vararg, + kwonlyargs, + kwarg, + range, + } = self; + let (posonlyargs, args, defaults) = + extract_positional_parameter_defaults(posonlyargs, args); + let (kwonlyargs, kw_defaults) = extract_keyword_parameter_defaults(kwonlyargs); + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeArguments::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item( + "posonlyargs", + posonlyargs.ast_to_object(vm, source_code), + vm, + ) + .unwrap(); + dict.set_item("args", args.ast_to_object(vm, source_code), vm) + .unwrap(); + dict.set_item("vararg", vararg.ast_to_object(vm, source_code), vm) + .unwrap(); + dict.set_item("kwonlyargs", kwonlyargs.ast_to_object(vm, source_code), vm) + .unwrap(); + dict.set_item( + "kw_defaults", + kw_defaults.ast_to_object(vm, source_code), + vm, + ) + .unwrap(); + dict.set_item("kwarg", kwarg.ast_to_object(vm, source_code), vm) + .unwrap(); + dict.set_item("defaults", defaults.ast_to_object(vm, source_code), vm) + .unwrap(); + node_add_location(&dict, range, vm, source_code); + node.into() + } + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + let kwonlyargs = Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "kwonlyargs", "arguments")?, + )?; + let kw_defaults = Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "kw_defaults", "arguments")?, + )?; + let kwonlyargs = merge_keyword_parameter_defaults(kwonlyargs, kw_defaults); + + let posonlyargs = Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "posonlyargs", "arguments")?, + )?; + let args = Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "args", "arguments")?, + )?; + let defaults = Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "defaults", "arguments")?, + )?; + let (posonlyargs, args) = merge_positional_parameter_defaults(posonlyargs, args, defaults); + + Ok(Self { + posonlyargs, + args, + vararg: get_node_field_opt(vm, &object, "vararg")? + .map(|obj| Node::ast_from_object(vm, source_code, obj)) + .transpose()?, + kwonlyargs, + kwarg: get_node_field_opt(vm, &object, "kwarg")? + .map(|obj| Node::ast_from_object(vm, source_code, obj)) + .transpose()?, + range: Default::default(), + }) + } + + fn is_none(&self) -> bool { + self.is_empty() + } +} +// product +impl Node for ruff::Parameter { + fn ast_to_object(self, _vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { + name, + annotation, + // type_comment, + range, + } = self; + + // ruff covers the ** in range but python expects it to start at the ident + let range = TextRange::new(name.start(), range.end()); + + let node = NodeAst + .into_ref_with_type(_vm, pyast::NodeArg::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("arg", name.ast_to_object(_vm, source_code), _vm) + .unwrap(); + dict.set_item( + "annotation", + annotation.ast_to_object(_vm, source_code), + _vm, + ) + .unwrap(); + // dict.set_item("type_comment", type_comment.ast_to_object(_vm), _vm) + // .unwrap(); + node_add_location(&dict, range, _vm, source_code); + node.into() + } + fn ast_from_object( + _vm: &VirtualMachine, + source_code: &SourceCodeOwned, + _object: PyObjectRef, + ) -> PyResult { + Ok(Self { + name: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "arg", "arg")?, + )?, + annotation: get_node_field_opt(_vm, &_object, "annotation")? + .map(|obj| Node::ast_from_object(_vm, source_code, obj)) + .transpose()?, + // type_comment: get_node_field_opt(_vm, &_object, "type_comment")? + // .map(|obj| Node::ast_from_object(_vm, obj)) + // .transpose()?, + range: range_from_object(_vm, source_code, _object, "arg")?, + }) + } +} + +// product +impl Node for ruff::Keyword { + fn ast_to_object(self, _vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { + arg, + value, + range: _range, + } = self; + let node = NodeAst + .into_ref_with_type(_vm, pyast::NodeKeyword::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("arg", arg.ast_to_object(_vm, source_code), _vm) + .unwrap(); + dict.set_item("value", value.ast_to_object(_vm, source_code), _vm) + .unwrap(); + node_add_location(&dict, _range, _vm, source_code); + node.into() + } + fn ast_from_object( + _vm: &VirtualMachine, + source_code: &SourceCodeOwned, + _object: PyObjectRef, + ) -> PyResult { + Ok(Self { + arg: get_node_field_opt(_vm, &_object, "arg")? + .map(|obj| Node::ast_from_object(_vm, source_code, obj)) + .transpose()?, + value: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "value", "keyword")?, + )?, + range: range_from_object(_vm, source_code, _object, "keyword")?, + }) + } +} + +struct PositionalParameters { + pub _range: TextRange, // TODO: Use this + pub args: Box<[ruff::Parameter]>, +} + +impl Node for PositionalParameters { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + BoxedSlice(self.args).ast_to_object(vm, source_code) + } + + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + let args: BoxedSlice<_> = Node::ast_from_object(vm, source_code, object)?; + Ok(Self { + args: args.0, + _range: TextRange::default(), // TODO + }) + } +} + +struct KeywordParameters { + pub _range: TextRange, // TODO: Use this + pub keywords: Box<[ruff::Parameter]>, +} + +impl Node for KeywordParameters { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + BoxedSlice(self.keywords).ast_to_object(vm, source_code) + } + + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + let keywords: BoxedSlice<_> = Node::ast_from_object(vm, source_code, object)?; + Ok(Self { + keywords: keywords.0, + _range: TextRange::default(), // TODO + }) + } +} + +struct ParameterDefaults { + pub _range: TextRange, // TODO: Use this + defaults: Box<[Option>]>, +} + +impl Node for ParameterDefaults { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + BoxedSlice(self.defaults).ast_to_object(vm, source_code) + } + + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + let defaults: BoxedSlice<_> = Node::ast_from_object(vm, source_code, object)?; + Ok(Self { + defaults: defaults.0, + _range: TextRange::default(), // TODO + }) + } +} + +fn extract_positional_parameter_defaults( + pos_only_args: Vec, + args: Vec, +) -> ( + PositionalParameters, + PositionalParameters, + ParameterDefaults, +) { + let mut defaults = vec![]; + defaults.extend(pos_only_args.iter().map(|item| item.default.clone())); + defaults.extend(args.iter().map(|item| item.default.clone())); + // If some positional parameters have no default value, + // the "defaults" list contains only the defaults of the last "n" parameters. + // Remove all positional parameters without a default value. + defaults.retain(Option::is_some); + let defaults = ParameterDefaults { + _range: defaults + .iter() + .flatten() + .map(|item| item.range()) + .reduce(|acc, next| acc.cover(next)) + .unwrap_or_default(), + defaults: defaults.into_boxed_slice(), + }; + + let pos_only_args = PositionalParameters { + _range: pos_only_args + .iter() + .map(|item| item.range()) + .reduce(|acc, next| acc.cover(next)) + .unwrap_or_default(), + args: { + let pos_only_args: Vec<_> = pos_only_args + .iter() + .map(|item| item.parameter.clone()) + .collect(); + pos_only_args.into_boxed_slice() + }, + }; + + let args = PositionalParameters { + _range: args + .iter() + .map(|item| item.range()) + .reduce(|acc, next| acc.cover(next)) + .unwrap_or_default(), + args: { + let args: Vec<_> = args.iter().map(|item| item.parameter.clone()).collect(); + args.into_boxed_slice() + }, + }; + + (pos_only_args, args, defaults) +} + +/// Merges the keyword parameters with their default values, opposite of [`extract_positional_parameter_defaults`]. +fn merge_positional_parameter_defaults( + posonlyargs: PositionalParameters, + args: PositionalParameters, + defaults: ParameterDefaults, +) -> ( + Vec, + Vec, +) { + let posonlyargs = posonlyargs.args; + let args = args.args; + let defaults = defaults.defaults; + + let mut posonlyargs: Vec<_> = as IntoIterator>::into_iter(posonlyargs) + .map(|parameter| ruff::ParameterWithDefault { + range: Default::default(), + parameter, + default: None, + }) + .collect(); + let mut args: Vec<_> = as IntoIterator>::into_iter(args) + .map(|parameter| ruff::ParameterWithDefault { + range: Default::default(), + parameter, + default: None, + }) + .collect(); + + // If an argument has a default value, insert it + // Note that "defaults" will only contain default values for the last "n" parameters + // so we need to skip the first "total_argument_count - n" arguments. + let default_argument_count = posonlyargs.len() + args.len() - defaults.len(); + for (arg, default) in posonlyargs + .iter_mut() + .chain(args.iter_mut()) + .skip(default_argument_count) + .zip(defaults) + { + arg.default = default; + } + + (posonlyargs, args) +} + +fn extract_keyword_parameter_defaults( + kw_only_args: Vec, +) -> (KeywordParameters, ParameterDefaults) { + let mut defaults = vec![]; + defaults.extend(kw_only_args.iter().map(|item| item.default.clone())); + let defaults = ParameterDefaults { + _range: defaults + .iter() + .flatten() + .map(|item| item.range()) + .reduce(|acc, next| acc.cover(next)) + .unwrap_or_default(), + defaults: defaults.into_boxed_slice(), + }; + + let kw_only_args = KeywordParameters { + _range: kw_only_args + .iter() + .map(|item| item.range()) + .reduce(|acc, next| acc.cover(next)) + .unwrap_or_default(), + keywords: { + let kw_only_args: Vec<_> = kw_only_args + .iter() + .map(|item| item.parameter.clone()) + .collect(); + kw_only_args.into_boxed_slice() + }, + }; + + (kw_only_args, defaults) +} + +/// Merges the keyword parameters with their default values, opposite of [`extract_keyword_parameter_defaults`]. +fn merge_keyword_parameter_defaults( + kw_only_args: KeywordParameters, + defaults: ParameterDefaults, +) -> Vec { + std::iter::zip(kw_only_args.keywords, defaults.defaults) + .map(|(parameter, default)| ruff::ParameterWithDefault { + parameter, + default, + range: Default::default(), + }) + .collect() +} diff --git a/vm/src/stdlib/ast/pattern.rs b/vm/src/stdlib/ast/pattern.rs new file mode 100644 index 0000000000..df5adefebf --- /dev/null +++ b/vm/src/stdlib/ast/pattern.rs @@ -0,0 +1,515 @@ +use super::*; + +// product +impl Node for ruff::MatchCase { + fn ast_to_object(self, _vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { + pattern, + guard, + body, + range: _range, + } = self; + let node = NodeAst + .into_ref_with_type(_vm, pyast::NodeMatchCase::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("pattern", pattern.ast_to_object(_vm, source_code), _vm) + .unwrap(); + dict.set_item("guard", guard.ast_to_object(_vm, source_code), _vm) + .unwrap(); + dict.set_item("body", body.ast_to_object(_vm, source_code), _vm) + .unwrap(); + node.into() + } + fn ast_from_object( + _vm: &VirtualMachine, + source_code: &SourceCodeOwned, + _object: PyObjectRef, + ) -> PyResult { + Ok(Self { + pattern: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "pattern", "match_case")?, + )?, + guard: get_node_field_opt(_vm, &_object, "guard")? + .map(|obj| Node::ast_from_object(_vm, source_code, obj)) + .transpose()?, + body: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "body", "match_case")?, + )?, + range: Default::default(), + }) + } +} +// sum +impl Node for ruff::Pattern { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + match self { + ruff::Pattern::MatchValue(cons) => cons.ast_to_object(vm, source_code), + ruff::Pattern::MatchSingleton(cons) => cons.ast_to_object(vm, source_code), + ruff::Pattern::MatchSequence(cons) => cons.ast_to_object(vm, source_code), + ruff::Pattern::MatchMapping(cons) => cons.ast_to_object(vm, source_code), + ruff::Pattern::MatchClass(cons) => cons.ast_to_object(vm, source_code), + ruff::Pattern::MatchStar(cons) => cons.ast_to_object(vm, source_code), + ruff::Pattern::MatchAs(cons) => cons.ast_to_object(vm, source_code), + ruff::Pattern::MatchOr(cons) => cons.ast_to_object(vm, source_code), + } + } + fn ast_from_object( + _vm: &VirtualMachine, + source_code: &SourceCodeOwned, + _object: PyObjectRef, + ) -> PyResult { + let _cls = _object.class(); + Ok(if _cls.is(pyast::NodePatternMatchValue::static_type()) { + ruff::Pattern::MatchValue(ruff::PatternMatchValue::ast_from_object( + _vm, + source_code, + _object, + )?) + } else if _cls.is(pyast::NodePatternMatchSingleton::static_type()) { + ruff::Pattern::MatchSingleton(ruff::PatternMatchSingleton::ast_from_object( + _vm, + source_code, + _object, + )?) + } else if _cls.is(pyast::NodePatternMatchSequence::static_type()) { + ruff::Pattern::MatchSequence(ruff::PatternMatchSequence::ast_from_object( + _vm, + source_code, + _object, + )?) + } else if _cls.is(pyast::NodePatternMatchMapping::static_type()) { + ruff::Pattern::MatchMapping(ruff::PatternMatchMapping::ast_from_object( + _vm, + source_code, + _object, + )?) + } else if _cls.is(pyast::NodePatternMatchClass::static_type()) { + ruff::Pattern::MatchClass(ruff::PatternMatchClass::ast_from_object( + _vm, + source_code, + _object, + )?) + } else if _cls.is(pyast::NodePatternMatchStar::static_type()) { + ruff::Pattern::MatchStar(ruff::PatternMatchStar::ast_from_object( + _vm, + source_code, + _object, + )?) + } else if _cls.is(pyast::NodePatternMatchAs::static_type()) { + ruff::Pattern::MatchAs(ruff::PatternMatchAs::ast_from_object( + _vm, + source_code, + _object, + )?) + } else if _cls.is(pyast::NodePatternMatchOr::static_type()) { + ruff::Pattern::MatchOr(ruff::PatternMatchOr::ast_from_object( + _vm, + source_code, + _object, + )?) + } else { + return Err(_vm.new_type_error(format!( + "expected some sort of pattern, but got {}", + _object.repr(_vm)? + ))); + }) + } +} +// constructor +impl Node for ruff::PatternMatchValue { + fn ast_to_object(self, _vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { + value, + range: _range, + } = self; + let node = NodeAst + .into_ref_with_type(_vm, pyast::NodePatternMatchValue::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("value", value.ast_to_object(_vm, source_code), _vm) + .unwrap(); + node_add_location(&dict, _range, _vm, source_code); + node.into() + } + fn ast_from_object( + _vm: &VirtualMachine, + source_code: &SourceCodeOwned, + _object: PyObjectRef, + ) -> PyResult { + Ok(Self { + value: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "value", "MatchValue")?, + )?, + range: range_from_object(_vm, source_code, _object, "MatchValue")?, + }) + } +} +// constructor +impl Node for ruff::PatternMatchSingleton { + fn ast_to_object(self, _vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { + value, + range: _range, + } = self; + let node = NodeAst + .into_ref_with_type( + _vm, + pyast::NodePatternMatchSingleton::static_type().to_owned(), + ) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("value", value.ast_to_object(_vm, source_code), _vm) + .unwrap(); + node_add_location(&dict, _range, _vm, source_code); + node.into() + } + fn ast_from_object( + _vm: &VirtualMachine, + source_code: &SourceCodeOwned, + _object: PyObjectRef, + ) -> PyResult { + Ok(Self { + value: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "value", "MatchSingleton")?, + )?, + range: range_from_object(_vm, source_code, _object, "MatchSingleton")?, + }) + } +} +impl Node for ruff::Singleton { + fn ast_to_object(self, _vm: &VirtualMachine, _source_code: &SourceCodeOwned) -> PyObjectRef { + todo!() + } + + fn ast_from_object( + _vm: &VirtualMachine, + _source_code: &SourceCodeOwned, + _object: PyObjectRef, + ) -> PyResult { + todo!() + } +} +// constructor +impl Node for ruff::PatternMatchSequence { + fn ast_to_object(self, _vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { + patterns, + range: _range, + } = self; + let node = NodeAst + .into_ref_with_type( + _vm, + pyast::NodePatternMatchSequence::static_type().to_owned(), + ) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("patterns", patterns.ast_to_object(_vm, source_code), _vm) + .unwrap(); + node_add_location(&dict, _range, _vm, source_code); + node.into() + } + fn ast_from_object( + _vm: &VirtualMachine, + source_code: &SourceCodeOwned, + _object: PyObjectRef, + ) -> PyResult { + Ok(Self { + patterns: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "patterns", "MatchSequence")?, + )?, + range: range_from_object(_vm, source_code, _object, "MatchSequence")?, + }) + } +} +// constructor +impl Node for ruff::PatternMatchMapping { + fn ast_to_object(self, _vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { + keys, + patterns, + rest, + range: _range, + } = self; + let node = NodeAst + .into_ref_with_type( + _vm, + pyast::NodePatternMatchMapping::static_type().to_owned(), + ) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("keys", keys.ast_to_object(_vm, source_code), _vm) + .unwrap(); + dict.set_item("patterns", patterns.ast_to_object(_vm, source_code), _vm) + .unwrap(); + dict.set_item("rest", rest.ast_to_object(_vm, source_code), _vm) + .unwrap(); + node_add_location(&dict, _range, _vm, source_code); + node.into() + } + fn ast_from_object( + _vm: &VirtualMachine, + source_code: &SourceCodeOwned, + _object: PyObjectRef, + ) -> PyResult { + Ok(Self { + keys: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "keys", "MatchMapping")?, + )?, + patterns: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "patterns", "MatchMapping")?, + )?, + rest: get_node_field_opt(_vm, &_object, "rest")? + .map(|obj| Node::ast_from_object(_vm, source_code, obj)) + .transpose()?, + range: range_from_object(_vm, source_code, _object, "MatchMapping")?, + }) + } +} +// constructor +impl Node for ruff::PatternMatchClass { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { + cls, + arguments, + range: _range, + } = self; + let (patterns, kwd_attrs, kwd_patterns) = split_pattern_match_class(arguments); + let node = NodeAst + .into_ref_with_type(vm, pyast::NodePatternMatchClass::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("cls", cls.ast_to_object(vm, source_code), vm) + .unwrap(); + dict.set_item("patterns", patterns.ast_to_object(vm, source_code), vm) + .unwrap(); + dict.set_item("kwd_attrs", kwd_attrs.ast_to_object(vm, source_code), vm) + .unwrap(); + dict.set_item( + "kwd_patterns", + kwd_patterns.ast_to_object(vm, source_code), + vm, + ) + .unwrap(); + node_add_location(&dict, _range, vm, source_code); + node.into() + } + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + let patterns = Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "patterns", "MatchClass")?, + )?; + let kwd_attrs = Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "kwd_attrs", "MatchClass")?, + )?; + let kwd_patterns = Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "kwd_patterns", "MatchClass")?, + )?; + let (patterns, keywords) = merge_pattern_match_class(patterns, kwd_attrs, kwd_patterns); + + Ok(Self { + cls: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "cls", "MatchClass")?, + )?, + range: range_from_object(vm, source_code, object, "MatchClass")?, + arguments: ruff::PatternArguments { + range: Default::default(), + patterns, + keywords, + }, + }) + } +} + +struct PatternMatchClassPatterns { + pub _range: TextRange, // TODO: Use this +} + +impl Node for PatternMatchClassPatterns { + fn ast_to_object(self, _vm: &VirtualMachine, _source_code: &SourceCodeOwned) -> PyObjectRef { + todo!() + } + + fn ast_from_object( + _vm: &VirtualMachine, + _source_code: &SourceCodeOwned, + _object: PyObjectRef, + ) -> PyResult { + todo!() + } +} + +struct PatternMatchClassKeywordAttributes { + pub _range: TextRange, // TODO: Use this +} + +impl Node for PatternMatchClassKeywordAttributes { + fn ast_to_object(self, _vm: &VirtualMachine, _source_code: &SourceCodeOwned) -> PyObjectRef { + todo!() + } + + fn ast_from_object( + _vm: &VirtualMachine, + _source_code: &SourceCodeOwned, + _object: PyObjectRef, + ) -> PyResult { + todo!() + } +} + +struct PatternMatchClassKeywordPatterns { + pub _range: TextRange, // TODO: Use this +} + +impl Node for PatternMatchClassKeywordPatterns { + fn ast_to_object(self, _vm: &VirtualMachine, _source_code: &SourceCodeOwned) -> PyObjectRef { + todo!() + } + + fn ast_from_object( + _vm: &VirtualMachine, + _source_code: &SourceCodeOwned, + _object: PyObjectRef, + ) -> PyResult { + todo!() + } +} +// constructor +impl Node for ruff::PatternMatchStar { + fn ast_to_object(self, _vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { + name, + range: _range, + } = self; + let node = NodeAst + .into_ref_with_type(_vm, pyast::NodePatternMatchStar::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("name", name.ast_to_object(_vm, source_code), _vm) + .unwrap(); + node_add_location(&dict, _range, _vm, source_code); + node.into() + } + fn ast_from_object( + _vm: &VirtualMachine, + source_code: &SourceCodeOwned, + _object: PyObjectRef, + ) -> PyResult { + Ok(Self { + name: get_node_field_opt(_vm, &_object, "name")? + .map(|obj| Node::ast_from_object(_vm, source_code, obj)) + .transpose()?, + range: range_from_object(_vm, source_code, _object, "MatchStar")?, + }) + } +} +// constructor +impl Node for ruff::PatternMatchAs { + fn ast_to_object(self, _vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { + pattern, + name, + range: _range, + } = self; + let node = NodeAst + .into_ref_with_type(_vm, pyast::NodePatternMatchAs::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("pattern", pattern.ast_to_object(_vm, source_code), _vm) + .unwrap(); + dict.set_item("name", name.ast_to_object(_vm, source_code), _vm) + .unwrap(); + node_add_location(&dict, _range, _vm, source_code); + node.into() + } + fn ast_from_object( + _vm: &VirtualMachine, + source_code: &SourceCodeOwned, + _object: PyObjectRef, + ) -> PyResult { + Ok(Self { + pattern: get_node_field_opt(_vm, &_object, "pattern")? + .map(|obj| Node::ast_from_object(_vm, source_code, obj)) + .transpose()?, + name: get_node_field_opt(_vm, &_object, "name")? + .map(|obj| Node::ast_from_object(_vm, source_code, obj)) + .transpose()?, + range: range_from_object(_vm, source_code, _object, "MatchAs")?, + }) + } +} +// constructor +impl Node for ruff::PatternMatchOr { + fn ast_to_object(self, _vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { + patterns, + range: _range, + } = self; + let node = NodeAst + .into_ref_with_type(_vm, pyast::NodePatternMatchOr::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("patterns", patterns.ast_to_object(_vm, source_code), _vm) + .unwrap(); + node_add_location(&dict, _range, _vm, source_code); + node.into() + } + fn ast_from_object( + _vm: &VirtualMachine, + source_code: &SourceCodeOwned, + _object: PyObjectRef, + ) -> PyResult { + Ok(Self { + patterns: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "patterns", "MatchOr")?, + )?, + range: range_from_object(_vm, source_code, _object, "MatchOr")?, + }) + } +} + +fn split_pattern_match_class( + _arguments: ruff::PatternArguments, +) -> ( + PatternMatchClassPatterns, + PatternMatchClassKeywordAttributes, + PatternMatchClassKeywordPatterns, +) { + todo!() +} + +/// Merges the pattern match class attributes and patterns, opposite of [`split_pattern_match_class`]. +fn merge_pattern_match_class( + _patterns: PatternMatchClassPatterns, + _kwd_attrs: PatternMatchClassKeywordAttributes, + _kwd_patterns: PatternMatchClassKeywordPatterns, +) -> (Vec, Vec) { + todo!() +} diff --git a/vm/src/stdlib/ast/pyast.rs b/vm/src/stdlib/ast/pyast.rs new file mode 100644 index 0000000000..3692b0a2c2 --- /dev/null +++ b/vm/src/stdlib/ast/pyast.rs @@ -0,0 +1,2448 @@ +#![allow(clippy::all)] + +use super::*; +use crate::common::ascii; +#[pyclass(module = "_ast", name = "mod", base = "NodeAst")] +pub(crate) struct NodeMod; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeMod {} +#[pyclass(module = "_ast", name = "Module", base = "NodeMod")] +pub(crate) struct NodeModModule; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeModModule { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("body")).into(), + ctx.new_str(ascii!("type_ignores")).into(), + ]) + .into(), + ); + class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); + } +} +#[pyclass(module = "_ast", name = "Interactive", base = "NodeMod")] +pub(crate) struct NodeModInteractive; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeModInteractive { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ctx.new_str(ascii!("body")).into()]) + .into(), + ); + class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); + } +} +#[pyclass(module = "_ast", name = "Expression", base = "NodeMod")] +pub(crate) struct NodeModExpression; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeModExpression { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ctx.new_str(ascii!("body")).into()]) + .into(), + ); + class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); + } +} +#[pyclass(module = "_ast", name = "FunctionType", base = "NodeMod")] +pub(crate) struct NodeModFunctionType; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeModFunctionType { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("argtypes")).into(), + ctx.new_str(ascii!("returns")).into(), + ]) + .into(), + ); + class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); + } +} +#[pyclass(module = "_ast", name = "stmt", base = "NodeAst")] +pub(crate) struct NodeStmt; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeStmt {} +#[pyclass(module = "_ast", name = "FunctionDef", base = "NodeStmt")] +pub(crate) struct NodeStmtFunctionDef; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeStmtFunctionDef { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("name")).into(), + ctx.new_str(ascii!("args")).into(), + ctx.new_str(ascii!("body")).into(), + ctx.new_str(ascii!("decorator_list")).into(), + ctx.new_str(ascii!("returns")).into(), + ctx.new_str(ascii!("type_comment")).into(), + ctx.new_str(ascii!("type_params")).into(), + ]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "AsyncFunctionDef", base = "NodeStmt")] +pub(crate) struct NodeStmtAsyncFunctionDef; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeStmtAsyncFunctionDef { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("name")).into(), + ctx.new_str(ascii!("args")).into(), + ctx.new_str(ascii!("body")).into(), + ctx.new_str(ascii!("decorator_list")).into(), + ctx.new_str(ascii!("returns")).into(), + ctx.new_str(ascii!("type_comment")).into(), + ctx.new_str(ascii!("type_params")).into(), + ]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "ClassDef", base = "NodeStmt")] +pub(crate) struct NodeStmtClassDef; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeStmtClassDef { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("name")).into(), + ctx.new_str(ascii!("bases")).into(), + ctx.new_str(ascii!("keywords")).into(), + ctx.new_str(ascii!("body")).into(), + ctx.new_str(ascii!("decorator_list")).into(), + ctx.new_str(ascii!("type_params")).into(), + ]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "Return", base = "NodeStmt")] +pub(crate) struct NodeStmtReturn; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeStmtReturn { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ctx.new_str(ascii!("value")).into()]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "Delete", base = "NodeStmt")] +pub(crate) struct NodeStmtDelete; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeStmtDelete { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ctx.new_str(ascii!("targets")).into()]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "Assign", base = "NodeStmt")] +pub(crate) struct NodeStmtAssign; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeStmtAssign { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("targets")).into(), + ctx.new_str(ascii!("value")).into(), + ctx.new_str(ascii!("type_comment")).into(), + ]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "TypeAlias", base = "NodeStmt")] +pub(crate) struct NodeStmtTypeAlias; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeStmtTypeAlias { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("name")).into(), + ctx.new_str(ascii!("type_params")).into(), + ctx.new_str(ascii!("value")).into(), + ]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "AugAssign", base = "NodeStmt")] +pub(crate) struct NodeStmtAugAssign; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeStmtAugAssign { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("target")).into(), + ctx.new_str(ascii!("op")).into(), + ctx.new_str(ascii!("value")).into(), + ]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "AnnAssign", base = "NodeStmt")] +pub(crate) struct NodeStmtAnnAssign; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeStmtAnnAssign { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("target")).into(), + ctx.new_str(ascii!("annotation")).into(), + ctx.new_str(ascii!("value")).into(), + ctx.new_str(ascii!("simple")).into(), + ]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "For", base = "NodeStmt")] +pub(crate) struct NodeStmtFor; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeStmtFor { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("target")).into(), + ctx.new_str(ascii!("iter")).into(), + ctx.new_str(ascii!("body")).into(), + ctx.new_str(ascii!("orelse")).into(), + ctx.new_str(ascii!("type_comment")).into(), + ]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "AsyncFor", base = "NodeStmt")] +pub(crate) struct NodeStmtAsyncFor; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeStmtAsyncFor { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("target")).into(), + ctx.new_str(ascii!("iter")).into(), + ctx.new_str(ascii!("body")).into(), + ctx.new_str(ascii!("orelse")).into(), + ctx.new_str(ascii!("type_comment")).into(), + ]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "While", base = "NodeStmt")] +pub(crate) struct NodeStmtWhile; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeStmtWhile { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("test")).into(), + ctx.new_str(ascii!("body")).into(), + ctx.new_str(ascii!("orelse")).into(), + ]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "If", base = "NodeStmt")] +pub(crate) struct NodeStmtIf; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeStmtIf { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("test")).into(), + ctx.new_str(ascii!("body")).into(), + ctx.new_str(ascii!("orelse")).into(), + ]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "With", base = "NodeStmt")] +pub(crate) struct NodeStmtWith; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeStmtWith { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("items")).into(), + ctx.new_str(ascii!("body")).into(), + ctx.new_str(ascii!("type_comment")).into(), + ]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "AsyncWith", base = "NodeStmt")] +pub(crate) struct NodeStmtAsyncWith; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeStmtAsyncWith { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("items")).into(), + ctx.new_str(ascii!("body")).into(), + ctx.new_str(ascii!("type_comment")).into(), + ]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "Match", base = "NodeStmt")] +pub(crate) struct NodeStmtMatch; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeStmtMatch { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("subject")).into(), + ctx.new_str(ascii!("cases")).into(), + ]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "Raise", base = "NodeStmt")] +pub(crate) struct NodeStmtRaise; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeStmtRaise { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("exc")).into(), + ctx.new_str(ascii!("cause")).into(), + ]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "Try", base = "NodeStmt")] +pub(crate) struct NodeStmtTry; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeStmtTry { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("body")).into(), + ctx.new_str(ascii!("handlers")).into(), + ctx.new_str(ascii!("orelse")).into(), + ctx.new_str(ascii!("finalbody")).into(), + ]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "TryStar", base = "NodeStmt")] +pub(crate) struct NodeStmtTryStar; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeStmtTryStar { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("body")).into(), + ctx.new_str(ascii!("handlers")).into(), + ctx.new_str(ascii!("orelse")).into(), + ctx.new_str(ascii!("finalbody")).into(), + ]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "Assert", base = "NodeStmt")] +pub(crate) struct NodeStmtAssert; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeStmtAssert { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("test")).into(), + ctx.new_str(ascii!("msg")).into(), + ]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "Import", base = "NodeStmt")] +pub(crate) struct NodeStmtImport; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeStmtImport { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ctx.new_str(ascii!("names")).into()]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "ImportFrom", base = "NodeStmt")] +pub(crate) struct NodeStmtImportFrom; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeStmtImportFrom { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("module")).into(), + ctx.new_str(ascii!("names")).into(), + ctx.new_str(ascii!("level")).into(), + ]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "Global", base = "NodeStmt")] +pub(crate) struct NodeStmtGlobal; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeStmtGlobal { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ctx.new_str(ascii!("names")).into()]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "Nonlocal", base = "NodeStmt")] +pub(crate) struct NodeStmtNonlocal; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeStmtNonlocal { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ctx.new_str(ascii!("names")).into()]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "Expr", base = "NodeStmt")] +pub(crate) struct NodeStmtExpr; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeStmtExpr { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ctx.new_str(ascii!("value")).into()]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "Pass", base = "NodeStmt")] +pub(crate) struct NodeStmtPass; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeStmtPass { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "Break", base = "NodeStmt")] +pub(crate) struct NodeStmtBreak; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeStmtBreak { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "Continue", base = "NodeStmt")] +pub(crate) struct NodeStmtContinue; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeStmtContinue { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "expr", base = "NodeAst")] +pub(crate) struct NodeExpr; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeExpr {} +#[pyclass(module = "_ast", name = "BoolOp", base = "NodeExpr")] +pub(crate) struct NodeExprBoolOp; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeExprBoolOp { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("op")).into(), + ctx.new_str(ascii!("values")).into(), + ]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "NamedExpr", base = "NodeExpr")] +pub(crate) struct NodeExprNamedExpr; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeExprNamedExpr { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("target")).into(), + ctx.new_str(ascii!("value")).into(), + ]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "BinOp", base = "NodeExpr")] +pub(crate) struct NodeExprBinOp; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeExprBinOp { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("left")).into(), + ctx.new_str(ascii!("op")).into(), + ctx.new_str(ascii!("right")).into(), + ]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "UnaryOp", base = "NodeExpr")] +pub(crate) struct NodeExprUnaryOp; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeExprUnaryOp { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("op")).into(), + ctx.new_str(ascii!("operand")).into(), + ]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "Lambda", base = "NodeExpr")] +pub(crate) struct NodeExprLambda; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeExprLambda { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("args")).into(), + ctx.new_str(ascii!("body")).into(), + ]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "IfExp", base = "NodeExpr")] +pub(crate) struct NodeExprIfExp; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeExprIfExp { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("test")).into(), + ctx.new_str(ascii!("body")).into(), + ctx.new_str(ascii!("orelse")).into(), + ]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "Dict", base = "NodeExpr")] +pub(crate) struct NodeExprDict; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeExprDict { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("keys")).into(), + ctx.new_str(ascii!("values")).into(), + ]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "Set", base = "NodeExpr")] +pub(crate) struct NodeExprSet; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeExprSet { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ctx.new_str(ascii!("elts")).into()]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "ListComp", base = "NodeExpr")] +pub(crate) struct NodeExprListComp; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeExprListComp { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("elt")).into(), + ctx.new_str(ascii!("generators")).into(), + ]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "SetComp", base = "NodeExpr")] +pub(crate) struct NodeExprSetComp; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeExprSetComp { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("elt")).into(), + ctx.new_str(ascii!("generators")).into(), + ]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "DictComp", base = "NodeExpr")] +pub(crate) struct NodeExprDictComp; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeExprDictComp { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("key")).into(), + ctx.new_str(ascii!("value")).into(), + ctx.new_str(ascii!("generators")).into(), + ]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "GeneratorExp", base = "NodeExpr")] +pub(crate) struct NodeExprGeneratorExp; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeExprGeneratorExp { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("elt")).into(), + ctx.new_str(ascii!("generators")).into(), + ]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "Await", base = "NodeExpr")] +pub(crate) struct NodeExprAwait; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeExprAwait { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ctx.new_str(ascii!("value")).into()]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "Yield", base = "NodeExpr")] +pub(crate) struct NodeExprYield; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeExprYield { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ctx.new_str(ascii!("value")).into()]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "YieldFrom", base = "NodeExpr")] +pub(crate) struct NodeExprYieldFrom; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeExprYieldFrom { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ctx.new_str(ascii!("value")).into()]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "Compare", base = "NodeExpr")] +pub(crate) struct NodeExprCompare; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeExprCompare { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("left")).into(), + ctx.new_str(ascii!("ops")).into(), + ctx.new_str(ascii!("comparators")).into(), + ]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "Call", base = "NodeExpr")] +pub(crate) struct NodeExprCall; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeExprCall { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("func")).into(), + ctx.new_str(ascii!("args")).into(), + ctx.new_str(ascii!("keywords")).into(), + ]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "FormattedValue", base = "NodeExpr")] +pub(crate) struct NodeExprFormattedValue; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeExprFormattedValue { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("value")).into(), + ctx.new_str(ascii!("conversion")).into(), + ctx.new_str(ascii!("format_spec")).into(), + ]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "JoinedStr", base = "NodeExpr")] +pub(crate) struct NodeExprJoinedStr; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeExprJoinedStr { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ctx.new_str(ascii!("values")).into()]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "Constant", base = "NodeExpr")] +pub(crate) struct NodeExprConstant; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeExprConstant { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("value")).into(), + ctx.new_str(ascii!("kind")).into(), + ]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "Attribute", base = "NodeExpr")] +pub(crate) struct NodeExprAttribute; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeExprAttribute { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("value")).into(), + ctx.new_str(ascii!("attr")).into(), + ctx.new_str(ascii!("ctx")).into(), + ]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "Subscript", base = "NodeExpr")] +pub(crate) struct NodeExprSubscript; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeExprSubscript { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("value")).into(), + ctx.new_str(ascii!("slice")).into(), + ctx.new_str(ascii!("ctx")).into(), + ]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "Starred", base = "NodeExpr")] +pub(crate) struct NodeExprStarred; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeExprStarred { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("value")).into(), + ctx.new_str(ascii!("ctx")).into(), + ]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "Name", base = "NodeExpr")] +pub(crate) struct NodeExprName; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeExprName { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("id")).into(), + ctx.new_str(ascii!("ctx")).into(), + ]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "List", base = "NodeExpr")] +pub(crate) struct NodeExprList; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeExprList { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("elts")).into(), + ctx.new_str(ascii!("ctx")).into(), + ]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "Tuple", base = "NodeExpr")] +pub(crate) struct NodeExprTuple; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeExprTuple { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("elts")).into(), + ctx.new_str(ascii!("ctx")).into(), + ]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "Slice", base = "NodeExpr")] +pub(crate) struct NodeExprSlice; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeExprSlice { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("lower")).into(), + ctx.new_str(ascii!("upper")).into(), + ctx.new_str(ascii!("step")).into(), + ]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "expr_context", base = "NodeAst")] +pub(crate) struct NodeExprContext; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeExprContext {} +#[pyclass(module = "_ast", name = "Load", base = "NodeExprContext")] +pub(crate) struct NodeExprContextLoad; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeExprContextLoad { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); + class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); + } +} +#[pyclass(module = "_ast", name = "Store", base = "NodeExprContext")] +pub(crate) struct NodeExprContextStore; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeExprContextStore { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); + class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); + } +} +#[pyclass(module = "_ast", name = "Del", base = "NodeExprContext")] +pub(crate) struct NodeExprContextDel; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeExprContextDel { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); + class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); + } +} +#[pyclass(module = "_ast", name = "boolop", base = "NodeAst")] +pub(crate) struct NodeBoolOp; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeBoolOp {} +#[pyclass(module = "_ast", name = "And", base = "NodeBoolOp")] +pub(crate) struct NodeBoolOpAnd; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeBoolOpAnd { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); + class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); + } +} +#[pyclass(module = "_ast", name = "Or", base = "NodeBoolOp")] +pub(crate) struct NodeBoolOpOr; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeBoolOpOr { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); + class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); + } +} +#[pyclass(module = "_ast", name = "operator", base = "NodeAst")] +pub(crate) struct NodeOperator; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeOperator {} +#[pyclass(module = "_ast", name = "Add", base = "NodeOperator")] +pub(crate) struct NodeOperatorAdd; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeOperatorAdd { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); + class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); + } +} +#[pyclass(module = "_ast", name = "Sub", base = "NodeOperator")] +pub(crate) struct NodeOperatorSub; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeOperatorSub { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); + class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); + } +} +#[pyclass(module = "_ast", name = "Mult", base = "NodeOperator")] +pub(crate) struct NodeOperatorMult; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeOperatorMult { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); + class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); + } +} +#[pyclass(module = "_ast", name = "MatMult", base = "NodeOperator")] +pub(crate) struct NodeOperatorMatMult; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeOperatorMatMult { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); + class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); + } +} +#[pyclass(module = "_ast", name = "Div", base = "NodeOperator")] +pub(crate) struct NodeOperatorDiv; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeOperatorDiv { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); + class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); + } +} +#[pyclass(module = "_ast", name = "Mod", base = "NodeOperator")] +pub(crate) struct NodeOperatorMod; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeOperatorMod { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); + class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); + } +} +#[pyclass(module = "_ast", name = "Pow", base = "NodeOperator")] +pub(crate) struct NodeOperatorPow; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeOperatorPow { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); + class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); + } +} +#[pyclass(module = "_ast", name = "LShift", base = "NodeOperator")] +pub(crate) struct NodeOperatorLShift; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeOperatorLShift { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); + class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); + } +} +#[pyclass(module = "_ast", name = "RShift", base = "NodeOperator")] +pub(crate) struct NodeOperatorRShift; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeOperatorRShift { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); + class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); + } +} +#[pyclass(module = "_ast", name = "BitOr", base = "NodeOperator")] +pub(crate) struct NodeOperatorBitOr; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeOperatorBitOr { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); + class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); + } +} +#[pyclass(module = "_ast", name = "BitXor", base = "NodeOperator")] +pub(crate) struct NodeOperatorBitXor; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeOperatorBitXor { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); + class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); + } +} +#[pyclass(module = "_ast", name = "BitAnd", base = "NodeOperator")] +pub(crate) struct NodeOperatorBitAnd; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeOperatorBitAnd { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); + class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); + } +} +#[pyclass(module = "_ast", name = "FloorDiv", base = "NodeOperator")] +pub(crate) struct NodeOperatorFloorDiv; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeOperatorFloorDiv { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); + class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); + } +} +#[pyclass(module = "_ast", name = "unaryop", base = "NodeAst")] +pub(crate) struct NodeUnaryOp; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeUnaryOp {} +#[pyclass(module = "_ast", name = "Invert", base = "NodeUnaryOp")] +pub(crate) struct NodeUnaryOpInvert; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeUnaryOpInvert { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); + class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); + } +} +#[pyclass(module = "_ast", name = "Not", base = "NodeUnaryOp")] +pub(crate) struct NodeUnaryOpNot; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeUnaryOpNot { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); + class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); + } +} +#[pyclass(module = "_ast", name = "UAdd", base = "NodeUnaryOp")] +pub(crate) struct NodeUnaryOpUAdd; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeUnaryOpUAdd { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); + class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); + } +} +#[pyclass(module = "_ast", name = "USub", base = "NodeUnaryOp")] +pub(crate) struct NodeUnaryOpUSub; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeUnaryOpUSub { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); + class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); + } +} +#[pyclass(module = "_ast", name = "cmpop", base = "NodeAst")] +pub(crate) struct NodeCmpOp; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeCmpOp {} +#[pyclass(module = "_ast", name = "Eq", base = "NodeCmpOp")] +pub(crate) struct NodeCmpOpEq; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeCmpOpEq { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); + class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); + } +} +#[pyclass(module = "_ast", name = "NotEq", base = "NodeCmpOp")] +pub(crate) struct NodeCmpOpNotEq; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeCmpOpNotEq { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); + class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); + } +} +#[pyclass(module = "_ast", name = "Lt", base = "NodeCmpOp")] +pub(crate) struct NodeCmpOpLt; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeCmpOpLt { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); + class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); + } +} +#[pyclass(module = "_ast", name = "LtE", base = "NodeCmpOp")] +pub(crate) struct NodeCmpOpLtE; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeCmpOpLtE { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); + class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); + } +} +#[pyclass(module = "_ast", name = "Gt", base = "NodeCmpOp")] +pub(crate) struct NodeCmpOpGt; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeCmpOpGt { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); + class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); + } +} +#[pyclass(module = "_ast", name = "GtE", base = "NodeCmpOp")] +pub(crate) struct NodeCmpOpGtE; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeCmpOpGtE { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); + class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); + } +} +#[pyclass(module = "_ast", name = "Is", base = "NodeCmpOp")] +pub(crate) struct NodeCmpOpIs; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeCmpOpIs { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); + class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); + } +} +#[pyclass(module = "_ast", name = "IsNot", base = "NodeCmpOp")] +pub(crate) struct NodeCmpOpIsNot; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeCmpOpIsNot { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); + class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); + } +} +#[pyclass(module = "_ast", name = "In", base = "NodeCmpOp")] +pub(crate) struct NodeCmpOpIn; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeCmpOpIn { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); + class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); + } +} +#[pyclass(module = "_ast", name = "NotIn", base = "NodeCmpOp")] +pub(crate) struct NodeCmpOpNotIn; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeCmpOpNotIn { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); + class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); + } +} +#[pyclass(module = "_ast", name = "comprehension", base = "NodeAst")] +pub(crate) struct NodeComprehension; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeComprehension { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("target")).into(), + ctx.new_str(ascii!("iter")).into(), + ctx.new_str(ascii!("ifs")).into(), + ctx.new_str(ascii!("is_async")).into(), + ]) + .into(), + ); + class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); + } +} +#[pyclass(module = "_ast", name = "excepthandler", base = "NodeAst")] +pub(crate) struct NodeExceptHandler; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeExceptHandler {} +#[pyclass(module = "_ast", name = "ExceptHandler", base = "NodeExceptHandler")] +pub(crate) struct NodeExceptHandlerExceptHandler; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeExceptHandlerExceptHandler { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("type")).into(), + ctx.new_str(ascii!("name")).into(), + ctx.new_str(ascii!("body")).into(), + ]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "arguments", base = "NodeAst")] +pub(crate) struct NodeArguments; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeArguments { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("posonlyargs")).into(), + ctx.new_str(ascii!("args")).into(), + ctx.new_str(ascii!("vararg")).into(), + ctx.new_str(ascii!("kwonlyargs")).into(), + ctx.new_str(ascii!("kw_defaults")).into(), + ctx.new_str(ascii!("kwarg")).into(), + ctx.new_str(ascii!("defaults")).into(), + ]) + .into(), + ); + class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); + } +} +#[pyclass(module = "_ast", name = "arg", base = "NodeAst")] +pub(crate) struct NodeArg; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeArg { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("arg")).into(), + ctx.new_str(ascii!("annotation")).into(), + ctx.new_str(ascii!("type_comment")).into(), + ]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "keyword", base = "NodeAst")] +pub(crate) struct NodeKeyword; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeKeyword { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("arg")).into(), + ctx.new_str(ascii!("value")).into(), + ]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "alias", base = "NodeAst")] +pub(crate) struct NodeAlias; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeAlias { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("name")).into(), + ctx.new_str(ascii!("asname")).into(), + ]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "withitem", base = "NodeAst")] +pub(crate) struct NodeWithItem; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeWithItem { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("context_expr")).into(), + ctx.new_str(ascii!("optional_vars")).into(), + ]) + .into(), + ); + class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); + } +} +#[pyclass(module = "_ast", name = "match_case", base = "NodeAst")] +pub(crate) struct NodeMatchCase; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeMatchCase { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("pattern")).into(), + ctx.new_str(ascii!("guard")).into(), + ctx.new_str(ascii!("body")).into(), + ]) + .into(), + ); + class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); + } +} +#[pyclass(module = "_ast", name = "pattern", base = "NodeAst")] +pub(crate) struct NodePattern; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodePattern {} +#[pyclass(module = "_ast", name = "MatchValue", base = "NodePattern")] +pub(crate) struct NodePatternMatchValue; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodePatternMatchValue { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ctx.new_str(ascii!("value")).into()]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "MatchSingleton", base = "NodePattern")] +pub(crate) struct NodePatternMatchSingleton; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodePatternMatchSingleton { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ctx.new_str(ascii!("value")).into()]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "MatchSequence", base = "NodePattern")] +pub(crate) struct NodePatternMatchSequence; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodePatternMatchSequence { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ctx.new_str(ascii!("patterns")).into()]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "MatchMapping", base = "NodePattern")] +pub(crate) struct NodePatternMatchMapping; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodePatternMatchMapping { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("keys")).into(), + ctx.new_str(ascii!("patterns")).into(), + ctx.new_str(ascii!("rest")).into(), + ]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "MatchClass", base = "NodePattern")] +pub(crate) struct NodePatternMatchClass; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodePatternMatchClass { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("cls")).into(), + ctx.new_str(ascii!("patterns")).into(), + ctx.new_str(ascii!("kwd_attrs")).into(), + ctx.new_str(ascii!("kwd_patterns")).into(), + ]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "MatchStar", base = "NodePattern")] +pub(crate) struct NodePatternMatchStar; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodePatternMatchStar { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ctx.new_str(ascii!("name")).into()]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "MatchAs", base = "NodePattern")] +pub(crate) struct NodePatternMatchAs; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodePatternMatchAs { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("pattern")).into(), + ctx.new_str(ascii!("name")).into(), + ]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "MatchOr", base = "NodePattern")] +pub(crate) struct NodePatternMatchOr; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodePatternMatchOr { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ctx.new_str(ascii!("patterns")).into()]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "type_ignore", base = "NodeAst")] +pub(crate) struct NodeTypeIgnore; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeTypeIgnore {} +#[pyclass(module = "_ast", name = "TypeIgnore", base = "NodeTypeIgnore")] +pub(crate) struct NodeTypeIgnoreTypeIgnore; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeTypeIgnoreTypeIgnore { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("tag")).into(), + ]) + .into(), + ); + class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); + } +} +#[pyclass(module = "_ast", name = "type_param", base = "NodeAst")] +pub(crate) struct NodeTypeParam; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeTypeParam {} +#[pyclass(module = "_ast", name = "TypeVar", base = "NodeTypeParam")] +pub(crate) struct NodeTypeParamTypeVar; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeTypeParamTypeVar { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("name")).into(), + ctx.new_str(ascii!("bound")).into(), + ]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "ParamSpec", base = "NodeTypeParam")] +pub(crate) struct NodeTypeParamParamSpec; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeTypeParamParamSpec { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ctx.new_str(ascii!("name")).into()]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} +#[pyclass(module = "_ast", name = "TypeVarTuple", base = "NodeTypeParam")] +pub(crate) struct NodeTypeParamTypeVarTuple; +#[pyclass(flags(HAS_DICT, BASETYPE))] +impl NodeTypeParamTypeVarTuple { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py) { + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ctx.new_str(ascii!("name")).into()]) + .into(), + ); + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_list(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} + +pub fn extend_module_nodes(vm: &VirtualMachine, module: &Py) { + extend_module!(vm, module, { + "mod" => NodeMod::make_class(&vm.ctx), + "Module" => NodeModModule::make_class(&vm.ctx), + "Interactive" => NodeModInteractive::make_class(&vm.ctx), + "Expression" => NodeModExpression::make_class(&vm.ctx), + "FunctionType" => NodeModFunctionType::make_class(&vm.ctx), + "stmt" => NodeStmt::make_class(&vm.ctx), + "FunctionDef" => NodeStmtFunctionDef::make_class(&vm.ctx), + "AsyncFunctionDef" => NodeStmtAsyncFunctionDef::make_class(&vm.ctx), + "ClassDef" => NodeStmtClassDef::make_class(&vm.ctx), + "Return" => NodeStmtReturn::make_class(&vm.ctx), + "Delete" => NodeStmtDelete::make_class(&vm.ctx), + "Assign" => NodeStmtAssign::make_class(&vm.ctx), + "TypeAlias" => NodeStmtTypeAlias::make_class(&vm.ctx), + "AugAssign" => NodeStmtAugAssign::make_class(&vm.ctx), + "AnnAssign" => NodeStmtAnnAssign::make_class(&vm.ctx), + "For" => NodeStmtFor::make_class(&vm.ctx), + "AsyncFor" => NodeStmtAsyncFor::make_class(&vm.ctx), + "While" => NodeStmtWhile::make_class(&vm.ctx), + "If" => NodeStmtIf::make_class(&vm.ctx), + "With" => NodeStmtWith::make_class(&vm.ctx), + "AsyncWith" => NodeStmtAsyncWith::make_class(&vm.ctx), + "Match" => NodeStmtMatch::make_class(&vm.ctx), + "Raise" => NodeStmtRaise::make_class(&vm.ctx), + "Try" => NodeStmtTry::make_class(&vm.ctx), + "TryStar" => NodeStmtTryStar::make_class(&vm.ctx), + "Assert" => NodeStmtAssert::make_class(&vm.ctx), + "Import" => NodeStmtImport::make_class(&vm.ctx), + "ImportFrom" => NodeStmtImportFrom::make_class(&vm.ctx), + "Global" => NodeStmtGlobal::make_class(&vm.ctx), + "Nonlocal" => NodeStmtNonlocal::make_class(&vm.ctx), + "Expr" => NodeStmtExpr::make_class(&vm.ctx), + "Pass" => NodeStmtPass::make_class(&vm.ctx), + "Break" => NodeStmtBreak::make_class(&vm.ctx), + "Continue" => NodeStmtContinue::make_class(&vm.ctx), + "expr" => NodeExpr::make_class(&vm.ctx), + "BoolOp" => NodeExprBoolOp::make_class(&vm.ctx), + "NamedExpr" => NodeExprNamedExpr::make_class(&vm.ctx), + "BinOp" => NodeExprBinOp::make_class(&vm.ctx), + "UnaryOp" => NodeExprUnaryOp::make_class(&vm.ctx), + "Lambda" => NodeExprLambda::make_class(&vm.ctx), + "IfExp" => NodeExprIfExp::make_class(&vm.ctx), + "Dict" => NodeExprDict::make_class(&vm.ctx), + "Set" => NodeExprSet::make_class(&vm.ctx), + "ListComp" => NodeExprListComp::make_class(&vm.ctx), + "SetComp" => NodeExprSetComp::make_class(&vm.ctx), + "DictComp" => NodeExprDictComp::make_class(&vm.ctx), + "GeneratorExp" => NodeExprGeneratorExp::make_class(&vm.ctx), + "Await" => NodeExprAwait::make_class(&vm.ctx), + "Yield" => NodeExprYield::make_class(&vm.ctx), + "YieldFrom" => NodeExprYieldFrom::make_class(&vm.ctx), + "Compare" => NodeExprCompare::make_class(&vm.ctx), + "Call" => NodeExprCall::make_class(&vm.ctx), + "FormattedValue" => NodeExprFormattedValue::make_class(&vm.ctx), + "JoinedStr" => NodeExprJoinedStr::make_class(&vm.ctx), + "Constant" => NodeExprConstant::make_class(&vm.ctx), + "Attribute" => NodeExprAttribute::make_class(&vm.ctx), + "Subscript" => NodeExprSubscript::make_class(&vm.ctx), + "Starred" => NodeExprStarred::make_class(&vm.ctx), + "Name" => NodeExprName::make_class(&vm.ctx), + "List" => NodeExprList::make_class(&vm.ctx), + "Tuple" => NodeExprTuple::make_class(&vm.ctx), + "Slice" => NodeExprSlice::make_class(&vm.ctx), + "expr_context" => NodeExprContext::make_class(&vm.ctx), + "Load" => NodeExprContextLoad::make_class(&vm.ctx), + "Store" => NodeExprContextStore::make_class(&vm.ctx), + "Del" => NodeExprContextDel::make_class(&vm.ctx), + "boolop" => NodeBoolOp::make_class(&vm.ctx), + "And" => NodeBoolOpAnd::make_class(&vm.ctx), + "Or" => NodeBoolOpOr::make_class(&vm.ctx), + "operator" => NodeOperator::make_class(&vm.ctx), + "Add" => NodeOperatorAdd::make_class(&vm.ctx), + "Sub" => NodeOperatorSub::make_class(&vm.ctx), + "Mult" => NodeOperatorMult::make_class(&vm.ctx), + "MatMult" => NodeOperatorMatMult::make_class(&vm.ctx), + "Div" => NodeOperatorDiv::make_class(&vm.ctx), + "Mod" => NodeOperatorMod::make_class(&vm.ctx), + "Pow" => NodeOperatorPow::make_class(&vm.ctx), + "LShift" => NodeOperatorLShift::make_class(&vm.ctx), + "RShift" => NodeOperatorRShift::make_class(&vm.ctx), + "BitOr" => NodeOperatorBitOr::make_class(&vm.ctx), + "BitXor" => NodeOperatorBitXor::make_class(&vm.ctx), + "BitAnd" => NodeOperatorBitAnd::make_class(&vm.ctx), + "FloorDiv" => NodeOperatorFloorDiv::make_class(&vm.ctx), + "unaryop" => NodeUnaryOp::make_class(&vm.ctx), + "Invert" => NodeUnaryOpInvert::make_class(&vm.ctx), + "Not" => NodeUnaryOpNot::make_class(&vm.ctx), + "UAdd" => NodeUnaryOpUAdd::make_class(&vm.ctx), + "USub" => NodeUnaryOpUSub::make_class(&vm.ctx), + "cmpop" => NodeCmpOp::make_class(&vm.ctx), + "Eq" => NodeCmpOpEq::make_class(&vm.ctx), + "NotEq" => NodeCmpOpNotEq::make_class(&vm.ctx), + "Lt" => NodeCmpOpLt::make_class(&vm.ctx), + "LtE" => NodeCmpOpLtE::make_class(&vm.ctx), + "Gt" => NodeCmpOpGt::make_class(&vm.ctx), + "GtE" => NodeCmpOpGtE::make_class(&vm.ctx), + "Is" => NodeCmpOpIs::make_class(&vm.ctx), + "IsNot" => NodeCmpOpIsNot::make_class(&vm.ctx), + "In" => NodeCmpOpIn::make_class(&vm.ctx), + "NotIn" => NodeCmpOpNotIn::make_class(&vm.ctx), + "comprehension" => NodeComprehension::make_class(&vm.ctx), + "excepthandler" => NodeExceptHandler::make_class(&vm.ctx), + "ExceptHandler" => NodeExceptHandlerExceptHandler::make_class(&vm.ctx), + "arguments" => NodeArguments::make_class(&vm.ctx), + "arg" => NodeArg::make_class(&vm.ctx), + "keyword" => NodeKeyword::make_class(&vm.ctx), + "alias" => NodeAlias::make_class(&vm.ctx), + "withitem" => NodeWithItem::make_class(&vm.ctx), + "match_case" => NodeMatchCase::make_class(&vm.ctx), + "pattern" => NodePattern::make_class(&vm.ctx), + "MatchValue" => NodePatternMatchValue::make_class(&vm.ctx), + "MatchSingleton" => NodePatternMatchSingleton::make_class(&vm.ctx), + "MatchSequence" => NodePatternMatchSequence::make_class(&vm.ctx), + "MatchMapping" => NodePatternMatchMapping::make_class(&vm.ctx), + "MatchClass" => NodePatternMatchClass::make_class(&vm.ctx), + "MatchStar" => NodePatternMatchStar::make_class(&vm.ctx), + "MatchAs" => NodePatternMatchAs::make_class(&vm.ctx), + "MatchOr" => NodePatternMatchOr::make_class(&vm.ctx), + "type_ignore" => NodeTypeIgnore::make_class(&vm.ctx), + "TypeIgnore" => NodeTypeIgnoreTypeIgnore::make_class(&vm.ctx), + "type_param" => NodeTypeParam::make_class(&vm.ctx), + "TypeVar" => NodeTypeParamTypeVar::make_class(&vm.ctx), + "ParamSpec" => NodeTypeParamParamSpec::make_class(&vm.ctx), + "TypeVarTuple" => NodeTypeParamTypeVarTuple::make_class(&vm.ctx), + }) +} diff --git a/vm/src/stdlib/ast/python.rs b/vm/src/stdlib/ast/python.rs new file mode 100644 index 0000000000..50f8294c76 --- /dev/null +++ b/vm/src/stdlib/ast/python.rs @@ -0,0 +1,57 @@ +use super::PY_COMPILE_FLAG_AST_ONLY; + +#[pymodule] +pub(crate) mod _ast { + use crate::{ + AsObject, Context, PyObjectRef, PyPayload, PyResult, VirtualMachine, + builtins::{PyStrRef, PyTupleRef}, + function::FuncArgs, + }; + #[pyattr] + #[pyclass(module = "_ast", name = "AST")] + #[derive(Debug, PyPayload)] + pub(crate) struct NodeAst; + + #[pyclass(flags(BASETYPE, HAS_DICT))] + impl NodeAst { + #[pyslot] + #[pymethod(magic)] + fn init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + let fields = zelf.get_attr("_fields", vm)?; + let fields: Vec = fields.try_to_value(vm)?; + let numargs = args.args.len(); + if numargs > fields.len() { + return Err(vm.new_type_error(format!( + "{} constructor takes at most {} positional argument{}", + zelf.class().name(), + fields.len(), + if fields.len() == 1 { "" } else { "s" }, + ))); + } + for (name, arg) in fields.iter().zip(args.args) { + zelf.set_attr(name, arg, vm)?; + } + for (key, value) in args.kwargs { + if let Some(pos) = fields.iter().position(|f| f.as_str() == key) { + if pos < numargs { + return Err(vm.new_type_error(format!( + "{} got multiple values for argument '{}'", + zelf.class().name(), + key + ))); + } + } + zelf.set_attr(vm.ctx.intern_str(key), value, vm)?; + } + Ok(()) + } + + #[pyattr(name = "_fields")] + fn fields(ctx: &Context) -> PyTupleRef { + ctx.empty_tuple.clone() + } + } + + #[pyattr(name = "PyCF_ONLY_AST")] + use super::PY_COMPILE_FLAG_AST_ONLY; +} diff --git a/vm/src/stdlib/ast/statement.rs b/vm/src/stdlib/ast/statement.rs new file mode 100644 index 0000000000..61b6c0eaff --- /dev/null +++ b/vm/src/stdlib/ast/statement.rs @@ -0,0 +1,1222 @@ +use super::*; +use crate::stdlib::ast::argument::{merge_class_def_args, split_class_def_args}; +// sum +impl Node for ruff::Stmt { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + match self { + ruff::Stmt::FunctionDef(cons) => cons.ast_to_object(vm, source_code), + ruff::Stmt::ClassDef(cons) => cons.ast_to_object(vm, source_code), + ruff::Stmt::Return(cons) => cons.ast_to_object(vm, source_code), + ruff::Stmt::Delete(cons) => cons.ast_to_object(vm, source_code), + ruff::Stmt::Assign(cons) => cons.ast_to_object(vm, source_code), + ruff::Stmt::TypeAlias(cons) => cons.ast_to_object(vm, source_code), + ruff::Stmt::AugAssign(cons) => cons.ast_to_object(vm, source_code), + ruff::Stmt::AnnAssign(cons) => cons.ast_to_object(vm, source_code), + ruff::Stmt::For(cons) => cons.ast_to_object(vm, source_code), + ruff::Stmt::While(cons) => cons.ast_to_object(vm, source_code), + ruff::Stmt::If(cons) => cons.ast_to_object(vm, source_code), + ruff::Stmt::With(cons) => cons.ast_to_object(vm, source_code), + ruff::Stmt::Match(cons) => cons.ast_to_object(vm, source_code), + ruff::Stmt::Raise(cons) => cons.ast_to_object(vm, source_code), + ruff::Stmt::Try(cons) => cons.ast_to_object(vm, source_code), + ruff::Stmt::Assert(cons) => cons.ast_to_object(vm, source_code), + ruff::Stmt::Import(cons) => cons.ast_to_object(vm, source_code), + ruff::Stmt::ImportFrom(cons) => cons.ast_to_object(vm, source_code), + ruff::Stmt::Global(cons) => cons.ast_to_object(vm, source_code), + ruff::Stmt::Nonlocal(cons) => cons.ast_to_object(vm, source_code), + ruff::Stmt::Expr(cons) => cons.ast_to_object(vm, source_code), + ruff::Stmt::Pass(cons) => cons.ast_to_object(vm, source_code), + ruff::Stmt::Break(cons) => cons.ast_to_object(vm, source_code), + ruff::Stmt::Continue(cons) => cons.ast_to_object(vm, source_code), + ruff::Stmt::IpyEscapeCommand(_) => { + unimplemented!("IPython escape command is not allowed in Python AST") + } + } + } + + #[allow(clippy::if_same_then_else)] + fn ast_from_object( + _vm: &VirtualMachine, + source_code: &SourceCodeOwned, + _object: PyObjectRef, + ) -> PyResult { + let _cls = _object.class(); + Ok(if _cls.is(pyast::NodeStmtFunctionDef::static_type()) { + ruff::Stmt::FunctionDef(ruff::StmtFunctionDef::ast_from_object( + _vm, + source_code, + _object, + )?) + } else if _cls.is(pyast::NodeStmtAsyncFunctionDef::static_type()) { + ruff::Stmt::FunctionDef(ruff::StmtFunctionDef::ast_from_object( + _vm, + source_code, + _object, + )?) + } else if _cls.is(pyast::NodeStmtClassDef::static_type()) { + ruff::Stmt::ClassDef(ruff::StmtClassDef::ast_from_object( + _vm, + source_code, + _object, + )?) + } else if _cls.is(pyast::NodeStmtReturn::static_type()) { + ruff::Stmt::Return(ruff::StmtReturn::ast_from_object( + _vm, + source_code, + _object, + )?) + } else if _cls.is(pyast::NodeStmtDelete::static_type()) { + ruff::Stmt::Delete(ruff::StmtDelete::ast_from_object( + _vm, + source_code, + _object, + )?) + } else if _cls.is(pyast::NodeStmtAssign::static_type()) { + ruff::Stmt::Assign(ruff::StmtAssign::ast_from_object( + _vm, + source_code, + _object, + )?) + } else if _cls.is(pyast::NodeStmtTypeAlias::static_type()) { + ruff::Stmt::TypeAlias(ruff::StmtTypeAlias::ast_from_object( + _vm, + source_code, + _object, + )?) + } else if _cls.is(pyast::NodeStmtAugAssign::static_type()) { + ruff::Stmt::AugAssign(ruff::StmtAugAssign::ast_from_object( + _vm, + source_code, + _object, + )?) + } else if _cls.is(pyast::NodeStmtAnnAssign::static_type()) { + ruff::Stmt::AnnAssign(ruff::StmtAnnAssign::ast_from_object( + _vm, + source_code, + _object, + )?) + } else if _cls.is(pyast::NodeStmtFor::static_type()) { + ruff::Stmt::For(ruff::StmtFor::ast_from_object(_vm, source_code, _object)?) + } else if _cls.is(pyast::NodeStmtAsyncFor::static_type()) { + ruff::Stmt::For(ruff::StmtFor::ast_from_object(_vm, source_code, _object)?) + } else if _cls.is(pyast::NodeStmtWhile::static_type()) { + ruff::Stmt::While(ruff::StmtWhile::ast_from_object(_vm, source_code, _object)?) + } else if _cls.is(pyast::NodeStmtIf::static_type()) { + ruff::Stmt::If(ruff::StmtIf::ast_from_object(_vm, source_code, _object)?) + } else if _cls.is(pyast::NodeStmtWith::static_type()) { + ruff::Stmt::With(ruff::StmtWith::ast_from_object(_vm, source_code, _object)?) + } else if _cls.is(pyast::NodeStmtAsyncWith::static_type()) { + ruff::Stmt::With(ruff::StmtWith::ast_from_object(_vm, source_code, _object)?) + } else if _cls.is(pyast::NodeStmtMatch::static_type()) { + ruff::Stmt::Match(ruff::StmtMatch::ast_from_object(_vm, source_code, _object)?) + } else if _cls.is(pyast::NodeStmtRaise::static_type()) { + ruff::Stmt::Raise(ruff::StmtRaise::ast_from_object(_vm, source_code, _object)?) + } else if _cls.is(pyast::NodeStmtTry::static_type()) { + ruff::Stmt::Try(ruff::StmtTry::ast_from_object(_vm, source_code, _object)?) + } else if _cls.is(pyast::NodeStmtTryStar::static_type()) { + ruff::Stmt::Try(ruff::StmtTry::ast_from_object(_vm, source_code, _object)?) + } else if _cls.is(pyast::NodeStmtAssert::static_type()) { + ruff::Stmt::Assert(ruff::StmtAssert::ast_from_object( + _vm, + source_code, + _object, + )?) + } else if _cls.is(pyast::NodeStmtImport::static_type()) { + ruff::Stmt::Import(ruff::StmtImport::ast_from_object( + _vm, + source_code, + _object, + )?) + } else if _cls.is(pyast::NodeStmtImportFrom::static_type()) { + ruff::Stmt::ImportFrom(ruff::StmtImportFrom::ast_from_object( + _vm, + source_code, + _object, + )?) + } else if _cls.is(pyast::NodeStmtGlobal::static_type()) { + ruff::Stmt::Global(ruff::StmtGlobal::ast_from_object( + _vm, + source_code, + _object, + )?) + } else if _cls.is(pyast::NodeStmtNonlocal::static_type()) { + ruff::Stmt::Nonlocal(ruff::StmtNonlocal::ast_from_object( + _vm, + source_code, + _object, + )?) + } else if _cls.is(pyast::NodeStmtExpr::static_type()) { + ruff::Stmt::Expr(ruff::StmtExpr::ast_from_object(_vm, source_code, _object)?) + } else if _cls.is(pyast::NodeStmtPass::static_type()) { + ruff::Stmt::Pass(ruff::StmtPass::ast_from_object(_vm, source_code, _object)?) + } else if _cls.is(pyast::NodeStmtBreak::static_type()) { + ruff::Stmt::Break(ruff::StmtBreak::ast_from_object(_vm, source_code, _object)?) + } else if _cls.is(pyast::NodeStmtContinue::static_type()) { + ruff::Stmt::Continue(ruff::StmtContinue::ast_from_object( + _vm, + source_code, + _object, + )?) + } else { + return Err(_vm.new_type_error(format!( + "expected some sort of stmt, but got {}", + _object.repr(_vm)? + ))); + }) + } +} +// constructor +impl Node for ruff::StmtFunctionDef { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { + name, + parameters, + body, + decorator_list, + returns, + // type_comment, + type_params, + is_async, + range: _range, + } = self; + + let cls = if !is_async { + pyast::NodeStmtFunctionDef::static_type().to_owned() + } else { + pyast::NodeStmtAsyncFunctionDef::static_type().to_owned() + }; + + let node = NodeAst.into_ref_with_type(vm, cls).unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("name", vm.ctx.new_str(name.as_str()).to_pyobject(vm), vm) + .unwrap(); + dict.set_item("args", parameters.ast_to_object(vm, source_code), vm) + .unwrap(); + dict.set_item("body", body.ast_to_object(vm, source_code), vm) + .unwrap(); + dict.set_item( + "decorator_list", + decorator_list.ast_to_object(vm, source_code), + vm, + ) + .unwrap(); + dict.set_item("returns", returns.ast_to_object(vm, source_code), vm) + .unwrap(); + // TODO: Ruff ignores type_comment during parsing + // dict.set_item("type_comment", type_comment.ast_to_object(_vm), _vm) + // .unwrap(); + dict.set_item( + "type_params", + type_params.ast_to_object(vm, source_code), + vm, + ) + .unwrap(); + node_add_location(&dict, _range, vm, source_code); + node.into() + } + fn ast_from_object( + _vm: &VirtualMachine, + source_code: &SourceCodeOwned, + _object: PyObjectRef, + ) -> PyResult { + let _cls = _object.class(); + let is_async = _cls.is(pyast::NodeStmtAsyncFunctionDef::static_type()); + Ok(Self { + name: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "name", "FunctionDef")?, + )?, + parameters: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "args", "FunctionDef")?, + )?, + body: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "body", "FunctionDef")?, + )?, + decorator_list: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "decorator_list", "FunctionDef")?, + )?, + returns: get_node_field_opt(_vm, &_object, "returns")? + .map(|obj| Node::ast_from_object(_vm, source_code, obj)) + .transpose()?, + // TODO: Ruff ignores type_comment during parsing + // type_comment: get_node_field_opt(_vm, &_object, "type_comment")? + // .map(|obj| Node::ast_from_object(_vm, obj)) + // .transpose()?, + type_params: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "type_params", "FunctionDef")?, + )?, + range: range_from_object(_vm, source_code, _object, "FunctionDef")?, + is_async, + }) + } +} +// constructor +impl Node for ruff::StmtClassDef { + fn ast_to_object(self, _vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { + name, + arguments, + body, + decorator_list, + type_params, + range: _range, + } = self; + let (bases, keywords) = split_class_def_args(arguments); + let node = NodeAst + .into_ref_with_type(_vm, pyast::NodeStmtClassDef::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("name", name.ast_to_object(_vm, source_code), _vm) + .unwrap(); + dict.set_item("bases", bases.ast_to_object(_vm, source_code), _vm) + .unwrap(); + dict.set_item("keywords", keywords.ast_to_object(_vm, source_code), _vm) + .unwrap(); + dict.set_item("body", body.ast_to_object(_vm, source_code), _vm) + .unwrap(); + dict.set_item( + "decorator_list", + decorator_list.ast_to_object(_vm, source_code), + _vm, + ) + .unwrap(); + dict.set_item( + "type_params", + type_params.ast_to_object(_vm, source_code), + _vm, + ) + .unwrap(); + node_add_location(&dict, _range, _vm, source_code); + node.into() + } + fn ast_from_object( + _vm: &VirtualMachine, + source_code: &SourceCodeOwned, + _object: PyObjectRef, + ) -> PyResult { + let bases = Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "bases", "ClassDef")?, + )?; + let keywords = Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "keywords", "ClassDef")?, + )?; + Ok(Self { + name: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "name", "ClassDef")?, + )?, + arguments: merge_class_def_args(bases, keywords), + body: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "body", "ClassDef")?, + )?, + decorator_list: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "decorator_list", "ClassDef")?, + )?, + type_params: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "type_params", "ClassDef")?, + )?, + range: range_from_object(_vm, source_code, _object, "ClassDef")?, + }) + } +} +// constructor +impl Node for ruff::StmtReturn { + fn ast_to_object(self, _vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let ruff::StmtReturn { + value, + range: _range, + } = self; + let node = NodeAst + .into_ref_with_type(_vm, pyast::NodeStmtReturn::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("value", value.ast_to_object(_vm, source_code), _vm) + .unwrap(); + node_add_location(&dict, _range, _vm, source_code); + node.into() + } + fn ast_from_object( + _vm: &VirtualMachine, + source_code: &SourceCodeOwned, + _object: PyObjectRef, + ) -> PyResult { + Ok(ruff::StmtReturn { + value: get_node_field_opt(_vm, &_object, "value")? + .map(|obj| Node::ast_from_object(_vm, source_code, obj)) + .transpose()?, + range: range_from_object(_vm, source_code, _object, "Return")?, + }) + } +} +// constructor +impl Node for ruff::StmtDelete { + fn ast_to_object(self, _vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let ruff::StmtDelete { + targets, + range: _range, + } = self; + let node = NodeAst + .into_ref_with_type(_vm, pyast::NodeStmtDelete::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("targets", targets.ast_to_object(_vm, source_code), _vm) + .unwrap(); + node_add_location(&dict, _range, _vm, source_code); + node.into() + } + fn ast_from_object( + _vm: &VirtualMachine, + source_code: &SourceCodeOwned, + _object: PyObjectRef, + ) -> PyResult { + Ok(ruff::StmtDelete { + targets: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "targets", "Delete")?, + )?, + range: range_from_object(_vm, source_code, _object, "Delete")?, + }) + } +} +// constructor +impl Node for ruff::StmtAssign { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { + targets, + value, + // type_comment, + range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeStmtAssign::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("targets", targets.ast_to_object(vm, source_code), vm) + .unwrap(); + dict.set_item("value", value.ast_to_object(vm, source_code), vm) + .unwrap(); + // TODO + dict.set_item("type_comment", vm.ctx.none(), vm).unwrap(); + node_add_location(&dict, range, vm, source_code); + node.into() + } + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + Ok(Self { + targets: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "targets", "Assign")?, + )?, + value: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "value", "Assign")?, + )?, + // type_comment: get_node_field_opt(_vm, &_object, "type_comment")? + // .map(|obj| Node::ast_from_object(_vm, obj)) + // .transpose()?, + range: range_from_object(vm, source_code, object, "Assign")?, + }) + } +} +// constructor +impl Node for ruff::StmtTypeAlias { + fn ast_to_object(self, _vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let ruff::StmtTypeAlias { + name, + type_params, + value, + range: _range, + } = self; + let node = NodeAst + .into_ref_with_type(_vm, pyast::NodeStmtTypeAlias::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("name", name.ast_to_object(_vm, source_code), _vm) + .unwrap(); + dict.set_item( + "type_params", + type_params.ast_to_object(_vm, source_code), + _vm, + ) + .unwrap(); + dict.set_item("value", value.ast_to_object(_vm, source_code), _vm) + .unwrap(); + node_add_location(&dict, _range, _vm, source_code); + node.into() + } + fn ast_from_object( + _vm: &VirtualMachine, + source_code: &SourceCodeOwned, + _object: PyObjectRef, + ) -> PyResult { + Ok(ruff::StmtTypeAlias { + name: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "name", "TypeAlias")?, + )?, + type_params: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "type_params", "TypeAlias")?, + )?, + value: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "value", "TypeAlias")?, + )?, + range: range_from_object(_vm, source_code, _object, "TypeAlias")?, + }) + } +} +// constructor +impl Node for ruff::StmtAugAssign { + fn ast_to_object(self, _vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { + target, + op, + value, + range: _range, + } = self; + let node = NodeAst + .into_ref_with_type(_vm, pyast::NodeStmtAugAssign::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("target", target.ast_to_object(_vm, source_code), _vm) + .unwrap(); + dict.set_item("op", op.ast_to_object(_vm, source_code), _vm) + .unwrap(); + dict.set_item("value", value.ast_to_object(_vm, source_code), _vm) + .unwrap(); + node_add_location(&dict, _range, _vm, source_code); + node.into() + } + fn ast_from_object( + _vm: &VirtualMachine, + source_code: &SourceCodeOwned, + _object: PyObjectRef, + ) -> PyResult { + Ok(Self { + target: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "target", "AugAssign")?, + )?, + op: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "op", "AugAssign")?, + )?, + value: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "value", "AugAssign")?, + )?, + range: range_from_object(_vm, source_code, _object, "AugAssign")?, + }) + } +} +// constructor +impl Node for ruff::StmtAnnAssign { + fn ast_to_object(self, _vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { + target, + annotation, + value, + simple, + range: _range, + } = self; + let node = NodeAst + .into_ref_with_type(_vm, pyast::NodeStmtAnnAssign::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("target", target.ast_to_object(_vm, source_code), _vm) + .unwrap(); + dict.set_item( + "annotation", + annotation.ast_to_object(_vm, source_code), + _vm, + ) + .unwrap(); + dict.set_item("value", value.ast_to_object(_vm, source_code), _vm) + .unwrap(); + dict.set_item("simple", simple.ast_to_object(_vm, source_code), _vm) + .unwrap(); + node_add_location(&dict, _range, _vm, source_code); + node.into() + } + fn ast_from_object( + _vm: &VirtualMachine, + source_code: &SourceCodeOwned, + _object: PyObjectRef, + ) -> PyResult { + Ok(Self { + target: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "target", "AnnAssign")?, + )?, + annotation: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "annotation", "AnnAssign")?, + )?, + value: get_node_field_opt(_vm, &_object, "value")? + .map(|obj| Node::ast_from_object(_vm, source_code, obj)) + .transpose()?, + simple: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "simple", "AnnAssign")?, + )?, + range: range_from_object(_vm, source_code, _object, "AnnAssign")?, + }) + } +} +// constructor +impl Node for ruff::StmtFor { + fn ast_to_object(self, _vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { + is_async, + target, + iter, + body, + orelse, + // type_comment, + range: _range, + } = self; + + let cls = if !is_async { + pyast::NodeStmtFor::static_type().to_owned() + } else { + pyast::NodeStmtAsyncFor::static_type().to_owned() + }; + + let node = NodeAst.into_ref_with_type(_vm, cls).unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("target", target.ast_to_object(_vm, source_code), _vm) + .unwrap(); + dict.set_item("iter", iter.ast_to_object(_vm, source_code), _vm) + .unwrap(); + dict.set_item("body", body.ast_to_object(_vm, source_code), _vm) + .unwrap(); + dict.set_item("orelse", orelse.ast_to_object(_vm, source_code), _vm) + .unwrap(); + // dict.set_item("type_comment", type_comment.ast_to_object(_vm), _vm) + // .unwrap(); + node_add_location(&dict, _range, _vm, source_code); + node.into() + } + fn ast_from_object( + _vm: &VirtualMachine, + source_code: &SourceCodeOwned, + _object: PyObjectRef, + ) -> PyResult { + let _cls = _object.class(); + debug_assert!( + _cls.is(pyast::NodeStmtFor::static_type()) + || _cls.is(pyast::NodeStmtAsyncFor::static_type()) + ); + let is_async = _cls.is(pyast::NodeStmtAsyncFor::static_type()); + Ok(Self { + target: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "target", "For")?, + )?, + iter: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "iter", "For")?, + )?, + body: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "body", "For")?, + )?, + orelse: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "orelse", "For")?, + )?, + // type_comment: get_node_field_opt(_vm, &_object, "type_comment")? + // .map(|obj| Node::ast_from_object(_vm, obj)) + // .transpose()?, + range: range_from_object(_vm, source_code, _object, "For")?, + is_async, + }) + } +} +// constructor +impl Node for ruff::StmtWhile { + fn ast_to_object(self, _vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { + test, + body, + orelse, + range: _range, + } = self; + let node = NodeAst + .into_ref_with_type(_vm, pyast::NodeStmtWhile::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("test", test.ast_to_object(_vm, source_code), _vm) + .unwrap(); + dict.set_item("body", body.ast_to_object(_vm, source_code), _vm) + .unwrap(); + dict.set_item("orelse", orelse.ast_to_object(_vm, source_code), _vm) + .unwrap(); + node_add_location(&dict, _range, _vm, source_code); + node.into() + } + fn ast_from_object( + _vm: &VirtualMachine, + source_code: &SourceCodeOwned, + _object: PyObjectRef, + ) -> PyResult { + Ok(Self { + test: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "test", "While")?, + )?, + body: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "body", "While")?, + )?, + orelse: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "orelse", "While")?, + )?, + range: range_from_object(_vm, source_code, _object, "While")?, + }) + } +} +// constructor +impl Node for ruff::StmtIf { + fn ast_to_object(self, _vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { + test, + body, + range, + elif_else_clauses, + } = self; + elif_else_clause::ast_to_object( + ruff::ElifElseClause { + range, + test: Some(*test), + body, + }, + elif_else_clauses.into_iter(), + _vm, + source_code, + ) + } + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + elif_else_clause::ast_from_object(vm, source_code, object) + } +} +// constructor +impl Node for ruff::StmtWith { + fn ast_to_object(self, _vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { + is_async, + items, + body, + // type_comment, + range: _range, + } = self; + + let cls = if !is_async { + pyast::NodeStmtWith::static_type().to_owned() + } else { + pyast::NodeStmtAsyncWith::static_type().to_owned() + }; + + let node = NodeAst.into_ref_with_type(_vm, cls).unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("items", items.ast_to_object(_vm, source_code), _vm) + .unwrap(); + dict.set_item("body", body.ast_to_object(_vm, source_code), _vm) + .unwrap(); + // dict.set_item("type_comment", type_comment.ast_to_object(_vm), _vm) + // .unwrap(); + node_add_location(&dict, _range, _vm, source_code); + node.into() + } + fn ast_from_object( + _vm: &VirtualMachine, + source_code: &SourceCodeOwned, + _object: PyObjectRef, + ) -> PyResult { + let _cls = _object.class(); + debug_assert!( + _cls.is(pyast::NodeStmtWith::static_type()) + || _cls.is(pyast::NodeStmtAsyncWith::static_type()) + ); + let is_async = _cls.is(pyast::NodeStmtAsyncWith::static_type()); + Ok(Self { + items: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "items", "With")?, + )?, + body: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "body", "With")?, + )?, + // type_comment: get_node_field_opt(_vm, &_object, "type_comment")? + // .map(|obj| Node::ast_from_object(_vm, obj)) + // .transpose()?, + range: range_from_object(_vm, source_code, _object, "With")?, + is_async, + }) + } +} +// constructor +impl Node for ruff::StmtMatch { + fn ast_to_object(self, _vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { + subject, + cases, + range: _range, + } = self; + let node = NodeAst + .into_ref_with_type(_vm, pyast::NodeStmtMatch::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("subject", subject.ast_to_object(_vm, source_code), _vm) + .unwrap(); + dict.set_item("cases", cases.ast_to_object(_vm, source_code), _vm) + .unwrap(); + node_add_location(&dict, _range, _vm, source_code); + node.into() + } + fn ast_from_object( + _vm: &VirtualMachine, + source_code: &SourceCodeOwned, + _object: PyObjectRef, + ) -> PyResult { + Ok(Self { + subject: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "subject", "Match")?, + )?, + cases: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "cases", "Match")?, + )?, + range: range_from_object(_vm, source_code, _object, "Match")?, + }) + } +} +// constructor +impl Node for ruff::StmtRaise { + fn ast_to_object(self, _vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { + exc, + cause, + range: _range, + } = self; + let node = NodeAst + .into_ref_with_type(_vm, pyast::NodeStmtRaise::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("exc", exc.ast_to_object(_vm, source_code), _vm) + .unwrap(); + dict.set_item("cause", cause.ast_to_object(_vm, source_code), _vm) + .unwrap(); + node_add_location(&dict, _range, _vm, source_code); + node.into() + } + fn ast_from_object( + _vm: &VirtualMachine, + source_code: &SourceCodeOwned, + _object: PyObjectRef, + ) -> PyResult { + Ok(Self { + exc: get_node_field_opt(_vm, &_object, "exc")? + .map(|obj| Node::ast_from_object(_vm, source_code, obj)) + .transpose()?, + cause: get_node_field_opt(_vm, &_object, "cause")? + .map(|obj| Node::ast_from_object(_vm, source_code, obj)) + .transpose()?, + range: range_from_object(_vm, source_code, _object, "Raise")?, + }) + } +} +// constructor +impl Node for ruff::StmtTry { + fn ast_to_object(self, _vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { + body, + handlers, + orelse, + finalbody, + range: _range, + is_star, + } = self; + + // let cls = gen::NodeStmtTry::static_type().to_owned(); + let cls = if is_star { + pyast::NodeStmtTryStar::static_type() + } else { + pyast::NodeStmtTry::static_type() + } + .to_owned(); + + let node = NodeAst.into_ref_with_type(_vm, cls).unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("body", body.ast_to_object(_vm, source_code), _vm) + .unwrap(); + dict.set_item("handlers", handlers.ast_to_object(_vm, source_code), _vm) + .unwrap(); + dict.set_item("orelse", orelse.ast_to_object(_vm, source_code), _vm) + .unwrap(); + dict.set_item("finalbody", finalbody.ast_to_object(_vm, source_code), _vm) + .unwrap(); + node_add_location(&dict, _range, _vm, source_code); + node.into() + } + fn ast_from_object( + _vm: &VirtualMachine, + source_code: &SourceCodeOwned, + _object: PyObjectRef, + ) -> PyResult { + let _cls = _object.class(); + let is_star = _cls.is(pyast::NodeStmtTryStar::static_type()); + let _cls = _object.class(); + debug_assert!( + _cls.is(pyast::NodeStmtTry::static_type()) + || _cls.is(pyast::NodeStmtTryStar::static_type()) + ); + + Ok(Self { + body: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "body", "Try")?, + )?, + handlers: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "handlers", "Try")?, + )?, + orelse: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "orelse", "Try")?, + )?, + finalbody: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "finalbody", "Try")?, + )?, + range: range_from_object(_vm, source_code, _object, "Try")?, + is_star, + }) + } +} +// constructor +impl Node for ruff::StmtAssert { + fn ast_to_object(self, _vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let ruff::StmtAssert { + test, + msg, + range: _range, + } = self; + let node = NodeAst + .into_ref_with_type(_vm, pyast::NodeStmtAssert::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("test", test.ast_to_object(_vm, source_code), _vm) + .unwrap(); + dict.set_item("msg", msg.ast_to_object(_vm, source_code), _vm) + .unwrap(); + node_add_location(&dict, _range, _vm, source_code); + node.into() + } + fn ast_from_object( + _vm: &VirtualMachine, + source_code: &SourceCodeOwned, + _object: PyObjectRef, + ) -> PyResult { + Ok(ruff::StmtAssert { + test: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "test", "Assert")?, + )?, + msg: get_node_field_opt(_vm, &_object, "msg")? + .map(|obj| Node::ast_from_object(_vm, source_code, obj)) + .transpose()?, + range: range_from_object(_vm, source_code, _object, "Assert")?, + }) + } +} +// constructor +impl Node for ruff::StmtImport { + fn ast_to_object(self, _vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let ruff::StmtImport { + names, + range: _range, + } = self; + let node = NodeAst + .into_ref_with_type(_vm, pyast::NodeStmtImport::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("names", names.ast_to_object(_vm, source_code), _vm) + .unwrap(); + node_add_location(&dict, _range, _vm, source_code); + node.into() + } + fn ast_from_object( + _vm: &VirtualMachine, + source_code: &SourceCodeOwned, + _object: PyObjectRef, + ) -> PyResult { + Ok(ruff::StmtImport { + names: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "names", "Import")?, + )?, + range: range_from_object(_vm, source_code, _object, "Import")?, + }) + } +} +// constructor +impl Node for ruff::StmtImportFrom { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { + module, + names, + level, + range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeStmtImportFrom::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("module", module.ast_to_object(vm, source_code), vm) + .unwrap(); + dict.set_item("names", names.ast_to_object(vm, source_code), vm) + .unwrap(); + dict.set_item("level", vm.ctx.new_int(level).to_pyobject(vm), vm) + .unwrap(); + node_add_location(&dict, range, vm, source_code); + node.into() + } + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + _object: PyObjectRef, + ) -> PyResult { + Ok(Self { + module: get_node_field_opt(vm, &_object, "module")? + .map(|obj| Node::ast_from_object(vm, source_code, obj)) + .transpose()?, + names: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &_object, "names", "ImportFrom")?, + )?, + level: get_node_field(vm, &_object, "level", "ImportFrom")? + .downcast_exact::(vm) + .unwrap() + .try_to_primitive::(vm)?, + range: range_from_object(vm, source_code, _object, "ImportFrom")?, + }) + } +} +// constructor +impl Node for ruff::StmtGlobal { + fn ast_to_object(self, _vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let ruff::StmtGlobal { + names, + range: _range, + } = self; + let node = NodeAst + .into_ref_with_type(_vm, pyast::NodeStmtGlobal::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("names", names.ast_to_object(_vm, source_code), _vm) + .unwrap(); + node_add_location(&dict, _range, _vm, source_code); + node.into() + } + fn ast_from_object( + _vm: &VirtualMachine, + source_code: &SourceCodeOwned, + _object: PyObjectRef, + ) -> PyResult { + Ok(ruff::StmtGlobal { + names: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "names", "Global")?, + )?, + range: range_from_object(_vm, source_code, _object, "Global")?, + }) + } +} +// constructor +impl Node for ruff::StmtNonlocal { + fn ast_to_object(self, _vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let ruff::StmtNonlocal { + names, + range: _range, + } = self; + let node = NodeAst + .into_ref_with_type(_vm, pyast::NodeStmtNonlocal::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("names", names.ast_to_object(_vm, source_code), _vm) + .unwrap(); + node_add_location(&dict, _range, _vm, source_code); + node.into() + } + fn ast_from_object( + _vm: &VirtualMachine, + source_code: &SourceCodeOwned, + _object: PyObjectRef, + ) -> PyResult { + Ok(ruff::StmtNonlocal { + names: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "names", "Nonlocal")?, + )?, + range: range_from_object(_vm, source_code, _object, "Nonlocal")?, + }) + } +} +// constructor +impl Node for ruff::StmtExpr { + fn ast_to_object(self, _vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let ruff::StmtExpr { + value, + range: _range, + } = self; + let node = NodeAst + .into_ref_with_type(_vm, pyast::NodeStmtExpr::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("value", value.ast_to_object(_vm, source_code), _vm) + .unwrap(); + node_add_location(&dict, _range, _vm, source_code); + node.into() + } + fn ast_from_object( + _vm: &VirtualMachine, + source_code: &SourceCodeOwned, + _object: PyObjectRef, + ) -> PyResult { + Ok(ruff::StmtExpr { + value: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "value", "Expr")?, + )?, + range: range_from_object(_vm, source_code, _object, "Expr")?, + }) + } +} +// constructor +impl Node for ruff::StmtPass { + fn ast_to_object(self, _vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let ruff::StmtPass { range: _range } = self; + let node = NodeAst + .into_ref_with_type(_vm, pyast::NodeStmtPass::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + node_add_location(&dict, _range, _vm, source_code); + node.into() + } + fn ast_from_object( + _vm: &VirtualMachine, + source_code: &SourceCodeOwned, + _object: PyObjectRef, + ) -> PyResult { + Ok(ruff::StmtPass { + range: range_from_object(_vm, source_code, _object, "Pass")?, + }) + } +} +// constructor +impl Node for ruff::StmtBreak { + fn ast_to_object(self, _vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let ruff::StmtBreak { range: _range } = self; + let node = NodeAst + .into_ref_with_type(_vm, pyast::NodeStmtBreak::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + node_add_location(&dict, _range, _vm, source_code); + node.into() + } + fn ast_from_object( + _vm: &VirtualMachine, + source_code: &SourceCodeOwned, + _object: PyObjectRef, + ) -> PyResult { + Ok(ruff::StmtBreak { + range: range_from_object(_vm, source_code, _object, "Break")?, + }) + } +} +// constructor +impl Node for ruff::StmtContinue { + fn ast_to_object(self, _vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let ruff::StmtContinue { range: _range } = self; + let node = NodeAst + .into_ref_with_type(_vm, pyast::NodeStmtContinue::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + node_add_location(&dict, _range, _vm, source_code); + node.into() + } + fn ast_from_object( + _vm: &VirtualMachine, + source_code: &SourceCodeOwned, + _object: PyObjectRef, + ) -> PyResult { + Ok(ruff::StmtContinue { + range: range_from_object(_vm, source_code, _object, "Continue")?, + }) + } +} diff --git a/vm/src/stdlib/ast/string.rs b/vm/src/stdlib/ast/string.rs new file mode 100644 index 0000000000..5b78967976 --- /dev/null +++ b/vm/src/stdlib/ast/string.rs @@ -0,0 +1,354 @@ +use super::constant::{Constant, ConstantLiteral}; +use super::*; + +fn ruff_fstring_value_into_iter( + mut fstring_value: ruff::FStringValue, +) -> impl Iterator + 'static { + let default = ruff::FStringPart::FString(ruff::FString { + range: Default::default(), + elements: Default::default(), + flags: Default::default(), + }); + (0..fstring_value.as_slice().len()).map(move |i| { + let fstring_value = &mut fstring_value; + let tmp = fstring_value.into_iter().nth(i).unwrap(); + std::mem::replace(tmp, default.clone()) + }) +} + +fn ruff_fstring_element_into_iter( + mut fstring_element: ruff::FStringElements, +) -> impl Iterator + 'static { + let default = ruff::FStringElement::Literal(ruff::FStringLiteralElement { + range: Default::default(), + value: Default::default(), + }); + (0..fstring_element.into_iter().len()).map(move |i| { + let fstring_element = &mut fstring_element; + let tmp = fstring_element.into_iter().nth(i).unwrap(); + std::mem::replace(tmp, default.clone()) + }) +} + +fn fstring_part_to_joined_str_part(fstring_part: ruff::FStringPart) -> Vec { + match fstring_part { + ruff::FStringPart::Literal(ruff::StringLiteral { + range, + value, + flags, + }) => { + vec![JoinedStrPart::Constant(Constant::new_str( + value, + flags.prefix(), + range, + ))] + } + ruff::FStringPart::FString(ruff::FString { + range: _, + elements, + flags: _, // TODO + }) => ruff_fstring_element_into_iter(elements) + .map(ruff_fstring_element_to_joined_str_part) + .collect(), + } +} + +fn ruff_fstring_element_to_joined_str_part(element: ruff::FStringElement) -> JoinedStrPart { + match element { + ruff::FStringElement::Literal(ruff::FStringLiteralElement { range, value }) => { + JoinedStrPart::Constant(Constant::new_str( + value, + ruff::str_prefix::StringLiteralPrefix::Empty, + range, + )) + } + ruff::FStringElement::Expression(ruff::FStringExpressionElement { + range, + expression, + debug_text: _, // TODO: What is this? + conversion, + format_spec, + }) => JoinedStrPart::FormattedValue(FormattedValue { + value: expression, + conversion, + format_spec: ruff_format_spec_to_joined_str(format_spec), + range, + }), + } +} + +fn ruff_format_spec_to_joined_str( + format_spec: Option>, +) -> Option> { + match format_spec { + None => None, + Some(format_spec) => { + let ruff::FStringFormatSpec { range, elements } = *format_spec; + let values: Vec<_> = ruff_fstring_element_into_iter(elements) + .map(ruff_fstring_element_to_joined_str_part) + .collect(); + let values = values.into_boxed_slice(); + Some(Box::new(JoinedStr { values, range })) + } + } +} + +fn ruff_fstring_element_to_ruff_fstring_part(element: ruff::FStringElement) -> ruff::FStringPart { + match element { + ruff::FStringElement::Literal(value) => { + let ruff::FStringLiteralElement { range, value } = value; + ruff::FStringPart::Literal(ruff::StringLiteral { + range, + value, + flags: Default::default(), + }) + } + ruff::FStringElement::Expression(value) => { + let ruff::FStringExpressionElement { + range, + expression, + debug_text, + conversion, + format_spec, + } = value; + ruff::FStringPart::FString(ruff::FString { + range, + elements: vec![ruff::FStringElement::Expression( + ruff::FStringExpressionElement { + range, + expression, + debug_text, + conversion, + format_spec, + }, + )] + .into(), + flags: Default::default(), + }) + } + } +} + +fn joined_str_to_ruff_format_spec( + joined_str: Option>, +) -> Option> { + match joined_str { + None => None, + Some(joined_str) => { + let JoinedStr { range, values } = *joined_str; + let elements: Vec<_> = Box::into_iter(values) + .map(joined_str_part_to_ruff_fstring_element) + .collect(); + let format_spec = ruff::FStringFormatSpec { + range, + elements: elements.into(), + }; + Some(Box::new(format_spec)) + } + } +} + +#[derive(Debug)] +pub(super) struct JoinedStr { + pub(super) range: TextRange, + pub(super) values: Box<[JoinedStrPart]>, +} + +impl JoinedStr { + pub(super) fn into_expr(self) -> ruff::Expr { + let Self { range, values } = self; + ruff::Expr::FString(ruff::ExprFString { + range: Default::default(), + value: match values.len() { + // ruff represents an empty fstring like this: + 0 => ruff::FStringValue::single(ruff::FString { + range, + elements: vec![].into(), + flags: Default::default(), + }), + 1 => ruff::FStringValue::single( + Box::<[_]>::into_iter(values) + .map(joined_str_part_to_ruff_fstring_element) + .map(|element| ruff::FString { + range, + elements: vec![element].into(), + flags: Default::default(), + }) + .next() + .expect("FString has exactly one part"), + ), + _ => ruff::FStringValue::concatenated( + Box::<[_]>::into_iter(values) + .map(joined_str_part_to_ruff_fstring_element) + .map(ruff_fstring_element_to_ruff_fstring_part) + .collect(), + ), + }, + }) + } +} + +fn joined_str_part_to_ruff_fstring_element(part: JoinedStrPart) -> ruff::FStringElement { + match part { + JoinedStrPart::FormattedValue(value) => { + ruff::FStringElement::Expression(ruff::FStringExpressionElement { + range: value.range, + expression: value.value.clone(), + debug_text: None, // TODO: What is this? + conversion: value.conversion, + format_spec: joined_str_to_ruff_format_spec(value.format_spec), + }) + } + JoinedStrPart::Constant(value) => { + ruff::FStringElement::Literal(ruff::FStringLiteralElement { + range: value.range, + value: match value.value { + ConstantLiteral::Str { value, .. } => value, + _ => todo!(), + }, + }) + } + } +} + +// constructor +impl Node for JoinedStr { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { values, range } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprJoinedStr::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item( + "values", + BoxedSlice(values).ast_to_object(vm, source_code), + vm, + ) + .unwrap(); + node_add_location(&dict, range, vm, source_code); + node.into() + } + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + let values: BoxedSlice<_> = Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "values", "JoinedStr")?, + )?; + Ok(Self { + values: values.0, + range: range_from_object(vm, source_code, object, "JoinedStr")?, + }) + } +} + +#[derive(Debug)] +pub(super) enum JoinedStrPart { + FormattedValue(FormattedValue), + Constant(Constant), +} + +// constructor +impl Node for JoinedStrPart { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + match self { + JoinedStrPart::FormattedValue(value) => value.ast_to_object(vm, source_code), + JoinedStrPart::Constant(value) => value.ast_to_object(vm, source_code), + } + } + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + let cls = object.class(); + if cls.is(pyast::NodeExprFormattedValue::static_type()) { + Ok(Self::FormattedValue(Node::ast_from_object( + vm, + source_code, + object, + )?)) + } else { + Ok(Self::Constant(Node::ast_from_object( + vm, + source_code, + object, + )?)) + } + } +} + +#[derive(Debug)] +pub(super) struct FormattedValue { + value: Box, + conversion: ruff::ConversionFlag, + format_spec: Option>, + range: TextRange, +} + +// constructor +impl Node for FormattedValue { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { + value, + conversion, + format_spec, + range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprFormattedValue::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("value", value.ast_to_object(vm, source_code), vm) + .unwrap(); + dict.set_item("conversion", conversion.ast_to_object(vm, source_code), vm) + .unwrap(); + dict.set_item( + "format_spec", + format_spec.ast_to_object(vm, source_code), + vm, + ) + .unwrap(); + node_add_location(&dict, range, vm, source_code); + node.into() + } + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + Ok(Self { + value: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "value", "FormattedValue")?, + )?, + conversion: Node::ast_from_object( + vm, + source_code, + get_node_field(vm, &object, "conversion", "FormattedValue")?, + )?, + format_spec: get_node_field_opt(vm, &object, "format_spec")? + .map(|obj| Node::ast_from_object(vm, source_code, obj)) + .transpose()?, + range: range_from_object(vm, source_code, object, "FormattedValue")?, + }) + } +} + +pub(super) fn fstring_to_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + expression: ruff::ExprFString, +) -> PyObjectRef { + let ruff::ExprFString { range, value } = expression; + let values: Vec<_> = ruff_fstring_value_into_iter(value) + .flat_map(fstring_part_to_joined_str_part) + .collect(); + let values = values.into_boxed_slice(); + let c = JoinedStr { range, values }; + c.ast_to_object(vm, source_code) +} diff --git a/vm/src/stdlib/ast/type_ignore.rs b/vm/src/stdlib/ast/type_ignore.rs new file mode 100644 index 0000000000..7e318f6949 --- /dev/null +++ b/vm/src/stdlib/ast/type_ignore.rs @@ -0,0 +1,73 @@ +use super::*; + +pub(super) enum TypeIgnore { + TypeIgnore(TypeIgnoreTypeIgnore), +} + +// sum +impl Node for TypeIgnore { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + match self { + TypeIgnore::TypeIgnore(cons) => cons.ast_to_object(vm, source_code), + } + } + fn ast_from_object( + _vm: &VirtualMachine, + source_code: &SourceCodeOwned, + _object: PyObjectRef, + ) -> PyResult { + let _cls = _object.class(); + Ok(if _cls.is(pyast::NodeTypeIgnoreTypeIgnore::static_type()) { + TypeIgnore::TypeIgnore(TypeIgnoreTypeIgnore::ast_from_object( + _vm, + source_code, + _object, + )?) + } else { + return Err(_vm.new_type_error(format!( + "expected some sort of type_ignore, but got {}", + _object.repr(_vm)? + ))); + }) + } +} + +pub(super) struct TypeIgnoreTypeIgnore { + range: TextRange, + lineno: PyRefExact, + tag: PyRefExact, +} + +// constructor +impl Node for TypeIgnoreTypeIgnore { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { lineno, tag, range } = self; + let node = NodeAst + .into_ref_with_type( + vm, + pyast::NodeTypeIgnoreTypeIgnore::static_type().to_owned(), + ) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("lineno", lineno.to_pyobject(vm), vm).unwrap(); + dict.set_item("tag", tag.to_pyobject(vm), vm).unwrap(); + node_add_location(&dict, range, vm, source_code); + node.into() + } + + fn ast_from_object( + vm: &VirtualMachine, + source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + Ok(Self { + lineno: get_node_field(vm, &object, "lineno", "TypeIgnore")? + .downcast_exact(vm) + .unwrap(), + tag: get_node_field(vm, &object, "tag", "TypeIgnore")? + .downcast_exact(vm) + .unwrap(), + range: range_from_object(vm, source_code, object, "TypeIgnore")?, + }) + } +} diff --git a/vm/src/stdlib/ast/type_parameters.rs b/vm/src/stdlib/ast/type_parameters.rs new file mode 100644 index 0000000000..686cee81f4 --- /dev/null +++ b/vm/src/stdlib/ast/type_parameters.rs @@ -0,0 +1,197 @@ +use super::*; + +impl Node for ruff::TypeParams { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + self.type_params.ast_to_object(vm, source_code) + } + + fn ast_from_object( + _vm: &VirtualMachine, + _source_code: &SourceCodeOwned, + _object: PyObjectRef, + ) -> PyResult { + let type_params: Vec = Node::ast_from_object(_vm, _source_code, _object)?; + let range = Option::zip(type_params.first(), type_params.last()) + .map(|(first, last)| first.range().cover(last.range())) + .unwrap_or_default(); + Ok(Self { type_params, range }) + } + + fn is_none(&self) -> bool { + self.type_params.is_empty() + } +} +// sum +impl Node for ruff::TypeParam { + fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + match self { + Self::TypeVar(cons) => cons.ast_to_object(vm, source_code), + Self::ParamSpec(cons) => cons.ast_to_object(vm, source_code), + Self::TypeVarTuple(cons) => cons.ast_to_object(vm, source_code), + } + } + + fn ast_from_object( + _vm: &VirtualMachine, + source_code: &SourceCodeOwned, + _object: PyObjectRef, + ) -> PyResult { + let _cls = _object.class(); + Ok(if _cls.is(pyast::NodeTypeParamTypeVar::static_type()) { + ruff::TypeParam::TypeVar(ruff::TypeParamTypeVar::ast_from_object( + _vm, + source_code, + _object, + )?) + } else if _cls.is(pyast::NodeTypeParamParamSpec::static_type()) { + ruff::TypeParam::ParamSpec(ruff::TypeParamParamSpec::ast_from_object( + _vm, + source_code, + _object, + )?) + } else if _cls.is(pyast::NodeTypeParamTypeVarTuple::static_type()) { + ruff::TypeParam::TypeVarTuple(ruff::TypeParamTypeVarTuple::ast_from_object( + _vm, + source_code, + _object, + )?) + } else { + return Err(_vm.new_type_error(format!( + "expected some sort of type_param, but got {}", + _object.repr(_vm)? + ))); + }) + } +} +// constructor +impl Node for ruff::TypeParamTypeVar { + fn ast_to_object(self, _vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { + name, + bound, + range: _range, + default: _, + } = self; + let node = NodeAst + .into_ref_with_type(_vm, pyast::NodeTypeParamTypeVar::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("name", name.ast_to_object(_vm, source_code), _vm) + .unwrap(); + dict.set_item("bound", bound.ast_to_object(_vm, source_code), _vm) + .unwrap(); + node_add_location(&dict, _range, _vm, source_code); + node.into() + } + fn ast_from_object( + _vm: &VirtualMachine, + source_code: &SourceCodeOwned, + _object: PyObjectRef, + ) -> PyResult { + Ok(Self { + name: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "name", "TypeVar")?, + )?, + bound: get_node_field_opt(_vm, &_object, "bound")? + .map(|obj| Node::ast_from_object(_vm, source_code, obj)) + .transpose()?, + default: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "default_value", "TypeVar")?, + )?, + range: range_from_object(_vm, source_code, _object, "TypeVar")?, + }) + } +} +// constructor +impl Node for ruff::TypeParamParamSpec { + fn ast_to_object(self, _vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { + name, + range: _range, + default, + } = self; + let node = NodeAst + .into_ref_with_type(_vm, pyast::NodeTypeParamParamSpec::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("name", name.ast_to_object(_vm, source_code), _vm) + .unwrap(); + dict.set_item( + "default_value", + default.ast_to_object(_vm, source_code), + _vm, + ) + .unwrap(); + node_add_location(&dict, _range, _vm, source_code); + node.into() + } + fn ast_from_object( + _vm: &VirtualMachine, + source_code: &SourceCodeOwned, + _object: PyObjectRef, + ) -> PyResult { + Ok(Self { + name: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "name", "ParamSpec")?, + )?, + default: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "default_value", "ParamSpec")?, + )?, + range: range_from_object(_vm, source_code, _object, "ParamSpec")?, + }) + } +} +// constructor +impl Node for ruff::TypeParamTypeVarTuple { + fn ast_to_object(self, _vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { + let Self { + name, + range: _range, + default, + } = self; + let node = NodeAst + .into_ref_with_type( + _vm, + pyast::NodeTypeParamTypeVarTuple::static_type().to_owned(), + ) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("name", name.ast_to_object(_vm, source_code), _vm) + .unwrap(); + dict.set_item( + "default_value", + default.ast_to_object(_vm, source_code), + _vm, + ) + .unwrap(); + node_add_location(&dict, _range, _vm, source_code); + node.into() + } + fn ast_from_object( + _vm: &VirtualMachine, + source_code: &SourceCodeOwned, + _object: PyObjectRef, + ) -> PyResult { + Ok(Self { + name: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "name", "TypeVarTuple")?, + )?, + default: Node::ast_from_object( + _vm, + source_code, + get_node_field(_vm, &_object, "default_value", "TypeVarTuple")?, + )?, + range: range_from_object(_vm, source_code, _object, "TypeVarTuple")?, + }) + } +} diff --git a/vm/src/stdlib/builtins.rs b/vm/src/stdlib/builtins.rs index 22aa11dc43..9c2826a1e9 100644 --- a/vm/src/stdlib/builtins.rs +++ b/vm/src/stdlib/builtins.rs @@ -110,10 +110,15 @@ mod builtins { _feature_version: OptionalArg, } - #[cfg(any(feature = "rustpython-parser", feature = "rustpython-codegen"))] + #[cfg(any(feature = "parser", feature = "compiler"))] #[pyfunction] fn compile(args: CompileArgs, vm: &VirtualMachine) -> PyResult { - #[cfg(feature = "rustpython-ast")] + #[cfg(not(feature = "ast"))] + { + _ = args; // to disable unused warning + return Err(vm.new_type_error("AST Not Supported".to_owned())); + } + #[cfg(feature = "ast")] { use crate::{class::PyClassImpl, stdlib::ast}; @@ -155,16 +160,16 @@ mod builtins { } } - #[cfg(not(feature = "rustpython-parser"))] + #[cfg(not(feature = "parser"))] { const PARSER_NOT_SUPPORTED: &str = "can't compile() source code when the `parser` feature of rustpython is disabled"; Err(vm.new_type_error(PARSER_NOT_SUPPORTED.to_owned())) } - #[cfg(feature = "rustpython-parser")] + #[cfg(feature = "parser")] { use crate::convert::ToPyException; use num_traits::Zero; - use rustpython_parser as parser; + use ruff_python_parser as parser; let source = ArgStrOrBytesLike::try_from_object(vm, args.source)?; let source = source.borrow_bytes(); @@ -180,11 +185,11 @@ mod builtins { } if (flags & ast::PY_COMPILE_FLAG_AST_ONLY).is_zero() { - #[cfg(not(feature = "rustpython-compiler"))] + #[cfg(not(feature = "compiler"))] { Err(vm.new_value_error(CODEGEN_NOT_SUPPORTED.to_owned())) } - #[cfg(feature = "rustpython-compiler")] + #[cfg(feature = "compiler")] { let mode = mode_str .parse::() diff --git a/vm/src/stdlib/mod.rs b/vm/src/stdlib/mod.rs index a3180c1c0f..382a8e0555 100644 --- a/vm/src/stdlib/mod.rs +++ b/vm/src/stdlib/mod.rs @@ -1,4 +1,4 @@ -#[cfg(feature = "rustpython-ast")] +#[cfg(feature = "ast")] pub(crate) mod ast; pub mod atexit; pub mod builtins; @@ -15,7 +15,7 @@ mod operator; // mod re; mod sre; mod string; -#[cfg(feature = "rustpython-compiler")] +#[cfg(feature = "compiler")] mod symtable; mod sysconfigdata; #[cfg(feature = "threading")] @@ -98,12 +98,12 @@ pub fn get_module_inits() -> StdlibMap { sys::sysconfigdata_name() => sysconfigdata::make_module, } // parser related modules: - #[cfg(feature = "rustpython-ast")] + #[cfg(feature = "ast")] { "_ast" => ast::make_module, } // compiler related modules: - #[cfg(feature = "rustpython-compiler")] + #[cfg(feature = "compiler")] { "symtable" => symtable::make_module, } diff --git a/vm/src/vm/compile.rs b/vm/src/vm/compile.rs index a14e986dac..f5145e4739 100644 --- a/vm/src/vm/compile.rs +++ b/vm/src/vm/compile.rs @@ -23,7 +23,7 @@ impl VirtualMachine { source_path: String, opts: CompileOpts, ) -> Result, CompileError> { - compiler::compile(source, mode, source_path, opts).map(|code| self.ctx.new_code(code)) + compiler::compile(source, mode, &source_path, opts).map(|code| self.ctx.new_code(code)) } pub fn run_script(&self, scope: Scope, path: &str) -> PyResult<()> { diff --git a/vm/src/vm/vm_new.rs b/vm/src/vm/vm_new.rs index bd42396299..01709b6873 100644 --- a/vm/src/vm/vm_new.rs +++ b/vm/src/vm/vm_new.rs @@ -256,22 +256,39 @@ impl VirtualMachine { self.new_exception_msg(overflow_error, msg) } - #[cfg(any(feature = "rustpython-parser", feature = "rustpython-codegen"))] + #[cfg(any(feature = "parser", feature = "compiler"))] pub fn new_syntax_error( &self, error: &crate::compiler::CompileError, source: Option<&str>, ) -> PyBaseExceptionRef { - use crate::source_code::SourceLocation; - - let syntax_error_type = match &error.error { - #[cfg(feature = "rustpython-parser")] - crate::compiler::CompileErrorType::Parse(p) if p.is_indentation_error() => { - self.ctx.exceptions.indentation_error - } - #[cfg(feature = "rustpython-parser")] - crate::compiler::CompileErrorType::Parse(p) if p.is_tab_error() => { - self.ctx.exceptions.tab_error + use crate::source::SourceLocation; + + let syntax_error_type = match &error { + #[cfg(feature = "parser")] + // FIXME: this condition will cause TabError even when the matching actual error is IndentationError + crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { + error: + ruff_python_parser::ParseErrorType::Lexical( + ruff_python_parser::lexer::LexicalErrorType::IndentationError, + ), + .. + }) => self.ctx.exceptions.tab_error, + #[cfg(feature = "parser")] + crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { + error: ruff_python_parser::ParseErrorType::UnexpectedIndentation, + .. + }) => self.ctx.exceptions.indentation_error, + #[cfg(feature = "parser")] + crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { + error: ruff_python_parser::ParseErrorType::OtherError(s), + .. + }) => { + if s.starts_with("Expected an indented block after") { + self.ctx.exceptions.indentation_error + } else { + self.ctx.exceptions.syntax_error + } } _ => self.ctx.exceptions.syntax_error, } @@ -281,18 +298,32 @@ impl VirtualMachine { fn get_statement(source: &str, loc: Option) -> Option { let line = source .split('\n') - .nth(loc?.row.to_zero_indexed_usize())? + .nth(loc?.row.to_zero_indexed())? .to_owned(); Some(line + "\n") } let statement = if let Some(source) = source { - get_statement(source, error.location) + get_statement(source, error.location()) } else { None }; - let syntax_error = self.new_exception_msg(syntax_error_type, error.error.to_string()); + let mut msg = error.to_string(); + if let Some(msg) = msg.get_mut(..1) { + msg.make_ascii_lowercase(); + } + match error { + #[cfg(feature = "parser")] + crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { + error: + ruff_python_parser::ParseErrorType::FStringError(_) + | ruff_python_parser::ParseErrorType::UnexpectedExpressionToken, + .. + }) => msg.insert_str(0, "invalid syntax: "), + _ => {} + } + let syntax_error = self.new_exception_msg(syntax_error_type, msg); let (lineno, offset) = error.python_location(); let lineno = self.ctx.new_int(lineno); let offset = self.ctx.new_int(offset); @@ -311,11 +342,7 @@ impl VirtualMachine { .unwrap(); syntax_error .as_object() - .set_attr( - "filename", - self.ctx.new_str(error.source_path.clone()), - self, - ) + .set_attr("filename", self.ctx.new_str(error.source_path()), self) .unwrap(); syntax_error } diff --git a/wasm/lib/Cargo.toml b/wasm/lib/Cargo.toml index 4703cb9f4a..0e35292a2c 100644 --- a/wasm/lib/Cargo.toml +++ b/wasm/lib/Cargo.toml @@ -23,7 +23,7 @@ rustpython-stdlib = { workspace = true, default-features = false, optional = tru # make sure no threading! otherwise wasm build will fail rustpython-vm = { workspace = true, features = ["compiler", "encodings", "serde", "wasmbind"] } -rustpython-parser = { workspace = true } +ruff_python_parser = { workspace = true } serde = { workspace = true } wasm-bindgen = { workspace = true } diff --git a/wasm/lib/src/convert.rs b/wasm/lib/src/convert.rs index d4fb068e6c..2f80770319 100644 --- a/wasm/lib/src/convert.rs +++ b/wasm/lib/src/convert.rs @@ -3,11 +3,10 @@ use crate::js_module; use crate::vm_class::{WASMVirtualMachine, stored_vm_from_wasm}; use js_sys::{Array, ArrayBuffer, Object, Promise, Reflect, SyntaxError, Uint8Array}; -use rustpython_parser::{ParseErrorType, lexer::LexicalErrorType}; use rustpython_vm::{ AsObject, PyObjectRef, PyPayload, PyResult, TryFromBorrowedObject, VirtualMachine, builtins::PyBaseExceptionRef, - compiler::{CompileError, CompileErrorType}, + compiler::{CompileError, ParseError, parser::ParseErrorType, parser::lexer::LexicalErrorType}, exceptions, function::{ArgBytesLike, FuncArgs}, py_serde, @@ -252,21 +251,21 @@ pub fn syntax_err(err: CompileError) -> SyntaxError { let _ = Reflect::set( &js_err, &"row".into(), - &(err.location.unwrap().row.get()).into(), + &(err.location().unwrap().row.get()).into(), ); let _ = Reflect::set( &js_err, &"col".into(), - &(err.location.unwrap().column.get()).into(), + &(err.location().unwrap().column.get()).into(), ); + // | ParseErrorType::UnrecognizedToken(Token::Dedent, _) let can_continue = matches!( - &err.error, - CompileErrorType::Parse( - ParseErrorType::Eof - | ParseErrorType::Lexical(LexicalErrorType::Eof) - | ParseErrorType::Lexical(LexicalErrorType::IndentationError) - | ParseErrorType::UnrecognizedToken(rustpython_parser::Tok::Dedent, _) - ) + &err, + CompileError::Parse(ParseError { + error: ParseErrorType::Lexical(LexicalErrorType::Eof) + | ParseErrorType::Lexical(LexicalErrorType::IndentationError), + .. + }) ); let _ = Reflect::set(&js_err, &"canContinue".into(), &can_continue.into()); js_err From 950a8d5694b6450ce957b1b6f45fdcc1152e6ece Mon Sep 17 00:00:00 2001 From: Noa Date: Wed, 19 Mar 2025 21:37:06 -0500 Subject: [PATCH 089/295] Update to ruff_python_parser 0.11 --- Cargo.lock | 72 ++++++++++++++++++--------------- Cargo.toml | 10 ++--- Lib/test/test_grammar.py | 2 - compiler/codegen/src/compile.rs | 14 +++---- compiler/src/lib.rs | 7 ++-- src/shell.rs | 2 +- vm/src/stdlib/ast.rs | 2 +- vm/src/stdlib/ast/constant.rs | 6 +-- vm/src/stdlib/ast/expression.rs | 2 +- vm/src/stdlib/ast/other.rs | 26 +++++++----- vm/src/stdlib/ast/string.rs | 10 ++--- vm/src/vm/vm_new.rs | 2 +- wasm/lib/src/convert.rs | 2 +- 13 files changed, 83 insertions(+), 74 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index acf545efa8..c8106856e6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -214,6 +214,15 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" +[[package]] +name = "castaway" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0abae9be0aaf9ea96a3b1b8b1b55c602ca751eba1b1500220cea4ecbafe7c0d5" +dependencies = [ + "rustversion", +] + [[package]] name = "cc" version = "1.2.14" @@ -269,6 +278,20 @@ dependencies = [ "error-code", ] +[[package]] +name = "compact_str" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b79c4069c6cad78e2e0cdfcbd26275770669fb39fd308a752dc110e83b9af32" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "static_assertions", +] + [[package]] name = "console" version = "0.15.10" @@ -360,7 +383,7 @@ dependencies = [ "hashbrown 0.14.5", "log", "regalloc2", - "rustc-hash 2.1.1", + "rustc-hash", "serde", "smallvec", "target-lexicon 0.13.2", @@ -1007,15 +1030,6 @@ dependencies = [ "either", ] -[[package]] -name = "itertools" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.14.0" @@ -1895,7 +1909,7 @@ dependencies = [ "bumpalo", "hashbrown 0.15.2", "log", - "rustc-hash 2.1.1", + "rustc-hash", "smallvec", ] @@ -1964,25 +1978,25 @@ dependencies = [ [[package]] name = "ruff_python_ast" version = "0.0.0" -source = "git+https://github.com/astral-sh/ruff.git?tag=v0.4.10#b54922fd7394c36cdc390fd21aaee99206ebc361" +source = "git+https://github.com/astral-sh/ruff.git?tag=0.11.0#2cd25ef6410fb5fca96af1578728a3d828d2d53a" dependencies = [ "aho-corasick", "bitflags 2.8.0", + "compact_str", "is-macro", - "itertools 0.13.0", - "once_cell", + "itertools 0.14.0", + "memchr", "ruff_python_trivia", "ruff_source_file", "ruff_text_size", - "rustc-hash 1.1.0", + "rustc-hash", ] [[package]] name = "ruff_python_codegen" version = "0.0.0" -source = "git+https://github.com/astral-sh/ruff.git?tag=v0.4.10#b54922fd7394c36cdc390fd21aaee99206ebc361" +source = "git+https://github.com/astral-sh/ruff.git?tag=0.11.0#2cd25ef6410fb5fca96af1578728a3d828d2d53a" dependencies = [ - "once_cell", "ruff_python_ast", "ruff_python_literal", "ruff_python_parser", @@ -1993,10 +2007,10 @@ dependencies = [ [[package]] name = "ruff_python_literal" version = "0.0.0" -source = "git+https://github.com/astral-sh/ruff.git?tag=v0.4.10#b54922fd7394c36cdc390fd21aaee99206ebc361" +source = "git+https://github.com/astral-sh/ruff.git?tag=0.11.0#2cd25ef6410fb5fca96af1578728a3d828d2d53a" dependencies = [ "bitflags 2.8.0", - "itertools 0.13.0", + "itertools 0.14.0", "ruff_python_ast", "unic-ucd-category", ] @@ -2004,15 +2018,16 @@ dependencies = [ [[package]] name = "ruff_python_parser" version = "0.0.0" -source = "git+https://github.com/astral-sh/ruff.git?tag=v0.4.10#b54922fd7394c36cdc390fd21aaee99206ebc361" +source = "git+https://github.com/astral-sh/ruff.git?tag=0.11.0#2cd25ef6410fb5fca96af1578728a3d828d2d53a" dependencies = [ "bitflags 2.8.0", "bstr", + "compact_str", "memchr", "ruff_python_ast", "ruff_python_trivia", "ruff_text_size", - "rustc-hash 1.1.0", + "rustc-hash", "static_assertions", "unicode-ident", "unicode-normalization", @@ -2022,9 +2037,9 @@ dependencies = [ [[package]] name = "ruff_python_trivia" version = "0.0.0" -source = "git+https://github.com/astral-sh/ruff.git?tag=v0.4.10#b54922fd7394c36cdc390fd21aaee99206ebc361" +source = "git+https://github.com/astral-sh/ruff.git?tag=0.11.0#2cd25ef6410fb5fca96af1578728a3d828d2d53a" dependencies = [ - "itertools 0.13.0", + "itertools 0.14.0", "ruff_source_file", "ruff_text_size", "unicode-ident", @@ -2033,23 +2048,16 @@ dependencies = [ [[package]] name = "ruff_source_file" version = "0.0.0" -source = "git+https://github.com/astral-sh/ruff.git?tag=v0.4.10#b54922fd7394c36cdc390fd21aaee99206ebc361" +source = "git+https://github.com/astral-sh/ruff.git?tag=0.11.0#2cd25ef6410fb5fca96af1578728a3d828d2d53a" dependencies = [ "memchr", - "once_cell", "ruff_text_size", ] [[package]] name = "ruff_text_size" version = "0.0.0" -source = "git+https://github.com/astral-sh/ruff.git?tag=v0.4.10#b54922fd7394c36cdc390fd21aaee99206ebc361" - -[[package]] -name = "rustc-hash" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +source = "git+https://github.com/astral-sh/ruff.git?tag=0.11.0#2cd25ef6410fb5fca96af1578728a3d828d2d53a" [[package]] name = "rustc-hash" diff --git a/Cargo.toml b/Cargo.toml index ab5e56fa0c..469f895c7d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -122,11 +122,11 @@ rustpython-stdlib = { path = "stdlib", default-features = false, version = "0.4. rustpython-sre_engine = { path = "vm/sre_engine", version = "0.4.0" } rustpython-doc = { git = "https://github.com/RustPython/__doc__", tag = "0.3.0", version = "0.3.0" } -ruff_python_parser = { git = "https://github.com/astral-sh/ruff.git", tag = "v0.4.10" } -ruff_python_ast = { git = "https://github.com/astral-sh/ruff.git", tag = "v0.4.10" } -ruff_text_size = { git = "https://github.com/astral-sh/ruff.git", tag = "v0.4.10" } -ruff_source_file = { git = "https://github.com/astral-sh/ruff.git", tag = "v0.4.10" } -ruff_python_codegen = { git = "https://github.com/astral-sh/ruff.git", tag = "v0.4.10" } +ruff_python_parser = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" } +ruff_python_ast = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" } +ruff_text_size = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" } +ruff_source_file = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" } +ruff_python_codegen = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" } # rustpython-literal = { version = "0.4.0" } # rustpython-parser-core = { version = "0.4.0" } # rustpython-parser = { version = "0.4.0" } diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index f94d36fa0b..d5c9250ab0 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -1619,8 +1619,6 @@ def test_nested_front(): self.assertEqual(x, [('Boeing', 'Airliner'), ('Boeing', 'Engine'), ('Ford', 'Engine'), ('Macdonalds', 'Cheeseburger')]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_genexps(self): # generator expression tests g = ([x for x in range(10)] for x in range(1)) diff --git a/compiler/codegen/src/compile.rs b/compiler/codegen/src/compile.rs index 21087fddd6..a6eb216e2a 100644 --- a/compiler/codegen/src/compile.rs +++ b/compiler/codegen/src/compile.rs @@ -2029,8 +2029,7 @@ impl Compiler<'_> { if self.future_annotations { // FIXME: codegen? let ident = Default::default(); - let codegen = - ruff_python_codegen::Generator::new(&ident, Default::default(), Default::default()); + let codegen = ruff_python_codegen::Generator::new(&ident, Default::default()); self.emit_load_const(ConstantData::Str { value: codegen.expr(annotation), }); @@ -3595,18 +3594,19 @@ impl ToU32 for usize { #[cfg(test)] mod tests { use super::*; + use ruff_python_ast::name::Name; use ruff_python_ast::*; /// Test if the compiler can correctly identify fstrings containing an `await` expression. #[test] fn test_fstring_contains_await() { let range = TextRange::default(); - let flags = FStringFlags::default(); + let flags = FStringFlags::empty(); // f'{x}' let expr_x = Expr::Name(ExprName { range, - id: "x".to_owned(), + id: Name::new("x"), ctx: ExprContext::Load, }); let not_present = &Expr::FString(ExprFString { @@ -3631,7 +3631,7 @@ mod tests { range, value: Box::new(Expr::Name(ExprName { range, - id: "x".to_owned(), + id: Name::new("x"), ctx: ExprContext::Load, })), }); @@ -3655,14 +3655,14 @@ mod tests { // f'{x:{await y}}' let expr_x = Expr::Name(ExprName { range, - id: "x".to_owned(), + id: Name::new("x"), ctx: ExprContext::Load, }); let expr_await_y = Expr::Await(ExprAwait { range, value: Box::new(Expr::Name(ExprName { range, - id: "y".to_owned(), + id: Name::new("y"), ctx: ExprContext::Load, })), }); diff --git a/compiler/src/lib.rs b/compiler/src/lib.rs index 9c64de2610..9df7a2c5a1 100644 --- a/compiler/src/lib.rs +++ b/compiler/src/lib.rs @@ -119,7 +119,7 @@ fn _compile( mode: Mode, opts: CompileOpts, ) -> Result { - let parsed = parser::parse(source_code.text, mode.into()) + let parsed = parser::parse(source_code.text, parser::Mode::from(mode).into()) .map_err(|err| CompileError::from_ruff_parse_error(err, &source_code))?; let ast = parsed.into_syntax(); compile::compile_top(ast, source_code, mode, opts).map_err(|e| e.into()) @@ -145,9 +145,8 @@ pub fn _compile_symtable( symboltable::SymbolTable::scan_program(&ast.into_syntax(), source_code.clone()) } Mode::Eval => { - let ast = - ruff_python_parser::parse(source_code.text, ruff_python_parser::Mode::Expression) - .map_err(|e| CompileError::from_ruff_parse_error(e, &source_code))?; + let ast = ruff_python_parser::parse(source_code.text, parser::Mode::Expression.into()) + .map_err(|e| CompileError::from_ruff_parse_error(e, &source_code))?; symboltable::SymbolTable::scan_expr( &ast.into_syntax().expect_expression(), source_code.clone(), diff --git a/src/shell.rs b/src/shell.rs index f920b4d011..98ee6eee21 100644 --- a/src/shell.rs +++ b/src/shell.rs @@ -1,7 +1,7 @@ mod helper; use rustpython_compiler::{ - CompileError, ParseError, parser::ParseErrorType, parser::lexer::LexicalErrorType, + CompileError, ParseError, parser::LexicalErrorType, parser::ParseErrorType, }; use rustpython_vm::{ AsObject, PyResult, VirtualMachine, diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs index b5445c65f5..13341c1b1e 100644 --- a/vm/src/stdlib/ast.rs +++ b/vm/src/stdlib/ast.rs @@ -242,7 +242,7 @@ pub(crate) fn parse( mode: parser::Mode, ) -> Result { let source_code = SourceCodeOwned::new("".to_owned(), source.to_owned()); - let top = parser::parse(source, mode) + let top = parser::parse(source, mode.into()) .map_err(|parse_error| ParseError { error: parse_error.error, location: text_range_to_source_range(&source_code, parse_error.location) diff --git a/vm/src/stdlib/ast/constant.rs b/vm/src/stdlib/ast/constant.rs index 29c29b4660..857a5a7c91 100644 --- a/vm/src/stdlib/ast/constant.rs +++ b/vm/src/stdlib/ast/constant.rs @@ -255,7 +255,7 @@ fn constant_to_ruff_expr(value: Constant) -> ruff::Expr { value: ruff::StringLiteralValue::single(ruff::StringLiteral { range, value, - flags: ruff::StringLiteralFlags::default().with_prefix(prefix), + flags: ruff::StringLiteralFlags::empty().with_prefix(prefix), }), }) } @@ -265,7 +265,7 @@ fn constant_to_ruff_expr(value: Constant) -> ruff::Expr { value: ruff::BytesLiteralValue::single(ruff::BytesLiteral { range, value, - flags: Default::default(), // TODO + flags: ruff::BytesLiteralFlags::empty(), // TODO }), }) } @@ -293,7 +293,7 @@ fn constant_to_ruff_expr(value: Constant) -> ruff::Expr { // idk lol func: Box::new(ruff::Expr::Name(ruff::ExprName { range: TextRange::default(), - id: "frozenset".to_owned(), + id: ruff::name::Name::new_static("frozenset"), ctx: ruff::ExprContext::Load, })), arguments: ruff::Arguments { diff --git a/vm/src/stdlib/ast/expression.rs b/vm/src/stdlib/ast/expression.rs index b4c9124ad7..ed42dd5d0a 100644 --- a/vm/src/stdlib/ast/expression.rs +++ b/vm/src/stdlib/ast/expression.rs @@ -977,7 +977,7 @@ impl Node for ruff::ExprName { object: PyObjectRef, ) -> PyResult { Ok(Self { - id: get_node_field(vm, &object, "id", "Name")?.try_into_value(vm)?, + id: Node::ast_from_object(vm, source_code, get_node_field(vm, &object, "id", "Name")?)?, ctx: Node::ast_from_object( vm, source_code, diff --git a/vm/src/stdlib/ast/other.rs b/vm/src/stdlib/ast/other.rs index 09ffbd4077..2b9d292c1c 100644 --- a/vm/src/stdlib/ast/other.rs +++ b/vm/src/stdlib/ast/other.rs @@ -19,18 +19,22 @@ impl Node for ruff::ConversionFlag { } // /// This is just a string, not strictly an AST node. But it makes AST conversions easier. -// impl Node for ruff::name::Name { -// fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { -// vm.ctx.new_str(self.as_str()).to_pyobject(vm) -// } +impl Node for ruff::name::Name { + fn ast_to_object(self, vm: &VirtualMachine, _source_code: &SourceCodeOwned) -> PyObjectRef { + vm.ctx.new_str(self.as_str()).to_pyobject(vm) + } -// fn ast_from_object(vm: &VirtualMachine, source_code: &SourceCodeOwned, object: PyObjectRef) -> PyResult { -// match object.downcast::() { -// Ok(name) => Ok(Self::new(name)), -// Err(_) => Err(vm.new_value_error("expected str for name".to_owned())), -// } -// } -// } + fn ast_from_object( + vm: &VirtualMachine, + _source_code: &SourceCodeOwned, + object: PyObjectRef, + ) -> PyResult { + match object.downcast::() { + Ok(name) => Ok(Self::new(name)), + Err(_) => Err(vm.new_value_error("expected str for name".to_owned())), + } + } +} impl Node for ruff::Decorator { fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef { diff --git a/vm/src/stdlib/ast/string.rs b/vm/src/stdlib/ast/string.rs index 5b78967976..0d55d6f1e2 100644 --- a/vm/src/stdlib/ast/string.rs +++ b/vm/src/stdlib/ast/string.rs @@ -7,7 +7,7 @@ fn ruff_fstring_value_into_iter( let default = ruff::FStringPart::FString(ruff::FString { range: Default::default(), elements: Default::default(), - flags: Default::default(), + flags: ruff::FStringFlags::empty(), }); (0..fstring_value.as_slice().len()).map(move |i| { let fstring_value = &mut fstring_value; @@ -100,7 +100,7 @@ fn ruff_fstring_element_to_ruff_fstring_part(element: ruff::FStringElement) -> r ruff::FStringPart::Literal(ruff::StringLiteral { range, value, - flags: Default::default(), + flags: ruff::StringLiteralFlags::empty(), }) } ruff::FStringElement::Expression(value) => { @@ -123,7 +123,7 @@ fn ruff_fstring_element_to_ruff_fstring_part(element: ruff::FStringElement) -> r }, )] .into(), - flags: Default::default(), + flags: ruff::FStringFlags::empty(), }) } } @@ -164,7 +164,7 @@ impl JoinedStr { 0 => ruff::FStringValue::single(ruff::FString { range, elements: vec![].into(), - flags: Default::default(), + flags: ruff::FStringFlags::empty(), }), 1 => ruff::FStringValue::single( Box::<[_]>::into_iter(values) @@ -172,7 +172,7 @@ impl JoinedStr { .map(|element| ruff::FString { range, elements: vec![element].into(), - flags: Default::default(), + flags: ruff::FStringFlags::empty(), }) .next() .expect("FString has exactly one part"), diff --git a/vm/src/vm/vm_new.rs b/vm/src/vm/vm_new.rs index 01709b6873..12241414a7 100644 --- a/vm/src/vm/vm_new.rs +++ b/vm/src/vm/vm_new.rs @@ -270,7 +270,7 @@ impl VirtualMachine { crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { error: ruff_python_parser::ParseErrorType::Lexical( - ruff_python_parser::lexer::LexicalErrorType::IndentationError, + ruff_python_parser::LexicalErrorType::IndentationError, ), .. }) => self.ctx.exceptions.tab_error, diff --git a/wasm/lib/src/convert.rs b/wasm/lib/src/convert.rs index 2f80770319..4f6e4db35c 100644 --- a/wasm/lib/src/convert.rs +++ b/wasm/lib/src/convert.rs @@ -6,7 +6,7 @@ use js_sys::{Array, ArrayBuffer, Object, Promise, Reflect, SyntaxError, Uint8Arr use rustpython_vm::{ AsObject, PyObjectRef, PyPayload, PyResult, TryFromBorrowedObject, VirtualMachine, builtins::PyBaseExceptionRef, - compiler::{CompileError, ParseError, parser::ParseErrorType, parser::lexer::LexicalErrorType}, + compiler::{CompileError, ParseError, parser::LexicalErrorType, parser::ParseErrorType}, exceptions, function::{ArgBytesLike, FuncArgs}, py_serde, From bfe72689fcdb06e16cd821307f7358261b417f7e Mon Sep 17 00:00:00 2001 From: Noa Date: Wed, 19 Mar 2025 21:42:33 -0500 Subject: [PATCH 090/295] Remove -merge from Cargo.lock gitattributes --- .gitattributes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitattributes b/.gitattributes index 0c4ae3d850..f54bcd3b72 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,5 +1,5 @@ Lib/** linguist-vendored -Cargo.lock linguist-generated -merge +Cargo.lock linguist-generated *.snap linguist-generated -merge vm/src/stdlib/ast/gen.rs linguist-generated -merge Lib/*.py text working-tree-encoding=UTF-8 eol=LF From a4466adf8bf16941ea2045dd942d4da1800b4394 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Fri, 14 Mar 2025 21:04:08 -0700 Subject: [PATCH 091/295] add _aix_support --- Lib/_aix_support.py | 108 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 Lib/_aix_support.py diff --git a/Lib/_aix_support.py b/Lib/_aix_support.py new file mode 100644 index 0000000000..dadc75c2bf --- /dev/null +++ b/Lib/_aix_support.py @@ -0,0 +1,108 @@ +"""Shared AIX support functions.""" + +import sys +import sysconfig + + +# Taken from _osx_support _read_output function +def _read_cmd_output(commandstring, capture_stderr=False): + """Output from successful command execution or None""" + # Similar to os.popen(commandstring, "r").read(), + # but without actually using os.popen because that + # function is not usable during python bootstrap. + import os + import contextlib + fp = open("/tmp/_aix_support.%s"%( + os.getpid(),), "w+b") + + with contextlib.closing(fp) as fp: + if capture_stderr: + cmd = "%s >'%s' 2>&1" % (commandstring, fp.name) + else: + cmd = "%s 2>/dev/null >'%s'" % (commandstring, fp.name) + return fp.read() if not os.system(cmd) else None + + +def _aix_tag(vrtl, bd): + # type: (List[int], int) -> str + # Infer the ABI bitwidth from maxsize (assuming 64 bit as the default) + _sz = 32 if sys.maxsize == (2**31-1) else 64 + _bd = bd if bd != 0 else 9988 + # vrtl[version, release, technology_level] + return "aix-{:1x}{:1d}{:02d}-{:04d}-{}".format(vrtl[0], vrtl[1], vrtl[2], _bd, _sz) + + +# extract version, release and technology level from a VRMF string +def _aix_vrtl(vrmf): + # type: (str) -> List[int] + v, r, tl = vrmf.split(".")[:3] + return [int(v[-1]), int(r), int(tl)] + + +def _aix_bos_rte(): + # type: () -> Tuple[str, int] + """ + Return a Tuple[str, int] e.g., ['7.1.4.34', 1806] + The fileset bos.rte represents the current AIX run-time level. It's VRMF and + builddate reflect the current ABI levels of the runtime environment. + If no builddate is found give a value that will satisfy pep425 related queries + """ + # All AIX systems to have lslpp installed in this location + # subprocess may not be available during python bootstrap + try: + import subprocess + out = subprocess.check_output(["/usr/bin/lslpp", "-Lqc", "bos.rte"]) + except ImportError: + out = _read_cmd_output("/usr/bin/lslpp -Lqc bos.rte") + out = out.decode("utf-8") + out = out.strip().split(":") # type: ignore + _bd = int(out[-1]) if out[-1] != '' else 9988 + return (str(out[2]), _bd) + + +def aix_platform(): + # type: () -> str + """ + AIX filesets are identified by four decimal values: V.R.M.F. + V (version) and R (release) can be retrieved using ``uname`` + Since 2007, starting with AIX 5.3 TL7, the M value has been + included with the fileset bos.rte and represents the Technology + Level (TL) of AIX. The F (Fix) value also increases, but is not + relevant for comparing releases and binary compatibility. + For binary compatibility the so-called builddate is needed. + Again, the builddate of an AIX release is associated with bos.rte. + AIX ABI compatibility is described as guaranteed at: https://www.ibm.com/\ + support/knowledgecenter/en/ssw_aix_72/install/binary_compatability.html + + For pep425 purposes the AIX platform tag becomes: + "aix-{:1x}{:1d}{:02d}-{:04d}-{}".format(v, r, tl, builddate, bitsize) + e.g., "aix-6107-1415-32" for AIX 6.1 TL7 bd 1415, 32-bit + and, "aix-6107-1415-64" for AIX 6.1 TL7 bd 1415, 64-bit + """ + vrmf, bd = _aix_bos_rte() + return _aix_tag(_aix_vrtl(vrmf), bd) + + +# extract vrtl from the BUILD_GNU_TYPE as an int +def _aix_bgt(): + # type: () -> List[int] + gnu_type = sysconfig.get_config_var("BUILD_GNU_TYPE") + if not gnu_type: + raise ValueError("BUILD_GNU_TYPE is not defined") + return _aix_vrtl(vrmf=gnu_type) + + +def aix_buildtag(): + # type: () -> str + """ + Return the platform_tag of the system Python was built on. + """ + # AIX_BUILDDATE is defined by configure with: + # lslpp -Lcq bos.rte | awk -F: '{ print $NF }' + build_date = sysconfig.get_config_var("AIX_BUILDDATE") + try: + build_date = int(build_date) + except (ValueError, TypeError): + raise ValueError(f"AIX_BUILDDATE is not defined or invalid: " + f"{build_date!r}") + return _aix_tag(_aix_bgt(), build_date) From 081dc4e8ca21dbf3e38acd6c9f0f8390e4562eb3 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Wed, 19 Mar 2025 22:28:53 -0700 Subject: [PATCH 092/295] removed cgib (#5609) --- Lib/cgitb.py | 332 ----------------------------------------- Lib/test/test_cgitb.py | 71 --------- 2 files changed, 403 deletions(-) delete mode 100644 Lib/cgitb.py delete mode 100644 Lib/test/test_cgitb.py diff --git a/Lib/cgitb.py b/Lib/cgitb.py deleted file mode 100644 index f6b97f25c5..0000000000 --- a/Lib/cgitb.py +++ /dev/null @@ -1,332 +0,0 @@ -"""More comprehensive traceback formatting for Python scripts. - -To enable this module, do: - - import cgitb; cgitb.enable() - -at the top of your script. The optional arguments to enable() are: - - display - if true, tracebacks are displayed in the web browser - logdir - if set, tracebacks are written to files in this directory - context - number of lines of source code to show for each stack frame - format - 'text' or 'html' controls the output format - -By default, tracebacks are displayed but not saved, the context is 5 lines -and the output format is 'html' (for backwards compatibility with the -original use of this module) - -Alternatively, if you have caught an exception and want cgitb to display it -for you, call cgitb.handler(). The optional argument to handler() is a -3-item tuple (etype, evalue, etb) just like the value of sys.exc_info(). -The default handler displays output as HTML. - -""" -import inspect -import keyword -import linecache -import os -import pydoc -import sys -import tempfile -import time -import tokenize -import traceback -import warnings -from html import escape as html_escape - -warnings._deprecated(__name__, remove=(3, 13)) - - -def reset(): - """Return a string that resets the CGI and browser to a known state.""" - return ''' - --> --> - - ''' - -__UNDEF__ = [] # a special sentinel object -def small(text): - if text: - return '' + text + '' - else: - return '' - -def strong(text): - if text: - return '' + text + '' - else: - return '' - -def grey(text): - if text: - return '' + text + '' - else: - return '' - -def lookup(name, frame, locals): - """Find the value for a given name in the given environment.""" - if name in locals: - return 'local', locals[name] - if name in frame.f_globals: - return 'global', frame.f_globals[name] - if '__builtins__' in frame.f_globals: - builtins = frame.f_globals['__builtins__'] - if isinstance(builtins, dict): - if name in builtins: - return 'builtin', builtins[name] - else: - if hasattr(builtins, name): - return 'builtin', getattr(builtins, name) - return None, __UNDEF__ - -def scanvars(reader, frame, locals): - """Scan one logical line of Python and look up values of variables used.""" - vars, lasttoken, parent, prefix, value = [], None, None, '', __UNDEF__ - for ttype, token, start, end, line in tokenize.generate_tokens(reader): - if ttype == tokenize.NEWLINE: break - if ttype == tokenize.NAME and token not in keyword.kwlist: - if lasttoken == '.': - if parent is not __UNDEF__: - value = getattr(parent, token, __UNDEF__) - vars.append((prefix + token, prefix, value)) - else: - where, value = lookup(token, frame, locals) - vars.append((token, where, value)) - elif token == '.': - prefix += lasttoken + '.' - parent = value - else: - parent, prefix = None, '' - lasttoken = token - return vars - -def html(einfo, context=5): - """Return a nice HTML document describing a given traceback.""" - etype, evalue, etb = einfo - if isinstance(etype, type): - etype = etype.__name__ - pyver = 'Python ' + sys.version.split()[0] + ': ' + sys.executable - date = time.ctime(time.time()) - head = f''' - - - - - -
 
- 
-{html_escape(str(etype))}		 -{pyver}
{date}
-

A problem occurred in a Python script. Here is the sequence of -function calls leading up to the error, in the order they occurred.

''' - - indent = '' + small(' ' * 5) + ' ' - frames = [] - records = inspect.getinnerframes(etb, context) - for frame, file, lnum, func, lines, index in records: - if file: - file = os.path.abspath(file) - link = '%s' % (file, pydoc.html.escape(file)) - else: - file = link = '?' - args, varargs, varkw, locals = inspect.getargvalues(frame) - call = '' - if func != '?': - call = 'in ' + strong(pydoc.html.escape(func)) - if func != "": - call += inspect.formatargvalues(args, varargs, varkw, locals, - formatvalue=lambda value: '=' + pydoc.html.repr(value)) - - highlight = {} - def reader(lnum=[lnum]): - highlight[lnum[0]] = 1 - try: return linecache.getline(file, lnum[0]) - finally: lnum[0] += 1 - vars = scanvars(reader, frame, locals) - - rows = ['%s%s %s' % - (' ', link, call)] - if index is not None: - i = lnum - index - for line in lines: - num = small(' ' * (5-len(str(i))) + str(i)) + ' ' - if i in highlight: - line = '=>%s%s' % (num, pydoc.html.preformat(line)) - rows.append('%s' % line) - else: - line = '  %s%s' % (num, pydoc.html.preformat(line)) - rows.append('%s' % grey(line)) - i += 1 - - done, dump = {}, [] - for name, where, value in vars: - if name in done: continue - done[name] = 1 - if value is not __UNDEF__: - if where in ('global', 'builtin'): - name = ('%s ' % where) + strong(name) - elif where == 'local': - name = strong(name) - else: - name = where + strong(name.split('.')[-1]) - dump.append('%s = %s' % (name, pydoc.html.repr(value))) - else: - dump.append(name + ' undefined') - - rows.append('%s' % small(grey(', '.join(dump)))) - frames.append(''' - -%s
''' % '\n'.join(rows)) - - exception = ['

%s: %s' % (strong(pydoc.html.escape(str(etype))), - pydoc.html.escape(str(evalue)))] - for name in dir(evalue): - if name[:1] == '_': continue - value = pydoc.html.repr(getattr(evalue, name)) - exception.append('\n
%s%s =\n%s' % (indent, name, value)) - - return head + ''.join(frames) + ''.join(exception) + ''' - - - -''' % pydoc.html.escape( - ''.join(traceback.format_exception(etype, evalue, etb))) - -def text(einfo, context=5): - """Return a plain text document describing a given traceback.""" - etype, evalue, etb = einfo - if isinstance(etype, type): - etype = etype.__name__ - pyver = 'Python ' + sys.version.split()[0] + ': ' + sys.executable - date = time.ctime(time.time()) - head = "%s\n%s\n%s\n" % (str(etype), pyver, date) + ''' -A problem occurred in a Python script. Here is the sequence of -function calls leading up to the error, in the order they occurred. -''' - - frames = [] - records = inspect.getinnerframes(etb, context) - for frame, file, lnum, func, lines, index in records: - file = file and os.path.abspath(file) or '?' - args, varargs, varkw, locals = inspect.getargvalues(frame) - call = '' - if func != '?': - call = 'in ' + func - if func != "": - call += inspect.formatargvalues(args, varargs, varkw, locals, - formatvalue=lambda value: '=' + pydoc.text.repr(value)) - - highlight = {} - def reader(lnum=[lnum]): - highlight[lnum[0]] = 1 - try: return linecache.getline(file, lnum[0]) - finally: lnum[0] += 1 - vars = scanvars(reader, frame, locals) - - rows = [' %s %s' % (file, call)] - if index is not None: - i = lnum - index - for line in lines: - num = '%5d ' % i - rows.append(num+line.rstrip()) - i += 1 - - done, dump = {}, [] - for name, where, value in vars: - if name in done: continue - done[name] = 1 - if value is not __UNDEF__: - if where == 'global': name = 'global ' + name - elif where != 'local': name = where + name.split('.')[-1] - dump.append('%s = %s' % (name, pydoc.text.repr(value))) - else: - dump.append(name + ' undefined') - - rows.append('\n'.join(dump)) - frames.append('\n%s\n' % '\n'.join(rows)) - - exception = ['%s: %s' % (str(etype), str(evalue))] - for name in dir(evalue): - value = pydoc.text.repr(getattr(evalue, name)) - exception.append('\n%s%s = %s' % (" "*4, name, value)) - - return head + ''.join(frames) + ''.join(exception) + ''' - -The above is a description of an error in a Python program. Here is -the original traceback: - -%s -''' % ''.join(traceback.format_exception(etype, evalue, etb)) - -class Hook: - """A hook to replace sys.excepthook that shows tracebacks in HTML.""" - - def __init__(self, display=1, logdir=None, context=5, file=None, - format="html"): - self.display = display # send tracebacks to browser if true - self.logdir = logdir # log tracebacks to files if not None - self.context = context # number of source code lines per frame - self.file = file or sys.stdout # place to send the output - self.format = format - - def __call__(self, etype, evalue, etb): - self.handle((etype, evalue, etb)) - - def handle(self, info=None): - info = info or sys.exc_info() - if self.format == "html": - self.file.write(reset()) - - formatter = (self.format=="html") and html or text - plain = False - try: - doc = formatter(info, self.context) - except: # just in case something goes wrong - doc = ''.join(traceback.format_exception(*info)) - plain = True - - if self.display: - if plain: - doc = pydoc.html.escape(doc) - self.file.write('

' + doc + '
\n') - else: - self.file.write(doc + '\n') - else: - self.file.write('

A problem occurred in a Python script.\n') - - if self.logdir is not None: - suffix = ['.txt', '.html'][self.format=="html"] - (fd, path) = tempfile.mkstemp(suffix=suffix, dir=self.logdir) - - try: - with os.fdopen(fd, 'w') as file: - file.write(doc) - msg = '%s contains the description of this error.' % path - except: - msg = 'Tried to save traceback to %s, but failed.' % path - - if self.format == 'html': - self.file.write('

%s

\n' % msg) - else: - self.file.write(msg + '\n') - try: - self.file.flush() - except: pass - -handler = Hook().handle -def enable(display=1, logdir=None, context=5, format="html"): - """Install an exception handler that formats tracebacks as HTML. - - The optional argument 'display' can be set to 0 to suppress sending the - traceback to the browser, and 'logdir' can be set to a directory to cause - tracebacks to be written to files there.""" - sys.excepthook = Hook(display=display, logdir=logdir, - context=context, format=format) diff --git a/Lib/test/test_cgitb.py b/Lib/test/test_cgitb.py deleted file mode 100644 index 501c7fcce2..0000000000 --- a/Lib/test/test_cgitb.py +++ /dev/null @@ -1,71 +0,0 @@ -from test.support.os_helper import temp_dir -from test.support.script_helper import assert_python_failure -from test.support.warnings_helper import import_deprecated -import unittest -import sys -cgitb = import_deprecated("cgitb") - -class TestCgitb(unittest.TestCase): - - def test_fonts(self): - text = "Hello Robbie!" - self.assertEqual(cgitb.small(text), "{}".format(text)) - self.assertEqual(cgitb.strong(text), "{}".format(text)) - self.assertEqual(cgitb.grey(text), - '{}'.format(text)) - - def test_blanks(self): - self.assertEqual(cgitb.small(""), "") - self.assertEqual(cgitb.strong(""), "") - self.assertEqual(cgitb.grey(""), "") - - def test_html(self): - try: - raise ValueError("Hello World") - except ValueError as err: - # If the html was templated we could do a bit more here. - # At least check that we get details on what we just raised. - html = cgitb.html(sys.exc_info()) - self.assertIn("ValueError", html) - self.assertIn(str(err), html) - - def test_text(self): - try: - raise ValueError("Hello World") - except ValueError: - text = cgitb.text(sys.exc_info()) - self.assertIn("ValueError", text) - self.assertIn("Hello World", text) - - def test_syshook_no_logdir_default_format(self): - with temp_dir() as tracedir: - rc, out, err = assert_python_failure( - '-c', - ('import cgitb; cgitb.enable(logdir=%s); ' - 'raise ValueError("Hello World")') % repr(tracedir), - PYTHONIOENCODING='utf-8') - out = out.decode() - self.assertIn("ValueError", out) - self.assertIn("Hello World", out) - self.assertIn("<module>", out) - # By default we emit HTML markup. - self.assertIn('

', out) - self.assertIn('

', out) - - def test_syshook_no_logdir_text_format(self): - # Issue 12890: we were emitting the

tag in text mode. - with temp_dir() as tracedir: - rc, out, err = assert_python_failure( - '-c', - ('import cgitb; cgitb.enable(format="text", logdir=%s); ' - 'raise ValueError("Hello World")') % repr(tracedir), - PYTHONIOENCODING='utf-8') - out = out.decode() - self.assertIn("ValueError", out) - self.assertIn("Hello World", out) - self.assertNotIn('

', out) - self.assertNotIn('

', out) - - -if __name__ == "__main__": - unittest.main() From 37bd49cf385212a66eaee0f85d7b54913c4ad60a Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Fri, 14 Mar 2025 23:19:47 -0700 Subject: [PATCH 093/295] add _pylong.py at 3.13.2 --- Lib/_pylong.py | 363 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 363 insertions(+) create mode 100644 Lib/_pylong.py diff --git a/Lib/_pylong.py b/Lib/_pylong.py new file mode 100644 index 0000000000..4970eb3fa6 --- /dev/null +++ b/Lib/_pylong.py @@ -0,0 +1,363 @@ +"""Python implementations of some algorithms for use by longobject.c. +The goal is to provide asymptotically faster algorithms that can be +used for operations on integers with many digits. In those cases, the +performance overhead of the Python implementation is not significant +since the asymptotic behavior is what dominates runtime. Functions +provided by this module should be considered private and not part of any +public API. + +Note: for ease of maintainability, please prefer clear code and avoid +"micro-optimizations". This module will only be imported and used for +integers with a huge number of digits. Saving a few microseconds with +tricky or non-obvious code is not worth it. For people looking for +maximum performance, they should use something like gmpy2.""" + +import re +import decimal +try: + import _decimal +except ImportError: + _decimal = None + +# A number of functions have this form, where `w` is a desired number of +# digits in base `base`: +# +# def inner(...w...): +# if w <= LIMIT: +# return something +# lo = w >> 1 +# hi = w - lo +# something involving base**lo, inner(...lo...), j, and inner(...hi...) +# figure out largest w needed +# result = inner(w) +# +# They all had some on-the-fly scheme to cache `base**lo` results for reuse. +# Power is costly. +# +# This routine aims to compute all amd only the needed powers in advance, as +# efficiently as reasonably possible. This isn't trivial, and all the +# on-the-fly methods did needless work in many cases. The driving code above +# changes to: +# +# figure out largest w needed +# mycache = compute_powers(w, base, LIMIT) +# result = inner(w) +# +# and `mycache[lo]` replaces `base**lo` in the inner function. +# +# While this does give minor speedups (a few percent at best), the primary +# intent is to simplify the functions using this, by eliminating the need for +# them to craft their own ad-hoc caching schemes. +def compute_powers(w, base, more_than, show=False): + seen = set() + need = set() + ws = {w} + while ws: + w = ws.pop() # any element is fine to use next + if w in seen or w <= more_than: + continue + seen.add(w) + lo = w >> 1 + # only _need_ lo here; some other path may, or may not, need hi + need.add(lo) + ws.add(lo) + if w & 1: + ws.add(lo + 1) + + d = {} + if not need: + return d + it = iter(sorted(need)) + first = next(it) + if show: + print("pow at", first) + d[first] = base ** first + for this in it: + if this - 1 in d: + if show: + print("* base at", this) + d[this] = d[this - 1] * base # cheap + else: + lo = this >> 1 + hi = this - lo + assert lo in d + if show: + print("square at", this) + # Multiplying a bigint by itself (same object!) is about twice + # as fast in CPython. + sq = d[lo] * d[lo] + if hi != lo: + assert hi == lo + 1 + if show: + print(" and * base") + sq *= base + d[this] = sq + return d + +_unbounded_dec_context = decimal.getcontext().copy() +_unbounded_dec_context.prec = decimal.MAX_PREC +_unbounded_dec_context.Emax = decimal.MAX_EMAX +_unbounded_dec_context.Emin = decimal.MIN_EMIN +_unbounded_dec_context.traps[decimal.Inexact] = 1 # sanity check + +def int_to_decimal(n): + """Asymptotically fast conversion of an 'int' to Decimal.""" + + # Function due to Tim Peters. See GH issue #90716 for details. + # https://github.com/python/cpython/issues/90716 + # + # The implementation in longobject.c of base conversion algorithms + # between power-of-2 and non-power-of-2 bases are quadratic time. + # This function implements a divide-and-conquer algorithm that is + # faster for large numbers. Builds an equal decimal.Decimal in a + # "clever" recursive way. If we want a string representation, we + # apply str to _that_. + + from decimal import Decimal as D + BITLIM = 200 + + # Don't bother caching the "lo" mask in this; the time to compute it is + # tiny compared to the multiply. + def inner(n, w): + if w <= BITLIM: + return D(n) + w2 = w >> 1 + hi = n >> w2 + lo = n & ((1 << w2) - 1) + return inner(lo, w2) + inner(hi, w - w2) * w2pow[w2] + + with decimal.localcontext(_unbounded_dec_context): + nbits = n.bit_length() + w2pow = compute_powers(nbits, D(2), BITLIM) + if n < 0: + negate = True + n = -n + else: + negate = False + result = inner(n, nbits) + if negate: + result = -result + return result + +def int_to_decimal_string(n): + """Asymptotically fast conversion of an 'int' to a decimal string.""" + w = n.bit_length() + if w > 450_000 and _decimal is not None: + # It is only usable with the C decimal implementation. + # _pydecimal.py calls str() on very large integers, which in its + # turn calls int_to_decimal_string(), causing very deep recursion. + return str(int_to_decimal(n)) + + # Fallback algorithm for the case when the C decimal module isn't + # available. This algorithm is asymptotically worse than the algorithm + # using the decimal module, but better than the quadratic time + # implementation in longobject.c. + + DIGLIM = 1000 + def inner(n, w): + if w <= DIGLIM: + return str(n) + w2 = w >> 1 + hi, lo = divmod(n, pow10[w2]) + return inner(hi, w - w2) + inner(lo, w2).zfill(w2) + + # The estimation of the number of decimal digits. + # There is no harm in small error. If we guess too large, there may + # be leading 0's that need to be stripped. If we guess too small, we + # may need to call str() recursively for the remaining highest digits, + # which can still potentially be a large integer. This is manifested + # only if the number has way more than 10**15 digits, that exceeds + # the 52-bit physical address limit in both Intel64 and AMD64. + w = int(w * 0.3010299956639812 + 1) # log10(2) + pow10 = compute_powers(w, 5, DIGLIM) + for k, v in pow10.items(): + pow10[k] = v << k # 5**k << k == 5**k * 2**k == 10**k + if n < 0: + n = -n + sign = '-' + else: + sign = '' + s = inner(n, w) + if s[0] == '0' and n: + # If our guess of w is too large, there may be leading 0's that + # need to be stripped. + s = s.lstrip('0') + return sign + s + +def _str_to_int_inner(s): + """Asymptotically fast conversion of a 'str' to an 'int'.""" + + # Function due to Bjorn Martinsson. See GH issue #90716 for details. + # https://github.com/python/cpython/issues/90716 + # + # The implementation in longobject.c of base conversion algorithms + # between power-of-2 and non-power-of-2 bases are quadratic time. + # This function implements a divide-and-conquer algorithm making use + # of Python's built in big int multiplication. Since Python uses the + # Karatsuba algorithm for multiplication, the time complexity + # of this function is O(len(s)**1.58). + + DIGLIM = 2048 + + def inner(a, b): + if b - a <= DIGLIM: + return int(s[a:b]) + mid = (a + b + 1) >> 1 + return (inner(mid, b) + + ((inner(a, mid) * w5pow[b - mid]) + << (b - mid))) + + w5pow = compute_powers(len(s), 5, DIGLIM) + return inner(0, len(s)) + + +def int_from_string(s): + """Asymptotically fast version of PyLong_FromString(), conversion + of a string of decimal digits into an 'int'.""" + # PyLong_FromString() has already removed leading +/-, checked for invalid + # use of underscore characters, checked that string consists of only digits + # and underscores, and stripped leading whitespace. The input can still + # contain underscores and have trailing whitespace. + s = s.rstrip().replace('_', '') + return _str_to_int_inner(s) + +def str_to_int(s): + """Asymptotically fast version of decimal string to 'int' conversion.""" + # FIXME: this doesn't support the full syntax that int() supports. + m = re.match(r'\s*([+-]?)([0-9_]+)\s*', s) + if not m: + raise ValueError('invalid literal for int() with base 10') + v = int_from_string(m.group(2)) + if m.group(1) == '-': + v = -v + return v + + +# Fast integer division, based on code from Mark Dickinson, fast_div.py +# GH-47701. Additional refinements and optimizations by Bjorn Martinsson. The +# algorithm is due to Burnikel and Ziegler, in their paper "Fast Recursive +# Division". + +_DIV_LIMIT = 4000 + + +def _div2n1n(a, b, n): + """Divide a 2n-bit nonnegative integer a by an n-bit positive integer + b, using a recursive divide-and-conquer algorithm. + + Inputs: + n is a positive integer + b is a positive integer with exactly n bits + a is a nonnegative integer such that a < 2**n * b + + Output: + (q, r) such that a = b*q+r and 0 <= r < b. + + """ + if a.bit_length() - n <= _DIV_LIMIT: + return divmod(a, b) + pad = n & 1 + if pad: + a <<= 1 + b <<= 1 + n += 1 + half_n = n >> 1 + mask = (1 << half_n) - 1 + b1, b2 = b >> half_n, b & mask + q1, r = _div3n2n(a >> n, (a >> half_n) & mask, b, b1, b2, half_n) + q2, r = _div3n2n(r, a & mask, b, b1, b2, half_n) + if pad: + r >>= 1 + return q1 << half_n | q2, r + + +def _div3n2n(a12, a3, b, b1, b2, n): + """Helper function for _div2n1n; not intended to be called directly.""" + if a12 >> n == b1: + q, r = (1 << n) - 1, a12 - (b1 << n) + b1 + else: + q, r = _div2n1n(a12, b1, n) + r = (r << n | a3) - q * b2 + while r < 0: + q -= 1 + r += b + return q, r + + +def _int2digits(a, n): + """Decompose non-negative int a into base 2**n + + Input: + a is a non-negative integer + + Output: + List of the digits of a in base 2**n in little-endian order, + meaning the most significant digit is last. The most + significant digit is guaranteed to be non-zero. + If a is 0 then the output is an empty list. + + """ + a_digits = [0] * ((a.bit_length() + n - 1) // n) + + def inner(x, L, R): + if L + 1 == R: + a_digits[L] = x + return + mid = (L + R) >> 1 + shift = (mid - L) * n + upper = x >> shift + lower = x ^ (upper << shift) + inner(lower, L, mid) + inner(upper, mid, R) + + if a: + inner(a, 0, len(a_digits)) + return a_digits + + +def _digits2int(digits, n): + """Combine base-2**n digits into an int. This function is the + inverse of `_int2digits`. For more details, see _int2digits. + """ + + def inner(L, R): + if L + 1 == R: + return digits[L] + mid = (L + R) >> 1 + shift = (mid - L) * n + return (inner(mid, R) << shift) + inner(L, mid) + + return inner(0, len(digits)) if digits else 0 + + +def _divmod_pos(a, b): + """Divide a non-negative integer a by a positive integer b, giving + quotient and remainder.""" + # Use grade-school algorithm in base 2**n, n = nbits(b) + n = b.bit_length() + a_digits = _int2digits(a, n) + + r = 0 + q_digits = [] + for a_digit in reversed(a_digits): + q_digit, r = _div2n1n((r << n) + a_digit, b, n) + q_digits.append(q_digit) + q_digits.reverse() + q = _digits2int(q_digits, n) + return q, r + + +def int_divmod(a, b): + """Asymptotically fast replacement for divmod, for 'int'. + Its time complexity is O(n**1.58), where n = #bits(a) + #bits(b). + """ + if b == 0: + raise ZeroDivisionError + elif b < 0: + q, r = int_divmod(-a, -b) + return q, -r + elif a < 0: + q, r = int_divmod(~a, b) + return ~q, b + ~r + else: + return _divmod_pos(a, b) From 70c36a48a8930a26fe0ceee725d9c7432d81d676 Mon Sep 17 00:00:00 2001 From: Myeongseon Choi Date: Sat, 15 Mar 2025 18:43:35 +0900 Subject: [PATCH 094/295] Copy test_class.py from cpython v3.13.2 --- Lib/test/test_class.py | 262 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 257 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_class.py b/Lib/test/test_class.py index 1bb761238b..c48726d44b 100644 --- a/Lib/test/test_class.py +++ b/Lib/test/test_class.py @@ -1,7 +1,7 @@ "Test the functionality of Python classes implementing operators." import unittest - +from test.support import cpython_only, import_helper, script_helper testmeths = [ @@ -448,15 +448,15 @@ def __delattr__(self, *args): def testHasAttrString(self): import sys from test.support import import_helper - _testcapi = import_helper.import_module('_testcapi') + _testlimitedcapi = import_helper.import_module('_testlimitedcapi') class A: def __init__(self): self.attr = 1 a = A() - self.assertEqual(_testcapi.object_hasattrstring(a, b"attr"), 1) - self.assertEqual(_testcapi.object_hasattrstring(a, b"noattr"), 0) + self.assertEqual(_testlimitedcapi.object_hasattrstring(a, b"attr"), 1) + self.assertEqual(_testlimitedcapi.object_hasattrstring(a, b"noattr"), 0) self.assertIsNone(sys.exception()) def testDel(self): @@ -503,6 +503,56 @@ def __eq__(self, other): return 1 self.assertRaises(TypeError, hash, C2()) + def testPredefinedAttrs(self): + o = object() + + class Custom: + pass + + c = Custom() + + methods = ( + '__class__', '__delattr__', '__dir__', '__eq__', '__format__', + '__ge__', '__getattribute__', '__getstate__', '__gt__', '__hash__', + '__init__', '__init_subclass__', '__le__', '__lt__', '__ne__', + '__new__', '__reduce__', '__reduce_ex__', '__repr__', + '__setattr__', '__sizeof__', '__str__', '__subclasshook__' + ) + for name in methods: + with self.subTest(name): + self.assertTrue(callable(getattr(object, name, None))) + self.assertTrue(callable(getattr(o, name, None))) + self.assertTrue(callable(getattr(Custom, name, None))) + self.assertTrue(callable(getattr(c, name, None))) + + not_defined = [ + '__abs__', '__aenter__', '__aexit__', '__aiter__', '__anext__', + '__await__', '__bool__', '__bytes__', '__ceil__', + '__complex__', '__contains__', '__del__', '__delete__', + '__delitem__', '__divmod__', '__enter__', '__exit__', + '__float__', '__floor__', '__get__', '__getattr__', '__getitem__', + '__index__', '__int__', '__invert__', '__iter__', '__len__', + '__length_hint__', '__missing__', '__neg__', '__next__', + '__objclass__', '__pos__', '__rdivmod__', '__reversed__', + '__round__', '__set__', '__setitem__', '__trunc__' + ] + augment = ( + 'add', 'and', 'floordiv', 'lshift', 'matmul', 'mod', 'mul', 'pow', + 'rshift', 'sub', 'truediv', 'xor' + ) + not_defined.extend(map("__{}__".format, augment)) + not_defined.extend(map("__r{}__".format, augment)) + not_defined.extend(map("__i{}__".format, augment)) + for name in not_defined: + with self.subTest(name): + self.assertFalse(hasattr(object, name)) + self.assertFalse(hasattr(o, name)) + self.assertFalse(hasattr(Custom, name)) + self.assertFalse(hasattr(c, name)) + + # __call__() is defined on the metaclass but not the class + self.assertFalse(hasattr(o, "__call__")) + self.assertFalse(hasattr(c, "__call__")) @unittest.skip("TODO: RUSTPYTHON, segmentation fault") def testSFBug532646(self): @@ -647,6 +697,14 @@ class A: class B: y = 0 __slots__ = ('z',) + class C: + __slots__ = ("y",) + + def __setattr__(self, name, value) -> None: + if name == "z": + super().__setattr__("y", 1) + else: + super().__setattr__(name, value) error_msg = "'A' object has no attribute 'x'" with self.assertRaisesRegex(AttributeError, error_msg): @@ -659,8 +717,16 @@ class B: B().x with self.assertRaisesRegex(AttributeError, error_msg): del B().x - with self.assertRaisesRegex(AttributeError, error_msg): + with self.assertRaisesRegex( + AttributeError, + "'B' object has no attribute 'x' and no __dict__ for setting new attributes" + ): B().x = 0 + with self.assertRaisesRegex( + AttributeError, + "'C' object has no attribute 'x'" + ): + C().x = 0 error_msg = "'B' object attribute 'y' is read-only" with self.assertRaisesRegex(AttributeError, error_msg): @@ -748,6 +814,192 @@ class A(0, 1, 2, 3, 4, 5, 6, 7, **d): pass class A(0, *range(1, 8), **d, foo='bar'): pass self.assertEqual(A, (tuple(range(8)), {'foo': 'bar'})) + def testClassCallRecursionLimit(self): + class C: + def __init__(self): + self.c = C() + + with self.assertRaises(RecursionError): + C() + + def add_one_level(): + #Each call to C() consumes 2 levels, so offset by 1. + C() + + with self.assertRaises(RecursionError): + add_one_level() + + def testMetaclassCallOptimization(self): + calls = 0 + + class TypeMetaclass(type): + def __call__(cls, *args, **kwargs): + nonlocal calls + calls += 1 + return type.__call__(cls, *args, **kwargs) + + class Type(metaclass=TypeMetaclass): + def __init__(self, obj): + self._obj = obj + + for i in range(100): + Type(i) + self.assertEqual(calls, 100) + +from _testinternalcapi import has_inline_values + +Py_TPFLAGS_MANAGED_DICT = (1 << 2) + +class Plain: + pass + + +class WithAttrs: + + def __init__(self): + self.a = 1 + self.b = 2 + self.c = 3 + self.d = 4 + + +class TestInlineValues(unittest.TestCase): + + def test_flags(self): + self.assertEqual(Plain.__flags__ & Py_TPFLAGS_MANAGED_DICT, Py_TPFLAGS_MANAGED_DICT) + self.assertEqual(WithAttrs.__flags__ & Py_TPFLAGS_MANAGED_DICT, Py_TPFLAGS_MANAGED_DICT) + def test_has_inline_values(self): + c = Plain() + self.assertTrue(has_inline_values(c)) + del c.__dict__ + self.assertFalse(has_inline_values(c)) + + def test_instances(self): + self.assertTrue(has_inline_values(Plain())) + self.assertTrue(has_inline_values(WithAttrs())) + def test_inspect_dict(self): + for cls in (Plain, WithAttrs): + c = cls() + c.__dict__ + self.assertTrue(has_inline_values(c)) + def test_update_dict(self): + d = { "e": 5, "f": 6 } + for cls in (Plain, WithAttrs): + c = cls() + c.__dict__.update(d) + self.assertTrue(has_inline_values(c)) + + @staticmethod + def set_100(obj): + for i in range(100): + setattr(obj, f"a{i}", i) + + def check_100(self, obj): + for i in range(100): + self.assertEqual(getattr(obj, f"a{i}"), i) + def test_many_attributes(self): + class C: pass + c = C() + self.assertTrue(has_inline_values(c)) + self.set_100(c) + self.assertFalse(has_inline_values(c)) + self.check_100(c) + c = C() + self.assertTrue(has_inline_values(c)) + def test_many_attributes_with_dict(self): + class C: pass + c = C() + d = c.__dict__ + self.assertTrue(has_inline_values(c)) + self.set_100(c) + self.assertFalse(has_inline_values(c)) + self.check_100(c) + + def test_bug_117750(self): + "Aborted on 3.13a6" + class C: + def __init__(self): + self.__dict__.clear() + + obj = C() + self.assertEqual(obj.__dict__, {}) + obj.foo = None # Aborted here + self.assertEqual(obj.__dict__, {"foo":None}) + + def test_store_attr_deleted_dict(self): + class Foo: + pass + + f = Foo() + del f.__dict__ + f.a = 3 + self.assertEqual(f.a, 3) + + def test_rematerialize_object_dict(self): + # gh-121860: rematerializing an object's managed dictionary after it + # had been deleted caused a crash. + class Foo: pass + f = Foo() + f.__dict__["attr"] = 1 + del f.__dict__ + + # Using a str subclass is a way to trigger the re-materialization + class StrSubclass(str): pass + self.assertFalse(hasattr(f, StrSubclass("attr"))) + + # Changing the __class__ also triggers the re-materialization + class Bar: pass + f.__class__ = Bar + self.assertIsInstance(f, Bar) + self.assertEqual(f.__dict__, {}) + + def test_store_attr_type_cache(self): + """Verifies that the type cache doesn't provide a value which is + inconsistent from the dict.""" + class X: + def __del__(inner_self): + v = C.a + self.assertEqual(v, C.__dict__['a']) + + class C: + a = X() + + # prime the cache + C.a + C.a + + # destructor shouldn't be able to see inconsistent state + C.a = X() + C.a = X() + def test_detach_materialized_dict_no_memory(self): + # Skip test if _testcapi is not available: + import_helper.import_module('_testcapi') + + code = """if 1: + import test.support + import _testcapi + + class A: + def __init__(self): + self.a = 1 + self.b = 2 + a = A() + d = a.__dict__ + with test.support.catch_unraisable_exception() as ex: + _testcapi.set_nomemory(0, 1) + del a + assert ex.unraisable.exc_type is MemoryError + try: + d["a"] + except KeyError: + pass + else: + assert False, "KeyError not raised" + """ + rc, out, err = script_helper.assert_python_ok("-c", code) + self.assertEqual(rc, 0) + self.assertFalse(out, msg=out.decode('utf-8')) + self.assertFalse(err, msg=err.decode('utf-8')) if __name__ == '__main__': unittest.main() From 4081c08b5aaee7962299a7d3b89993bf5444733d Mon Sep 17 00:00:00 2001 From: Myeongseon Choi Date: Sat, 15 Mar 2025 18:44:58 +0900 Subject: [PATCH 095/295] add cpython_only tag for tests required `_testlimitedcapi` --- Lib/test/test_class.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_class.py b/Lib/test/test_class.py index c48726d44b..8cccb1c705 100644 --- a/Lib/test/test_class.py +++ b/Lib/test/test_class.py @@ -445,6 +445,7 @@ def __delattr__(self, *args): del testme.cardinal self.assertCallStack([('__delattr__', (testme, "cardinal"))]) + @cpython_only def testHasAttrString(self): import sys from test.support import import_helper @@ -846,8 +847,10 @@ def __init__(self, obj): Type(i) self.assertEqual(calls, 100) - -from _testinternalcapi import has_inline_values +try: + from _testinternalcapi import has_inline_values +except ImportError: + has_inline_values = None Py_TPFLAGS_MANAGED_DICT = (1 << 2) @@ -869,20 +872,27 @@ class TestInlineValues(unittest.TestCase): def test_flags(self): self.assertEqual(Plain.__flags__ & Py_TPFLAGS_MANAGED_DICT, Py_TPFLAGS_MANAGED_DICT) self.assertEqual(WithAttrs.__flags__ & Py_TPFLAGS_MANAGED_DICT, Py_TPFLAGS_MANAGED_DICT) + + @cpython_only def test_has_inline_values(self): c = Plain() self.assertTrue(has_inline_values(c)) del c.__dict__ self.assertFalse(has_inline_values(c)) + @cpython_only def test_instances(self): self.assertTrue(has_inline_values(Plain())) self.assertTrue(has_inline_values(WithAttrs())) + + @cpython_only def test_inspect_dict(self): for cls in (Plain, WithAttrs): c = cls() c.__dict__ self.assertTrue(has_inline_values(c)) + + @cpython_only def test_update_dict(self): d = { "e": 5, "f": 6 } for cls in (Plain, WithAttrs): @@ -898,6 +908,8 @@ def set_100(obj): def check_100(self, obj): for i in range(100): self.assertEqual(getattr(obj, f"a{i}"), i) + + @cpython_only def test_many_attributes(self): class C: pass c = C() @@ -907,6 +919,8 @@ class C: pass self.check_100(c) c = C() self.assertTrue(has_inline_values(c)) + + @cpython_only def test_many_attributes_with_dict(self): class C: pass c = C() @@ -972,6 +986,8 @@ class C: # destructor shouldn't be able to see inconsistent state C.a = X() C.a = X() + + @cpython_only def test_detach_materialized_dict_no_memory(self): # Skip test if _testcapi is not available: import_helper.import_module('_testcapi') @@ -1001,5 +1017,6 @@ def __init__(self): self.assertEqual(rc, 0) self.assertFalse(out, msg=out.decode('utf-8')) self.assertFalse(err, msg=err.decode('utf-8')) + if __name__ == '__main__': unittest.main() From 2d3b125d511427dbdbe1d6706d71c566327c0692 Mon Sep 17 00:00:00 2001 From: Myeongseon Choi Date: Sat, 15 Mar 2025 18:53:24 +0900 Subject: [PATCH 096/295] Add expectedFailure and skip decorators for RUSTPYTHON tests in test_class.py --- Lib/test/test_class.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Lib/test/test_class.py b/Lib/test/test_class.py index 8cccb1c705..3288e011a3 100644 --- a/Lib/test/test_class.py +++ b/Lib/test/test_class.py @@ -869,6 +869,8 @@ def __init__(self): class TestInlineValues(unittest.TestCase): + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_flags(self): self.assertEqual(Plain.__flags__ & Py_TPFLAGS_MANAGED_DICT, Py_TPFLAGS_MANAGED_DICT) self.assertEqual(WithAttrs.__flags__ & Py_TPFLAGS_MANAGED_DICT, Py_TPFLAGS_MANAGED_DICT) @@ -941,6 +943,8 @@ def __init__(self): obj.foo = None # Aborted here self.assertEqual(obj.__dict__, {"foo":None}) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_store_attr_deleted_dict(self): class Foo: pass @@ -950,6 +954,8 @@ class Foo: f.a = 3 self.assertEqual(f.a, 3) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_rematerialize_object_dict(self): # gh-121860: rematerializing an object's managed dictionary after it # had been deleted caused a crash. @@ -968,6 +974,7 @@ class Bar: pass self.assertIsInstance(f, Bar) self.assertEqual(f.__dict__, {}) + @unittest.skip("TODO: RUSTPYTHON, unexpectedly long runtime") def test_store_attr_type_cache(self): """Verifies that the type cache doesn't provide a value which is inconsistent from the dict.""" From 70a5774737cdbd1144e7d853616168ad3cb21600 Mon Sep 17 00:00:00 2001 From: Myeongseon Choi Date: Sat, 15 Mar 2025 22:43:12 +0900 Subject: [PATCH 097/295] Update CPYTHON_SPECIFIC_MODS to include '_testlimitedcapi' --- whats_left.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/whats_left.py b/whats_left.py index 30b1de088e..7c1c30ba6c 100755 --- a/whats_left.py +++ b/whats_left.py @@ -97,8 +97,8 @@ def parse_args(): # CPython specific modules (mostly consisting of templates/tests) CPYTHON_SPECIFIC_MODS = { - 'xxmodule', 'xxsubtype', 'xxlimited', '_xxtestfuzz' - '_testbuffer', '_testcapi', '_testimportmultiple', '_testinternalcapi', '_testmultiphase', + 'xxmodule', 'xxsubtype', 'xxlimited', '_xxtestfuzz', + '_testbuffer', '_testcapi', '_testimportmultiple', '_testinternalcapi', '_testmultiphase', '_testlimitedcapi' } IGNORED_MODULES = {"this", "antigravity"} | PEP_594_MODULES | CPYTHON_SPECIFIC_MODS From a9bfaa96c5ed7a5cf050ebbc695399df2a6be015 Mon Sep 17 00:00:00 2001 From: Myeongseon Choi Date: Mon, 17 Mar 2025 00:33:30 +0900 Subject: [PATCH 098/295] remove unnecessary cpython_only --- Lib/test/test_class.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/test_class.py b/Lib/test/test_class.py index 3288e011a3..662de752b7 100644 --- a/Lib/test/test_class.py +++ b/Lib/test/test_class.py @@ -445,7 +445,6 @@ def __delattr__(self, *args): del testme.cardinal self.assertCallStack([('__delattr__', (testme, "cardinal"))]) - @cpython_only def testHasAttrString(self): import sys from test.support import import_helper From 0717d5a33146136727ac315deea3f967fc2c00f6 Mon Sep 17 00:00:00 2001 From: Myeongseon Choi Date: Mon, 17 Mar 2025 01:08:05 +0900 Subject: [PATCH 099/295] remove wrong cpython_only tag --- Lib/test/test_class.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_class.py b/Lib/test/test_class.py index 662de752b7..29215f0600 100644 --- a/Lib/test/test_class.py +++ b/Lib/test/test_class.py @@ -874,26 +874,30 @@ def test_flags(self): self.assertEqual(Plain.__flags__ & Py_TPFLAGS_MANAGED_DICT, Py_TPFLAGS_MANAGED_DICT) self.assertEqual(WithAttrs.__flags__ & Py_TPFLAGS_MANAGED_DICT, Py_TPFLAGS_MANAGED_DICT) - @cpython_only + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_has_inline_values(self): c = Plain() self.assertTrue(has_inline_values(c)) del c.__dict__ self.assertFalse(has_inline_values(c)) - @cpython_only + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_instances(self): self.assertTrue(has_inline_values(Plain())) self.assertTrue(has_inline_values(WithAttrs())) - @cpython_only + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_inspect_dict(self): for cls in (Plain, WithAttrs): c = cls() c.__dict__ self.assertTrue(has_inline_values(c)) - @cpython_only + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_update_dict(self): d = { "e": 5, "f": 6 } for cls in (Plain, WithAttrs): @@ -910,7 +914,8 @@ def check_100(self, obj): for i in range(100): self.assertEqual(getattr(obj, f"a{i}"), i) - @cpython_only + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_many_attributes(self): class C: pass c = C() @@ -921,7 +926,8 @@ class C: pass c = C() self.assertTrue(has_inline_values(c)) - @cpython_only + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_many_attributes_with_dict(self): class C: pass c = C() From 948368fdb471eaec88498d9307e9f1e72c7f6e60 Mon Sep 17 00:00:00 2001 From: Noa Date: Wed, 19 Mar 2025 22:54:27 -0500 Subject: [PATCH 100/295] Try fix windows long paths error --- .github/workflows/ci.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 73b5cc1cd6..1dac8d47c7 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -132,6 +132,7 @@ jobs: - name: Set up the Windows environment shell: bash run: | + git config --system core.longpaths true cargo install --target-dir=target -v cargo-vcpkg cargo vcpkg -v build if: runner.os == 'Windows' @@ -255,6 +256,7 @@ jobs: - name: Set up the Windows environment shell: bash run: | + git config --system core.longpaths true cargo install cargo-vcpkg cargo vcpkg build if: runner.os == 'Windows' From c585678ec925f2da6fdf224eb8ced93d65c82a90 Mon Sep 17 00:00:00 2001 From: Noa Date: Sat, 22 Mar 2025 19:27:13 -0500 Subject: [PATCH 101/295] Merge format and literal back into this repo (and update malachite) (#5618) * Merge format and literal back into this repo * Update format and literal to work * Update malachite * Remove RustPython/Parser from Cargo.toml --- Cargo.lock | 61 +- Cargo.toml | 24 +- common/Cargo.toml | 4 +- common/src/cformat.rs | 1044 +++++++++++++++++++++++++ common/src/format.rs | 1328 ++++++++++++++++++++++++++++++++ common/src/lib.rs | 2 + common/src/str.rs | 2 +- compiler/literal/Cargo.toml | 19 + compiler/literal/src/char.rs | 15 + compiler/literal/src/escape.rs | 441 +++++++++++ compiler/literal/src/float.rs | 342 ++++++++ compiler/literal/src/format.rs | 5 + compiler/literal/src/lib.rs | 4 + deny.toml | 2 +- vm/Cargo.toml | 1 - vm/src/builtins/bool.rs | 2 +- vm/src/builtins/float.rs | 3 +- vm/src/builtins/int.rs | 2 +- vm/src/builtins/str.rs | 2 +- vm/src/cformat.rs | 2 +- vm/src/format.rs | 2 +- vm/src/stdlib/string.rs | 6 +- 22 files changed, 3240 insertions(+), 73 deletions(-) create mode 100644 common/src/cformat.rs create mode 100644 common/src/format.rs create mode 100644 compiler/literal/Cargo.toml create mode 100644 compiler/literal/src/char.rs create mode 100644 compiler/literal/src/escape.rs create mode 100644 compiler/literal/src/float.rs create mode 100644 compiler/literal/src/format.rs create mode 100644 compiler/literal/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index c8106856e6..94ae260f94 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -787,6 +787,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + [[package]] name = "foreign-types" version = "0.3.2" @@ -896,15 +902,15 @@ name = "hashbrown" version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" -dependencies = [ - "ahash", -] [[package]] name = "hashbrown" version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" +dependencies = [ + "foldhash", +] [[package]] name = "heck" @@ -1255,24 +1261,13 @@ dependencies = [ "libc", ] -[[package]] -name = "malachite" -version = "0.4.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fbdf9cb251732db30a7200ebb6ae5d22fe8e11397364416617d2c2cf0c51cb5" -dependencies = [ - "malachite-base", - "malachite-nz", - "malachite-q", -] - [[package]] name = "malachite-base" -version = "0.4.22" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ea0ed76adf7defc1a92240b5c36d5368cfe9251640dcce5bd2d0b7c1fd87aeb" +checksum = "5063891d2cec8fd20cabccbd3fc277bf8d5666f481fb3f79d999559b39a62713" dependencies = [ - "hashbrown 0.14.5", + "hashbrown 0.15.2", "itertools 0.11.0", "libm", "ryu", @@ -1280,12 +1275,13 @@ dependencies = [ [[package]] name = "malachite-bigint" -version = "0.2.3" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d149aaa2965d70381709d9df4c7ee1fc0de1c614a4efc2ee356f5e43d68749f8" +checksum = "7b1b1fec8b370139968919a5b77071c94b282eaba3da1cf179ae5299060d4e75" dependencies = [ "derive_more", - "malachite", + "malachite-base", + "malachite-nz", "num-integer", "num-traits", "paste", @@ -1293,9 +1289,9 @@ dependencies = [ [[package]] name = "malachite-nz" -version = "0.4.22" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34a79feebb2bc9aa7762047c8e5495269a367da6b5a90a99882a0aeeac1841f7" +checksum = "175263cd5b846c552b9afb9d4b03ca465b4ad10717d789cad7dac24441c4fcbe" dependencies = [ "itertools 0.11.0", "libm", @@ -1304,9 +1300,9 @@ dependencies = [ [[package]] name = "malachite-q" -version = "0.4.22" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50f235d5747b1256b47620f5640c2a17a88c7569eebdf27cd9cb130e1a619191" +checksum = "5261ba8feb1ad20cddab3d625af28206c663c08014b2e5c5f9bd54b0f230234f" dependencies = [ "itertools 0.11.0", "malachite-base", @@ -2152,7 +2148,7 @@ dependencies = [ "parking_lot", "radium", "rand 0.9.0", - "rustpython-format", + "rustpython-literal", "siphasher 0.3.11", "volatile", "widestring", @@ -2229,18 +2225,6 @@ dependencies = [ "once_cell", ] -[[package]] -name = "rustpython-format" -version = "0.4.0" -source = "git+https://github.com/RustPython/Parser.git?rev=d2f137b372ec08ce4a243564a80f8f9153c45a23#d2f137b372ec08ce4a243564a80f8f9153c45a23" -dependencies = [ - "bitflags 2.8.0", - "itertools 0.11.0", - "malachite-bigint", - "num-traits", - "rustpython-literal", -] - [[package]] name = "rustpython-jit" version = "0.4.0" @@ -2259,12 +2243,12 @@ dependencies = [ [[package]] name = "rustpython-literal" version = "0.4.0" -source = "git+https://github.com/RustPython/Parser.git?rev=d2f137b372ec08ce4a243564a80f8f9153c45a23#d2f137b372ec08ce4a243564a80f8f9153c45a23" dependencies = [ "hexf-parse", "is-macro", "lexical-parse-float", "num-traits", + "rand 0.9.0", "unic-ucd-category", ] @@ -2416,7 +2400,6 @@ dependencies = [ "rustpython-compiler-core", "rustpython-compiler-source", "rustpython-derive", - "rustpython-format", "rustpython-jit", "rustpython-literal", "rustpython-sre_engine", diff --git a/Cargo.toml b/Cargo.toml index 469f895c7d..1da72c9dc5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -94,7 +94,7 @@ x86_64-pc-windows-msvc = { triplet = "x64-windows-static-md", dev-dependencies = [workspace] resolver = "2" members = [ - "compiler", "compiler/core", "compiler/codegen", "compiler/source", + "compiler", "compiler/core", "compiler/codegen", "compiler/literal", "compiler/source", ".", "common", "derive", "jit", "vm", "vm/sre_engine", "pylib", "stdlib", "derive-impl", "wasm/lib", ] @@ -116,6 +116,7 @@ rustpython-common = { path = "common", version = "0.4.0" } rustpython-derive = { path = "derive", version = "0.4.0" } rustpython-derive-impl = { path = "derive-impl", version = "0.4.0" } rustpython-jit = { path = "jit", version = "0.4.0" } +rustpython-literal = { path = "compiler/literal", version = "0.4.0" } rustpython-vm = { path = "vm", default-features = false, version = "0.4.0" } rustpython-pylib = { path = "pylib", version = "0.4.0" } rustpython-stdlib = { path = "stdlib", default-features = false, version = "0.4.0" } @@ -127,21 +128,6 @@ ruff_python_ast = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0 ruff_text_size = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" } ruff_source_file = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" } ruff_python_codegen = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" } -# rustpython-literal = { version = "0.4.0" } -# rustpython-parser-core = { version = "0.4.0" } -# rustpython-parser = { version = "0.4.0" } -# rustpython-ast = { version = "0.4.0" } -# rustpython-format= { version = "0.4.0" } -rustpython-literal = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "d2f137b372ec08ce4a243564a80f8f9153c45a23" } -# rustpython-parser-core = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "d2f137b372ec08ce4a243564a80f8f9153c45a23" } -# rustpython-parser = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "d2f137b372ec08ce4a243564a80f8f9153c45a23" } -# rustpython-ast = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "d2f137b372ec08ce4a243564a80f8f9153c45a23" } -rustpython-format = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "d2f137b372ec08ce4a243564a80f8f9153c45a23" } -# rustpython-literal = { path = "../RustPython-parser/literal" } -# rustpython-parser-core = { path = "../RustPython-parser/core" } -# rustpython-parser = { path = "../RustPython-parser/parser" } -# rustpython-ast = { path = "../RustPython-parser/ast" } -# rustpython-format = { path = "../RustPython-parser/format" } ahash = "0.8.11" ascii = "1.1" @@ -163,9 +149,9 @@ junction = "1.2.0" libc = "0.2.169" log = "0.4.25" nix = { version = "0.29", features = ["fs", "user", "process", "term", "time", "signal", "ioctl", "socket", "sched", "zerocopy", "dir", "hostname", "net", "poll"] } -malachite-bigint = "0.2.3" -malachite-q = "0.4.22" -malachite-base = "0.4.22" +malachite-bigint = "0.5" +malachite-q = "0.5" +malachite-base = "0.5" memchr = "2.7.4" num-complex = "0.4.6" num-integer = "0.1.46" diff --git a/common/Cargo.toml b/common/Cargo.toml index 8ba07f4be8..589170064c 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -12,7 +12,7 @@ license.workspace = true threading = ["parking_lot"] [dependencies] -rustpython-format = { workspace = true } +rustpython-literal = { workspace = true } ascii = { workspace = true } bitflags = { workspace = true } @@ -45,4 +45,4 @@ windows-sys = { workspace = true, features = [ ] } [lints] -workspace = true \ No newline at end of file +workspace = true diff --git a/common/src/cformat.rs b/common/src/cformat.rs new file mode 100644 index 0000000000..cd282eb4cb --- /dev/null +++ b/common/src/cformat.rs @@ -0,0 +1,1044 @@ +//! Implementation of Printf-Style string formatting +//! as per the [Python Docs](https://docs.python.org/3/library/stdtypes.html#printf-style-string-formatting). +use bitflags::bitflags; +use malachite_bigint::{BigInt, Sign}; +use num_traits::Signed; +use rustpython_literal::{float, format::Case}; +use std::{ + cmp, fmt, + iter::{Enumerate, Peekable}, + str::FromStr, +}; + +#[derive(Debug, PartialEq)] +pub enum CFormatErrorType { + UnmatchedKeyParentheses, + MissingModuloSign, + UnsupportedFormatChar(char), + IncompleteFormat, + IntTooBig, + // Unimplemented, +} + +// also contains how many chars the parsing function consumed +pub type ParsingError = (CFormatErrorType, usize); + +#[derive(Debug, PartialEq)] +pub struct CFormatError { + pub typ: CFormatErrorType, // FIXME + pub index: usize, +} + +impl fmt::Display for CFormatError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + use CFormatErrorType::*; + match self.typ { + UnmatchedKeyParentheses => write!(f, "incomplete format key"), + IncompleteFormat => write!(f, "incomplete format"), + UnsupportedFormatChar(c) => write!( + f, + "unsupported format character '{}' ({:#x}) at index {}", + c, c as u32, self.index + ), + IntTooBig => write!(f, "width/precision too big"), + _ => write!(f, "unexpected error parsing format string"), + } + } +} + +pub type CFormatConversion = super::format::FormatConversion; + +#[derive(Debug, PartialEq)] +pub enum CNumberType { + Decimal, + Octal, + Hex(Case), +} + +#[derive(Debug, PartialEq)] +pub enum CFloatType { + Exponent(Case), + PointDecimal(Case), + General(Case), +} + +#[derive(Debug, PartialEq)] +pub enum CFormatType { + Number(CNumberType), + Float(CFloatType), + Character, + String(CFormatConversion), +} + +#[derive(Debug, PartialEq)] +pub enum CFormatPrecision { + Quantity(CFormatQuantity), + Dot, +} + +impl From for CFormatPrecision { + fn from(quantity: CFormatQuantity) -> Self { + CFormatPrecision::Quantity(quantity) + } +} + +bitflags! { + #[derive(Copy, Clone, Debug, PartialEq)] + pub struct CConversionFlags: u32 { + const ALTERNATE_FORM = 0b0000_0001; + const ZERO_PAD = 0b0000_0010; + const LEFT_ADJUST = 0b0000_0100; + const BLANK_SIGN = 0b0000_1000; + const SIGN_CHAR = 0b0001_0000; + } +} + +impl CConversionFlags { + #[inline] + pub fn sign_string(&self) -> &'static str { + if self.contains(CConversionFlags::SIGN_CHAR) { + "+" + } else if self.contains(CConversionFlags::BLANK_SIGN) { + " " + } else { + "" + } + } +} + +#[derive(Debug, PartialEq)] +pub enum CFormatQuantity { + Amount(usize), + FromValuesTuple, +} + +#[derive(Debug, PartialEq)] +pub struct CFormatSpec { + pub mapping_key: Option, + pub flags: CConversionFlags, + pub min_field_width: Option, + pub precision: Option, + pub format_type: CFormatType, + pub format_char: char, + // chars_consumed: usize, +} + +impl FromStr for CFormatSpec { + type Err = ParsingError; + + fn from_str(text: &str) -> Result { + let mut chars = text.chars().enumerate().peekable(); + if chars.next().map(|x| x.1) != Some('%') { + return Err((CFormatErrorType::MissingModuloSign, 1)); + } + + CFormatSpec::parse(&mut chars) + } +} + +pub type ParseIter = Peekable>; + +impl CFormatSpec { + pub fn parse(iter: &mut ParseIter) -> Result + where + T: Into + Copy, + I: Iterator, + { + let mapping_key = parse_spec_mapping_key(iter)?; + let flags = parse_flags(iter); + let min_field_width = parse_quantity(iter)?; + let precision = parse_precision(iter)?; + consume_length(iter); + let (format_type, format_char) = parse_format_type(iter)?; + + Ok(CFormatSpec { + mapping_key, + flags, + min_field_width, + precision, + format_type, + format_char, + }) + } + + fn compute_fill_string(fill_char: char, fill_chars_needed: usize) -> String { + (0..fill_chars_needed) + .map(|_| fill_char) + .collect::() + } + + fn fill_string( + &self, + string: String, + fill_char: char, + num_prefix_chars: Option, + ) -> String { + let mut num_chars = string.chars().count(); + if let Some(num_prefix_chars) = num_prefix_chars { + num_chars += num_prefix_chars; + } + let num_chars = num_chars; + + let width = match &self.min_field_width { + Some(CFormatQuantity::Amount(width)) => cmp::max(width, &num_chars), + _ => &num_chars, + }; + let fill_chars_needed = width.saturating_sub(num_chars); + let fill_string = CFormatSpec::compute_fill_string(fill_char, fill_chars_needed); + + if !fill_string.is_empty() { + if self.flags.contains(CConversionFlags::LEFT_ADJUST) { + format!("{string}{fill_string}") + } else { + format!("{fill_string}{string}") + } + } else { + string + } + } + + fn fill_string_with_precision(&self, string: String, fill_char: char) -> String { + let num_chars = string.chars().count(); + + let width = match &self.precision { + Some(CFormatPrecision::Quantity(CFormatQuantity::Amount(width))) => { + cmp::max(width, &num_chars) + } + _ => &num_chars, + }; + let fill_chars_needed = width.saturating_sub(num_chars); + let fill_string = CFormatSpec::compute_fill_string(fill_char, fill_chars_needed); + + if !fill_string.is_empty() { + // Don't left-adjust if precision-filling: that will always be prepending 0s to %d + // arguments, the LEFT_ADJUST flag will be used by a later call to fill_string with + // the 0-filled string as the string param. + format!("{fill_string}{string}") + } else { + string + } + } + + fn format_string_with_precision( + &self, + string: String, + precision: Option<&CFormatPrecision>, + ) -> String { + // truncate if needed + let string = match precision { + Some(CFormatPrecision::Quantity(CFormatQuantity::Amount(precision))) + if string.chars().count() > *precision => + { + string.chars().take(*precision).collect::() + } + Some(CFormatPrecision::Dot) => { + // truncate to 0 + String::new() + } + _ => string, + }; + self.fill_string(string, ' ', None) + } + + #[inline] + pub fn format_string(&self, string: String) -> String { + self.format_string_with_precision(string, self.precision.as_ref()) + } + + #[inline] + pub fn format_char(&self, ch: char) -> String { + self.format_string_with_precision( + ch.to_string(), + Some(&(CFormatQuantity::Amount(1).into())), + ) + } + + pub fn format_bytes(&self, bytes: &[u8]) -> Vec { + let bytes = if let Some(CFormatPrecision::Quantity(CFormatQuantity::Amount(precision))) = + self.precision + { + &bytes[..cmp::min(bytes.len(), precision)] + } else { + bytes + }; + if let Some(CFormatQuantity::Amount(width)) = self.min_field_width { + let fill = cmp::max(0, width - bytes.len()); + let mut v = Vec::with_capacity(bytes.len() + fill); + if self.flags.contains(CConversionFlags::LEFT_ADJUST) { + v.extend_from_slice(bytes); + v.append(&mut vec![b' '; fill]); + } else { + v.append(&mut vec![b' '; fill]); + v.extend_from_slice(bytes); + } + v + } else { + bytes.to_vec() + } + } + + pub fn format_number(&self, num: &BigInt) -> String { + use CNumberType::*; + let magnitude = num.abs(); + let prefix = if self.flags.contains(CConversionFlags::ALTERNATE_FORM) { + match self.format_type { + CFormatType::Number(Octal) => "0o", + CFormatType::Number(Hex(Case::Lower)) => "0x", + CFormatType::Number(Hex(Case::Upper)) => "0X", + _ => "", + } + } else { + "" + }; + + let magnitude_string: String = match self.format_type { + CFormatType::Number(Decimal) => magnitude.to_str_radix(10), + CFormatType::Number(Octal) => magnitude.to_str_radix(8), + CFormatType::Number(Hex(Case::Lower)) => magnitude.to_str_radix(16), + CFormatType::Number(Hex(Case::Upper)) => { + let mut result = magnitude.to_str_radix(16); + result.make_ascii_uppercase(); + result + } + _ => unreachable!(), // Should not happen because caller has to make sure that this is a number + }; + + let sign_string = match num.sign() { + Sign::Minus => "-", + _ => self.flags.sign_string(), + }; + + let padded_magnitude_string = self.fill_string_with_precision(magnitude_string, '0'); + + if self.flags.contains(CConversionFlags::ZERO_PAD) { + let fill_char = if !self.flags.contains(CConversionFlags::LEFT_ADJUST) { + '0' + } else { + ' ' // '-' overrides the '0' conversion if both are given + }; + let signed_prefix = format!("{sign_string}{prefix}"); + format!( + "{}{}", + signed_prefix, + self.fill_string( + padded_magnitude_string, + fill_char, + Some(signed_prefix.chars().count()), + ), + ) + } else { + self.fill_string( + format!("{sign_string}{prefix}{padded_magnitude_string}"), + ' ', + None, + ) + } + } + + pub fn format_float(&self, num: f64) -> String { + let sign_string = if num.is_sign_negative() && !num.is_nan() { + "-" + } else { + self.flags.sign_string() + }; + + let precision = match &self.precision { + Some(CFormatPrecision::Quantity(quantity)) => match quantity { + CFormatQuantity::Amount(amount) => *amount, + CFormatQuantity::FromValuesTuple => 6, + }, + Some(CFormatPrecision::Dot) => 0, + None => 6, + }; + + let magnitude_string = match &self.format_type { + CFormatType::Float(CFloatType::PointDecimal(case)) => { + let magnitude = num.abs(); + float::format_fixed( + precision, + magnitude, + *case, + self.flags.contains(CConversionFlags::ALTERNATE_FORM), + ) + } + CFormatType::Float(CFloatType::Exponent(case)) => { + let magnitude = num.abs(); + float::format_exponent( + precision, + magnitude, + *case, + self.flags.contains(CConversionFlags::ALTERNATE_FORM), + ) + } + CFormatType::Float(CFloatType::General(case)) => { + let precision = if precision == 0 { 1 } else { precision }; + let magnitude = num.abs(); + float::format_general( + precision, + magnitude, + *case, + self.flags.contains(CConversionFlags::ALTERNATE_FORM), + false, + ) + } + _ => unreachable!(), + }; + + if self.flags.contains(CConversionFlags::ZERO_PAD) { + let fill_char = if !self.flags.contains(CConversionFlags::LEFT_ADJUST) { + '0' + } else { + ' ' + }; + format!( + "{}{}", + sign_string, + self.fill_string( + magnitude_string, + fill_char, + Some(sign_string.chars().count()), + ) + ) + } else { + self.fill_string(format!("{sign_string}{magnitude_string}"), ' ', None) + } + } +} + +fn parse_spec_mapping_key(iter: &mut ParseIter) -> Result, ParsingError> +where + T: Into + Copy, + I: Iterator, +{ + if let Some(&(index, c)) = iter.peek() { + if c.into() == '(' { + iter.next().unwrap(); + return match parse_text_inside_parentheses(iter) { + Some(key) => Ok(Some(key)), + None => Err((CFormatErrorType::UnmatchedKeyParentheses, index)), + }; + } + } + Ok(None) +} + +fn parse_flags(iter: &mut ParseIter) -> CConversionFlags +where + T: Into + Copy, + I: Iterator, +{ + let mut flags = CConversionFlags::empty(); + while let Some(&(_, c)) = iter.peek() { + let flag = match c.into() { + '#' => CConversionFlags::ALTERNATE_FORM, + '0' => CConversionFlags::ZERO_PAD, + '-' => CConversionFlags::LEFT_ADJUST, + ' ' => CConversionFlags::BLANK_SIGN, + '+' => CConversionFlags::SIGN_CHAR, + _ => break, + }; + iter.next().unwrap(); + flags |= flag; + } + flags +} + +fn consume_length(iter: &mut ParseIter) +where + T: Into + Copy, + I: Iterator, +{ + if let Some(&(_, c)) = iter.peek() { + let c = c.into(); + if c == 'h' || c == 'l' || c == 'L' { + iter.next().unwrap(); + } + } +} + +fn parse_format_type(iter: &mut ParseIter) -> Result<(CFormatType, char), ParsingError> +where + T: Into, + I: Iterator, +{ + use CFloatType::*; + use CNumberType::*; + let (index, c) = match iter.next() { + Some((index, c)) => (index, c.into()), + None => { + return Err(( + CFormatErrorType::IncompleteFormat, + iter.peek().map(|x| x.0).unwrap_or(0), + )); + } + }; + let format_type = match c { + 'd' | 'i' | 'u' => CFormatType::Number(Decimal), + 'o' => CFormatType::Number(Octal), + 'x' => CFormatType::Number(Hex(Case::Lower)), + 'X' => CFormatType::Number(Hex(Case::Upper)), + 'e' => CFormatType::Float(Exponent(Case::Lower)), + 'E' => CFormatType::Float(Exponent(Case::Upper)), + 'f' => CFormatType::Float(PointDecimal(Case::Lower)), + 'F' => CFormatType::Float(PointDecimal(Case::Upper)), + 'g' => CFormatType::Float(General(Case::Lower)), + 'G' => CFormatType::Float(General(Case::Upper)), + 'c' => CFormatType::Character, + 'r' => CFormatType::String(CFormatConversion::Repr), + 's' => CFormatType::String(CFormatConversion::Str), + 'b' => CFormatType::String(CFormatConversion::Bytes), + 'a' => CFormatType::String(CFormatConversion::Ascii), + _ => return Err((CFormatErrorType::UnsupportedFormatChar(c), index)), + }; + Ok((format_type, c)) +} + +fn parse_quantity(iter: &mut ParseIter) -> Result, ParsingError> +where + T: Into + Copy, + I: Iterator, +{ + if let Some(&(_, c)) = iter.peek() { + let c: char = c.into(); + if c == '*' { + iter.next().unwrap(); + return Ok(Some(CFormatQuantity::FromValuesTuple)); + } + if let Some(i) = c.to_digit(10) { + let mut num = i as i32; + iter.next().unwrap(); + while let Some(&(index, c)) = iter.peek() { + if let Some(i) = c.into().to_digit(10) { + num = num + .checked_mul(10) + .and_then(|num| num.checked_add(i as i32)) + .ok_or((CFormatErrorType::IntTooBig, index))?; + iter.next().unwrap(); + } else { + break; + } + } + return Ok(Some(CFormatQuantity::Amount(num.unsigned_abs() as usize))); + } + } + Ok(None) +} + +fn parse_precision(iter: &mut ParseIter) -> Result, ParsingError> +where + T: Into + Copy, + I: Iterator, +{ + if let Some(&(_, c)) = iter.peek() { + if c.into() == '.' { + iter.next().unwrap(); + let quantity = parse_quantity(iter)?; + let precision = quantity.map_or(CFormatPrecision::Dot, CFormatPrecision::Quantity); + return Ok(Some(precision)); + } + } + Ok(None) +} + +fn parse_text_inside_parentheses(iter: &mut ParseIter) -> Option +where + T: Into, + I: Iterator, +{ + let mut counter: i32 = 1; + let mut contained_text = String::new(); + loop { + let (_, c) = iter.next()?; + let c = c.into(); + match c { + _ if c == '(' => { + counter += 1; + } + _ if c == ')' => { + counter -= 1; + } + _ => (), + } + + if counter > 0 { + contained_text.push(c); + } else { + break; + } + } + + Some(contained_text) +} + +#[derive(Debug, PartialEq)] +pub enum CFormatPart { + Literal(T), + Spec(CFormatSpec), +} + +impl CFormatPart { + #[inline] + pub fn is_specifier(&self) -> bool { + matches!(self, CFormatPart::Spec(_)) + } + + #[inline] + pub fn has_key(&self) -> bool { + match self { + CFormatPart::Spec(s) => s.mapping_key.is_some(), + _ => false, + } + } +} + +#[derive(Debug, PartialEq)] +pub struct CFormatStrOrBytes { + parts: Vec<(usize, CFormatPart)>, +} + +impl CFormatStrOrBytes { + pub fn check_specifiers(&self) -> Option<(usize, bool)> { + let mut count = 0; + let mut mapping_required = false; + for (_, part) in &self.parts { + if part.is_specifier() { + let has_key = part.has_key(); + if count == 0 { + mapping_required = has_key; + } else if mapping_required != has_key { + return None; + } + count += 1; + } + } + Some((count, mapping_required)) + } + + #[inline] + pub fn iter(&self) -> impl Iterator)> { + self.parts.iter() + } + + #[inline] + pub fn iter_mut(&mut self) -> impl Iterator)> { + self.parts.iter_mut() + } +} + +pub type CFormatBytes = CFormatStrOrBytes>; + +impl CFormatBytes { + pub fn parse>(iter: &mut ParseIter) -> Result { + let mut parts = vec![]; + let mut literal = vec![]; + let mut part_index = 0; + while let Some((index, c)) = iter.next() { + if c == b'%' { + if let Some(&(_, second)) = iter.peek() { + if second == b'%' { + iter.next().unwrap(); + literal.push(b'%'); + continue; + } else { + if !literal.is_empty() { + parts.push(( + part_index, + CFormatPart::Literal(std::mem::take(&mut literal)), + )); + } + let spec = CFormatSpec::parse(iter).map_err(|err| CFormatError { + typ: err.0, + index: err.1, + })?; + parts.push((index, CFormatPart::Spec(spec))); + if let Some(&(index, _)) = iter.peek() { + part_index = index; + } + } + } else { + return Err(CFormatError { + typ: CFormatErrorType::IncompleteFormat, + index: index + 1, + }); + } + } else { + literal.push(c); + } + } + if !literal.is_empty() { + parts.push((part_index, CFormatPart::Literal(literal))); + } + Ok(Self { parts }) + } + + pub fn parse_from_bytes(bytes: &[u8]) -> Result { + let mut iter = bytes.iter().cloned().enumerate().peekable(); + Self::parse(&mut iter) + } +} + +pub type CFormatString = CFormatStrOrBytes; + +impl FromStr for CFormatString { + type Err = CFormatError; + + fn from_str(text: &str) -> Result { + let mut iter = text.chars().enumerate().peekable(); + Self::parse(&mut iter) + } +} + +impl CFormatString { + pub(crate) fn parse>( + iter: &mut ParseIter, + ) -> Result { + let mut parts = vec![]; + let mut literal = String::new(); + let mut part_index = 0; + while let Some((index, c)) = iter.next() { + if c == '%' { + if let Some(&(_, second)) = iter.peek() { + if second == '%' { + iter.next().unwrap(); + literal.push('%'); + continue; + } else { + if !literal.is_empty() { + parts.push(( + part_index, + CFormatPart::Literal(std::mem::take(&mut literal)), + )); + } + let spec = CFormatSpec::parse(iter).map_err(|err| CFormatError { + typ: err.0, + index: err.1, + })?; + parts.push((index, CFormatPart::Spec(spec))); + if let Some(&(index, _)) = iter.peek() { + part_index = index; + } + } + } else { + return Err(CFormatError { + typ: CFormatErrorType::IncompleteFormat, + index: index + 1, + }); + } + } else { + literal.push(c); + } + } + if !literal.is_empty() { + parts.push((part_index, CFormatPart::Literal(literal))); + } + Ok(Self { parts }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_fill_and_align() { + assert_eq!( + "%10s" + .parse::() + .unwrap() + .format_string("test".to_owned()), + " test".to_owned() + ); + assert_eq!( + "%-10s" + .parse::() + .unwrap() + .format_string("test".to_owned()), + "test ".to_owned() + ); + assert_eq!( + "%#10x" + .parse::() + .unwrap() + .format_number(&BigInt::from(0x1337)), + " 0x1337".to_owned() + ); + assert_eq!( + "%-#10x" + .parse::() + .unwrap() + .format_number(&BigInt::from(0x1337)), + "0x1337 ".to_owned() + ); + } + + #[test] + fn test_parse_key() { + let expected = Ok(CFormatSpec { + mapping_key: Some("amount".to_owned()), + format_type: CFormatType::Number(CNumberType::Decimal), + format_char: 'd', + min_field_width: None, + precision: None, + flags: CConversionFlags::empty(), + }); + assert_eq!("%(amount)d".parse::(), expected); + + let expected = Ok(CFormatSpec { + mapping_key: Some("m((u(((l((((ti))))p)))l))e".to_owned()), + format_type: CFormatType::Number(CNumberType::Decimal), + format_char: 'd', + min_field_width: None, + precision: None, + flags: CConversionFlags::empty(), + }); + assert_eq!( + "%(m((u(((l((((ti))))p)))l))e)d".parse::(), + expected + ); + } + + #[test] + fn test_format_parse_key_fail() { + assert_eq!( + "%(aged".parse::(), + Err(CFormatError { + typ: CFormatErrorType::UnmatchedKeyParentheses, + index: 1 + }) + ); + } + + #[test] + fn test_format_parse_type_fail() { + assert_eq!( + "Hello %n".parse::(), + Err(CFormatError { + typ: CFormatErrorType::UnsupportedFormatChar('n'), + index: 7 + }) + ); + } + + #[test] + fn test_incomplete_format_fail() { + assert_eq!( + "Hello %".parse::(), + Err(CFormatError { + typ: CFormatErrorType::IncompleteFormat, + index: 7 + }) + ); + } + + #[test] + fn test_parse_flags() { + let expected = Ok(CFormatSpec { + format_type: CFormatType::Number(CNumberType::Decimal), + format_char: 'd', + min_field_width: Some(CFormatQuantity::Amount(10)), + precision: None, + mapping_key: None, + flags: CConversionFlags::all(), + }); + let parsed = "% 0 -+++###10d".parse::(); + assert_eq!(parsed, expected); + assert_eq!( + parsed.unwrap().format_number(&BigInt::from(12)), + "+12 ".to_owned() + ); + } + + #[test] + fn test_parse_and_format_string() { + assert_eq!( + "%5.4s" + .parse::() + .unwrap() + .format_string("Hello, World!".to_owned()), + " Hell".to_owned() + ); + assert_eq!( + "%-5.4s" + .parse::() + .unwrap() + .format_string("Hello, World!".to_owned()), + "Hell ".to_owned() + ); + assert_eq!( + "%.s" + .parse::() + .unwrap() + .format_string("Hello, World!".to_owned()), + "".to_owned() + ); + assert_eq!( + "%5.s" + .parse::() + .unwrap() + .format_string("Hello, World!".to_owned()), + " ".to_owned() + ); + } + + #[test] + fn test_parse_and_format_unicode_string() { + assert_eq!( + "%.2s" + .parse::() + .unwrap() + .format_string("❤❤❤❤❤❤❤❤".to_owned()), + "❤❤".to_owned() + ); + } + + #[test] + fn test_parse_and_format_number() { + assert_eq!( + "%5d" + .parse::() + .unwrap() + .format_number(&BigInt::from(27)), + " 27".to_owned() + ); + assert_eq!( + "%05d" + .parse::() + .unwrap() + .format_number(&BigInt::from(27)), + "00027".to_owned() + ); + assert_eq!( + "%.5d" + .parse::() + .unwrap() + .format_number(&BigInt::from(27)), + "00027".to_owned() + ); + assert_eq!( + "%+05d" + .parse::() + .unwrap() + .format_number(&BigInt::from(27)), + "+0027".to_owned() + ); + assert_eq!( + "%-d" + .parse::() + .unwrap() + .format_number(&BigInt::from(-27)), + "-27".to_owned() + ); + assert_eq!( + "% d" + .parse::() + .unwrap() + .format_number(&BigInt::from(27)), + " 27".to_owned() + ); + assert_eq!( + "% d" + .parse::() + .unwrap() + .format_number(&BigInt::from(-27)), + "-27".to_owned() + ); + assert_eq!( + "%08x" + .parse::() + .unwrap() + .format_number(&BigInt::from(0x1337)), + "00001337".to_owned() + ); + assert_eq!( + "%#010x" + .parse::() + .unwrap() + .format_number(&BigInt::from(0x1337)), + "0x00001337".to_owned() + ); + assert_eq!( + "%-#010x" + .parse::() + .unwrap() + .format_number(&BigInt::from(0x1337)), + "0x1337 ".to_owned() + ); + } + + #[test] + fn test_parse_and_format_float() { + assert_eq!( + "%f".parse::().unwrap().format_float(1.2345), + "1.234500" + ); + assert_eq!( + "%.2f".parse::().unwrap().format_float(1.2345), + "1.23" + ); + assert_eq!( + "%.f".parse::().unwrap().format_float(1.2345), + "1" + ); + assert_eq!( + "%+.f".parse::().unwrap().format_float(1.2345), + "+1" + ); + assert_eq!( + "%+f".parse::().unwrap().format_float(1.2345), + "+1.234500" + ); + assert_eq!( + "% f".parse::().unwrap().format_float(1.2345), + " 1.234500" + ); + assert_eq!( + "%f".parse::().unwrap().format_float(-1.2345), + "-1.234500" + ); + assert_eq!( + "%f".parse::() + .unwrap() + .format_float(1.2345678901), + "1.234568" + ); + } + + #[test] + fn test_format_parse() { + let fmt = "Hello, my name is %s and I'm %d years old"; + let expected = Ok(CFormatString { + parts: vec![ + (0, CFormatPart::Literal("Hello, my name is ".to_owned())), + ( + 18, + CFormatPart::Spec(CFormatSpec { + format_type: CFormatType::String(CFormatConversion::Str), + format_char: 's', + mapping_key: None, + min_field_width: None, + precision: None, + flags: CConversionFlags::empty(), + }), + ), + (20, CFormatPart::Literal(" and I'm ".to_owned())), + ( + 29, + CFormatPart::Spec(CFormatSpec { + format_type: CFormatType::Number(CNumberType::Decimal), + format_char: 'd', + mapping_key: None, + min_field_width: None, + precision: None, + flags: CConversionFlags::empty(), + }), + ), + (31, CFormatPart::Literal(" years old".to_owned())), + ], + }); + let result = fmt.parse::(); + assert_eq!( + result, expected, + "left = {result:#?} \n\n\n right = {expected:#?}" + ); + } +} diff --git a/common/src/format.rs b/common/src/format.rs new file mode 100644 index 0000000000..1c41df38c9 --- /dev/null +++ b/common/src/format.rs @@ -0,0 +1,1328 @@ +use itertools::{Itertools, PeekingNext}; +use malachite_bigint::{BigInt, Sign}; +use num_traits::FromPrimitive; +use num_traits::{Signed, cast::ToPrimitive}; +use rustpython_literal::float; +use rustpython_literal::format::Case; +use std::ops::Deref; +use std::{cmp, str::FromStr}; + +trait FormatParse { + fn parse(text: &str) -> (Option, &str) + where + Self: Sized; +} + +#[derive(Debug, Copy, Clone, PartialEq)] +pub enum FormatConversion { + Str, + Repr, + Ascii, + Bytes, +} + +impl FormatParse for FormatConversion { + fn parse(text: &str) -> (Option, &str) { + let Some(conversion) = Self::from_string(text) else { + return (None, text); + }; + let mut chars = text.chars(); + chars.next(); // Consume the bang + chars.next(); // Consume one r,s,a char + (Some(conversion), chars.as_str()) + } +} + +impl FormatConversion { + pub fn from_char(c: char) -> Option { + match c { + 's' => Some(FormatConversion::Str), + 'r' => Some(FormatConversion::Repr), + 'a' => Some(FormatConversion::Ascii), + 'b' => Some(FormatConversion::Bytes), + _ => None, + } + } + + fn from_string(text: &str) -> Option { + let mut chars = text.chars(); + if chars.next() != Some('!') { + return None; + } + + FormatConversion::from_char(chars.next()?) + } +} + +#[derive(Debug, Copy, Clone, PartialEq)] +pub enum FormatAlign { + Left, + Right, + AfterSign, + Center, +} + +impl FormatAlign { + fn from_char(c: char) -> Option { + match c { + '<' => Some(FormatAlign::Left), + '>' => Some(FormatAlign::Right), + '=' => Some(FormatAlign::AfterSign), + '^' => Some(FormatAlign::Center), + _ => None, + } + } +} + +impl FormatParse for FormatAlign { + fn parse(text: &str) -> (Option, &str) { + let mut chars = text.chars(); + if let Some(maybe_align) = chars.next().and_then(Self::from_char) { + (Some(maybe_align), chars.as_str()) + } else { + (None, text) + } + } +} + +#[derive(Debug, Copy, Clone, PartialEq)] +pub enum FormatSign { + Plus, + Minus, + MinusOrSpace, +} + +impl FormatParse for FormatSign { + fn parse(text: &str) -> (Option, &str) { + let mut chars = text.chars(); + match chars.next() { + Some('-') => (Some(Self::Minus), chars.as_str()), + Some('+') => (Some(Self::Plus), chars.as_str()), + Some(' ') => (Some(Self::MinusOrSpace), chars.as_str()), + _ => (None, text), + } + } +} + +#[derive(Debug, PartialEq)] +pub enum FormatGrouping { + Comma, + Underscore, +} + +impl FormatParse for FormatGrouping { + fn parse(text: &str) -> (Option, &str) { + let mut chars = text.chars(); + match chars.next() { + Some('_') => (Some(Self::Underscore), chars.as_str()), + Some(',') => (Some(Self::Comma), chars.as_str()), + _ => (None, text), + } + } +} + +#[derive(Debug, PartialEq)] +pub enum FormatType { + String, + Binary, + Character, + Decimal, + Octal, + Number(Case), + Hex(Case), + Exponent(Case), + GeneralFormat(Case), + FixedPoint(Case), + Percentage, +} + +impl From<&FormatType> for char { + fn from(from: &FormatType) -> char { + match from { + FormatType::String => 's', + FormatType::Binary => 'b', + FormatType::Character => 'c', + FormatType::Decimal => 'd', + FormatType::Octal => 'o', + FormatType::Number(Case::Lower) => 'n', + FormatType::Number(Case::Upper) => 'N', + FormatType::Hex(Case::Lower) => 'x', + FormatType::Hex(Case::Upper) => 'X', + FormatType::Exponent(Case::Lower) => 'e', + FormatType::Exponent(Case::Upper) => 'E', + FormatType::GeneralFormat(Case::Lower) => 'g', + FormatType::GeneralFormat(Case::Upper) => 'G', + FormatType::FixedPoint(Case::Lower) => 'f', + FormatType::FixedPoint(Case::Upper) => 'F', + FormatType::Percentage => '%', + } + } +} + +impl FormatParse for FormatType { + fn parse(text: &str) -> (Option, &str) { + let mut chars = text.chars(); + match chars.next() { + Some('s') => (Some(Self::String), chars.as_str()), + Some('b') => (Some(Self::Binary), chars.as_str()), + Some('c') => (Some(Self::Character), chars.as_str()), + Some('d') => (Some(Self::Decimal), chars.as_str()), + Some('o') => (Some(Self::Octal), chars.as_str()), + Some('n') => (Some(Self::Number(Case::Lower)), chars.as_str()), + Some('N') => (Some(Self::Number(Case::Upper)), chars.as_str()), + Some('x') => (Some(Self::Hex(Case::Lower)), chars.as_str()), + Some('X') => (Some(Self::Hex(Case::Upper)), chars.as_str()), + Some('e') => (Some(Self::Exponent(Case::Lower)), chars.as_str()), + Some('E') => (Some(Self::Exponent(Case::Upper)), chars.as_str()), + Some('f') => (Some(Self::FixedPoint(Case::Lower)), chars.as_str()), + Some('F') => (Some(Self::FixedPoint(Case::Upper)), chars.as_str()), + Some('g') => (Some(Self::GeneralFormat(Case::Lower)), chars.as_str()), + Some('G') => (Some(Self::GeneralFormat(Case::Upper)), chars.as_str()), + Some('%') => (Some(Self::Percentage), chars.as_str()), + _ => (None, text), + } + } +} + +#[derive(Debug, PartialEq)] +pub struct FormatSpec { + conversion: Option, + fill: Option, + align: Option, + sign: Option, + alternate_form: bool, + width: Option, + grouping_option: Option, + precision: Option, + format_type: Option, +} + +fn get_num_digits(text: &str) -> usize { + for (index, character) in text.char_indices() { + if !character.is_ascii_digit() { + return index; + } + } + text.len() +} + +fn parse_fill_and_align(text: &str) -> (Option, Option, &str) { + let char_indices: Vec<(usize, char)> = text.char_indices().take(3).collect(); + if char_indices.is_empty() { + (None, None, text) + } else if char_indices.len() == 1 { + let (maybe_align, remaining) = FormatAlign::parse(text); + (None, maybe_align, remaining) + } else { + let (maybe_align, remaining) = FormatAlign::parse(&text[char_indices[1].0..]); + if maybe_align.is_some() { + (Some(char_indices[0].1), maybe_align, remaining) + } else { + let (only_align, only_align_remaining) = FormatAlign::parse(text); + (None, only_align, only_align_remaining) + } + } +} + +fn parse_number(text: &str) -> Result<(Option, &str), FormatSpecError> { + let num_digits: usize = get_num_digits(text); + if num_digits == 0 { + return Ok((None, text)); + } + if let Ok(num) = text[..num_digits].parse::() { + Ok((Some(num), &text[num_digits..])) + } else { + // NOTE: this condition is different from CPython + Err(FormatSpecError::DecimalDigitsTooMany) + } +} + +fn parse_alternate_form(text: &str) -> (bool, &str) { + let mut chars = text.chars(); + match chars.next() { + Some('#') => (true, chars.as_str()), + _ => (false, text), + } +} + +fn parse_zero(text: &str) -> (bool, &str) { + let mut chars = text.chars(); + match chars.next() { + Some('0') => (true, chars.as_str()), + _ => (false, text), + } +} + +fn parse_precision(text: &str) -> Result<(Option, &str), FormatSpecError> { + let mut chars = text.chars(); + Ok(match chars.next() { + Some('.') => { + let (size, remaining) = parse_number(chars.as_str())?; + if let Some(size) = size { + if size > i32::MAX as usize { + return Err(FormatSpecError::PrecisionTooBig); + } + (Some(size), remaining) + } else { + (None, text) + } + } + _ => (None, text), + }) +} + +impl FormatSpec { + pub fn parse(text: &str) -> Result { + // get_integer in CPython + let (conversion, text) = FormatConversion::parse(text); + let (mut fill, mut align, text) = parse_fill_and_align(text); + let (sign, text) = FormatSign::parse(text); + let (alternate_form, text) = parse_alternate_form(text); + let (zero, text) = parse_zero(text); + let (width, text) = parse_number(text)?; + let (grouping_option, text) = FormatGrouping::parse(text); + let (precision, text) = parse_precision(text)?; + let (format_type, text) = FormatType::parse(text); + if !text.is_empty() { + return Err(FormatSpecError::InvalidFormatSpecifier); + } + + if zero && fill.is_none() { + fill.replace('0'); + align = align.or(Some(FormatAlign::AfterSign)); + } + + Ok(FormatSpec { + conversion, + fill, + align, + sign, + alternate_form, + width, + grouping_option, + precision, + format_type, + }) + } + + fn compute_fill_string(fill_char: char, fill_chars_needed: i32) -> String { + (0..fill_chars_needed) + .map(|_| fill_char) + .collect::() + } + + fn add_magnitude_separators_for_char( + magnitude_str: String, + inter: i32, + sep: char, + disp_digit_cnt: i32, + ) -> String { + // Don't add separators to the floating decimal point of numbers + let mut parts = magnitude_str.splitn(2, '.'); + let magnitude_int_str = parts.next().unwrap().to_string(); + let dec_digit_cnt = magnitude_str.len() as i32 - magnitude_int_str.len() as i32; + let int_digit_cnt = disp_digit_cnt - dec_digit_cnt; + let mut result = FormatSpec::separate_integer(magnitude_int_str, inter, sep, int_digit_cnt); + if let Some(part) = parts.next() { + result.push_str(&format!(".{part}")) + } + result + } + + fn separate_integer( + magnitude_str: String, + inter: i32, + sep: char, + disp_digit_cnt: i32, + ) -> String { + let magnitude_len = magnitude_str.len() as i32; + let offset = (disp_digit_cnt % (inter + 1) == 0) as i32; + let disp_digit_cnt = disp_digit_cnt + offset; + let pad_cnt = disp_digit_cnt - magnitude_len; + let sep_cnt = disp_digit_cnt / (inter + 1); + let diff = pad_cnt - sep_cnt; + if pad_cnt > 0 && diff > 0 { + // separate with 0 padding + let padding = "0".repeat(diff as usize); + let padded_num = format!("{padding}{magnitude_str}"); + FormatSpec::insert_separator(padded_num, inter, sep, sep_cnt) + } else { + // separate without padding + let sep_cnt = (magnitude_len - 1) / inter; + FormatSpec::insert_separator(magnitude_str, inter, sep, sep_cnt) + } + } + + fn insert_separator(mut magnitude_str: String, inter: i32, sep: char, sep_cnt: i32) -> String { + let magnitude_len = magnitude_str.len() as i32; + for i in 1..sep_cnt + 1 { + magnitude_str.insert((magnitude_len - inter * i) as usize, sep); + } + magnitude_str + } + + fn validate_format(&self, default_format_type: FormatType) -> Result<(), FormatSpecError> { + let format_type = self.format_type.as_ref().unwrap_or(&default_format_type); + match (&self.grouping_option, format_type) { + ( + Some(FormatGrouping::Comma), + FormatType::String + | FormatType::Character + | FormatType::Binary + | FormatType::Octal + | FormatType::Hex(_) + | FormatType::Number(_), + ) => { + let ch = char::from(format_type); + Err(FormatSpecError::UnspecifiedFormat(',', ch)) + } + ( + Some(FormatGrouping::Underscore), + FormatType::String | FormatType::Character | FormatType::Number(_), + ) => { + let ch = char::from(format_type); + Err(FormatSpecError::UnspecifiedFormat('_', ch)) + } + _ => Ok(()), + } + } + + fn get_separator_interval(&self) -> usize { + match self.format_type { + Some(FormatType::Binary | FormatType::Octal | FormatType::Hex(_)) => 4, + Some(FormatType::Decimal | FormatType::Number(_) | FormatType::FixedPoint(_)) => 3, + None => 3, + _ => panic!("Separators only valid for numbers!"), + } + } + + fn add_magnitude_separators(&self, magnitude_str: String, prefix: &str) -> String { + match &self.grouping_option { + Some(fg) => { + let sep = match fg { + FormatGrouping::Comma => ',', + FormatGrouping::Underscore => '_', + }; + let inter = self.get_separator_interval().try_into().unwrap(); + let magnitude_len = magnitude_str.len(); + let width = self.width.unwrap_or(magnitude_len) as i32 - prefix.len() as i32; + let disp_digit_cnt = cmp::max(width, magnitude_len as i32); + FormatSpec::add_magnitude_separators_for_char( + magnitude_str, + inter, + sep, + disp_digit_cnt, + ) + } + None => magnitude_str, + } + } + + pub fn format_bool(&self, input: bool) -> Result { + let x = u8::from(input); + match &self.format_type { + Some( + FormatType::Binary + | FormatType::Decimal + | FormatType::Octal + | FormatType::Number(Case::Lower) + | FormatType::Hex(_) + | FormatType::GeneralFormat(_) + | FormatType::Character, + ) => self.format_int(&BigInt::from_u8(x).unwrap()), + Some(FormatType::Exponent(_) | FormatType::FixedPoint(_) | FormatType::Percentage) => { + self.format_float(x as f64) + } + None => { + let first_letter = (input.to_string().as_bytes()[0] as char).to_uppercase(); + Ok(first_letter.collect::() + &input.to_string()[1..]) + } + _ => Err(FormatSpecError::InvalidFormatSpecifier), + } + } + + pub fn format_float(&self, num: f64) -> Result { + self.validate_format(FormatType::FixedPoint(Case::Lower))?; + let precision = self.precision.unwrap_or(6); + let magnitude = num.abs(); + let raw_magnitude_str: Result = match &self.format_type { + Some(FormatType::FixedPoint(case)) => Ok(float::format_fixed( + precision, + magnitude, + *case, + self.alternate_form, + )), + Some(FormatType::Decimal) + | Some(FormatType::Binary) + | Some(FormatType::Octal) + | Some(FormatType::Hex(_)) + | Some(FormatType::String) + | Some(FormatType::Character) + | Some(FormatType::Number(Case::Upper)) => { + let ch = char::from(self.format_type.as_ref().unwrap()); + Err(FormatSpecError::UnknownFormatCode(ch, "float")) + } + Some(FormatType::GeneralFormat(case)) | Some(FormatType::Number(case)) => { + let precision = if precision == 0 { 1 } else { precision }; + Ok(float::format_general( + precision, + magnitude, + *case, + self.alternate_form, + false, + )) + } + Some(FormatType::Exponent(case)) => Ok(float::format_exponent( + precision, + magnitude, + *case, + self.alternate_form, + )), + Some(FormatType::Percentage) => match magnitude { + magnitude if magnitude.is_nan() => Ok("nan%".to_owned()), + magnitude if magnitude.is_infinite() => Ok("inf%".to_owned()), + _ => { + let result = format!("{:.*}", precision, magnitude * 100.0); + let point = float::decimal_point_or_empty(precision, self.alternate_form); + Ok(format!("{result}{point}%")) + } + }, + None => match magnitude { + magnitude if magnitude.is_nan() => Ok("nan".to_owned()), + magnitude if magnitude.is_infinite() => Ok("inf".to_owned()), + _ => match self.precision { + Some(precision) => Ok(float::format_general( + precision, + magnitude, + Case::Lower, + self.alternate_form, + true, + )), + None => Ok(float::to_string(magnitude)), + }, + }, + }; + let format_sign = self.sign.unwrap_or(FormatSign::Minus); + let sign_str = if num.is_sign_negative() && !num.is_nan() { + "-" + } else { + match format_sign { + FormatSign::Plus => "+", + FormatSign::Minus => "", + FormatSign::MinusOrSpace => " ", + } + }; + let magnitude_str = self.add_magnitude_separators(raw_magnitude_str?, sign_str); + self.format_sign_and_align(&AsciiStr::new(&magnitude_str), sign_str, FormatAlign::Right) + } + + #[inline] + fn format_int_radix(&self, magnitude: BigInt, radix: u32) -> Result { + match self.precision { + Some(_) => Err(FormatSpecError::PrecisionNotAllowed), + None => Ok(magnitude.to_str_radix(radix)), + } + } + + pub fn format_int(&self, num: &BigInt) -> Result { + self.validate_format(FormatType::Decimal)?; + let magnitude = num.abs(); + let prefix = if self.alternate_form { + match self.format_type { + Some(FormatType::Binary) => "0b", + Some(FormatType::Octal) => "0o", + Some(FormatType::Hex(Case::Lower)) => "0x", + Some(FormatType::Hex(Case::Upper)) => "0X", + _ => "", + } + } else { + "" + }; + let raw_magnitude_str = match self.format_type { + Some(FormatType::Binary) => self.format_int_radix(magnitude, 2), + Some(FormatType::Decimal) => self.format_int_radix(magnitude, 10), + Some(FormatType::Octal) => self.format_int_radix(magnitude, 8), + Some(FormatType::Hex(Case::Lower)) => self.format_int_radix(magnitude, 16), + Some(FormatType::Hex(Case::Upper)) => match self.precision { + Some(_) => Err(FormatSpecError::PrecisionNotAllowed), + None => { + let mut result = magnitude.to_str_radix(16); + result.make_ascii_uppercase(); + Ok(result) + } + }, + Some(FormatType::Number(Case::Lower)) => self.format_int_radix(magnitude, 10), + Some(FormatType::Number(Case::Upper)) => { + Err(FormatSpecError::UnknownFormatCode('N', "int")) + } + Some(FormatType::String) => Err(FormatSpecError::UnknownFormatCode('s', "int")), + Some(FormatType::Character) => match (self.sign, self.alternate_form) { + (Some(_), _) => Err(FormatSpecError::NotAllowed("Sign")), + (_, true) => Err(FormatSpecError::NotAllowed("Alternate form (#)")), + (_, _) => match num.to_u32() { + Some(n) if n <= 0x10ffff => Ok(std::char::from_u32(n).unwrap().to_string()), + Some(_) | None => Err(FormatSpecError::CodeNotInRange), + }, + }, + Some(FormatType::GeneralFormat(_)) + | Some(FormatType::FixedPoint(_)) + | Some(FormatType::Exponent(_)) + | Some(FormatType::Percentage) => match num.to_f64() { + Some(float) => return self.format_float(float), + _ => Err(FormatSpecError::UnableToConvert), + }, + None => self.format_int_radix(magnitude, 10), + }?; + let format_sign = self.sign.unwrap_or(FormatSign::Minus); + let sign_str = match num.sign() { + Sign::Minus => "-", + _ => match format_sign { + FormatSign::Plus => "+", + FormatSign::Minus => "", + FormatSign::MinusOrSpace => " ", + }, + }; + let sign_prefix = format!("{sign_str}{prefix}"); + let magnitude_str = self.add_magnitude_separators(raw_magnitude_str, &sign_prefix); + self.format_sign_and_align( + &AsciiStr::new(&magnitude_str), + &sign_prefix, + FormatAlign::Right, + ) + } + + pub fn format_string(&self, s: &T) -> Result + where + T: CharLen + Deref, + { + self.validate_format(FormatType::String)?; + match self.format_type { + Some(FormatType::String) | None => self + .format_sign_and_align(s, "", FormatAlign::Left) + .map(|mut value| { + if let Some(precision) = self.precision { + value.truncate(precision); + } + value + }), + _ => { + let ch = char::from(self.format_type.as_ref().unwrap()); + Err(FormatSpecError::UnknownFormatCode(ch, "str")) + } + } + } + + fn format_sign_and_align( + &self, + magnitude_str: &T, + sign_str: &str, + default_align: FormatAlign, + ) -> Result + where + T: CharLen + Deref, + { + let align = self.align.unwrap_or(default_align); + + let num_chars = magnitude_str.char_len(); + let fill_char = self.fill.unwrap_or(' '); + let fill_chars_needed: i32 = self.width.map_or(0, |w| { + cmp::max(0, (w as i32) - (num_chars as i32) - (sign_str.len() as i32)) + }); + + let magnitude_str = magnitude_str.deref(); + Ok(match align { + FormatAlign::Left => format!( + "{}{}{}", + sign_str, + magnitude_str, + FormatSpec::compute_fill_string(fill_char, fill_chars_needed) + ), + FormatAlign::Right => format!( + "{}{}{}", + FormatSpec::compute_fill_string(fill_char, fill_chars_needed), + sign_str, + magnitude_str + ), + FormatAlign::AfterSign => format!( + "{}{}{}", + sign_str, + FormatSpec::compute_fill_string(fill_char, fill_chars_needed), + magnitude_str + ), + FormatAlign::Center => { + let left_fill_chars_needed = fill_chars_needed / 2; + let right_fill_chars_needed = fill_chars_needed - left_fill_chars_needed; + let left_fill_string = + FormatSpec::compute_fill_string(fill_char, left_fill_chars_needed); + let right_fill_string = + FormatSpec::compute_fill_string(fill_char, right_fill_chars_needed); + format!("{left_fill_string}{sign_str}{magnitude_str}{right_fill_string}") + } + }) + } +} + +pub trait CharLen { + /// Returns the number of characters in the text + fn char_len(&self) -> usize; +} + +struct AsciiStr<'a> { + inner: &'a str, +} + +impl<'a> AsciiStr<'a> { + fn new(inner: &'a str) -> Self { + Self { inner } + } +} + +impl CharLen for AsciiStr<'_> { + fn char_len(&self) -> usize { + self.inner.len() + } +} + +impl Deref for AsciiStr<'_> { + type Target = str; + fn deref(&self) -> &Self::Target { + self.inner + } +} + +#[derive(Debug, PartialEq)] +pub enum FormatSpecError { + DecimalDigitsTooMany, + PrecisionTooBig, + InvalidFormatSpecifier, + UnspecifiedFormat(char, char), + UnknownFormatCode(char, &'static str), + PrecisionNotAllowed, + NotAllowed(&'static str), + UnableToConvert, + CodeNotInRange, + NotImplemented(char, &'static str), +} + +#[derive(Debug, PartialEq)] +pub enum FormatParseError { + UnmatchedBracket, + MissingStartBracket, + UnescapedStartBracketInLiteral, + InvalidFormatSpecifier, + UnknownConversion, + EmptyAttribute, + MissingRightBracket, + InvalidCharacterAfterRightBracket, +} + +impl FromStr for FormatSpec { + type Err = FormatSpecError; + fn from_str(s: &str) -> Result { + FormatSpec::parse(s) + } +} + +#[derive(Debug, PartialEq)] +pub enum FieldNamePart { + Attribute(String), + Index(usize), + StringIndex(String), +} + +impl FieldNamePart { + fn parse_part( + chars: &mut impl PeekingNext, + ) -> Result, FormatParseError> { + chars + .next() + .map(|ch| match ch { + '.' => { + let mut attribute = String::new(); + for ch in chars.peeking_take_while(|ch| *ch != '.' && *ch != '[') { + attribute.push(ch); + } + if attribute.is_empty() { + Err(FormatParseError::EmptyAttribute) + } else { + Ok(FieldNamePart::Attribute(attribute)) + } + } + '[' => { + let mut index = String::new(); + for ch in chars { + if ch == ']' { + return if index.is_empty() { + Err(FormatParseError::EmptyAttribute) + } else if let Ok(index) = index.parse::() { + Ok(FieldNamePart::Index(index)) + } else { + Ok(FieldNamePart::StringIndex(index)) + }; + } + index.push(ch); + } + Err(FormatParseError::MissingRightBracket) + } + _ => Err(FormatParseError::InvalidCharacterAfterRightBracket), + }) + .transpose() + } +} + +#[derive(Debug, PartialEq)] +pub enum FieldType { + Auto, + Index(usize), + Keyword(String), +} + +#[derive(Debug, PartialEq)] +pub struct FieldName { + pub field_type: FieldType, + pub parts: Vec, +} + +impl FieldName { + pub fn parse(text: &str) -> Result { + let mut chars = text.chars().peekable(); + let mut first = String::new(); + for ch in chars.peeking_take_while(|ch| *ch != '.' && *ch != '[') { + first.push(ch); + } + + let field_type = if first.is_empty() { + FieldType::Auto + } else if let Ok(index) = first.parse::() { + FieldType::Index(index) + } else { + FieldType::Keyword(first) + }; + + let mut parts = Vec::new(); + while let Some(part) = FieldNamePart::parse_part(&mut chars)? { + parts.push(part) + } + + Ok(FieldName { field_type, parts }) + } +} + +#[derive(Debug, PartialEq)] +pub enum FormatPart { + Field { + field_name: String, + conversion_spec: Option, + format_spec: String, + }, + Literal(String), +} + +#[derive(Debug, PartialEq)] +pub struct FormatString { + pub format_parts: Vec, +} + +impl FormatString { + fn parse_literal_single(text: &str) -> Result<(char, &str), FormatParseError> { + let mut chars = text.chars(); + // This should never be called with an empty str + let first_char = chars.next().unwrap(); + // isn't this detectable only with bytes operation? + if first_char == '{' || first_char == '}' { + let maybe_next_char = chars.next(); + // if we see a bracket, it has to be escaped by doubling up to be in a literal + return if maybe_next_char.is_none() || maybe_next_char.unwrap() != first_char { + Err(FormatParseError::UnescapedStartBracketInLiteral) + } else { + Ok((first_char, chars.as_str())) + }; + } + Ok((first_char, chars.as_str())) + } + + fn parse_literal(text: &str) -> Result<(FormatPart, &str), FormatParseError> { + let mut cur_text = text; + let mut result_string = String::new(); + while !cur_text.is_empty() { + match FormatString::parse_literal_single(cur_text) { + Ok((next_char, remaining)) => { + result_string.push(next_char); + cur_text = remaining; + } + Err(err) => { + return if !result_string.is_empty() { + Ok((FormatPart::Literal(result_string), cur_text)) + } else { + Err(err) + }; + } + } + } + Ok((FormatPart::Literal(result_string), "")) + } + + fn parse_part_in_brackets(text: &str) -> Result { + let mut chars = text.chars().peekable(); + + let mut left = String::new(); + let mut right = String::new(); + + let mut split = false; + let mut selected = &mut left; + let mut inside_brackets = false; + + while let Some(char) = chars.next() { + if char == '[' { + inside_brackets = true; + + selected.push(char); + + while let Some(next_char) = chars.next() { + selected.push(next_char); + + if next_char == ']' { + inside_brackets = false; + break; + } + if chars.peek().is_none() { + return Err(FormatParseError::MissingRightBracket); + } + } + } else if char == ':' && !split && !inside_brackets { + split = true; + selected = &mut right; + } else { + selected.push(char); + } + } + + // before the comma is a keyword or arg index, after the comma is maybe a spec. + let arg_part: &str = &left; + + let format_spec = if split { right } else { String::new() }; + + // left can still be the conversion (!r, !s, !a) + let parts: Vec<&str> = arg_part.splitn(2, '!').collect(); + // before the bang is a keyword or arg index, after the comma is maybe a conversion spec. + let arg_part = parts[0]; + + let conversion_spec = parts + .get(1) + .map(|conversion| { + // conversions are only every one character + conversion + .chars() + .exactly_one() + .map_err(|_| FormatParseError::UnknownConversion) + }) + .transpose()?; + + Ok(FormatPart::Field { + field_name: arg_part.to_owned(), + conversion_spec, + format_spec, + }) + } + + fn parse_spec(text: &str) -> Result<(FormatPart, &str), FormatParseError> { + let mut nested = false; + let mut end_bracket_pos = None; + let mut left = String::new(); + + // There may be one layer nesting brackets in spec + for (idx, c) in text.char_indices() { + if idx == 0 { + if c != '{' { + return Err(FormatParseError::MissingStartBracket); + } + } else if c == '{' { + if nested { + return Err(FormatParseError::InvalidFormatSpecifier); + } else { + nested = true; + left.push(c); + continue; + } + } else if c == '}' { + if nested { + nested = false; + left.push(c); + continue; + } else { + end_bracket_pos = Some(idx); + break; + } + } else { + left.push(c); + } + } + if let Some(pos) = end_bracket_pos { + let (_, right) = text.split_at(pos); + let format_part = FormatString::parse_part_in_brackets(&left)?; + Ok((format_part, &right[1..])) + } else { + Err(FormatParseError::UnmatchedBracket) + } + } +} + +pub trait FromTemplate<'a>: Sized { + type Err; + fn from_str(s: &'a str) -> Result; +} + +impl<'a> FromTemplate<'a> for FormatString { + type Err = FormatParseError; + + fn from_str(text: &'a str) -> Result { + let mut cur_text: &str = text; + let mut parts: Vec = Vec::new(); + while !cur_text.is_empty() { + // Try to parse both literals and bracketed format parts until we + // run out of text + cur_text = FormatString::parse_literal(cur_text) + .or_else(|_| FormatString::parse_spec(cur_text)) + .map(|(part, new_text)| { + parts.push(part); + new_text + })?; + } + Ok(FormatString { + format_parts: parts, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_fill_and_align() { + assert_eq!( + parse_fill_and_align(" <"), + (Some(' '), Some(FormatAlign::Left), "") + ); + assert_eq!( + parse_fill_and_align(" <22"), + (Some(' '), Some(FormatAlign::Left), "22") + ); + assert_eq!( + parse_fill_and_align("<22"), + (None, Some(FormatAlign::Left), "22") + ); + assert_eq!( + parse_fill_and_align(" ^^"), + (Some(' '), Some(FormatAlign::Center), "^") + ); + assert_eq!( + parse_fill_and_align("==="), + (Some('='), Some(FormatAlign::AfterSign), "=") + ); + } + + #[test] + fn test_width_only() { + let expected = Ok(FormatSpec { + conversion: None, + fill: None, + align: None, + sign: None, + alternate_form: false, + width: Some(33), + grouping_option: None, + precision: None, + format_type: None, + }); + assert_eq!(FormatSpec::parse("33"), expected); + } + + #[test] + fn test_fill_and_width() { + let expected = Ok(FormatSpec { + conversion: None, + fill: Some('<'), + align: Some(FormatAlign::Right), + sign: None, + alternate_form: false, + width: Some(33), + grouping_option: None, + precision: None, + format_type: None, + }); + assert_eq!(FormatSpec::parse("<>33"), expected); + } + + #[test] + fn test_all() { + let expected = Ok(FormatSpec { + conversion: None, + fill: Some('<'), + align: Some(FormatAlign::Right), + sign: Some(FormatSign::Minus), + alternate_form: true, + width: Some(23), + grouping_option: Some(FormatGrouping::Comma), + precision: Some(11), + format_type: Some(FormatType::Binary), + }); + assert_eq!(FormatSpec::parse("<>-#23,.11b"), expected); + } + + fn format_bool(text: &str, value: bool) -> Result { + FormatSpec::parse(text).and_then(|spec| spec.format_bool(value)) + } + + #[test] + fn test_format_bool() { + assert_eq!(format_bool("b", true), Ok("1".to_owned())); + assert_eq!(format_bool("b", false), Ok("0".to_owned())); + assert_eq!(format_bool("d", true), Ok("1".to_owned())); + assert_eq!(format_bool("d", false), Ok("0".to_owned())); + assert_eq!(format_bool("o", true), Ok("1".to_owned())); + assert_eq!(format_bool("o", false), Ok("0".to_owned())); + assert_eq!(format_bool("n", true), Ok("1".to_owned())); + assert_eq!(format_bool("n", false), Ok("0".to_owned())); + assert_eq!(format_bool("x", true), Ok("1".to_owned())); + assert_eq!(format_bool("x", false), Ok("0".to_owned())); + assert_eq!(format_bool("X", true), Ok("1".to_owned())); + assert_eq!(format_bool("X", false), Ok("0".to_owned())); + assert_eq!(format_bool("g", true), Ok("1".to_owned())); + assert_eq!(format_bool("g", false), Ok("0".to_owned())); + assert_eq!(format_bool("G", true), Ok("1".to_owned())); + assert_eq!(format_bool("G", false), Ok("0".to_owned())); + assert_eq!(format_bool("c", true), Ok("\x01".to_owned())); + assert_eq!(format_bool("c", false), Ok("\x00".to_owned())); + assert_eq!(format_bool("e", true), Ok("1.000000e+00".to_owned())); + assert_eq!(format_bool("e", false), Ok("0.000000e+00".to_owned())); + assert_eq!(format_bool("E", true), Ok("1.000000E+00".to_owned())); + assert_eq!(format_bool("E", false), Ok("0.000000E+00".to_owned())); + assert_eq!(format_bool("f", true), Ok("1.000000".to_owned())); + assert_eq!(format_bool("f", false), Ok("0.000000".to_owned())); + assert_eq!(format_bool("F", true), Ok("1.000000".to_owned())); + assert_eq!(format_bool("F", false), Ok("0.000000".to_owned())); + assert_eq!(format_bool("%", true), Ok("100.000000%".to_owned())); + assert_eq!(format_bool("%", false), Ok("0.000000%".to_owned())); + } + + #[test] + fn test_format_int() { + assert_eq!( + FormatSpec::parse("d") + .unwrap() + .format_int(&BigInt::from_bytes_be(Sign::Plus, b"\x10")), + Ok("16".to_owned()) + ); + assert_eq!( + FormatSpec::parse("x") + .unwrap() + .format_int(&BigInt::from_bytes_be(Sign::Plus, b"\x10")), + Ok("10".to_owned()) + ); + assert_eq!( + FormatSpec::parse("b") + .unwrap() + .format_int(&BigInt::from_bytes_be(Sign::Plus, b"\x10")), + Ok("10000".to_owned()) + ); + assert_eq!( + FormatSpec::parse("o") + .unwrap() + .format_int(&BigInt::from_bytes_be(Sign::Plus, b"\x10")), + Ok("20".to_owned()) + ); + assert_eq!( + FormatSpec::parse("+d") + .unwrap() + .format_int(&BigInt::from_bytes_be(Sign::Plus, b"\x10")), + Ok("+16".to_owned()) + ); + assert_eq!( + FormatSpec::parse("^ 5d") + .unwrap() + .format_int(&BigInt::from_bytes_be(Sign::Minus, b"\x10")), + Ok(" -16 ".to_owned()) + ); + assert_eq!( + FormatSpec::parse("0>+#10x") + .unwrap() + .format_int(&BigInt::from_bytes_be(Sign::Plus, b"\x10")), + Ok("00000+0x10".to_owned()) + ); + } + + #[test] + fn test_format_int_sep() { + let spec = FormatSpec::parse(",").expect(""); + assert_eq!(spec.grouping_option, Some(FormatGrouping::Comma)); + assert_eq!( + spec.format_int(&BigInt::from_str("1234567890123456789012345678").unwrap()), + Ok("1,234,567,890,123,456,789,012,345,678".to_owned()) + ); + } + + #[test] + fn test_format_parse() { + let expected = Ok(FormatString { + format_parts: vec![ + FormatPart::Literal("abcd".to_owned()), + FormatPart::Field { + field_name: "1".to_owned(), + conversion_spec: None, + format_spec: String::new(), + }, + FormatPart::Literal(":".to_owned()), + FormatPart::Field { + field_name: "key".to_owned(), + conversion_spec: None, + format_spec: String::new(), + }, + ], + }); + + assert_eq!(FormatString::from_str("abcd{1}:{key}"), expected); + } + + #[test] + fn test_format_parse_multi_byte_char() { + assert!(FormatString::from_str("{a:%ЫйЯЧ}").is_ok()); + } + + #[test] + fn test_format_parse_fail() { + assert_eq!( + FormatString::from_str("{s"), + Err(FormatParseError::UnmatchedBracket) + ); + } + + #[test] + fn test_square_brackets_inside_format() { + assert_eq!( + FormatString::from_str("{[:123]}"), + Ok(FormatString { + format_parts: vec![FormatPart::Field { + field_name: "[:123]".to_owned(), + conversion_spec: None, + format_spec: "".to_owned(), + }], + }), + ); + + assert_eq!(FormatString::from_str("{asdf[:123]asdf}"), { + Ok(FormatString { + format_parts: vec![FormatPart::Field { + field_name: "asdf[:123]asdf".to_owned(), + conversion_spec: None, + format_spec: "".to_owned(), + }], + }) + }); + + assert_eq!(FormatString::from_str("{[1234}"), { + Err(FormatParseError::MissingRightBracket) + }); + } + + #[test] + fn test_format_parse_escape() { + let expected = Ok(FormatString { + format_parts: vec![ + FormatPart::Literal("{".to_owned()), + FormatPart::Field { + field_name: "key".to_owned(), + conversion_spec: None, + format_spec: String::new(), + }, + FormatPart::Literal("}ddfe".to_owned()), + ], + }); + + assert_eq!(FormatString::from_str("{{{key}}}ddfe"), expected); + } + + #[test] + fn test_format_invalid_specification() { + assert_eq!( + FormatSpec::parse("%3"), + Err(FormatSpecError::InvalidFormatSpecifier) + ); + assert_eq!( + FormatSpec::parse(".2fa"), + Err(FormatSpecError::InvalidFormatSpecifier) + ); + assert_eq!( + FormatSpec::parse("ds"), + Err(FormatSpecError::InvalidFormatSpecifier) + ); + assert_eq!( + FormatSpec::parse("x+"), + Err(FormatSpecError::InvalidFormatSpecifier) + ); + assert_eq!( + FormatSpec::parse("b4"), + Err(FormatSpecError::InvalidFormatSpecifier) + ); + assert_eq!( + FormatSpec::parse("o!"), + Err(FormatSpecError::InvalidFormatSpecifier) + ); + assert_eq!( + FormatSpec::parse("d "), + Err(FormatSpecError::InvalidFormatSpecifier) + ); + } + + #[test] + fn test_parse_field_name() { + assert_eq!( + FieldName::parse(""), + Ok(FieldName { + field_type: FieldType::Auto, + parts: Vec::new(), + }) + ); + assert_eq!( + FieldName::parse("0"), + Ok(FieldName { + field_type: FieldType::Index(0), + parts: Vec::new(), + }) + ); + assert_eq!( + FieldName::parse("key"), + Ok(FieldName { + field_type: FieldType::Keyword("key".to_owned()), + parts: Vec::new(), + }) + ); + assert_eq!( + FieldName::parse("key.attr[0][string]"), + Ok(FieldName { + field_type: FieldType::Keyword("key".to_owned()), + parts: vec![ + FieldNamePart::Attribute("attr".to_owned()), + FieldNamePart::Index(0), + FieldNamePart::StringIndex("string".to_owned()) + ], + }) + ); + assert_eq!( + FieldName::parse("key.."), + Err(FormatParseError::EmptyAttribute) + ); + assert_eq!( + FieldName::parse("key[]"), + Err(FormatParseError::EmptyAttribute) + ); + assert_eq!( + FieldName::parse("key["), + Err(FormatParseError::MissingRightBracket) + ); + assert_eq!( + FieldName::parse("key[0]after"), + Err(FormatParseError::InvalidCharacterAfterRightBracket) + ); + } +} diff --git a/common/src/lib.rs b/common/src/lib.rs index cf2286c1cd..760ba8c55f 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -9,6 +9,7 @@ pub use macros::*; pub mod atomic; pub mod borrow; pub mod boxvec; +pub mod cformat; pub mod cmp; #[cfg(any(unix, windows, target_os = "wasi"))] pub mod crt_fd; @@ -16,6 +17,7 @@ pub mod encodings; #[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))] pub mod fileutils; pub mod float_ops; +pub mod format; pub mod hash; pub mod int; pub mod linked_list; diff --git a/common/src/str.rs b/common/src/str.rs index b4f7a1a636..89d2381d3e 100644 --- a/common/src/str.rs +++ b/common/src/str.rs @@ -1,9 +1,9 @@ use crate::{ atomic::{PyAtomic, Radium}, + format::CharLen, hash::PyHash, }; use ascii::AsciiString; -use rustpython_format::CharLen; use std::ops::{Bound, RangeBounds}; #[cfg(not(target_arch = "wasm32"))] diff --git a/compiler/literal/Cargo.toml b/compiler/literal/Cargo.toml new file mode 100644 index 0000000000..de7e89c0b4 --- /dev/null +++ b/compiler/literal/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "rustpython-literal" +description = "Common literal handling utilities mostly useful for unparse and repr." +edition = { workspace = true } +version = { workspace = true } +authors = { workspace = true } +repository = { workspace = true } +license = { workspace = true } +rust-version = { workspace = true } + +[dependencies] +hexf-parse = "0.2.1" +is-macro.workspace = true +lexical-parse-float = { version = "0.8.0", features = ["format"] } +num-traits = { workspace = true } +unic-ucd-category = "0.9" + +[dev-dependencies] +rand = { workspace = true } diff --git a/compiler/literal/src/char.rs b/compiler/literal/src/char.rs new file mode 100644 index 0000000000..cd64f6dfa9 --- /dev/null +++ b/compiler/literal/src/char.rs @@ -0,0 +1,15 @@ +use unic_ucd_category::GeneralCategory; + +/// According to python following categories aren't printable: +/// * Cc (Other, Control) +/// * Cf (Other, Format) +/// * Cs (Other, Surrogate) +/// * Co (Other, Private Use) +/// * Cn (Other, Not Assigned) +/// * Zl Separator, Line ('\u2028', LINE SEPARATOR) +/// * Zp Separator, Paragraph ('\u2029', PARAGRAPH SEPARATOR) +/// * Zs (Separator, Space) other than ASCII space('\x20'). +pub fn is_printable(c: char) -> bool { + let cat = GeneralCategory::of(c); + !(cat.is_other() || cat.is_separator()) +} diff --git a/compiler/literal/src/escape.rs b/compiler/literal/src/escape.rs new file mode 100644 index 0000000000..ba8e3ecf17 --- /dev/null +++ b/compiler/literal/src/escape.rs @@ -0,0 +1,441 @@ +#[derive(Debug, PartialEq, Eq, Copy, Clone, Hash, is_macro::Is)] +pub enum Quote { + Single, + Double, +} + +impl Quote { + #[inline] + pub const fn swap(self) -> Quote { + match self { + Quote::Single => Quote::Double, + Quote::Double => Quote::Single, + } + } + + #[inline] + pub const fn to_byte(&self) -> u8 { + match self { + Quote::Single => b'\'', + Quote::Double => b'"', + } + } + + #[inline] + pub const fn to_char(&self) -> char { + match self { + Quote::Single => '\'', + Quote::Double => '"', + } + } +} + +pub struct EscapeLayout { + pub quote: Quote, + pub len: Option, +} + +pub trait Escape { + fn source_len(&self) -> usize; + fn layout(&self) -> &EscapeLayout; + fn changed(&self) -> bool { + self.layout().len != Some(self.source_len()) + } + + fn write_source(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result; + fn write_body_slow(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result; + fn write_body(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result { + if self.changed() { + self.write_body_slow(formatter) + } else { + self.write_source(formatter) + } + } +} + +/// Returns the outer quotes to use and the number of quotes that need to be +/// escaped. +pub(crate) const fn choose_quote( + single_count: usize, + double_count: usize, + preferred_quote: Quote, +) -> (Quote, usize) { + let (primary_count, secondary_count) = match preferred_quote { + Quote::Single => (single_count, double_count), + Quote::Double => (double_count, single_count), + }; + + // always use primary unless we have primary but no secondary + let use_secondary = primary_count > 0 && secondary_count == 0; + if use_secondary { + (preferred_quote.swap(), secondary_count) + } else { + (preferred_quote, primary_count) + } +} + +pub struct UnicodeEscape<'a> { + source: &'a str, + layout: EscapeLayout, +} + +impl<'a> UnicodeEscape<'a> { + #[inline] + pub fn with_forced_quote(source: &'a str, quote: Quote) -> Self { + let layout = EscapeLayout { quote, len: None }; + Self { source, layout } + } + #[inline] + pub fn with_preferred_quote(source: &'a str, quote: Quote) -> Self { + let layout = Self::repr_layout(source, quote); + Self { source, layout } + } + #[inline] + pub fn new_repr(source: &'a str) -> Self { + Self::with_preferred_quote(source, Quote::Single) + } + #[inline] + pub fn str_repr<'r>(&'a self) -> StrRepr<'r, 'a> { + StrRepr(self) + } +} + +pub struct StrRepr<'r, 'a>(&'r UnicodeEscape<'a>); + +impl StrRepr<'_, '_> { + pub fn write(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result { + let quote = self.0.layout().quote.to_char(); + formatter.write_char(quote)?; + self.0.write_body(formatter)?; + formatter.write_char(quote) + } + + pub fn to_string(&self) -> Option { + let mut s = String::with_capacity(self.0.layout().len?); + self.write(&mut s).unwrap(); + Some(s) + } +} + +impl std::fmt::Display for StrRepr<'_, '_> { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.write(formatter) + } +} + +impl UnicodeEscape<'_> { + const REPR_RESERVED_LEN: usize = 2; // for quotes + + pub fn repr_layout(source: &str, preferred_quote: Quote) -> EscapeLayout { + Self::output_layout_with_checker(source, preferred_quote, |a, b| { + Some((a as isize).checked_add(b as isize)? as usize) + }) + } + + fn output_layout_with_checker( + source: &str, + preferred_quote: Quote, + length_add: impl Fn(usize, usize) -> Option, + ) -> EscapeLayout { + let mut out_len = Self::REPR_RESERVED_LEN; + let mut single_count = 0; + let mut double_count = 0; + + for ch in source.chars() { + let incr = match ch { + '\'' => { + single_count += 1; + 1 + } + '"' => { + double_count += 1; + 1 + } + c => Self::escaped_char_len(c), + }; + let Some(new_len) = length_add(out_len, incr) else { + #[cold] + fn stop( + single_count: usize, + double_count: usize, + preferred_quote: Quote, + ) -> EscapeLayout { + EscapeLayout { + quote: choose_quote(single_count, double_count, preferred_quote).0, + len: None, + } + } + return stop(single_count, double_count, preferred_quote); + }; + out_len = new_len; + } + + let (quote, num_escaped_quotes) = choose_quote(single_count, double_count, preferred_quote); + // we'll be adding backslashes in front of the existing inner quotes + let Some(out_len) = length_add(out_len, num_escaped_quotes) else { + return EscapeLayout { quote, len: None }; + }; + + EscapeLayout { + quote, + len: Some(out_len - Self::REPR_RESERVED_LEN), + } + } + + fn escaped_char_len(ch: char) -> usize { + match ch { + '\\' | '\t' | '\r' | '\n' => 2, + ch if ch < ' ' || ch as u32 == 0x7f => 4, // \xHH + ch if ch.is_ascii() => 1, + ch if crate::char::is_printable(ch) => { + // max = std::cmp::max(ch, max); + ch.len_utf8() + } + ch if (ch as u32) < 0x100 => 4, // \xHH + ch if (ch as u32) < 0x10000 => 6, // \uHHHH + _ => 10, // \uHHHHHHHH + } + } + + fn write_char( + ch: char, + quote: Quote, + formatter: &mut impl std::fmt::Write, + ) -> std::fmt::Result { + match ch { + '\n' => formatter.write_str("\\n"), + '\t' => formatter.write_str("\\t"), + '\r' => formatter.write_str("\\r"), + // these 2 branches *would* be handled below, but we shouldn't have to do a + // unicodedata lookup just for ascii characters + '\x20'..='\x7e' => { + // printable ascii range + if ch == quote.to_char() || ch == '\\' { + formatter.write_char('\\')?; + } + formatter.write_char(ch) + } + ch if ch.is_ascii() => { + write!(formatter, "\\x{:02x}", ch as u8) + } + ch if crate::char::is_printable(ch) => formatter.write_char(ch), + '\0'..='\u{ff}' => { + write!(formatter, "\\x{:02x}", ch as u32) + } + '\0'..='\u{ffff}' => { + write!(formatter, "\\u{:04x}", ch as u32) + } + _ => { + write!(formatter, "\\U{:08x}", ch as u32) + } + } + } +} + +impl Escape for UnicodeEscape<'_> { + fn source_len(&self) -> usize { + self.source.len() + } + + fn layout(&self) -> &EscapeLayout { + &self.layout + } + + fn write_source(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result { + formatter.write_str(self.source) + } + + #[cold] + fn write_body_slow(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result { + for ch in self.source.chars() { + Self::write_char(ch, self.layout().quote, formatter)?; + } + Ok(()) + } +} + +pub struct AsciiEscape<'a> { + source: &'a [u8], + layout: EscapeLayout, +} + +impl<'a> AsciiEscape<'a> { + #[inline] + pub fn new(source: &'a [u8], layout: EscapeLayout) -> Self { + Self { source, layout } + } + #[inline] + pub fn with_forced_quote(source: &'a [u8], quote: Quote) -> Self { + let layout = EscapeLayout { quote, len: None }; + Self { source, layout } + } + #[inline] + pub fn with_preferred_quote(source: &'a [u8], quote: Quote) -> Self { + let layout = Self::repr_layout(source, quote); + Self { source, layout } + } + #[inline] + pub fn new_repr(source: &'a [u8]) -> Self { + Self::with_preferred_quote(source, Quote::Single) + } + #[inline] + pub fn bytes_repr<'r>(&'a self) -> BytesRepr<'r, 'a> { + BytesRepr(self) + } +} + +impl AsciiEscape<'_> { + pub fn repr_layout(source: &[u8], preferred_quote: Quote) -> EscapeLayout { + Self::output_layout_with_checker(source, preferred_quote, 3, |a, b| { + Some((a as isize).checked_add(b as isize)? as usize) + }) + } + + pub fn named_repr_layout(source: &[u8], name: &str) -> EscapeLayout { + Self::output_layout_with_checker(source, Quote::Single, name.len() + 2 + 3, |a, b| { + Some((a as isize).checked_add(b as isize)? as usize) + }) + } + + fn output_layout_with_checker( + source: &[u8], + preferred_quote: Quote, + reserved_len: usize, + length_add: impl Fn(usize, usize) -> Option, + ) -> EscapeLayout { + let mut out_len = reserved_len; + let mut single_count = 0; + let mut double_count = 0; + + for ch in source.iter() { + let incr = match ch { + b'\'' => { + single_count += 1; + 1 + } + b'"' => { + double_count += 1; + 1 + } + c => Self::escaped_char_len(*c), + }; + let Some(new_len) = length_add(out_len, incr) else { + #[cold] + fn stop( + single_count: usize, + double_count: usize, + preferred_quote: Quote, + ) -> EscapeLayout { + EscapeLayout { + quote: choose_quote(single_count, double_count, preferred_quote).0, + len: None, + } + } + return stop(single_count, double_count, preferred_quote); + }; + out_len = new_len; + } + + let (quote, num_escaped_quotes) = choose_quote(single_count, double_count, preferred_quote); + // we'll be adding backslashes in front of the existing inner quotes + let Some(out_len) = length_add(out_len, num_escaped_quotes) else { + return EscapeLayout { quote, len: None }; + }; + + EscapeLayout { + quote, + len: Some(out_len - reserved_len), + } + } + + fn escaped_char_len(ch: u8) -> usize { + match ch { + b'\\' | b'\t' | b'\r' | b'\n' => 2, + 0x20..=0x7e => 1, + _ => 4, // \xHH + } + } + + fn write_char(ch: u8, quote: Quote, formatter: &mut impl std::fmt::Write) -> std::fmt::Result { + match ch { + b'\t' => formatter.write_str("\\t"), + b'\n' => formatter.write_str("\\n"), + b'\r' => formatter.write_str("\\r"), + 0x20..=0x7e => { + // printable ascii range + if ch == quote.to_byte() || ch == b'\\' { + formatter.write_char('\\')?; + } + formatter.write_char(ch as char) + } + ch => write!(formatter, "\\x{ch:02x}"), + } + } +} + +impl Escape for AsciiEscape<'_> { + fn source_len(&self) -> usize { + self.source.len() + } + + fn layout(&self) -> &EscapeLayout { + &self.layout + } + + fn write_source(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result { + formatter.write_str(unsafe { + // SAFETY: this function must be called only when source is printable ascii characters + std::str::from_utf8_unchecked(self.source) + }) + } + + #[cold] + fn write_body_slow(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result { + for ch in self.source.iter() { + Self::write_char(*ch, self.layout().quote, formatter)?; + } + Ok(()) + } +} + +pub struct BytesRepr<'r, 'a>(&'r AsciiEscape<'a>); + +impl BytesRepr<'_, '_> { + pub fn write(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result { + let quote = self.0.layout().quote.to_char(); + formatter.write_char('b')?; + formatter.write_char(quote)?; + self.0.write_body(formatter)?; + formatter.write_char(quote) + } + + pub fn to_string(&self) -> Option { + let mut s = String::with_capacity(self.0.layout().len?); + self.write(&mut s).unwrap(); + Some(s) + } +} + +impl std::fmt::Display for BytesRepr<'_, '_> { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.write(formatter) + } +} + +#[cfg(test)] +mod unicode_escape_tests { + use super::*; + + #[test] + fn changed() { + fn test(s: &str) -> bool { + UnicodeEscape::new_repr(s).changed() + } + assert!(!test("hello")); + assert!(!test("'hello'")); + assert!(!test("\"hello\"")); + + assert!(test("'\"hello")); + assert!(test("hello\n")); + } +} diff --git a/compiler/literal/src/float.rs b/compiler/literal/src/float.rs new file mode 100644 index 0000000000..a58a3f48a7 --- /dev/null +++ b/compiler/literal/src/float.rs @@ -0,0 +1,342 @@ +use crate::format::Case; +use num_traits::{Float, Zero}; +use std::f64; + +pub fn parse_str(literal: &str) -> Option { + parse_inner(literal.trim().as_bytes()) +} + +fn strip_underlines(literal: &[u8]) -> Option> { + let mut prev = b'\0'; + let mut dup = Vec::::new(); + for p in literal { + if *p == b'_' { + // Underscores are only allowed after digits. + if !prev.is_ascii_digit() { + return None; + } + } else { + dup.push(*p); + // Underscores are only allowed before digits. + if prev == b'_' && !p.is_ascii_digit() { + return None; + } + } + prev = *p; + } + + // Underscores are not allowed at the end. + if prev == b'_' { + return None; + } + + Some(dup) +} + +pub fn parse_bytes(literal: &[u8]) -> Option { + parse_inner(trim_slice(literal, |b| b.is_ascii_whitespace())) +} + +fn trim_slice(v: &[T], mut trim: impl FnMut(&T) -> bool) -> &[T] { + let mut it = v.iter(); + // it.take_while_ref(&mut trim).for_each(drop); + // hmm.. `&mut slice::Iter<_>` is not `Clone` + // it.by_ref().rev().take_while_ref(&mut trim).for_each(drop); + while it.clone().next().is_some_and(&mut trim) { + it.next(); + } + while it.clone().next_back().is_some_and(&mut trim) { + it.next_back(); + } + it.as_slice() +} + +fn parse_inner(literal: &[u8]) -> Option { + use lexical_parse_float::{ + FromLexicalWithOptions, NumberFormatBuilder, Options, format::PYTHON3_LITERAL, + }; + + // Use custom function for underline handling for now. + // For further information see https://github.com/Alexhuszagh/rust-lexical/issues/96. + let stripped = strip_underlines(literal)?; + + // lexical-core's format::PYTHON_STRING is inaccurate + const PYTHON_STRING: u128 = NumberFormatBuilder::rebuild(PYTHON3_LITERAL) + .no_special(false) + .build(); + f64::from_lexical_with_options::(&stripped, &Options::new()).ok() +} + +pub fn is_integer(v: f64) -> bool { + (v - v.round()).abs() < f64::EPSILON +} + +fn format_nan(case: Case) -> String { + let nan = match case { + Case::Lower => "nan", + Case::Upper => "NAN", + }; + + nan.to_string() +} + +fn format_inf(case: Case) -> String { + let inf = match case { + Case::Lower => "inf", + Case::Upper => "INF", + }; + + inf.to_string() +} + +pub fn decimal_point_or_empty(precision: usize, alternate_form: bool) -> &'static str { + match (precision, alternate_form) { + (0, true) => ".", + _ => "", + } +} + +pub fn format_fixed(precision: usize, magnitude: f64, case: Case, alternate_form: bool) -> String { + match magnitude { + magnitude if magnitude.is_finite() => { + let point = decimal_point_or_empty(precision, alternate_form); + format!("{magnitude:.precision$}{point}") + } + magnitude if magnitude.is_nan() => format_nan(case), + magnitude if magnitude.is_infinite() => format_inf(case), + _ => "".to_string(), + } +} + +// Formats floats into Python style exponent notation, by first formatting in Rust style +// exponent notation (`1.0000e0`), then convert to Python style (`1.0000e+00`). +pub fn format_exponent( + precision: usize, + magnitude: f64, + case: Case, + alternate_form: bool, +) -> String { + match magnitude { + magnitude if magnitude.is_finite() => { + let r_exp = format!("{magnitude:.precision$e}"); + let mut parts = r_exp.splitn(2, 'e'); + let base = parts.next().unwrap(); + let exponent = parts.next().unwrap().parse::().unwrap(); + let e = match case { + Case::Lower => 'e', + Case::Upper => 'E', + }; + let point = decimal_point_or_empty(precision, alternate_form); + format!("{base}{point}{e}{exponent:+#03}") + } + magnitude if magnitude.is_nan() => format_nan(case), + magnitude if magnitude.is_infinite() => format_inf(case), + _ => "".to_string(), + } +} + +/// If s represents a floating point value, trailing zeros and a possibly trailing +/// decimal point will be removed. +/// This function does NOT work with decimal commas. +fn maybe_remove_trailing_redundant_chars(s: String, alternate_form: bool) -> String { + if !alternate_form && s.contains('.') { + // only truncate floating point values when not in alternate form + let s = remove_trailing_zeros(s); + remove_trailing_decimal_point(s) + } else { + s + } +} + +fn remove_trailing_zeros(s: String) -> String { + let mut s = s; + while s.ends_with('0') { + s.pop(); + } + s +} + +fn remove_trailing_decimal_point(s: String) -> String { + let mut s = s; + if s.ends_with('.') { + s.pop(); + } + s +} + +pub fn format_general( + precision: usize, + magnitude: f64, + case: Case, + alternate_form: bool, + always_shows_fract: bool, +) -> String { + match magnitude { + magnitude if magnitude.is_finite() => { + let r_exp = format!("{:.*e}", precision.saturating_sub(1), magnitude); + let mut parts = r_exp.splitn(2, 'e'); + let base = parts.next().unwrap(); + let exponent = parts.next().unwrap().parse::().unwrap(); + if exponent < -4 || exponent + (always_shows_fract as i64) >= (precision as i64) { + let e = match case { + Case::Lower => 'e', + Case::Upper => 'E', + }; + let magnitude = format!("{:.*}", precision + 1, base); + let base = maybe_remove_trailing_redundant_chars(magnitude, alternate_form); + let point = decimal_point_or_empty(precision.saturating_sub(1), alternate_form); + format!("{base}{point}{e}{exponent:+#03}") + } else { + let precision = ((precision as i64) - 1 - exponent) as usize; + let magnitude = format!("{magnitude:.precision$}"); + let base = maybe_remove_trailing_redundant_chars(magnitude, alternate_form); + let point = decimal_point_or_empty(precision, alternate_form); + format!("{base}{point}") + } + } + magnitude if magnitude.is_nan() => format_nan(case), + magnitude if magnitude.is_infinite() => format_inf(case), + _ => "".to_string(), + } +} + +// TODO: rewrite using format_general +pub fn to_string(value: f64) -> String { + let lit = format!("{value:e}"); + if let Some(position) = lit.find('e') { + let significand = &lit[..position]; + let exponent = &lit[position + 1..]; + let exponent = exponent.parse::().unwrap(); + if exponent < 16 && exponent > -5 { + if is_integer(value) { + format!("{value:.1?}") + } else { + value.to_string() + } + } else { + format!("{significand}e{exponent:+#03}") + } + } else { + let mut s = value.to_string(); + s.make_ascii_lowercase(); + s + } +} + +pub fn from_hex(s: &str) -> Option { + if let Ok(f) = hexf_parse::parse_hexf64(s, false) { + return Some(f); + } + match s.to_ascii_lowercase().as_str() { + "nan" | "+nan" | "-nan" => Some(f64::NAN), + "inf" | "infinity" | "+inf" | "+infinity" => Some(f64::INFINITY), + "-inf" | "-infinity" => Some(f64::NEG_INFINITY), + value => { + let mut hex = String::with_capacity(value.len()); + let has_0x = value.contains("0x"); + let has_p = value.contains('p'); + let has_dot = value.contains('.'); + let mut start = 0; + + if !has_0x && value.starts_with('-') { + hex.push_str("-0x"); + start += 1; + } else if !has_0x { + hex.push_str("0x"); + if value.starts_with('+') { + start += 1; + } + } + + for (index, ch) in value.chars().enumerate() { + if ch == 'p' { + if has_dot { + hex.push('p'); + } else { + hex.push_str(".p"); + } + } else if index >= start { + hex.push(ch); + } + } + + if !has_p && has_dot { + hex.push_str("p0"); + } else if !has_p && !has_dot { + hex.push_str(".p0") + } + + hexf_parse::parse_hexf64(hex.as_str(), false).ok() + } + } +} + +pub fn to_hex(value: f64) -> String { + let (mantissa, exponent, sign) = value.integer_decode(); + let sign_fmt = if sign < 0 { "-" } else { "" }; + match value { + value if value.is_zero() => format!("{sign_fmt}0x0.0p+0"), + value if value.is_infinite() => format!("{sign_fmt}inf"), + value if value.is_nan() => "nan".to_owned(), + _ => { + const BITS: i16 = 52; + const FRACT_MASK: u64 = 0xf_ffff_ffff_ffff; + format!( + "{}{:#x}.{:013x}p{:+}", + sign_fmt, + mantissa >> BITS, + mantissa & FRACT_MASK, + exponent + BITS + ) + } + } +} + +#[test] +fn test_to_hex() { + use rand::Rng; + for _ in 0..20000 { + let bytes = rand::rng().random::(); + let f = f64::from_bits(bytes); + if !f.is_finite() { + continue; + } + let hex = to_hex(f); + // println!("{} -> {}", f, hex); + let roundtrip = hexf_parse::parse_hexf64(&hex, false).unwrap(); + // println!(" -> {}", roundtrip); + assert!(f == roundtrip, "{} {} {}", f, hex, roundtrip); + } +} + +#[test] +fn test_remove_trailing_zeros() { + assert!(remove_trailing_zeros(String::from("100")) == *"1"); + assert!(remove_trailing_zeros(String::from("100.00")) == *"100."); + + // leave leading zeros untouched + assert!(remove_trailing_zeros(String::from("001")) == *"001"); + + // leave strings untouched if they don't end with 0 + assert!(remove_trailing_zeros(String::from("101")) == *"101"); +} + +#[test] +fn test_remove_trailing_decimal_point() { + assert!(remove_trailing_decimal_point(String::from("100.")) == *"100"); + assert!(remove_trailing_decimal_point(String::from("1.")) == *"1"); + + // leave leading decimal points untouched + assert!(remove_trailing_decimal_point(String::from(".5")) == *".5"); +} + +#[test] +fn test_maybe_remove_trailing_redundant_chars() { + assert!(maybe_remove_trailing_redundant_chars(String::from("100."), true) == *"100."); + assert!(maybe_remove_trailing_redundant_chars(String::from("100."), false) == *"100"); + assert!(maybe_remove_trailing_redundant_chars(String::from("1."), false) == *"1"); + assert!(maybe_remove_trailing_redundant_chars(String::from("10.0"), false) == *"10"); + + // don't truncate integers + assert!(maybe_remove_trailing_redundant_chars(String::from("1000"), false) == *"1000"); +} diff --git a/compiler/literal/src/format.rs b/compiler/literal/src/format.rs new file mode 100644 index 0000000000..4ce21ad781 --- /dev/null +++ b/compiler/literal/src/format.rs @@ -0,0 +1,5 @@ +#[derive(Debug, PartialEq, Eq, Clone, Copy, is_macro::Is, Hash)] +pub enum Case { + Lower, + Upper, +} diff --git a/compiler/literal/src/lib.rs b/compiler/literal/src/lib.rs new file mode 100644 index 0000000000..9b9620573d --- /dev/null +++ b/compiler/literal/src/lib.rs @@ -0,0 +1,4 @@ +pub mod char; +pub mod escape; +pub mod float; +pub mod format; diff --git a/deny.toml b/deny.toml index 4063d1fec1..85513455fc 100644 --- a/deny.toml +++ b/deny.toml @@ -242,4 +242,4 @@ unknown-git = "warn" # if not specified. If it is specified but empty, no registries are allowed. allow-registry = ["https://github.com/rust-lang/crates.io-index"] # List of URLs for allowed Git repositories -allow-git = ["https://github.com/RustPython/Parser.git", "https://github.com/RustPython/__doc__"] +allow-git = ["https://github.com/RustPython/__doc__"] diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 49b0883bbe..559390d278 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -42,7 +42,6 @@ rustpython-compiler-core = { workspace = true } rustpython-compiler-source = { workspace = true } # rustpython-parser-core = { workspace = true } rustpython-literal = { workspace = true } -rustpython-format = { workspace = true } rustpython-sre_engine = { workspace = true } ascii = { workspace = true } diff --git a/vm/src/builtins/bool.rs b/vm/src/builtins/bool.rs index 93f5f4f1de..006fc4d1eb 100644 --- a/vm/src/builtins/bool.rs +++ b/vm/src/builtins/bool.rs @@ -1,4 +1,5 @@ use super::{PyInt, PyStrRef, PyType, PyTypeRef}; +use crate::common::format::FormatSpec; use crate::{ AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyResult, TryFromBorrowedObject, VirtualMachine, @@ -11,7 +12,6 @@ use crate::{ }; use malachite_bigint::Sign; use num_traits::Zero; -use rustpython_format::FormatSpec; use std::fmt::{Debug, Formatter}; impl ToPyObject for bool { diff --git a/vm/src/builtins/float.rs b/vm/src/builtins/float.rs index e33c25cb56..b4601fbb92 100644 --- a/vm/src/builtins/float.rs +++ b/vm/src/builtins/float.rs @@ -7,7 +7,7 @@ use crate::{ AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromBorrowedObject, TryFromObject, VirtualMachine, class::PyClassImpl, - common::{float_ops, hash}, + common::{float_ops, format::FormatSpec, hash}, convert::{IntoPyException, ToPyObject, ToPyResult}, function::{ ArgBytesLike, OptionalArg, OptionalOption, @@ -21,7 +21,6 @@ use malachite_bigint::{BigInt, ToBigInt}; use num_complex::Complex64; use num_traits::{Signed, ToPrimitive, Zero}; use rustpython_common::int::float_to_ratio; -use rustpython_format::FormatSpec; #[pyclass(module = false, name = "float")] #[derive(Debug, Copy, Clone, PartialEq)] diff --git a/vm/src/builtins/int.rs b/vm/src/builtins/int.rs index db9c5ea4ed..aa9613e9d7 100644 --- a/vm/src/builtins/int.rs +++ b/vm/src/builtins/int.rs @@ -6,6 +6,7 @@ use crate::{ bytesinner::PyBytesInner, class::PyClassImpl, common::{ + format::FormatSpec, hash, int::{bigint_to_finite_float, bytes_to_int, true_div}, }, @@ -20,7 +21,6 @@ use crate::{ use malachite_bigint::{BigInt, Sign}; use num_integer::Integer; use num_traits::{One, Pow, PrimInt, Signed, ToPrimitive, Zero}; -use rustpython_format::FormatSpec; use std::fmt; use std::ops::{Neg, Not}; diff --git a/vm/src/builtins/str.rs b/vm/src/builtins/str.rs index 3446a3d54c..554fa1de7f 100644 --- a/vm/src/builtins/str.rs +++ b/vm/src/builtins/str.rs @@ -31,10 +31,10 @@ use once_cell::sync::Lazy; use rustpython_common::{ ascii, atomic::{self, PyAtomic, Radium}, + format::{FormatSpec, FormatString, FromTemplate}, hash, lock::PyMutex, }; -use rustpython_format::{FormatSpec, FormatString, FromTemplate}; use std::{char, fmt, ops::Range, string::ToString}; use unic_ucd_bidi::BidiClass; use unic_ucd_category::GeneralCategory; diff --git a/vm/src/cformat.rs b/vm/src/cformat.rs index 6e14034d0b..a11c06a252 100644 --- a/vm/src/cformat.rs +++ b/vm/src/cformat.rs @@ -1,6 +1,7 @@ //! Implementation of Printf-Style string formatting //! as per the [Python Docs](https://docs.python.org/3/library/stdtypes.html#printf-style-string-formatting). +use crate::common::cformat::*; use crate::{ AsObject, PyObjectRef, PyResult, TryFromBorrowedObject, TryFromObject, VirtualMachine, builtins::{ @@ -12,7 +13,6 @@ use crate::{ }; use itertools::Itertools; use num_traits::cast::ToPrimitive; -use rustpython_format::cformat::*; use std::str::FromStr; fn spec_format_bytes( diff --git a/vm/src/format.rs b/vm/src/format.rs index 7e9bb54265..1dbfd46779 100644 --- a/vm/src/format.rs +++ b/vm/src/format.rs @@ -6,7 +6,7 @@ use crate::{ stdlib::builtins, }; -use rustpython_format::*; +use crate::common::format::*; impl IntoPyException for FormatSpecError { fn into_pyexception(self, vm: &VirtualMachine) -> PyBaseExceptionRef { diff --git a/vm/src/stdlib/string.rs b/vm/src/stdlib/string.rs index 8a8a182732..3c399e1c37 100644 --- a/vm/src/stdlib/string.rs +++ b/vm/src/stdlib/string.rs @@ -6,15 +6,15 @@ pub(crate) use _string::make_module; #[pymodule] mod _string { use crate::common::ascii; + use crate::common::format::{ + FieldName, FieldNamePart, FieldType, FormatPart, FormatString, FromTemplate, + }; use crate::{ PyObjectRef, PyResult, VirtualMachine, builtins::{PyList, PyStrRef}, convert::ToPyException, convert::ToPyObject, }; - use rustpython_format::{ - FieldName, FieldNamePart, FieldType, FormatPart, FormatString, FromTemplate, - }; use std::mem; fn create_format_part( From 9779de98b86e456350ce6d9b155a23370348d1c0 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sat, 22 Mar 2025 20:25:52 -0700 Subject: [PATCH 102/295] _tkinter pt. 1 (#5583) * Add _tkinter module and gate * add tkinter module * add tcl_error * fix tk setup and add demo * fix TK_VERSION * create and TkApp --- Cargo.lock | 396 ++- Cargo.toml | 1 + Lib/tkinter/__init__.py | 4984 +++++++++++++++++++++++++++++++++++ Lib/tkinter/__main__.py | 7 + Lib/tkinter/colorchooser.py | 86 + Lib/tkinter/commondialog.py | 53 + Lib/tkinter/constants.py | 110 + Lib/tkinter/dialog.py | 49 + Lib/tkinter/dnd.py | 324 +++ Lib/tkinter/filedialog.py | 492 ++++ Lib/tkinter/font.py | 239 ++ Lib/tkinter/messagebox.py | 146 + Lib/tkinter/scrolledtext.py | 56 + Lib/tkinter/simpledialog.py | 441 ++++ Lib/tkinter/ttk.py | 1647 ++++++++++++ stdlib/Cargo.toml | 5 + stdlib/src/lib.rs | 7 + stdlib/src/tkinter.rs | 84 + 18 files changed, 9112 insertions(+), 15 deletions(-) create mode 100644 Lib/tkinter/__init__.py create mode 100644 Lib/tkinter/__main__.py create mode 100644 Lib/tkinter/colorchooser.py create mode 100644 Lib/tkinter/commondialog.py create mode 100644 Lib/tkinter/constants.py create mode 100644 Lib/tkinter/dialog.py create mode 100644 Lib/tkinter/dnd.py create mode 100644 Lib/tkinter/filedialog.py create mode 100644 Lib/tkinter/font.py create mode 100644 Lib/tkinter/messagebox.py create mode 100644 Lib/tkinter/scrolledtext.py create mode 100644 Lib/tkinter/simpledialog.py create mode 100644 Lib/tkinter/ttk.py create mode 100644 stdlib/src/tkinter.rs diff --git a/Cargo.lock b/Cargo.lock index 94ae260f94..b214780cb4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -116,6 +116,39 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" +[[package]] +name = "bind_syn" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d6608ba072b4bc847774fac76963956592b5cdfa3751afcefa252fb61cb85b9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.98", +] + +[[package]] +name = "bindgen" +version = "0.64.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4243e6031260db77ede97ad86c27e501d646a27ab57b59a574f725d98ab1fb4" +dependencies = [ + "bitflags 1.3.2", + "cexpr", + "clang-sys", + "lazy_static 1.5.0", + "lazycell", + "log", + "peeking_take_while", + "proc-macro2", + "quote", + "regex", + "rustc-hash 1.1.0", + "shlex", + "syn 1.0.109", + "which 4.4.2", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -232,6 +265,37 @@ dependencies = [ "shlex", ] +[[package]] +name = "cex" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b0114a3f232423fadbbdbb692688e3e68c3b58b4b063ac3a7d0190d561080da" +dependencies = [ + "cex_derive", + "enumx", +] + +[[package]] +name = "cex_derive" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b5048cd656d7d2e739960fa33d9f95693005792dc8aad0af8b8f0b7d76c938d" +dependencies = [ + "indexmap 1.9.3", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + [[package]] name = "cfg-if" version = "1.0.0" @@ -258,6 +322,17 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + [[package]] name = "clap" version = "2.34.0" @@ -269,6 +344,19 @@ dependencies = [ "unicode-width 0.1.14", ] +[[package]] +name = "clib" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fda1a698cd341f055d3ae1fdbfb1cc441e7f9bacce795356ecc685e69134e957" +dependencies = [ + "anyhow", + "bindgen", + "inwelling", + "pkg-config", + "toml", +] + [[package]] name = "clipboard-win" version = "5.4.0" @@ -383,7 +471,7 @@ dependencies = [ "hashbrown 0.14.5", "log", "regalloc2", - "rustc-hash", + "rustc-hash 2.1.1", "serde", "smallvec", "target-lexicon 0.13.2", @@ -678,6 +766,27 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" +[[package]] +name = "enumx" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32875abeb14f7fe2c2b8ad15e58f41701f455d124d0a03bc88132d5face2663f" +dependencies = [ + "enumx_derive", +] + +[[package]] +name = "enumx_derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa5d66efdd1eab6ea85ba31bdb58bed1e4ce218c1361061384ece88f40ebeb49" +dependencies = [ + "indexmap 1.9.3", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "env_logger" version = "0.9.3" @@ -765,7 +874,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8168cbad48fdda10be94de9c6319f9e8ac5d3cf0a1abda1864269dfcca3d302a" dependencies = [ "flame", - "indexmap", + "indexmap 2.7.1", "serde", "serde_json", ] @@ -871,7 +980,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" dependencies = [ "fallible-iterator", - "indexmap", + "indexmap 2.7.1", "stable_deref_trait", ] @@ -897,6 +1006,12 @@ dependencies = [ "crunchy", ] +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "hashbrown" version = "0.14.5" @@ -912,12 +1027,30 @@ dependencies = [ "foldhash", ] +[[package]] +name = "heck" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "heck" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "heredom" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7a4d76fa670b51cfb56e908ad8bfd44a14fee853ea764790e46634d3fcdf4d" +dependencies = [ + "tuplex", +] + [[package]] name = "hermit-abi" version = "0.1.19" @@ -977,6 +1110,16 @@ dependencies = [ "cc", ] +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + [[package]] name = "indexmap" version = "2.7.1" @@ -1006,13 +1149,23 @@ dependencies = [ "similar", ] +[[package]] +name = "inwelling" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f6292c68ffca1fa94ca8f95ca5ad2885d79d96377f1d37ced6a47cd26cfaf8c" +dependencies = [ + "toml", + "walkdir", +] + [[package]] name = "is-macro" version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d57a3e447e24c22647738e4607f1df1e0ec6f72e16182c4cd199f647cdfb0e4" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "quote", "syn 2.0.98", @@ -1098,6 +1251,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "lexical-parse-float" version = "0.8.5" @@ -1355,6 +1514,12 @@ dependencies = [ "autocfg", ] +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.4" @@ -1373,6 +1538,12 @@ dependencies = [ "rand_core 0.9.1", ] +[[package]] +name = "mutf8" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97b444426a4c188e9ad33560853ebd52309ab72811f536a9e6f37907fd12cf45" + [[package]] name = "nibble_vec" version = "0.1.0" @@ -1395,6 +1566,16 @@ dependencies = [ "memoffset", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "num-complex" version = "0.4.6" @@ -1447,6 +1628,7 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af1844ef2428cc3e1cb900be36181049ef3d3193c63e43026cfe202983b27a56" dependencies = [ + "proc-macro-crate", "proc-macro2", "quote", "syn 2.0.98", @@ -1563,6 +1745,12 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "peeking_take_while" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" + [[package]] name = "phf" version = "0.11.3" @@ -1681,6 +1869,15 @@ dependencies = [ "zerocopy 0.7.35", ] +[[package]] +name = "proc-macro-crate" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edce586971a4dfaa28950c6f18ed55e0406c1ab88bbce2c6f6293a7aaba73d35" +dependencies = [ + "toml_edit 0.22.24", +] + [[package]] name = "proc-macro2" version = "1.0.93" @@ -1756,7 +1953,7 @@ version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "36c011a03ba1e50152b4b394b479826cad97e7a21eb52df179cd91ac411cbfbe" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "pyo3-build-config", "quote", @@ -1905,7 +2102,7 @@ dependencies = [ "bumpalo", "hashbrown 0.15.2", "log", - "rustc-hash", + "rustc-hash 2.1.1", "smallvec", ] @@ -1985,7 +2182,7 @@ dependencies = [ "ruff_python_trivia", "ruff_source_file", "ruff_text_size", - "rustc-hash", + "rustc-hash 2.1.1", ] [[package]] @@ -2023,7 +2220,7 @@ dependencies = [ "ruff_python_ast", "ruff_python_trivia", "ruff_text_size", - "rustc-hash", + "rustc-hash 2.1.1", "static_assertions", "unicode-ident", "unicode-normalization", @@ -2055,6 +2252,12 @@ name = "ruff_text_size" version = "0.0.0" source = "git+https://github.com/astral-sh/ruff.git?tag=0.11.0#2cd25ef6410fb5fca96af1578728a3d828d2d53a" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc-hash" version = "2.1.1" @@ -2111,7 +2314,7 @@ version = "0.4.0" dependencies = [ "ahash", "bitflags 2.8.0", - "indexmap", + "indexmap 2.7.1", "insta", "itertools 0.14.0", "log", @@ -2292,7 +2495,7 @@ dependencies = [ "foreign-types-shared", "gethostname", "hex", - "indexmap", + "indexmap 2.7.1", "itertools 0.14.0", "junction", "libc", @@ -2328,8 +2531,10 @@ dependencies = [ "sha3", "socket2", "system-configuration", + "tcl", "termios", "thread_local", + "tk", "ucd", "unic-char-property", "unic-normal", @@ -2365,7 +2570,7 @@ dependencies = [ "glob", "half 2.4.1", "hex", - "indexmap", + "indexmap 2.7.1", "is-macro", "itertools 0.14.0", "junction", @@ -2407,8 +2612,8 @@ dependencies = [ "schannel", "serde", "static_assertions", - "strum", - "strum_macros", + "strum 0.27.1", + "strum_macros 0.27.1", "thiserror 2.0.11", "thread_local", "timsort", @@ -2419,7 +2624,7 @@ dependencies = [ "unicode-casing", "unicode_names2", "wasm-bindgen", - "which", + "which 6.0.3", "widestring", "windows", "windows-sys 0.59.0", @@ -2563,6 +2768,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" +dependencies = [ + "serde", +] + [[package]] name = "sha-1" version = "0.10.1" @@ -2647,19 +2861,37 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "strum" +version = "0.19.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b89a286a7e3b5720b9a477b23253bc50debac207c8d21505f8e70b36792f11b5" + [[package]] name = "strum" version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f64def088c51c9510a8579e3c5d67c65349dcf755e5479ad3d010aa6454e2c32" +[[package]] +name = "strum_macros" +version = "0.19.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e61bb0be289045cb80bfce000512e32d09f8337e54c186725da381377ad1f8d5" +dependencies = [ + "heck 0.3.3", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "strum_macros" version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c77a8c5abcaf0f9ce05d62342b7d298c346515365c36b673df4ebe3ced01fde8" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "quote", "rustversion", @@ -2738,6 +2970,36 @@ version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e502f78cdbb8ba4718f566c418c52bc729126ffd16baee5baa718cf25dd5a69a" +[[package]] +name = "tcl" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e0d0928e8b4dca8ebd485f687f725bb34e454c7a28c1d353bf7d1b8060581bf" +dependencies = [ + "cex", + "clib", + "enumx", + "inwelling", + "mutf8", + "serde", + "serde_derive", + "tcl_derive", + "tuplex", +] + +[[package]] +name = "tcl_derive" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625d95e672231bbf31dead6861b0ad72bcb71a2891b26b0c4924cd1cc9687b93" +dependencies = [ + "bind_syn", + "proc-macro2", + "quote", + "syn 2.0.98", + "uuid", +] + [[package]] name = "termcolor" version = "1.4.1" @@ -2863,6 +3125,79 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +[[package]] +name = "tk" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6fbe29c813c9eee5e0d4d996a4a615f6538220f0ad181269a413f21c13eb077" +dependencies = [ + "bitflags 1.3.2", + "cex", + "clib", + "enumx", + "heredom", + "inwelling", + "num_enum", + "once_cell", + "serde", + "strum 0.19.5", + "strum_macros 0.19.4", + "tcl", + "tcl_derive", + "tuplex", +] + +[[package]] +name = "toml" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd79e69d3b627db300ff956027cc6c3798cef26d22526befdfcd12feeb6d2257" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit 0.19.15", +] + +[[package]] +name = "toml_datetime" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.19.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" +dependencies = [ + "indexmap 2.7.1", + "serde", + "serde_spanned", + "toml_datetime", + "winnow 0.5.40", +] + +[[package]] +name = "toml_edit" +version = "0.22.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474" +dependencies = [ + "indexmap 2.7.1", + "toml_datetime", + "winnow 0.7.4", +] + +[[package]] +name = "tuplex" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "676ac81d5454c4dcf37955d34fa8626ede3490f744b86ca14a7b90168d2a08aa" + [[package]] name = "twox-hash" version = "1.6.3" @@ -3085,6 +3420,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c1f41ffb7cf259f1ecc2876861a17e7142e63ead296f671f81f6ae85903e0d6" dependencies = [ "atomic", + "getrandom 0.3.1", ] [[package]] @@ -3223,6 +3559,18 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix", +] + [[package]] name = "which" version = "6.0.3" @@ -3439,6 +3787,24 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "0.5.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876" +dependencies = [ + "memchr", +] + +[[package]] +name = "winnow" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e97b544156e9bebe1a0ffbc03484fc1ffe3100cbce3ffb17eac35f7cdd7ab36" +dependencies = [ + "memchr", +] + [[package]] name = "winreg" version = "0.55.0" diff --git a/Cargo.toml b/Cargo.toml index 1da72c9dc5..2795c94979 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,7 @@ bz2 = ["stdlib", "rustpython-stdlib/bz2"] sqlite = ["rustpython-stdlib/sqlite"] ssl = ["rustpython-stdlib/ssl"] ssl-vendor = ["ssl", "rustpython-stdlib/ssl-vendor"] +tkinter = ["rustpython-stdlib/tkinter"] [dependencies] rustpython-compiler = { workspace = true } diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py new file mode 100644 index 0000000000..8b2502b4c0 --- /dev/null +++ b/Lib/tkinter/__init__.py @@ -0,0 +1,4984 @@ +"""Wrapper functions for Tcl/Tk. + +Tkinter provides classes which allow the display, positioning and +control of widgets. Toplevel widgets are Tk and Toplevel. Other +widgets are Frame, Label, Entry, Text, Canvas, Button, Radiobutton, +Checkbutton, Scale, Listbox, Scrollbar, OptionMenu, Spinbox +LabelFrame and PanedWindow. + +Properties of the widgets are specified with keyword arguments. +Keyword arguments have the same name as the corresponding resource +under Tk. + +Widgets are positioned with one of the geometry managers Place, Pack +or Grid. These managers can be called with methods place, pack, grid +available in every Widget. + +Actions are bound to events by resources (e.g. keyword argument +command) or with the method bind. + +Example (Hello, World): +import tkinter +from tkinter.constants import * +tk = tkinter.Tk() +frame = tkinter.Frame(tk, relief=RIDGE, borderwidth=2) +frame.pack(fill=BOTH,expand=1) +label = tkinter.Label(frame, text="Hello, World") +label.pack(fill=X, expand=1) +button = tkinter.Button(frame,text="Exit",command=tk.destroy) +button.pack(side=BOTTOM) +tk.mainloop() +""" + +import collections +import enum +import sys +import types + +import _tkinter # If this fails your Python may not be configured for Tk +TclError = _tkinter.TclError +from tkinter.constants import * +import re + +wantobjects = 1 +_debug = False # set to True to print executed Tcl/Tk commands + +TkVersion = float(_tkinter.TK_VERSION) +TclVersion = float(_tkinter.TCL_VERSION) + +READABLE = _tkinter.READABLE +WRITABLE = _tkinter.WRITABLE +EXCEPTION = _tkinter.EXCEPTION + + +_magic_re = re.compile(r'([\\{}])') +_space_re = re.compile(r'([\s])', re.ASCII) + + +def _join(value): + """Internal function.""" + return ' '.join(map(_stringify, value)) + + +def _stringify(value): + """Internal function.""" + if isinstance(value, (list, tuple)): + if len(value) == 1: + value = _stringify(value[0]) + if _magic_re.search(value): + value = '{%s}' % value + else: + value = '{%s}' % _join(value) + else: + if isinstance(value, bytes): + value = str(value, 'latin1') + else: + value = str(value) + if not value: + value = '{}' + elif _magic_re.search(value): + # add '\' before special characters and spaces + value = _magic_re.sub(r'\\\1', value) + value = value.replace('\n', r'\n') + value = _space_re.sub(r'\\\1', value) + if value[0] == '"': + value = '\\' + value + elif value[0] == '"' or _space_re.search(value): + value = '{%s}' % value + return value + + +def _flatten(seq): + """Internal function.""" + res = () + for item in seq: + if isinstance(item, (tuple, list)): + res = res + _flatten(item) + elif item is not None: + res = res + (item,) + return res + + +try: _flatten = _tkinter._flatten +except AttributeError: pass + + +def _cnfmerge(cnfs): + """Internal function.""" + if isinstance(cnfs, dict): + return cnfs + elif isinstance(cnfs, (type(None), str)): + return cnfs + else: + cnf = {} + for c in _flatten(cnfs): + try: + cnf.update(c) + except (AttributeError, TypeError) as msg: + print("_cnfmerge: fallback due to:", msg) + for k, v in c.items(): + cnf[k] = v + return cnf + + +try: _cnfmerge = _tkinter._cnfmerge +except AttributeError: pass + + +def _splitdict(tk, v, cut_minus=True, conv=None): + """Return a properly formatted dict built from Tcl list pairs. + + If cut_minus is True, the supposed '-' prefix will be removed from + keys. If conv is specified, it is used to convert values. + + Tcl list is expected to contain an even number of elements. + """ + t = tk.splitlist(v) + if len(t) % 2: + raise RuntimeError('Tcl list representing a dict is expected ' + 'to contain an even number of elements') + it = iter(t) + dict = {} + for key, value in zip(it, it): + key = str(key) + if cut_minus and key[0] == '-': + key = key[1:] + if conv: + value = conv(value) + dict[key] = value + return dict + +class _VersionInfoType(collections.namedtuple('_VersionInfoType', + ('major', 'minor', 'micro', 'releaselevel', 'serial'))): + def __str__(self): + if self.releaselevel == 'final': + return f'{self.major}.{self.minor}.{self.micro}' + else: + return f'{self.major}.{self.minor}{self.releaselevel[0]}{self.serial}' + +def _parse_version(version): + import re + m = re.fullmatch(r'(\d+)\.(\d+)([ab.])(\d+)', version) + major, minor, releaselevel, serial = m.groups() + major, minor, serial = int(major), int(minor), int(serial) + if releaselevel == '.': + micro = serial + serial = 0 + releaselevel = 'final' + else: + micro = 0 + releaselevel = {'a': 'alpha', 'b': 'beta'}[releaselevel] + return _VersionInfoType(major, minor, micro, releaselevel, serial) + + +@enum._simple_enum(enum.StrEnum) +class EventType: + KeyPress = '2' + Key = KeyPress + KeyRelease = '3' + ButtonPress = '4' + Button = ButtonPress + ButtonRelease = '5' + Motion = '6' + Enter = '7' + Leave = '8' + FocusIn = '9' + FocusOut = '10' + Keymap = '11' # undocumented + Expose = '12' + GraphicsExpose = '13' # undocumented + NoExpose = '14' # undocumented + Visibility = '15' + Create = '16' + Destroy = '17' + Unmap = '18' + Map = '19' + MapRequest = '20' + Reparent = '21' + Configure = '22' + ConfigureRequest = '23' + Gravity = '24' + ResizeRequest = '25' + Circulate = '26' + CirculateRequest = '27' + Property = '28' + SelectionClear = '29' # undocumented + SelectionRequest = '30' # undocumented + Selection = '31' # undocumented + Colormap = '32' + ClientMessage = '33' # undocumented + Mapping = '34' # undocumented + VirtualEvent = '35' # undocumented + Activate = '36' + Deactivate = '37' + MouseWheel = '38' + + +class Event: + """Container for the properties of an event. + + Instances of this type are generated if one of the following events occurs: + + KeyPress, KeyRelease - for keyboard events + ButtonPress, ButtonRelease, Motion, Enter, Leave, MouseWheel - for mouse events + Visibility, Unmap, Map, Expose, FocusIn, FocusOut, Circulate, + Colormap, Gravity, Reparent, Property, Destroy, Activate, + Deactivate - for window events. + + If a callback function for one of these events is registered + using bind, bind_all, bind_class, or tag_bind, the callback is + called with an Event as first argument. It will have the + following attributes (in braces are the event types for which + the attribute is valid): + + serial - serial number of event + num - mouse button pressed (ButtonPress, ButtonRelease) + focus - whether the window has the focus (Enter, Leave) + height - height of the exposed window (Configure, Expose) + width - width of the exposed window (Configure, Expose) + keycode - keycode of the pressed key (KeyPress, KeyRelease) + state - state of the event as a number (ButtonPress, ButtonRelease, + Enter, KeyPress, KeyRelease, + Leave, Motion) + state - state as a string (Visibility) + time - when the event occurred + x - x-position of the mouse + y - y-position of the mouse + x_root - x-position of the mouse on the screen + (ButtonPress, ButtonRelease, KeyPress, KeyRelease, Motion) + y_root - y-position of the mouse on the screen + (ButtonPress, ButtonRelease, KeyPress, KeyRelease, Motion) + char - pressed character (KeyPress, KeyRelease) + send_event - see X/Windows documentation + keysym - keysym of the event as a string (KeyPress, KeyRelease) + keysym_num - keysym of the event as a number (KeyPress, KeyRelease) + type - type of the event as a number + widget - widget in which the event occurred + delta - delta of wheel movement (MouseWheel) + """ + + def __repr__(self): + attrs = {k: v for k, v in self.__dict__.items() if v != '??'} + if not self.char: + del attrs['char'] + elif self.char != '??': + attrs['char'] = repr(self.char) + if not getattr(self, 'send_event', True): + del attrs['send_event'] + if self.state == 0: + del attrs['state'] + elif isinstance(self.state, int): + state = self.state + mods = ('Shift', 'Lock', 'Control', + 'Mod1', 'Mod2', 'Mod3', 'Mod4', 'Mod5', + 'Button1', 'Button2', 'Button3', 'Button4', 'Button5') + s = [] + for i, n in enumerate(mods): + if state & (1 << i): + s.append(n) + state = state & ~((1<< len(mods)) - 1) + if state or not s: + s.append(hex(state)) + attrs['state'] = '|'.join(s) + if self.delta == 0: + del attrs['delta'] + # widget usually is known + # serial and time are not very interesting + # keysym_num duplicates keysym + # x_root and y_root mostly duplicate x and y + keys = ('send_event', + 'state', 'keysym', 'keycode', 'char', + 'num', 'delta', 'focus', + 'x', 'y', 'width', 'height') + return '<%s event%s>' % ( + getattr(self.type, 'name', self.type), + ''.join(' %s=%s' % (k, attrs[k]) for k in keys if k in attrs) + ) + + +_support_default_root = True +_default_root = None + + +def NoDefaultRoot(): + """Inhibit setting of default root window. + + Call this function to inhibit that the first instance of + Tk is used for windows without an explicit parent window. + """ + global _support_default_root, _default_root + _support_default_root = False + # Delete, so any use of _default_root will immediately raise an exception. + # Rebind before deletion, so repeated calls will not fail. + _default_root = None + del _default_root + + +def _get_default_root(what=None): + if not _support_default_root: + raise RuntimeError("No master specified and tkinter is " + "configured to not support default root") + if _default_root is None: + if what: + raise RuntimeError(f"Too early to {what}: no default root window") + root = Tk() + assert _default_root is root + return _default_root + + +def _get_temp_root(): + global _support_default_root + if not _support_default_root: + raise RuntimeError("No master specified and tkinter is " + "configured to not support default root") + root = _default_root + if root is None: + assert _support_default_root + _support_default_root = False + root = Tk() + _support_default_root = True + assert _default_root is None + root.withdraw() + root._temporary = True + return root + + +def _destroy_temp_root(master): + if getattr(master, '_temporary', False): + try: + master.destroy() + except TclError: + pass + + +def _tkerror(err): + """Internal function.""" + pass + + +def _exit(code=0): + """Internal function. Calling it will raise the exception SystemExit.""" + try: + code = int(code) + except ValueError: + pass + raise SystemExit(code) + + +_varnum = 0 + + +class Variable: + """Class to define value holders for e.g. buttons. + + Subclasses StringVar, IntVar, DoubleVar, BooleanVar are specializations + that constrain the type of the value returned from get().""" + _default = "" + _tk = None + _tclCommands = None + + def __init__(self, master=None, value=None, name=None): + """Construct a variable + + MASTER can be given as master widget. + VALUE is an optional value (defaults to "") + NAME is an optional Tcl name (defaults to PY_VARnum). + + If NAME matches an existing variable and VALUE is omitted + then the existing value is retained. + """ + # check for type of NAME parameter to override weird error message + # raised from Modules/_tkinter.c:SetVar like: + # TypeError: setvar() takes exactly 3 arguments (2 given) + if name is not None and not isinstance(name, str): + raise TypeError("name must be a string") + global _varnum + if master is None: + master = _get_default_root('create variable') + self._root = master._root() + self._tk = master.tk + if name: + self._name = name + else: + self._name = 'PY_VAR' + repr(_varnum) + _varnum += 1 + if value is not None: + self.initialize(value) + elif not self._tk.getboolean(self._tk.call("info", "exists", self._name)): + self.initialize(self._default) + + def __del__(self): + """Unset the variable in Tcl.""" + if self._tk is None: + return + if self._tk.getboolean(self._tk.call("info", "exists", self._name)): + self._tk.globalunsetvar(self._name) + if self._tclCommands is not None: + for name in self._tclCommands: + self._tk.deletecommand(name) + self._tclCommands = None + + def __str__(self): + """Return the name of the variable in Tcl.""" + return self._name + + def set(self, value): + """Set the variable to VALUE.""" + return self._tk.globalsetvar(self._name, value) + + initialize = set + + def get(self): + """Return value of variable.""" + return self._tk.globalgetvar(self._name) + + def _register(self, callback): + f = CallWrapper(callback, None, self._root).__call__ + cbname = repr(id(f)) + try: + callback = callback.__func__ + except AttributeError: + pass + try: + cbname = cbname + callback.__name__ + except AttributeError: + pass + self._tk.createcommand(cbname, f) + if self._tclCommands is None: + self._tclCommands = [] + self._tclCommands.append(cbname) + return cbname + + def trace_add(self, mode, callback): + """Define a trace callback for the variable. + + Mode is one of "read", "write", "unset", or a list or tuple of + such strings. + Callback must be a function which is called when the variable is + read, written or unset. + + Return the name of the callback. + """ + cbname = self._register(callback) + self._tk.call('trace', 'add', 'variable', + self._name, mode, (cbname,)) + return cbname + + def trace_remove(self, mode, cbname): + """Delete the trace callback for a variable. + + Mode is one of "read", "write", "unset" or a list or tuple of + such strings. Must be same as were specified in trace_add(). + cbname is the name of the callback returned from trace_add(). + """ + self._tk.call('trace', 'remove', 'variable', + self._name, mode, cbname) + for m, ca in self.trace_info(): + if self._tk.splitlist(ca)[0] == cbname: + break + else: + self._tk.deletecommand(cbname) + try: + self._tclCommands.remove(cbname) + except ValueError: + pass + + def trace_info(self): + """Return all trace callback information.""" + splitlist = self._tk.splitlist + return [(splitlist(k), v) for k, v in map(splitlist, + splitlist(self._tk.call('trace', 'info', 'variable', self._name)))] + + def trace_variable(self, mode, callback): + """Define a trace callback for the variable. + + MODE is one of "r", "w", "u" for read, write, undefine. + CALLBACK must be a function which is called when + the variable is read, written or undefined. + + Return the name of the callback. + + This deprecated method wraps a deprecated Tcl method that will + likely be removed in the future. Use trace_add() instead. + """ + # TODO: Add deprecation warning + cbname = self._register(callback) + self._tk.call("trace", "variable", self._name, mode, cbname) + return cbname + + trace = trace_variable + + def trace_vdelete(self, mode, cbname): + """Delete the trace callback for a variable. + + MODE is one of "r", "w", "u" for read, write, undefine. + CBNAME is the name of the callback returned from trace_variable or trace. + + This deprecated method wraps a deprecated Tcl method that will + likely be removed in the future. Use trace_remove() instead. + """ + # TODO: Add deprecation warning + self._tk.call("trace", "vdelete", self._name, mode, cbname) + cbname = self._tk.splitlist(cbname)[0] + for m, ca in self.trace_info(): + if self._tk.splitlist(ca)[0] == cbname: + break + else: + self._tk.deletecommand(cbname) + try: + self._tclCommands.remove(cbname) + except ValueError: + pass + + def trace_vinfo(self): + """Return all trace callback information. + + This deprecated method wraps a deprecated Tcl method that will + likely be removed in the future. Use trace_info() instead. + """ + # TODO: Add deprecation warning + return [self._tk.splitlist(x) for x in self._tk.splitlist( + self._tk.call("trace", "vinfo", self._name))] + + def __eq__(self, other): + if not isinstance(other, Variable): + return NotImplemented + return (self._name == other._name + and self.__class__.__name__ == other.__class__.__name__ + and self._tk == other._tk) + + +class StringVar(Variable): + """Value holder for strings variables.""" + _default = "" + + def __init__(self, master=None, value=None, name=None): + """Construct a string variable. + + MASTER can be given as master widget. + VALUE is an optional value (defaults to "") + NAME is an optional Tcl name (defaults to PY_VARnum). + + If NAME matches an existing variable and VALUE is omitted + then the existing value is retained. + """ + Variable.__init__(self, master, value, name) + + def get(self): + """Return value of variable as string.""" + value = self._tk.globalgetvar(self._name) + if isinstance(value, str): + return value + return str(value) + + +class IntVar(Variable): + """Value holder for integer variables.""" + _default = 0 + + def __init__(self, master=None, value=None, name=None): + """Construct an integer variable. + + MASTER can be given as master widget. + VALUE is an optional value (defaults to 0) + NAME is an optional Tcl name (defaults to PY_VARnum). + + If NAME matches an existing variable and VALUE is omitted + then the existing value is retained. + """ + Variable.__init__(self, master, value, name) + + def get(self): + """Return the value of the variable as an integer.""" + value = self._tk.globalgetvar(self._name) + try: + return self._tk.getint(value) + except (TypeError, TclError): + return int(self._tk.getdouble(value)) + + +class DoubleVar(Variable): + """Value holder for float variables.""" + _default = 0.0 + + def __init__(self, master=None, value=None, name=None): + """Construct a float variable. + + MASTER can be given as master widget. + VALUE is an optional value (defaults to 0.0) + NAME is an optional Tcl name (defaults to PY_VARnum). + + If NAME matches an existing variable and VALUE is omitted + then the existing value is retained. + """ + Variable.__init__(self, master, value, name) + + def get(self): + """Return the value of the variable as a float.""" + return self._tk.getdouble(self._tk.globalgetvar(self._name)) + + +class BooleanVar(Variable): + """Value holder for boolean variables.""" + _default = False + + def __init__(self, master=None, value=None, name=None): + """Construct a boolean variable. + + MASTER can be given as master widget. + VALUE is an optional value (defaults to False) + NAME is an optional Tcl name (defaults to PY_VARnum). + + If NAME matches an existing variable and VALUE is omitted + then the existing value is retained. + """ + Variable.__init__(self, master, value, name) + + def set(self, value): + """Set the variable to VALUE.""" + return self._tk.globalsetvar(self._name, self._tk.getboolean(value)) + + initialize = set + + def get(self): + """Return the value of the variable as a bool.""" + try: + return self._tk.getboolean(self._tk.globalgetvar(self._name)) + except TclError: + raise ValueError("invalid literal for getboolean()") + + +def mainloop(n=0): + """Run the main loop of Tcl.""" + _get_default_root('run the main loop').tk.mainloop(n) + + +getint = int + +getdouble = float + + +def getboolean(s): + """Convert Tcl object to True or False.""" + try: + return _get_default_root('use getboolean()').tk.getboolean(s) + except TclError: + raise ValueError("invalid literal for getboolean()") + + +# Methods defined on both toplevel and interior widgets + +class Misc: + """Internal class. + + Base class which defines methods common for interior widgets.""" + + # used for generating child widget names + _last_child_ids = None + + # XXX font command? + _tclCommands = None + + def destroy(self): + """Internal function. + + Delete all Tcl commands created for + this widget in the Tcl interpreter.""" + if self._tclCommands is not None: + for name in self._tclCommands: + self.tk.deletecommand(name) + self._tclCommands = None + + def deletecommand(self, name): + """Internal function. + + Delete the Tcl command provided in NAME.""" + self.tk.deletecommand(name) + try: + self._tclCommands.remove(name) + except ValueError: + pass + + def tk_strictMotif(self, boolean=None): + """Set Tcl internal variable, whether the look and feel + should adhere to Motif. + + A parameter of 1 means adhere to Motif (e.g. no color + change if mouse passes over slider). + Returns the set value.""" + return self.tk.getboolean(self.tk.call( + 'set', 'tk_strictMotif', boolean)) + + def tk_bisque(self): + """Change the color scheme to light brown as used in Tk 3.6 and before.""" + self.tk.call('tk_bisque') + + def tk_setPalette(self, *args, **kw): + """Set a new color scheme for all widget elements. + + A single color as argument will cause that all colors of Tk + widget elements are derived from this. + Alternatively several keyword parameters and its associated + colors can be given. The following keywords are valid: + activeBackground, foreground, selectColor, + activeForeground, highlightBackground, selectBackground, + background, highlightColor, selectForeground, + disabledForeground, insertBackground, troughColor.""" + self.tk.call(('tk_setPalette',) + + _flatten(args) + _flatten(list(kw.items()))) + + def wait_variable(self, name='PY_VAR'): + """Wait until the variable is modified. + + A parameter of type IntVar, StringVar, DoubleVar or + BooleanVar must be given.""" + self.tk.call('tkwait', 'variable', name) + waitvar = wait_variable # XXX b/w compat + + def wait_window(self, window=None): + """Wait until a WIDGET is destroyed. + + If no parameter is given self is used.""" + if window is None: + window = self + self.tk.call('tkwait', 'window', window._w) + + def wait_visibility(self, window=None): + """Wait until the visibility of a WIDGET changes + (e.g. it appears). + + If no parameter is given self is used.""" + if window is None: + window = self + self.tk.call('tkwait', 'visibility', window._w) + + def setvar(self, name='PY_VAR', value='1'): + """Set Tcl variable NAME to VALUE.""" + self.tk.setvar(name, value) + + def getvar(self, name='PY_VAR'): + """Return value of Tcl variable NAME.""" + return self.tk.getvar(name) + + def getint(self, s): + try: + return self.tk.getint(s) + except TclError as exc: + raise ValueError(str(exc)) + + def getdouble(self, s): + try: + return self.tk.getdouble(s) + except TclError as exc: + raise ValueError(str(exc)) + + def getboolean(self, s): + """Return a boolean value for Tcl boolean values true and false given as parameter.""" + try: + return self.tk.getboolean(s) + except TclError: + raise ValueError("invalid literal for getboolean()") + + def focus_set(self): + """Direct input focus to this widget. + + If the application currently does not have the focus + this widget will get the focus if the application gets + the focus through the window manager.""" + self.tk.call('focus', self._w) + focus = focus_set # XXX b/w compat? + + def focus_force(self): + """Direct input focus to this widget even if the + application does not have the focus. Use with + caution!""" + self.tk.call('focus', '-force', self._w) + + def focus_get(self): + """Return the widget which has currently the focus in the + application. + + Use focus_displayof to allow working with several + displays. Return None if application does not have + the focus.""" + name = self.tk.call('focus') + if name == 'none' or not name: return None + return self._nametowidget(name) + + def focus_displayof(self): + """Return the widget which has currently the focus on the + display where this widget is located. + + Return None if the application does not have the focus.""" + name = self.tk.call('focus', '-displayof', self._w) + if name == 'none' or not name: return None + return self._nametowidget(name) + + def focus_lastfor(self): + """Return the widget which would have the focus if top level + for this widget gets the focus from the window manager.""" + name = self.tk.call('focus', '-lastfor', self._w) + if name == 'none' or not name: return None + return self._nametowidget(name) + + def tk_focusFollowsMouse(self): + """The widget under mouse will get automatically focus. Can not + be disabled easily.""" + self.tk.call('tk_focusFollowsMouse') + + def tk_focusNext(self): + """Return the next widget in the focus order which follows + widget which has currently the focus. + + The focus order first goes to the next child, then to + the children of the child recursively and then to the + next sibling which is higher in the stacking order. A + widget is omitted if it has the takefocus resource set + to 0.""" + name = self.tk.call('tk_focusNext', self._w) + if not name: return None + return self._nametowidget(name) + + def tk_focusPrev(self): + """Return previous widget in the focus order. See tk_focusNext for details.""" + name = self.tk.call('tk_focusPrev', self._w) + if not name: return None + return self._nametowidget(name) + + def after(self, ms, func=None, *args): + """Call function once after given time. + + MS specifies the time in milliseconds. FUNC gives the + function which shall be called. Additional parameters + are given as parameters to the function call. Return + identifier to cancel scheduling with after_cancel.""" + if func is None: + # I'd rather use time.sleep(ms*0.001) + self.tk.call('after', ms) + return None + else: + def callit(): + try: + func(*args) + finally: + try: + self.deletecommand(name) + except TclError: + pass + try: + callit.__name__ = func.__name__ + except AttributeError: + # Required for callable classes (bpo-44404) + callit.__name__ = type(func).__name__ + name = self._register(callit) + return self.tk.call('after', ms, name) + + def after_idle(self, func, *args): + """Call FUNC once if the Tcl main loop has no event to + process. + + Return an identifier to cancel the scheduling with + after_cancel.""" + return self.after('idle', func, *args) + + def after_cancel(self, id): + """Cancel scheduling of function identified with ID. + + Identifier returned by after or after_idle must be + given as first parameter. + """ + if not id: + raise ValueError('id must be a valid identifier returned from ' + 'after or after_idle') + try: + data = self.tk.call('after', 'info', id) + script = self.tk.splitlist(data)[0] + self.deletecommand(script) + except TclError: + pass + self.tk.call('after', 'cancel', id) + + def after_info(self, id=None): + """Return information about existing event handlers. + + With no argument, return a tuple of the identifiers for all existing + event handlers created by the after and after_idle commands for this + interpreter. If id is supplied, it specifies an existing handler; id + must have been the return value from some previous call to after or + after_idle and it must not have triggered yet or been canceled. If the + id doesn't exist, a TclError is raised. Otherwise, the return value is + a tuple containing (script, type) where script is a reference to the + function to be called by the event handler and type is either 'idle' + or 'timer' to indicate what kind of event handler it is. + """ + return self.tk.splitlist(self.tk.call('after', 'info', id)) + + def bell(self, displayof=0): + """Ring a display's bell.""" + self.tk.call(('bell',) + self._displayof(displayof)) + + def tk_busy_cget(self, option): + """Return the value of busy configuration option. + + The widget must have been previously made busy by + tk_busy_hold(). Option may have any of the values accepted by + tk_busy_hold(). + """ + return self.tk.call('tk', 'busy', 'cget', self._w, '-'+option) + busy_cget = tk_busy_cget + + def tk_busy_configure(self, cnf=None, **kw): + """Query or modify the busy configuration options. + + The widget must have been previously made busy by + tk_busy_hold(). Options may have any of the values accepted by + tk_busy_hold(). + + Please note that the option database is referenced by the widget + name or class. For example, if a Frame widget with name "frame" + is to be made busy, the busy cursor can be specified for it by + either call: + + w.option_add('*frame.busyCursor', 'gumby') + w.option_add('*Frame.BusyCursor', 'gumby') + """ + if kw: + cnf = _cnfmerge((cnf, kw)) + elif cnf: + cnf = _cnfmerge(cnf) + if cnf is None: + return self._getconfigure( + 'tk', 'busy', 'configure', self._w) + if isinstance(cnf, str): + return self._getconfigure1( + 'tk', 'busy', 'configure', self._w, '-'+cnf) + self.tk.call('tk', 'busy', 'configure', self._w, *self._options(cnf)) + busy_config = busy_configure = tk_busy_config = tk_busy_configure + + def tk_busy_current(self, pattern=None): + """Return a list of widgets that are currently busy. + + If a pattern is given, only busy widgets whose path names match + a pattern are returned. + """ + return [self._nametowidget(x) for x in + self.tk.splitlist(self.tk.call( + 'tk', 'busy', 'current', pattern))] + busy_current = tk_busy_current + + def tk_busy_forget(self): + """Make this widget no longer busy. + + User events will again be received by the widget. + """ + self.tk.call('tk', 'busy', 'forget', self._w) + busy_forget = tk_busy_forget + + def tk_busy_hold(self, **kw): + """Make this widget appear busy. + + The specified widget and its descendants will be blocked from + user interactions. Normally update() should be called + immediately afterward to insure that the hold operation is in + effect before the application starts its processing. + + The only supported configuration option is: + + cursor: the cursor to be displayed when the widget is made + busy. + """ + self.tk.call('tk', 'busy', 'hold', self._w, *self._options(kw)) + busy = busy_hold = tk_busy = tk_busy_hold + + def tk_busy_status(self): + """Return True if the widget is busy, False otherwise.""" + return self.tk.getboolean(self.tk.call( + 'tk', 'busy', 'status', self._w)) + busy_status = tk_busy_status + + # Clipboard handling: + def clipboard_get(self, **kw): + """Retrieve data from the clipboard on window's display. + + The window keyword defaults to the root window of the Tkinter + application. + + The type keyword specifies the form in which the data is + to be returned and should be an atom name such as STRING + or FILE_NAME. Type defaults to STRING, except on X11, where the default + is to try UTF8_STRING and fall back to STRING. + + This command is equivalent to: + + selection_get(CLIPBOARD) + """ + if 'type' not in kw and self._windowingsystem == 'x11': + try: + kw['type'] = 'UTF8_STRING' + return self.tk.call(('clipboard', 'get') + self._options(kw)) + except TclError: + del kw['type'] + return self.tk.call(('clipboard', 'get') + self._options(kw)) + + def clipboard_clear(self, **kw): + """Clear the data in the Tk clipboard. + + A widget specified for the optional displayof keyword + argument specifies the target display.""" + if 'displayof' not in kw: kw['displayof'] = self._w + self.tk.call(('clipboard', 'clear') + self._options(kw)) + + def clipboard_append(self, string, **kw): + """Append STRING to the Tk clipboard. + + A widget specified at the optional displayof keyword + argument specifies the target display. The clipboard + can be retrieved with selection_get.""" + if 'displayof' not in kw: kw['displayof'] = self._w + self.tk.call(('clipboard', 'append') + self._options(kw) + + ('--', string)) + # XXX grab current w/o window argument + + def grab_current(self): + """Return widget which has currently the grab in this application + or None.""" + name = self.tk.call('grab', 'current', self._w) + if not name: return None + return self._nametowidget(name) + + def grab_release(self): + """Release grab for this widget if currently set.""" + self.tk.call('grab', 'release', self._w) + + def grab_set(self): + """Set grab for this widget. + + A grab directs all events to this and descendant + widgets in the application.""" + self.tk.call('grab', 'set', self._w) + + def grab_set_global(self): + """Set global grab for this widget. + + A global grab directs all events to this and + descendant widgets on the display. Use with caution - + other applications do not get events anymore.""" + self.tk.call('grab', 'set', '-global', self._w) + + def grab_status(self): + """Return None, "local" or "global" if this widget has + no, a local or a global grab.""" + status = self.tk.call('grab', 'status', self._w) + if status == 'none': status = None + return status + + def option_add(self, pattern, value, priority = None): + """Set a VALUE (second parameter) for an option + PATTERN (first parameter). + + An optional third parameter gives the numeric priority + (defaults to 80).""" + self.tk.call('option', 'add', pattern, value, priority) + + def option_clear(self): + """Clear the option database. + + It will be reloaded if option_add is called.""" + self.tk.call('option', 'clear') + + def option_get(self, name, className): + """Return the value for an option NAME for this widget + with CLASSNAME. + + Values with higher priority override lower values.""" + return self.tk.call('option', 'get', self._w, name, className) + + def option_readfile(self, fileName, priority = None): + """Read file FILENAME into the option database. + + An optional second parameter gives the numeric + priority.""" + self.tk.call('option', 'readfile', fileName, priority) + + def selection_clear(self, **kw): + """Clear the current X selection.""" + if 'displayof' not in kw: kw['displayof'] = self._w + self.tk.call(('selection', 'clear') + self._options(kw)) + + def selection_get(self, **kw): + """Return the contents of the current X selection. + + A keyword parameter selection specifies the name of + the selection and defaults to PRIMARY. A keyword + parameter displayof specifies a widget on the display + to use. A keyword parameter type specifies the form of data to be + fetched, defaulting to STRING except on X11, where UTF8_STRING is tried + before STRING.""" + if 'displayof' not in kw: kw['displayof'] = self._w + if 'type' not in kw and self._windowingsystem == 'x11': + try: + kw['type'] = 'UTF8_STRING' + return self.tk.call(('selection', 'get') + self._options(kw)) + except TclError: + del kw['type'] + return self.tk.call(('selection', 'get') + self._options(kw)) + + def selection_handle(self, command, **kw): + """Specify a function COMMAND to call if the X + selection owned by this widget is queried by another + application. + + This function must return the contents of the + selection. The function will be called with the + arguments OFFSET and LENGTH which allows the chunking + of very long selections. The following keyword + parameters can be provided: + selection - name of the selection (default PRIMARY), + type - type of the selection (e.g. STRING, FILE_NAME).""" + name = self._register(command) + self.tk.call(('selection', 'handle') + self._options(kw) + + (self._w, name)) + + def selection_own(self, **kw): + """Become owner of X selection. + + A keyword parameter selection specifies the name of + the selection (default PRIMARY).""" + self.tk.call(('selection', 'own') + + self._options(kw) + (self._w,)) + + def selection_own_get(self, **kw): + """Return owner of X selection. + + The following keyword parameter can + be provided: + selection - name of the selection (default PRIMARY), + type - type of the selection (e.g. STRING, FILE_NAME).""" + if 'displayof' not in kw: kw['displayof'] = self._w + name = self.tk.call(('selection', 'own') + self._options(kw)) + if not name: return None + return self._nametowidget(name) + + def send(self, interp, cmd, *args): + """Send Tcl command CMD to different interpreter INTERP to be executed.""" + return self.tk.call(('send', interp, cmd) + args) + + def lower(self, belowThis=None): + """Lower this widget in the stacking order.""" + self.tk.call('lower', self._w, belowThis) + + def tkraise(self, aboveThis=None): + """Raise this widget in the stacking order.""" + self.tk.call('raise', self._w, aboveThis) + + lift = tkraise + + def info_patchlevel(self): + """Returns the exact version of the Tcl library.""" + patchlevel = self.tk.call('info', 'patchlevel') + return _parse_version(patchlevel) + + def winfo_atom(self, name, displayof=0): + """Return integer which represents atom NAME.""" + args = ('winfo', 'atom') + self._displayof(displayof) + (name,) + return self.tk.getint(self.tk.call(args)) + + def winfo_atomname(self, id, displayof=0): + """Return name of atom with identifier ID.""" + args = ('winfo', 'atomname') \ + + self._displayof(displayof) + (id,) + return self.tk.call(args) + + def winfo_cells(self): + """Return number of cells in the colormap for this widget.""" + return self.tk.getint( + self.tk.call('winfo', 'cells', self._w)) + + def winfo_children(self): + """Return a list of all widgets which are children of this widget.""" + result = [] + for child in self.tk.splitlist( + self.tk.call('winfo', 'children', self._w)): + try: + # Tcl sometimes returns extra windows, e.g. for + # menus; those need to be skipped + result.append(self._nametowidget(child)) + except KeyError: + pass + return result + + def winfo_class(self): + """Return window class name of this widget.""" + return self.tk.call('winfo', 'class', self._w) + + def winfo_colormapfull(self): + """Return True if at the last color request the colormap was full.""" + return self.tk.getboolean( + self.tk.call('winfo', 'colormapfull', self._w)) + + def winfo_containing(self, rootX, rootY, displayof=0): + """Return the widget which is at the root coordinates ROOTX, ROOTY.""" + args = ('winfo', 'containing') \ + + self._displayof(displayof) + (rootX, rootY) + name = self.tk.call(args) + if not name: return None + return self._nametowidget(name) + + def winfo_depth(self): + """Return the number of bits per pixel.""" + return self.tk.getint(self.tk.call('winfo', 'depth', self._w)) + + def winfo_exists(self): + """Return true if this widget exists.""" + return self.tk.getint( + self.tk.call('winfo', 'exists', self._w)) + + def winfo_fpixels(self, number): + """Return the number of pixels for the given distance NUMBER + (e.g. "3c") as float.""" + return self.tk.getdouble(self.tk.call( + 'winfo', 'fpixels', self._w, number)) + + def winfo_geometry(self): + """Return geometry string for this widget in the form "widthxheight+X+Y".""" + return self.tk.call('winfo', 'geometry', self._w) + + def winfo_height(self): + """Return height of this widget.""" + return self.tk.getint( + self.tk.call('winfo', 'height', self._w)) + + def winfo_id(self): + """Return identifier ID for this widget.""" + return int(self.tk.call('winfo', 'id', self._w), 0) + + def winfo_interps(self, displayof=0): + """Return the name of all Tcl interpreters for this display.""" + args = ('winfo', 'interps') + self._displayof(displayof) + return self.tk.splitlist(self.tk.call(args)) + + def winfo_ismapped(self): + """Return true if this widget is mapped.""" + return self.tk.getint( + self.tk.call('winfo', 'ismapped', self._w)) + + def winfo_manager(self): + """Return the window manager name for this widget.""" + return self.tk.call('winfo', 'manager', self._w) + + def winfo_name(self): + """Return the name of this widget.""" + return self.tk.call('winfo', 'name', self._w) + + def winfo_parent(self): + """Return the name of the parent of this widget.""" + return self.tk.call('winfo', 'parent', self._w) + + def winfo_pathname(self, id, displayof=0): + """Return the pathname of the widget given by ID.""" + if isinstance(id, int): + id = hex(id) + args = ('winfo', 'pathname') \ + + self._displayof(displayof) + (id,) + return self.tk.call(args) + + def winfo_pixels(self, number): + """Rounded integer value of winfo_fpixels.""" + return self.tk.getint( + self.tk.call('winfo', 'pixels', self._w, number)) + + def winfo_pointerx(self): + """Return the x coordinate of the pointer on the root window.""" + return self.tk.getint( + self.tk.call('winfo', 'pointerx', self._w)) + + def winfo_pointerxy(self): + """Return a tuple of x and y coordinates of the pointer on the root window.""" + return self._getints( + self.tk.call('winfo', 'pointerxy', self._w)) + + def winfo_pointery(self): + """Return the y coordinate of the pointer on the root window.""" + return self.tk.getint( + self.tk.call('winfo', 'pointery', self._w)) + + def winfo_reqheight(self): + """Return requested height of this widget.""" + return self.tk.getint( + self.tk.call('winfo', 'reqheight', self._w)) + + def winfo_reqwidth(self): + """Return requested width of this widget.""" + return self.tk.getint( + self.tk.call('winfo', 'reqwidth', self._w)) + + def winfo_rgb(self, color): + """Return a tuple of integer RGB values in range(65536) for color in this widget.""" + return self._getints( + self.tk.call('winfo', 'rgb', self._w, color)) + + def winfo_rootx(self): + """Return x coordinate of upper left corner of this widget on the + root window.""" + return self.tk.getint( + self.tk.call('winfo', 'rootx', self._w)) + + def winfo_rooty(self): + """Return y coordinate of upper left corner of this widget on the + root window.""" + return self.tk.getint( + self.tk.call('winfo', 'rooty', self._w)) + + def winfo_screen(self): + """Return the screen name of this widget.""" + return self.tk.call('winfo', 'screen', self._w) + + def winfo_screencells(self): + """Return the number of the cells in the colormap of the screen + of this widget.""" + return self.tk.getint( + self.tk.call('winfo', 'screencells', self._w)) + + def winfo_screendepth(self): + """Return the number of bits per pixel of the root window of the + screen of this widget.""" + return self.tk.getint( + self.tk.call('winfo', 'screendepth', self._w)) + + def winfo_screenheight(self): + """Return the number of pixels of the height of the screen of this widget + in pixel.""" + return self.tk.getint( + self.tk.call('winfo', 'screenheight', self._w)) + + def winfo_screenmmheight(self): + """Return the number of pixels of the height of the screen of + this widget in mm.""" + return self.tk.getint( + self.tk.call('winfo', 'screenmmheight', self._w)) + + def winfo_screenmmwidth(self): + """Return the number of pixels of the width of the screen of + this widget in mm.""" + return self.tk.getint( + self.tk.call('winfo', 'screenmmwidth', self._w)) + + def winfo_screenvisual(self): + """Return one of the strings directcolor, grayscale, pseudocolor, + staticcolor, staticgray, or truecolor for the default + colormodel of this screen.""" + return self.tk.call('winfo', 'screenvisual', self._w) + + def winfo_screenwidth(self): + """Return the number of pixels of the width of the screen of + this widget in pixel.""" + return self.tk.getint( + self.tk.call('winfo', 'screenwidth', self._w)) + + def winfo_server(self): + """Return information of the X-Server of the screen of this widget in + the form "XmajorRminor vendor vendorVersion".""" + return self.tk.call('winfo', 'server', self._w) + + def winfo_toplevel(self): + """Return the toplevel widget of this widget.""" + return self._nametowidget(self.tk.call( + 'winfo', 'toplevel', self._w)) + + def winfo_viewable(self): + """Return true if the widget and all its higher ancestors are mapped.""" + return self.tk.getint( + self.tk.call('winfo', 'viewable', self._w)) + + def winfo_visual(self): + """Return one of the strings directcolor, grayscale, pseudocolor, + staticcolor, staticgray, or truecolor for the + colormodel of this widget.""" + return self.tk.call('winfo', 'visual', self._w) + + def winfo_visualid(self): + """Return the X identifier for the visual for this widget.""" + return self.tk.call('winfo', 'visualid', self._w) + + def winfo_visualsavailable(self, includeids=False): + """Return a list of all visuals available for the screen + of this widget. + + Each item in the list consists of a visual name (see winfo_visual), a + depth and if includeids is true is given also the X identifier.""" + data = self.tk.call('winfo', 'visualsavailable', self._w, + 'includeids' if includeids else None) + data = [self.tk.splitlist(x) for x in self.tk.splitlist(data)] + return [self.__winfo_parseitem(x) for x in data] + + def __winfo_parseitem(self, t): + """Internal function.""" + return t[:1] + tuple(map(self.__winfo_getint, t[1:])) + + def __winfo_getint(self, x): + """Internal function.""" + return int(x, 0) + + def winfo_vrootheight(self): + """Return the height of the virtual root window associated with this + widget in pixels. If there is no virtual root window return the + height of the screen.""" + return self.tk.getint( + self.tk.call('winfo', 'vrootheight', self._w)) + + def winfo_vrootwidth(self): + """Return the width of the virtual root window associated with this + widget in pixel. If there is no virtual root window return the + width of the screen.""" + return self.tk.getint( + self.tk.call('winfo', 'vrootwidth', self._w)) + + def winfo_vrootx(self): + """Return the x offset of the virtual root relative to the root + window of the screen of this widget.""" + return self.tk.getint( + self.tk.call('winfo', 'vrootx', self._w)) + + def winfo_vrooty(self): + """Return the y offset of the virtual root relative to the root + window of the screen of this widget.""" + return self.tk.getint( + self.tk.call('winfo', 'vrooty', self._w)) + + def winfo_width(self): + """Return the width of this widget.""" + return self.tk.getint( + self.tk.call('winfo', 'width', self._w)) + + def winfo_x(self): + """Return the x coordinate of the upper left corner of this widget + in the parent.""" + return self.tk.getint( + self.tk.call('winfo', 'x', self._w)) + + def winfo_y(self): + """Return the y coordinate of the upper left corner of this widget + in the parent.""" + return self.tk.getint( + self.tk.call('winfo', 'y', self._w)) + + def update(self): + """Enter event loop until all pending events have been processed by Tcl.""" + self.tk.call('update') + + def update_idletasks(self): + """Enter event loop until all idle callbacks have been called. This + will update the display of windows but not process events caused by + the user.""" + self.tk.call('update', 'idletasks') + + def bindtags(self, tagList=None): + """Set or get the list of bindtags for this widget. + + With no argument return the list of all bindtags associated with + this widget. With a list of strings as argument the bindtags are + set to this list. The bindtags determine in which order events are + processed (see bind).""" + if tagList is None: + return self.tk.splitlist( + self.tk.call('bindtags', self._w)) + else: + self.tk.call('bindtags', self._w, tagList) + + def _bind(self, what, sequence, func, add, needcleanup=1): + """Internal function.""" + if isinstance(func, str): + self.tk.call(what + (sequence, func)) + elif func: + funcid = self._register(func, self._substitute, + needcleanup) + cmd = ('%sif {"[%s %s]" == "break"} break\n' + % + (add and '+' or '', + funcid, self._subst_format_str)) + self.tk.call(what + (sequence, cmd)) + return funcid + elif sequence: + return self.tk.call(what + (sequence,)) + else: + return self.tk.splitlist(self.tk.call(what)) + + def bind(self, sequence=None, func=None, add=None): + """Bind to this widget at event SEQUENCE a call to function FUNC. + + SEQUENCE is a string of concatenated event + patterns. An event pattern is of the form + where MODIFIER is one + of Control, Mod2, M2, Shift, Mod3, M3, Lock, Mod4, M4, + Button1, B1, Mod5, M5 Button2, B2, Meta, M, Button3, + B3, Alt, Button4, B4, Double, Button5, B5 Triple, + Mod1, M1. TYPE is one of Activate, Enter, Map, + ButtonPress, Button, Expose, Motion, ButtonRelease + FocusIn, MouseWheel, Circulate, FocusOut, Property, + Colormap, Gravity Reparent, Configure, KeyPress, Key, + Unmap, Deactivate, KeyRelease Visibility, Destroy, + Leave and DETAIL is the button number for ButtonPress, + ButtonRelease and DETAIL is the Keysym for KeyPress and + KeyRelease. Examples are + for pressing Control and mouse button 1 or + for pressing A and the Alt key (KeyPress can be omitted). + An event pattern can also be a virtual event of the form + <> where AString can be arbitrary. This + event can be generated by event_generate. + If events are concatenated they must appear shortly + after each other. + + FUNC will be called if the event sequence occurs with an + instance of Event as argument. If the return value of FUNC is + "break" no further bound function is invoked. + + An additional boolean parameter ADD specifies whether FUNC will + be called additionally to the other bound function or whether + it will replace the previous function. + + Bind will return an identifier to allow deletion of the bound function with + unbind without memory leak. + + If FUNC or SEQUENCE is omitted the bound function or list + of bound events are returned.""" + + return self._bind(('bind', self._w), sequence, func, add) + + def unbind(self, sequence, funcid=None): + """Unbind for this widget the event SEQUENCE. + + If FUNCID is given, only unbind the function identified with FUNCID + and also delete the corresponding Tcl command. + + Otherwise destroy the current binding for SEQUENCE, leaving SEQUENCE + unbound. + """ + self._unbind(('bind', self._w, sequence), funcid) + + def _unbind(self, what, funcid=None): + if funcid is None: + self.tk.call(*what, '') + else: + lines = self.tk.call(what).split('\n') + prefix = f'if {{"[{funcid} ' + keep = '\n'.join(line for line in lines + if not line.startswith(prefix)) + if not keep.strip(): + keep = '' + self.tk.call(*what, keep) + self.deletecommand(funcid) + + def bind_all(self, sequence=None, func=None, add=None): + """Bind to all widgets at an event SEQUENCE a call to function FUNC. + An additional boolean parameter ADD specifies whether FUNC will + be called additionally to the other bound function or whether + it will replace the previous function. See bind for the return value.""" + return self._root()._bind(('bind', 'all'), sequence, func, add, True) + + def unbind_all(self, sequence): + """Unbind for all widgets for event SEQUENCE all functions.""" + self._root()._unbind(('bind', 'all', sequence)) + + def bind_class(self, className, sequence=None, func=None, add=None): + """Bind to widgets with bindtag CLASSNAME at event + SEQUENCE a call of function FUNC. An additional + boolean parameter ADD specifies whether FUNC will be + called additionally to the other bound function or + whether it will replace the previous function. See bind for + the return value.""" + + return self._root()._bind(('bind', className), sequence, func, add, True) + + def unbind_class(self, className, sequence): + """Unbind for all widgets with bindtag CLASSNAME for event SEQUENCE + all functions.""" + self._root()._unbind(('bind', className, sequence)) + + def mainloop(self, n=0): + """Call the mainloop of Tk.""" + self.tk.mainloop(n) + + def quit(self): + """Quit the Tcl interpreter. All widgets will be destroyed.""" + self.tk.quit() + + def _getints(self, string): + """Internal function.""" + if string: + return tuple(map(self.tk.getint, self.tk.splitlist(string))) + + def _getdoubles(self, string): + """Internal function.""" + if string: + return tuple(map(self.tk.getdouble, self.tk.splitlist(string))) + + def _getboolean(self, string): + """Internal function.""" + if string: + return self.tk.getboolean(string) + + def _displayof(self, displayof): + """Internal function.""" + if displayof: + return ('-displayof', displayof) + if displayof is None: + return ('-displayof', self._w) + return () + + @property + def _windowingsystem(self): + """Internal function.""" + try: + return self._root()._windowingsystem_cached + except AttributeError: + ws = self._root()._windowingsystem_cached = \ + self.tk.call('tk', 'windowingsystem') + return ws + + def _options(self, cnf, kw = None): + """Internal function.""" + if kw: + cnf = _cnfmerge((cnf, kw)) + else: + cnf = _cnfmerge(cnf) + res = () + for k, v in cnf.items(): + if v is not None: + if k[-1] == '_': k = k[:-1] + if callable(v): + v = self._register(v) + elif isinstance(v, (tuple, list)): + nv = [] + for item in v: + if isinstance(item, int): + nv.append(str(item)) + elif isinstance(item, str): + nv.append(_stringify(item)) + else: + break + else: + v = ' '.join(nv) + res = res + ('-'+k, v) + return res + + def nametowidget(self, name): + """Return the Tkinter instance of a widget identified by + its Tcl name NAME.""" + name = str(name).split('.') + w = self + + if not name[0]: + w = w._root() + name = name[1:] + + for n in name: + if not n: + break + w = w.children[n] + + return w + + _nametowidget = nametowidget + + def _register(self, func, subst=None, needcleanup=1): + """Return a newly created Tcl function. If this + function is called, the Python function FUNC will + be executed. An optional function SUBST can + be given which will be executed before FUNC.""" + f = CallWrapper(func, subst, self).__call__ + name = repr(id(f)) + try: + func = func.__func__ + except AttributeError: + pass + try: + name = name + func.__name__ + except AttributeError: + pass + self.tk.createcommand(name, f) + if needcleanup: + if self._tclCommands is None: + self._tclCommands = [] + self._tclCommands.append(name) + return name + + register = _register + + def _root(self): + """Internal function.""" + w = self + while w.master is not None: w = w.master + return w + _subst_format = ('%#', '%b', '%f', '%h', '%k', + '%s', '%t', '%w', '%x', '%y', + '%A', '%E', '%K', '%N', '%W', '%T', '%X', '%Y', '%D') + _subst_format_str = " ".join(_subst_format) + + def _substitute(self, *args): + """Internal function.""" + if len(args) != len(self._subst_format): return args + getboolean = self.tk.getboolean + + getint = self.tk.getint + def getint_event(s): + """Tk changed behavior in 8.4.2, returning "??" rather more often.""" + try: + return getint(s) + except (ValueError, TclError): + return s + + if any(isinstance(s, tuple) for s in args): + args = [s[0] if isinstance(s, tuple) and len(s) == 1 else s + for s in args] + nsign, b, f, h, k, s, t, w, x, y, A, E, K, N, W, T, X, Y, D = args + # Missing: (a, c, d, m, o, v, B, R) + e = Event() + # serial field: valid for all events + # number of button: ButtonPress and ButtonRelease events only + # height field: Configure, ConfigureRequest, Create, + # ResizeRequest, and Expose events only + # keycode field: KeyPress and KeyRelease events only + # time field: "valid for events that contain a time field" + # width field: Configure, ConfigureRequest, Create, ResizeRequest, + # and Expose events only + # x field: "valid for events that contain an x field" + # y field: "valid for events that contain a y field" + # keysym as decimal: KeyPress and KeyRelease events only + # x_root, y_root fields: ButtonPress, ButtonRelease, KeyPress, + # KeyRelease, and Motion events + e.serial = getint(nsign) + e.num = getint_event(b) + try: e.focus = getboolean(f) + except TclError: pass + e.height = getint_event(h) + e.keycode = getint_event(k) + e.state = getint_event(s) + e.time = getint_event(t) + e.width = getint_event(w) + e.x = getint_event(x) + e.y = getint_event(y) + e.char = A + try: e.send_event = getboolean(E) + except TclError: pass + e.keysym = K + e.keysym_num = getint_event(N) + try: + e.type = EventType(T) + except ValueError: + try: + e.type = EventType(str(T)) # can be int + except ValueError: + e.type = T + try: + e.widget = self._nametowidget(W) + except KeyError: + e.widget = W + e.x_root = getint_event(X) + e.y_root = getint_event(Y) + try: + e.delta = getint(D) + except (ValueError, TclError): + e.delta = 0 + return (e,) + + def _report_exception(self): + """Internal function.""" + exc, val, tb = sys.exc_info() + root = self._root() + root.report_callback_exception(exc, val, tb) + + def _getconfigure(self, *args): + """Call Tcl configure command and return the result as a dict.""" + cnf = {} + for x in self.tk.splitlist(self.tk.call(*args)): + x = self.tk.splitlist(x) + cnf[x[0][1:]] = (x[0][1:],) + x[1:] + return cnf + + def _getconfigure1(self, *args): + x = self.tk.splitlist(self.tk.call(*args)) + return (x[0][1:],) + x[1:] + + def _configure(self, cmd, cnf, kw): + """Internal function.""" + if kw: + cnf = _cnfmerge((cnf, kw)) + elif cnf: + cnf = _cnfmerge(cnf) + if cnf is None: + return self._getconfigure(_flatten((self._w, cmd))) + if isinstance(cnf, str): + return self._getconfigure1(_flatten((self._w, cmd, '-'+cnf))) + self.tk.call(_flatten((self._w, cmd)) + self._options(cnf)) + # These used to be defined in Widget: + + def configure(self, cnf=None, **kw): + """Configure resources of a widget. + + The values for resources are specified as keyword + arguments. To get an overview about + the allowed keyword arguments call the method keys. + """ + return self._configure('configure', cnf, kw) + + config = configure + + def cget(self, key): + """Return the resource value for a KEY given as string.""" + return self.tk.call(self._w, 'cget', '-' + key) + + __getitem__ = cget + + def __setitem__(self, key, value): + self.configure({key: value}) + + def keys(self): + """Return a list of all resource names of this widget.""" + splitlist = self.tk.splitlist + return [splitlist(x)[0][1:] for x in + splitlist(self.tk.call(self._w, 'configure'))] + + def __str__(self): + """Return the window path name of this widget.""" + return self._w + + def __repr__(self): + return '<%s.%s object %s>' % ( + self.__class__.__module__, self.__class__.__qualname__, self._w) + + # Pack methods that apply to the master + _noarg_ = ['_noarg_'] + + def pack_propagate(self, flag=_noarg_): + """Set or get the status for propagation of geometry information. + + A boolean argument specifies whether the geometry information + of the slaves will determine the size of this widget. If no argument + is given the current setting will be returned. + """ + if flag is Misc._noarg_: + return self._getboolean(self.tk.call( + 'pack', 'propagate', self._w)) + else: + self.tk.call('pack', 'propagate', self._w, flag) + + propagate = pack_propagate + + def pack_slaves(self): + """Return a list of all slaves of this widget + in its packing order.""" + return [self._nametowidget(x) for x in + self.tk.splitlist( + self.tk.call('pack', 'slaves', self._w))] + + slaves = pack_slaves + + # Place method that applies to the master + def place_slaves(self): + """Return a list of all slaves of this widget + in its packing order.""" + return [self._nametowidget(x) for x in + self.tk.splitlist( + self.tk.call( + 'place', 'slaves', self._w))] + + # Grid methods that apply to the master + + def grid_anchor(self, anchor=None): # new in Tk 8.5 + """The anchor value controls how to place the grid within the + master when no row/column has any weight. + + The default anchor is nw.""" + self.tk.call('grid', 'anchor', self._w, anchor) + + anchor = grid_anchor + + def grid_bbox(self, column=None, row=None, col2=None, row2=None): + """Return a tuple of integer coordinates for the bounding + box of this widget controlled by the geometry manager grid. + + If COLUMN, ROW is given the bounding box applies from + the cell with row and column 0 to the specified + cell. If COL2 and ROW2 are given the bounding box + starts at that cell. + + The returned integers specify the offset of the upper left + corner in the master widget and the width and height. + """ + args = ('grid', 'bbox', self._w) + if column is not None and row is not None: + args = args + (column, row) + if col2 is not None and row2 is not None: + args = args + (col2, row2) + return self._getints(self.tk.call(*args)) or None + + bbox = grid_bbox + + def _gridconvvalue(self, value): + if isinstance(value, (str, _tkinter.Tcl_Obj)): + try: + svalue = str(value) + if not svalue: + return None + elif '.' in svalue: + return self.tk.getdouble(svalue) + else: + return self.tk.getint(svalue) + except (ValueError, TclError): + pass + return value + + def _grid_configure(self, command, index, cnf, kw): + """Internal function.""" + if isinstance(cnf, str) and not kw: + if cnf[-1:] == '_': + cnf = cnf[:-1] + if cnf[:1] != '-': + cnf = '-'+cnf + options = (cnf,) + else: + options = self._options(cnf, kw) + if not options: + return _splitdict( + self.tk, + self.tk.call('grid', command, self._w, index), + conv=self._gridconvvalue) + res = self.tk.call( + ('grid', command, self._w, index) + + options) + if len(options) == 1: + return self._gridconvvalue(res) + + def grid_columnconfigure(self, index, cnf={}, **kw): + """Configure column INDEX of a grid. + + Valid resources are minsize (minimum size of the column), + weight (how much does additional space propagate to this column) + and pad (how much space to let additionally).""" + return self._grid_configure('columnconfigure', index, cnf, kw) + + columnconfigure = grid_columnconfigure + + def grid_location(self, x, y): + """Return a tuple of column and row which identify the cell + at which the pixel at position X and Y inside the master + widget is located.""" + return self._getints( + self.tk.call( + 'grid', 'location', self._w, x, y)) or None + + def grid_propagate(self, flag=_noarg_): + """Set or get the status for propagation of geometry information. + + A boolean argument specifies whether the geometry information + of the slaves will determine the size of this widget. If no argument + is given, the current setting will be returned. + """ + if flag is Misc._noarg_: + return self._getboolean(self.tk.call( + 'grid', 'propagate', self._w)) + else: + self.tk.call('grid', 'propagate', self._w, flag) + + def grid_rowconfigure(self, index, cnf={}, **kw): + """Configure row INDEX of a grid. + + Valid resources are minsize (minimum size of the row), + weight (how much does additional space propagate to this row) + and pad (how much space to let additionally).""" + return self._grid_configure('rowconfigure', index, cnf, kw) + + rowconfigure = grid_rowconfigure + + def grid_size(self): + """Return a tuple of the number of column and rows in the grid.""" + return self._getints( + self.tk.call('grid', 'size', self._w)) or None + + size = grid_size + + def grid_slaves(self, row=None, column=None): + """Return a list of all slaves of this widget + in its packing order.""" + args = () + if row is not None: + args = args + ('-row', row) + if column is not None: + args = args + ('-column', column) + return [self._nametowidget(x) for x in + self.tk.splitlist(self.tk.call( + ('grid', 'slaves', self._w) + args))] + + # Support for the "event" command, new in Tk 4.2. + # By Case Roole. + + def event_add(self, virtual, *sequences): + """Bind a virtual event VIRTUAL (of the form <>) + to an event SEQUENCE such that the virtual event is triggered + whenever SEQUENCE occurs.""" + args = ('event', 'add', virtual) + sequences + self.tk.call(args) + + def event_delete(self, virtual, *sequences): + """Unbind a virtual event VIRTUAL from SEQUENCE.""" + args = ('event', 'delete', virtual) + sequences + self.tk.call(args) + + def event_generate(self, sequence, **kw): + """Generate an event SEQUENCE. Additional + keyword arguments specify parameter of the event + (e.g. x, y, rootx, rooty).""" + args = ('event', 'generate', self._w, sequence) + for k, v in kw.items(): + args = args + ('-%s' % k, str(v)) + self.tk.call(args) + + def event_info(self, virtual=None): + """Return a list of all virtual events or the information + about the SEQUENCE bound to the virtual event VIRTUAL.""" + return self.tk.splitlist( + self.tk.call('event', 'info', virtual)) + + # Image related commands + + def image_names(self): + """Return a list of all existing image names.""" + return self.tk.splitlist(self.tk.call('image', 'names')) + + def image_types(self): + """Return a list of all available image types (e.g. photo bitmap).""" + return self.tk.splitlist(self.tk.call('image', 'types')) + + +class CallWrapper: + """Internal class. Stores function to call when some user + defined Tcl function is called e.g. after an event occurred.""" + + def __init__(self, func, subst, widget): + """Store FUNC, SUBST and WIDGET as members.""" + self.func = func + self.subst = subst + self.widget = widget + + def __call__(self, *args): + """Apply first function SUBST to arguments, than FUNC.""" + try: + if self.subst: + args = self.subst(*args) + return self.func(*args) + except SystemExit: + raise + except: + self.widget._report_exception() + + +class XView: + """Mix-in class for querying and changing the horizontal position + of a widget's window.""" + + def xview(self, *args): + """Query and change the horizontal position of the view.""" + res = self.tk.call(self._w, 'xview', *args) + if not args: + return self._getdoubles(res) + + def xview_moveto(self, fraction): + """Adjusts the view in the window so that FRACTION of the + total width of the canvas is off-screen to the left.""" + self.tk.call(self._w, 'xview', 'moveto', fraction) + + def xview_scroll(self, number, what): + """Shift the x-view according to NUMBER which is measured in "units" + or "pages" (WHAT).""" + self.tk.call(self._w, 'xview', 'scroll', number, what) + + +class YView: + """Mix-in class for querying and changing the vertical position + of a widget's window.""" + + def yview(self, *args): + """Query and change the vertical position of the view.""" + res = self.tk.call(self._w, 'yview', *args) + if not args: + return self._getdoubles(res) + + def yview_moveto(self, fraction): + """Adjusts the view in the window so that FRACTION of the + total height of the canvas is off-screen to the top.""" + self.tk.call(self._w, 'yview', 'moveto', fraction) + + def yview_scroll(self, number, what): + """Shift the y-view according to NUMBER which is measured in + "units" or "pages" (WHAT).""" + self.tk.call(self._w, 'yview', 'scroll', number, what) + + +class Wm: + """Provides functions for the communication with the window manager.""" + + def wm_aspect(self, + minNumer=None, minDenom=None, + maxNumer=None, maxDenom=None): + """Instruct the window manager to set the aspect ratio (width/height) + of this widget to be between MINNUMER/MINDENOM and MAXNUMER/MAXDENOM. Return a tuple + of the actual values if no argument is given.""" + return self._getints( + self.tk.call('wm', 'aspect', self._w, + minNumer, minDenom, + maxNumer, maxDenom)) + + aspect = wm_aspect + + def wm_attributes(self, *args, return_python_dict=False, **kwargs): + """Return or sets platform specific attributes. + + When called with a single argument return_python_dict=True, + return a dict of the platform specific attributes and their values. + When called without arguments or with a single argument + return_python_dict=False, return a tuple containing intermixed + attribute names with the minus prefix and their values. + + When called with a single string value, return the value for the + specific option. When called with keyword arguments, set the + corresponding attributes. + """ + if not kwargs: + if not args: + res = self.tk.call('wm', 'attributes', self._w) + if return_python_dict: + return _splitdict(self.tk, res) + else: + return self.tk.splitlist(res) + if len(args) == 1 and args[0] is not None: + option = args[0] + if option[0] == '-': + # TODO: deprecate + option = option[1:] + return self.tk.call('wm', 'attributes', self._w, '-' + option) + # TODO: deprecate + return self.tk.call('wm', 'attributes', self._w, *args) + elif args: + raise TypeError('wm_attribute() options have been specified as ' + 'positional and keyword arguments') + else: + self.tk.call('wm', 'attributes', self._w, *self._options(kwargs)) + + attributes = wm_attributes + + def wm_client(self, name=None): + """Store NAME in WM_CLIENT_MACHINE property of this widget. Return + current value.""" + return self.tk.call('wm', 'client', self._w, name) + + client = wm_client + + def wm_colormapwindows(self, *wlist): + """Store list of window names (WLIST) into WM_COLORMAPWINDOWS property + of this widget. This list contains windows whose colormaps differ from their + parents. Return current list of widgets if WLIST is empty.""" + if len(wlist) > 1: + wlist = (wlist,) # Tk needs a list of windows here + args = ('wm', 'colormapwindows', self._w) + wlist + if wlist: + self.tk.call(args) + else: + return [self._nametowidget(x) + for x in self.tk.splitlist(self.tk.call(args))] + + colormapwindows = wm_colormapwindows + + def wm_command(self, value=None): + """Store VALUE in WM_COMMAND property. It is the command + which shall be used to invoke the application. Return current + command if VALUE is None.""" + return self.tk.call('wm', 'command', self._w, value) + + command = wm_command + + def wm_deiconify(self): + """Deiconify this widget. If it was never mapped it will not be mapped. + On Windows it will raise this widget and give it the focus.""" + return self.tk.call('wm', 'deiconify', self._w) + + deiconify = wm_deiconify + + def wm_focusmodel(self, model=None): + """Set focus model to MODEL. "active" means that this widget will claim + the focus itself, "passive" means that the window manager shall give + the focus. Return current focus model if MODEL is None.""" + return self.tk.call('wm', 'focusmodel', self._w, model) + + focusmodel = wm_focusmodel + + def wm_forget(self, window): # new in Tk 8.5 + """The window will be unmapped from the screen and will no longer + be managed by wm. toplevel windows will be treated like frame + windows once they are no longer managed by wm, however, the menu + option configuration will be remembered and the menus will return + once the widget is managed again.""" + self.tk.call('wm', 'forget', window) + + forget = wm_forget + + def wm_frame(self): + """Return identifier for decorative frame of this widget if present.""" + return self.tk.call('wm', 'frame', self._w) + + frame = wm_frame + + def wm_geometry(self, newGeometry=None): + """Set geometry to NEWGEOMETRY of the form =widthxheight+x+y. Return + current value if None is given.""" + return self.tk.call('wm', 'geometry', self._w, newGeometry) + + geometry = wm_geometry + + def wm_grid(self, + baseWidth=None, baseHeight=None, + widthInc=None, heightInc=None): + """Instruct the window manager that this widget shall only be + resized on grid boundaries. WIDTHINC and HEIGHTINC are the width and + height of a grid unit in pixels. BASEWIDTH and BASEHEIGHT are the + number of grid units requested in Tk_GeometryRequest.""" + return self._getints(self.tk.call( + 'wm', 'grid', self._w, + baseWidth, baseHeight, widthInc, heightInc)) + + grid = wm_grid + + def wm_group(self, pathName=None): + """Set the group leader widgets for related widgets to PATHNAME. Return + the group leader of this widget if None is given.""" + return self.tk.call('wm', 'group', self._w, pathName) + + group = wm_group + + def wm_iconbitmap(self, bitmap=None, default=None): + """Set bitmap for the iconified widget to BITMAP. Return + the bitmap if None is given. + + Under Windows, the DEFAULT parameter can be used to set the icon + for the widget and any descendants that don't have an icon set + explicitly. DEFAULT can be the relative path to a .ico file + (example: root.iconbitmap(default='myicon.ico') ). See Tk + documentation for more information.""" + if default is not None: + return self.tk.call('wm', 'iconbitmap', self._w, '-default', default) + else: + return self.tk.call('wm', 'iconbitmap', self._w, bitmap) + + iconbitmap = wm_iconbitmap + + def wm_iconify(self): + """Display widget as icon.""" + return self.tk.call('wm', 'iconify', self._w) + + iconify = wm_iconify + + def wm_iconmask(self, bitmap=None): + """Set mask for the icon bitmap of this widget. Return the + mask if None is given.""" + return self.tk.call('wm', 'iconmask', self._w, bitmap) + + iconmask = wm_iconmask + + def wm_iconname(self, newName=None): + """Set the name of the icon for this widget. Return the name if + None is given.""" + return self.tk.call('wm', 'iconname', self._w, newName) + + iconname = wm_iconname + + def wm_iconphoto(self, default=False, *args): # new in Tk 8.5 + """Sets the titlebar icon for this window based on the named photo + images passed through args. If default is True, this is applied to + all future created toplevels as well. + + The data in the images is taken as a snapshot at the time of + invocation. If the images are later changed, this is not reflected + to the titlebar icons. Multiple images are accepted to allow + different images sizes to be provided. The window manager may scale + provided icons to an appropriate size. + + On Windows, the images are packed into a Windows icon structure. + This will override an icon specified to wm_iconbitmap, and vice + versa. + + On X, the images are arranged into the _NET_WM_ICON X property, + which most modern window managers support. An icon specified by + wm_iconbitmap may exist simultaneously. + + On Macintosh, this currently does nothing.""" + if default: + self.tk.call('wm', 'iconphoto', self._w, "-default", *args) + else: + self.tk.call('wm', 'iconphoto', self._w, *args) + + iconphoto = wm_iconphoto + + def wm_iconposition(self, x=None, y=None): + """Set the position of the icon of this widget to X and Y. Return + a tuple of the current values of X and X if None is given.""" + return self._getints(self.tk.call( + 'wm', 'iconposition', self._w, x, y)) + + iconposition = wm_iconposition + + def wm_iconwindow(self, pathName=None): + """Set widget PATHNAME to be displayed instead of icon. Return the current + value if None is given.""" + return self.tk.call('wm', 'iconwindow', self._w, pathName) + + iconwindow = wm_iconwindow + + def wm_manage(self, widget): # new in Tk 8.5 + """The widget specified will become a stand alone top-level window. + The window will be decorated with the window managers title bar, + etc.""" + self.tk.call('wm', 'manage', widget) + + manage = wm_manage + + def wm_maxsize(self, width=None, height=None): + """Set max WIDTH and HEIGHT for this widget. If the window is gridded + the values are given in grid units. Return the current values if None + is given.""" + return self._getints(self.tk.call( + 'wm', 'maxsize', self._w, width, height)) + + maxsize = wm_maxsize + + def wm_minsize(self, width=None, height=None): + """Set min WIDTH and HEIGHT for this widget. If the window is gridded + the values are given in grid units. Return the current values if None + is given.""" + return self._getints(self.tk.call( + 'wm', 'minsize', self._w, width, height)) + + minsize = wm_minsize + + def wm_overrideredirect(self, boolean=None): + """Instruct the window manager to ignore this widget + if BOOLEAN is given with 1. Return the current value if None + is given.""" + return self._getboolean(self.tk.call( + 'wm', 'overrideredirect', self._w, boolean)) + + overrideredirect = wm_overrideredirect + + def wm_positionfrom(self, who=None): + """Instruct the window manager that the position of this widget shall + be defined by the user if WHO is "user", and by its own policy if WHO is + "program".""" + return self.tk.call('wm', 'positionfrom', self._w, who) + + positionfrom = wm_positionfrom + + def wm_protocol(self, name=None, func=None): + """Bind function FUNC to command NAME for this widget. + Return the function bound to NAME if None is given. NAME could be + e.g. "WM_SAVE_YOURSELF" or "WM_DELETE_WINDOW".""" + if callable(func): + command = self._register(func) + else: + command = func + return self.tk.call( + 'wm', 'protocol', self._w, name, command) + + protocol = wm_protocol + + def wm_resizable(self, width=None, height=None): + """Instruct the window manager whether this width can be resized + in WIDTH or HEIGHT. Both values are boolean values.""" + return self.tk.call('wm', 'resizable', self._w, width, height) + + resizable = wm_resizable + + def wm_sizefrom(self, who=None): + """Instruct the window manager that the size of this widget shall + be defined by the user if WHO is "user", and by its own policy if WHO is + "program".""" + return self.tk.call('wm', 'sizefrom', self._w, who) + + sizefrom = wm_sizefrom + + def wm_state(self, newstate=None): + """Query or set the state of this widget as one of normal, icon, + iconic (see wm_iconwindow), withdrawn, or zoomed (Windows only).""" + return self.tk.call('wm', 'state', self._w, newstate) + + state = wm_state + + def wm_title(self, string=None): + """Set the title of this widget.""" + return self.tk.call('wm', 'title', self._w, string) + + title = wm_title + + def wm_transient(self, master=None): + """Instruct the window manager that this widget is transient + with regard to widget MASTER.""" + return self.tk.call('wm', 'transient', self._w, master) + + transient = wm_transient + + def wm_withdraw(self): + """Withdraw this widget from the screen such that it is unmapped + and forgotten by the window manager. Re-draw it with wm_deiconify.""" + return self.tk.call('wm', 'withdraw', self._w) + + withdraw = wm_withdraw + + +class Tk(Misc, Wm): + """Toplevel widget of Tk which represents mostly the main window + of an application. It has an associated Tcl interpreter.""" + _w = '.' + + def __init__(self, screenName=None, baseName=None, className='Tk', + useTk=True, sync=False, use=None): + """Return a new top level widget on screen SCREENNAME. A new Tcl interpreter will + be created. BASENAME will be used for the identification of the profile file (see + readprofile). + It is constructed from sys.argv[0] without extensions if None is given. CLASSNAME + is the name of the widget class.""" + self.master = None + self.children = {} + self._tkloaded = False + # to avoid recursions in the getattr code in case of failure, we + # ensure that self.tk is always _something_. + self.tk = None + if baseName is None: + import os + baseName = os.path.basename(sys.argv[0]) + baseName, ext = os.path.splitext(baseName) + if ext not in ('.py', '.pyc'): + baseName = baseName + ext + interactive = False + self.tk = _tkinter.create(screenName, baseName, className, interactive, wantobjects, useTk, sync, use) + if _debug: + self.tk.settrace(_print_command) + if useTk: + self._loadtk() + if not sys.flags.ignore_environment: + # Issue #16248: Honor the -E flag to avoid code injection. + self.readprofile(baseName, className) + + def loadtk(self): + if not self._tkloaded: + self.tk.loadtk() + self._loadtk() + + def _loadtk(self): + self._tkloaded = True + global _default_root + # Version sanity checks + tk_version = self.tk.getvar('tk_version') + if tk_version != _tkinter.TK_VERSION: + raise RuntimeError("tk.h version (%s) doesn't match libtk.a version (%s)" + % (_tkinter.TK_VERSION, tk_version)) + # Under unknown circumstances, tcl_version gets coerced to float + tcl_version = str(self.tk.getvar('tcl_version')) + if tcl_version != _tkinter.TCL_VERSION: + raise RuntimeError("tcl.h version (%s) doesn't match libtcl.a version (%s)" \ + % (_tkinter.TCL_VERSION, tcl_version)) + # Create and register the tkerror and exit commands + # We need to inline parts of _register here, _ register + # would register differently-named commands. + if self._tclCommands is None: + self._tclCommands = [] + self.tk.createcommand('tkerror', _tkerror) + self.tk.createcommand('exit', _exit) + self._tclCommands.append('tkerror') + self._tclCommands.append('exit') + if _support_default_root and _default_root is None: + _default_root = self + self.protocol("WM_DELETE_WINDOW", self.destroy) + + def destroy(self): + """Destroy this and all descendants widgets. This will + end the application of this Tcl interpreter.""" + for c in list(self.children.values()): c.destroy() + self.tk.call('destroy', self._w) + Misc.destroy(self) + global _default_root + if _support_default_root and _default_root is self: + _default_root = None + + def readprofile(self, baseName, className): + """Internal function. It reads .BASENAME.tcl and .CLASSNAME.tcl into + the Tcl Interpreter and calls exec on the contents of .BASENAME.py and + .CLASSNAME.py if such a file exists in the home directory.""" + import os + if 'HOME' in os.environ: home = os.environ['HOME'] + else: home = os.curdir + class_tcl = os.path.join(home, '.%s.tcl' % className) + class_py = os.path.join(home, '.%s.py' % className) + base_tcl = os.path.join(home, '.%s.tcl' % baseName) + base_py = os.path.join(home, '.%s.py' % baseName) + dir = {'self': self} + exec('from tkinter import *', dir) + if os.path.isfile(class_tcl): + self.tk.call('source', class_tcl) + if os.path.isfile(class_py): + exec(open(class_py).read(), dir) + if os.path.isfile(base_tcl): + self.tk.call('source', base_tcl) + if os.path.isfile(base_py): + exec(open(base_py).read(), dir) + + def report_callback_exception(self, exc, val, tb): + """Report callback exception on sys.stderr. + + Applications may want to override this internal function, and + should when sys.stderr is None.""" + import traceback + print("Exception in Tkinter callback", file=sys.stderr) + sys.last_exc = val + sys.last_type = exc + sys.last_value = val + sys.last_traceback = tb + traceback.print_exception(exc, val, tb) + + def __getattr__(self, attr): + "Delegate attribute access to the interpreter object" + return getattr(self.tk, attr) + + +def _print_command(cmd, *, file=sys.stderr): + # Print executed Tcl/Tk commands. + assert isinstance(cmd, tuple) + cmd = _join(cmd) + print(cmd, file=file) + + +# Ideally, the classes Pack, Place and Grid disappear, the +# pack/place/grid methods are defined on the Widget class, and +# everybody uses w.pack_whatever(...) instead of Pack.whatever(w, +# ...), with pack(), place() and grid() being short for +# pack_configure(), place_configure() and grid_columnconfigure(), and +# forget() being short for pack_forget(). As a practical matter, I'm +# afraid that there is too much code out there that may be using the +# Pack, Place or Grid class, so I leave them intact -- but only as +# backwards compatibility features. Also note that those methods that +# take a master as argument (e.g. pack_propagate) have been moved to +# the Misc class (which now incorporates all methods common between +# toplevel and interior widgets). Again, for compatibility, these are +# copied into the Pack, Place or Grid class. + + +def Tcl(screenName=None, baseName=None, className='Tk', useTk=False): + return Tk(screenName, baseName, className, useTk) + + +class Pack: + """Geometry manager Pack. + + Base class to use the methods pack_* in every widget.""" + + def pack_configure(self, cnf={}, **kw): + """Pack a widget in the parent widget. Use as options: + after=widget - pack it after you have packed widget + anchor=NSEW (or subset) - position widget according to + given direction + before=widget - pack it before you will pack widget + expand=bool - expand widget if parent size grows + fill=NONE or X or Y or BOTH - fill widget if widget grows + in=master - use master to contain this widget + in_=master - see 'in' option description + ipadx=amount - add internal padding in x direction + ipady=amount - add internal padding in y direction + padx=amount - add padding in x direction + pady=amount - add padding in y direction + side=TOP or BOTTOM or LEFT or RIGHT - where to add this widget. + """ + self.tk.call( + ('pack', 'configure', self._w) + + self._options(cnf, kw)) + + pack = configure = config = pack_configure + + def pack_forget(self): + """Unmap this widget and do not use it for the packing order.""" + self.tk.call('pack', 'forget', self._w) + + forget = pack_forget + + def pack_info(self): + """Return information about the packing options + for this widget.""" + d = _splitdict(self.tk, self.tk.call('pack', 'info', self._w)) + if 'in' in d: + d['in'] = self.nametowidget(d['in']) + return d + + info = pack_info + propagate = pack_propagate = Misc.pack_propagate + slaves = pack_slaves = Misc.pack_slaves + + +class Place: + """Geometry manager Place. + + Base class to use the methods place_* in every widget.""" + + def place_configure(self, cnf={}, **kw): + """Place a widget in the parent widget. Use as options: + in=master - master relative to which the widget is placed + in_=master - see 'in' option description + x=amount - locate anchor of this widget at position x of master + y=amount - locate anchor of this widget at position y of master + relx=amount - locate anchor of this widget between 0.0 and 1.0 + relative to width of master (1.0 is right edge) + rely=amount - locate anchor of this widget between 0.0 and 1.0 + relative to height of master (1.0 is bottom edge) + anchor=NSEW (or subset) - position anchor according to given direction + width=amount - width of this widget in pixel + height=amount - height of this widget in pixel + relwidth=amount - width of this widget between 0.0 and 1.0 + relative to width of master (1.0 is the same width + as the master) + relheight=amount - height of this widget between 0.0 and 1.0 + relative to height of master (1.0 is the same + height as the master) + bordermode="inside" or "outside" - whether to take border width of + master widget into account + """ + self.tk.call( + ('place', 'configure', self._w) + + self._options(cnf, kw)) + + place = configure = config = place_configure + + def place_forget(self): + """Unmap this widget.""" + self.tk.call('place', 'forget', self._w) + + forget = place_forget + + def place_info(self): + """Return information about the placing options + for this widget.""" + d = _splitdict(self.tk, self.tk.call('place', 'info', self._w)) + if 'in' in d: + d['in'] = self.nametowidget(d['in']) + return d + + info = place_info + slaves = place_slaves = Misc.place_slaves + + +class Grid: + """Geometry manager Grid. + + Base class to use the methods grid_* in every widget.""" + # Thanks to Masazumi Yoshikawa (yosikawa@isi.edu) + + def grid_configure(self, cnf={}, **kw): + """Position a widget in the parent widget in a grid. Use as options: + column=number - use cell identified with given column (starting with 0) + columnspan=number - this widget will span several columns + in=master - use master to contain this widget + in_=master - see 'in' option description + ipadx=amount - add internal padding in x direction + ipady=amount - add internal padding in y direction + padx=amount - add padding in x direction + pady=amount - add padding in y direction + row=number - use cell identified with given row (starting with 0) + rowspan=number - this widget will span several rows + sticky=NSEW - if cell is larger on which sides will this + widget stick to the cell boundary + """ + self.tk.call( + ('grid', 'configure', self._w) + + self._options(cnf, kw)) + + grid = configure = config = grid_configure + bbox = grid_bbox = Misc.grid_bbox + columnconfigure = grid_columnconfigure = Misc.grid_columnconfigure + + def grid_forget(self): + """Unmap this widget.""" + self.tk.call('grid', 'forget', self._w) + + forget = grid_forget + + def grid_remove(self): + """Unmap this widget but remember the grid options.""" + self.tk.call('grid', 'remove', self._w) + + def grid_info(self): + """Return information about the options + for positioning this widget in a grid.""" + d = _splitdict(self.tk, self.tk.call('grid', 'info', self._w)) + if 'in' in d: + d['in'] = self.nametowidget(d['in']) + return d + + info = grid_info + location = grid_location = Misc.grid_location + propagate = grid_propagate = Misc.grid_propagate + rowconfigure = grid_rowconfigure = Misc.grid_rowconfigure + size = grid_size = Misc.grid_size + slaves = grid_slaves = Misc.grid_slaves + + +class BaseWidget(Misc): + """Internal class.""" + + def _setup(self, master, cnf): + """Internal function. Sets up information about children.""" + if master is None: + master = _get_default_root() + self.master = master + self.tk = master.tk + name = None + if 'name' in cnf: + name = cnf['name'] + del cnf['name'] + if not name: + name = self.__class__.__name__.lower() + if name[-1].isdigit(): + name += "!" # Avoid duplication when calculating names below + if master._last_child_ids is None: + master._last_child_ids = {} + count = master._last_child_ids.get(name, 0) + 1 + master._last_child_ids[name] = count + if count == 1: + name = '!%s' % (name,) + else: + name = '!%s%d' % (name, count) + self._name = name + if master._w=='.': + self._w = '.' + name + else: + self._w = master._w + '.' + name + self.children = {} + if self._name in self.master.children: + self.master.children[self._name].destroy() + self.master.children[self._name] = self + + def __init__(self, master, widgetName, cnf={}, kw={}, extra=()): + """Construct a widget with the parent widget MASTER, a name WIDGETNAME + and appropriate options.""" + if kw: + cnf = _cnfmerge((cnf, kw)) + self.widgetName = widgetName + self._setup(master, cnf) + if self._tclCommands is None: + self._tclCommands = [] + classes = [(k, v) for k, v in cnf.items() if isinstance(k, type)] + for k, v in classes: + del cnf[k] + self.tk.call( + (widgetName, self._w) + extra + self._options(cnf)) + for k, v in classes: + k.configure(self, v) + + def destroy(self): + """Destroy this and all descendants widgets.""" + for c in list(self.children.values()): c.destroy() + self.tk.call('destroy', self._w) + if self._name in self.master.children: + del self.master.children[self._name] + Misc.destroy(self) + + def _do(self, name, args=()): + # XXX Obsolete -- better use self.tk.call directly! + return self.tk.call((self._w, name) + args) + + +class Widget(BaseWidget, Pack, Place, Grid): + """Internal class. + + Base class for a widget which can be positioned with the geometry managers + Pack, Place or Grid.""" + pass + + +class Toplevel(BaseWidget, Wm): + """Toplevel widget, e.g. for dialogs.""" + + def __init__(self, master=None, cnf={}, **kw): + """Construct a toplevel widget with the parent MASTER. + + Valid resource names: background, bd, bg, borderwidth, class, + colormap, container, cursor, height, highlightbackground, + highlightcolor, highlightthickness, menu, relief, screen, takefocus, + use, visual, width.""" + if kw: + cnf = _cnfmerge((cnf, kw)) + extra = () + for wmkey in ['screen', 'class_', 'class', 'visual', + 'colormap']: + if wmkey in cnf: + val = cnf[wmkey] + # TBD: a hack needed because some keys + # are not valid as keyword arguments + if wmkey[-1] == '_': opt = '-'+wmkey[:-1] + else: opt = '-'+wmkey + extra = extra + (opt, val) + del cnf[wmkey] + BaseWidget.__init__(self, master, 'toplevel', cnf, {}, extra) + root = self._root() + self.iconname(root.iconname()) + self.title(root.title()) + self.protocol("WM_DELETE_WINDOW", self.destroy) + + +class Button(Widget): + """Button widget.""" + + def __init__(self, master=None, cnf={}, **kw): + """Construct a button widget with the parent MASTER. + + STANDARD OPTIONS + + activebackground, activeforeground, anchor, + background, bitmap, borderwidth, cursor, + disabledforeground, font, foreground + highlightbackground, highlightcolor, + highlightthickness, image, justify, + padx, pady, relief, repeatdelay, + repeatinterval, takefocus, text, + textvariable, underline, wraplength + + WIDGET-SPECIFIC OPTIONS + + command, compound, default, height, + overrelief, state, width + """ + Widget.__init__(self, master, 'button', cnf, kw) + + def flash(self): + """Flash the button. + + This is accomplished by redisplaying + the button several times, alternating between active and + normal colors. At the end of the flash the button is left + in the same normal/active state as when the command was + invoked. This command is ignored if the button's state is + disabled. + """ + self.tk.call(self._w, 'flash') + + def invoke(self): + """Invoke the command associated with the button. + + The return value is the return value from the command, + or an empty string if there is no command associated with + the button. This command is ignored if the button's state + is disabled. + """ + return self.tk.call(self._w, 'invoke') + + +class Canvas(Widget, XView, YView): + """Canvas widget to display graphical elements like lines or text.""" + + def __init__(self, master=None, cnf={}, **kw): + """Construct a canvas widget with the parent MASTER. + + Valid resource names: background, bd, bg, borderwidth, closeenough, + confine, cursor, height, highlightbackground, highlightcolor, + highlightthickness, insertbackground, insertborderwidth, + insertofftime, insertontime, insertwidth, offset, relief, + scrollregion, selectbackground, selectborderwidth, selectforeground, + state, takefocus, width, xscrollcommand, xscrollincrement, + yscrollcommand, yscrollincrement.""" + Widget.__init__(self, master, 'canvas', cnf, kw) + + def addtag(self, *args): + """Internal function.""" + self.tk.call((self._w, 'addtag') + args) + + def addtag_above(self, newtag, tagOrId): + """Add tag NEWTAG to all items above TAGORID.""" + self.addtag(newtag, 'above', tagOrId) + + def addtag_all(self, newtag): + """Add tag NEWTAG to all items.""" + self.addtag(newtag, 'all') + + def addtag_below(self, newtag, tagOrId): + """Add tag NEWTAG to all items below TAGORID.""" + self.addtag(newtag, 'below', tagOrId) + + def addtag_closest(self, newtag, x, y, halo=None, start=None): + """Add tag NEWTAG to item which is closest to pixel at X, Y. + If several match take the top-most. + All items closer than HALO are considered overlapping (all are + closest). If START is specified the next below this tag is taken.""" + self.addtag(newtag, 'closest', x, y, halo, start) + + def addtag_enclosed(self, newtag, x1, y1, x2, y2): + """Add tag NEWTAG to all items in the rectangle defined + by X1,Y1,X2,Y2.""" + self.addtag(newtag, 'enclosed', x1, y1, x2, y2) + + def addtag_overlapping(self, newtag, x1, y1, x2, y2): + """Add tag NEWTAG to all items which overlap the rectangle + defined by X1,Y1,X2,Y2.""" + self.addtag(newtag, 'overlapping', x1, y1, x2, y2) + + def addtag_withtag(self, newtag, tagOrId): + """Add tag NEWTAG to all items with TAGORID.""" + self.addtag(newtag, 'withtag', tagOrId) + + def bbox(self, *args): + """Return a tuple of X1,Y1,X2,Y2 coordinates for a rectangle + which encloses all items with tags specified as arguments.""" + return self._getints( + self.tk.call((self._w, 'bbox') + args)) or None + + def tag_unbind(self, tagOrId, sequence, funcid=None): + """Unbind for all items with TAGORID for event SEQUENCE the + function identified with FUNCID.""" + self._unbind((self._w, 'bind', tagOrId, sequence), funcid) + + def tag_bind(self, tagOrId, sequence=None, func=None, add=None): + """Bind to all items with TAGORID at event SEQUENCE a call to function FUNC. + + An additional boolean parameter ADD specifies whether FUNC will be + called additionally to the other bound function or whether it will + replace the previous function. See bind for the return value.""" + return self._bind((self._w, 'bind', tagOrId), + sequence, func, add) + + def canvasx(self, screenx, gridspacing=None): + """Return the canvas x coordinate of pixel position SCREENX rounded + to nearest multiple of GRIDSPACING units.""" + return self.tk.getdouble(self.tk.call( + self._w, 'canvasx', screenx, gridspacing)) + + def canvasy(self, screeny, gridspacing=None): + """Return the canvas y coordinate of pixel position SCREENY rounded + to nearest multiple of GRIDSPACING units.""" + return self.tk.getdouble(self.tk.call( + self._w, 'canvasy', screeny, gridspacing)) + + def coords(self, *args): + """Return a list of coordinates for the item given in ARGS.""" + args = _flatten(args) + return [self.tk.getdouble(x) for x in + self.tk.splitlist( + self.tk.call((self._w, 'coords') + args))] + + def _create(self, itemType, args, kw): # Args: (val, val, ..., cnf={}) + """Internal function.""" + args = _flatten(args) + cnf = args[-1] + if isinstance(cnf, (dict, tuple)): + args = args[:-1] + else: + cnf = {} + return self.tk.getint(self.tk.call( + self._w, 'create', itemType, + *(args + self._options(cnf, kw)))) + + def create_arc(self, *args, **kw): + """Create arc shaped region with coordinates x1,y1,x2,y2.""" + return self._create('arc', args, kw) + + def create_bitmap(self, *args, **kw): + """Create bitmap with coordinates x1,y1.""" + return self._create('bitmap', args, kw) + + def create_image(self, *args, **kw): + """Create image item with coordinates x1,y1.""" + return self._create('image', args, kw) + + def create_line(self, *args, **kw): + """Create line with coordinates x1,y1,...,xn,yn.""" + return self._create('line', args, kw) + + def create_oval(self, *args, **kw): + """Create oval with coordinates x1,y1,x2,y2.""" + return self._create('oval', args, kw) + + def create_polygon(self, *args, **kw): + """Create polygon with coordinates x1,y1,...,xn,yn.""" + return self._create('polygon', args, kw) + + def create_rectangle(self, *args, **kw): + """Create rectangle with coordinates x1,y1,x2,y2.""" + return self._create('rectangle', args, kw) + + def create_text(self, *args, **kw): + """Create text with coordinates x1,y1.""" + return self._create('text', args, kw) + + def create_window(self, *args, **kw): + """Create window with coordinates x1,y1,x2,y2.""" + return self._create('window', args, kw) + + def dchars(self, *args): + """Delete characters of text items identified by tag or id in ARGS (possibly + several times) from FIRST to LAST character (including).""" + self.tk.call((self._w, 'dchars') + args) + + def delete(self, *args): + """Delete items identified by all tag or ids contained in ARGS.""" + self.tk.call((self._w, 'delete') + args) + + def dtag(self, *args): + """Delete tag or id given as last arguments in ARGS from items + identified by first argument in ARGS.""" + self.tk.call((self._w, 'dtag') + args) + + def find(self, *args): + """Internal function.""" + return self._getints( + self.tk.call((self._w, 'find') + args)) or () + + def find_above(self, tagOrId): + """Return items above TAGORID.""" + return self.find('above', tagOrId) + + def find_all(self): + """Return all items.""" + return self.find('all') + + def find_below(self, tagOrId): + """Return all items below TAGORID.""" + return self.find('below', tagOrId) + + def find_closest(self, x, y, halo=None, start=None): + """Return item which is closest to pixel at X, Y. + If several match take the top-most. + All items closer than HALO are considered overlapping (all are + closest). If START is specified the next below this tag is taken.""" + return self.find('closest', x, y, halo, start) + + def find_enclosed(self, x1, y1, x2, y2): + """Return all items in rectangle defined + by X1,Y1,X2,Y2.""" + return self.find('enclosed', x1, y1, x2, y2) + + def find_overlapping(self, x1, y1, x2, y2): + """Return all items which overlap the rectangle + defined by X1,Y1,X2,Y2.""" + return self.find('overlapping', x1, y1, x2, y2) + + def find_withtag(self, tagOrId): + """Return all items with TAGORID.""" + return self.find('withtag', tagOrId) + + def focus(self, *args): + """Set focus to the first item specified in ARGS.""" + return self.tk.call((self._w, 'focus') + args) + + def gettags(self, *args): + """Return tags associated with the first item specified in ARGS.""" + return self.tk.splitlist( + self.tk.call((self._w, 'gettags') + args)) + + def icursor(self, *args): + """Set cursor at position POS in the item identified by TAGORID. + In ARGS TAGORID must be first.""" + self.tk.call((self._w, 'icursor') + args) + + def index(self, *args): + """Return position of cursor as integer in item specified in ARGS.""" + return self.tk.getint(self.tk.call((self._w, 'index') + args)) + + def insert(self, *args): + """Insert TEXT in item TAGORID at position POS. ARGS must + be TAGORID POS TEXT.""" + self.tk.call((self._w, 'insert') + args) + + def itemcget(self, tagOrId, option): + """Return the resource value for an OPTION for item TAGORID.""" + return self.tk.call( + (self._w, 'itemcget') + (tagOrId, '-'+option)) + + def itemconfigure(self, tagOrId, cnf=None, **kw): + """Configure resources of an item TAGORID. + + The values for resources are specified as keyword + arguments. To get an overview about + the allowed keyword arguments call the method without arguments. + """ + return self._configure(('itemconfigure', tagOrId), cnf, kw) + + itemconfig = itemconfigure + + # lower, tkraise/lift hide Misc.lower, Misc.tkraise/lift, + # so the preferred name for them is tag_lower, tag_raise + # (similar to tag_bind, and similar to the Text widget); + # unfortunately can't delete the old ones yet (maybe in 1.6) + def tag_lower(self, *args): + """Lower an item TAGORID given in ARGS + (optional below another item).""" + self.tk.call((self._w, 'lower') + args) + + lower = tag_lower + + def move(self, *args): + """Move an item TAGORID given in ARGS.""" + self.tk.call((self._w, 'move') + args) + + def moveto(self, tagOrId, x='', y=''): + """Move the items given by TAGORID in the canvas coordinate + space so that the first coordinate pair of the bottommost + item with tag TAGORID is located at position (X,Y). + X and Y may be the empty string, in which case the + corresponding coordinate will be unchanged. All items matching + TAGORID remain in the same positions relative to each other.""" + self.tk.call(self._w, 'moveto', tagOrId, x, y) + + def postscript(self, cnf={}, **kw): + """Print the contents of the canvas to a postscript + file. Valid options: colormap, colormode, file, fontmap, + height, pageanchor, pageheight, pagewidth, pagex, pagey, + rotate, width, x, y.""" + return self.tk.call((self._w, 'postscript') + + self._options(cnf, kw)) + + def tag_raise(self, *args): + """Raise an item TAGORID given in ARGS + (optional above another item).""" + self.tk.call((self._w, 'raise') + args) + + lift = tkraise = tag_raise + + def scale(self, *args): + """Scale item TAGORID with XORIGIN, YORIGIN, XSCALE, YSCALE.""" + self.tk.call((self._w, 'scale') + args) + + def scan_mark(self, x, y): + """Remember the current X, Y coordinates.""" + self.tk.call(self._w, 'scan', 'mark', x, y) + + def scan_dragto(self, x, y, gain=10): + """Adjust the view of the canvas to GAIN times the + difference between X and Y and the coordinates given in + scan_mark.""" + self.tk.call(self._w, 'scan', 'dragto', x, y, gain) + + def select_adjust(self, tagOrId, index): + """Adjust the end of the selection near the cursor of an item TAGORID to index.""" + self.tk.call(self._w, 'select', 'adjust', tagOrId, index) + + def select_clear(self): + """Clear the selection if it is in this widget.""" + self.tk.call(self._w, 'select', 'clear') + + def select_from(self, tagOrId, index): + """Set the fixed end of a selection in item TAGORID to INDEX.""" + self.tk.call(self._w, 'select', 'from', tagOrId, index) + + def select_item(self): + """Return the item which has the selection.""" + return self.tk.call(self._w, 'select', 'item') or None + + def select_to(self, tagOrId, index): + """Set the variable end of a selection in item TAGORID to INDEX.""" + self.tk.call(self._w, 'select', 'to', tagOrId, index) + + def type(self, tagOrId): + """Return the type of the item TAGORID.""" + return self.tk.call(self._w, 'type', tagOrId) or None + + +_checkbutton_count = 0 + +class Checkbutton(Widget): + """Checkbutton widget which is either in on- or off-state.""" + + def __init__(self, master=None, cnf={}, **kw): + """Construct a checkbutton widget with the parent MASTER. + + Valid resource names: activebackground, activeforeground, anchor, + background, bd, bg, bitmap, borderwidth, command, cursor, + disabledforeground, fg, font, foreground, height, + highlightbackground, highlightcolor, highlightthickness, image, + indicatoron, justify, offvalue, onvalue, padx, pady, relief, + selectcolor, selectimage, state, takefocus, text, textvariable, + underline, variable, width, wraplength.""" + Widget.__init__(self, master, 'checkbutton', cnf, kw) + + def _setup(self, master, cnf): + # Because Checkbutton defaults to a variable with the same name as + # the widget, Checkbutton default names must be globally unique, + # not just unique within the parent widget. + if not cnf.get('name'): + global _checkbutton_count + name = self.__class__.__name__.lower() + _checkbutton_count += 1 + # To avoid collisions with ttk.Checkbutton, use the different + # name template. + cnf['name'] = f'!{name}-{_checkbutton_count}' + super()._setup(master, cnf) + + def deselect(self): + """Put the button in off-state.""" + self.tk.call(self._w, 'deselect') + + def flash(self): + """Flash the button.""" + self.tk.call(self._w, 'flash') + + def invoke(self): + """Toggle the button and invoke a command if given as resource.""" + return self.tk.call(self._w, 'invoke') + + def select(self): + """Put the button in on-state.""" + self.tk.call(self._w, 'select') + + def toggle(self): + """Toggle the button.""" + self.tk.call(self._w, 'toggle') + + +class Entry(Widget, XView): + """Entry widget which allows displaying simple text.""" + + def __init__(self, master=None, cnf={}, **kw): + """Construct an entry widget with the parent MASTER. + + Valid resource names: background, bd, bg, borderwidth, cursor, + exportselection, fg, font, foreground, highlightbackground, + highlightcolor, highlightthickness, insertbackground, + insertborderwidth, insertofftime, insertontime, insertwidth, + invalidcommand, invcmd, justify, relief, selectbackground, + selectborderwidth, selectforeground, show, state, takefocus, + textvariable, validate, validatecommand, vcmd, width, + xscrollcommand.""" + Widget.__init__(self, master, 'entry', cnf, kw) + + def delete(self, first, last=None): + """Delete text from FIRST to LAST (not included).""" + self.tk.call(self._w, 'delete', first, last) + + def get(self): + """Return the text.""" + return self.tk.call(self._w, 'get') + + def icursor(self, index): + """Insert cursor at INDEX.""" + self.tk.call(self._w, 'icursor', index) + + def index(self, index): + """Return position of cursor.""" + return self.tk.getint(self.tk.call( + self._w, 'index', index)) + + def insert(self, index, string): + """Insert STRING at INDEX.""" + self.tk.call(self._w, 'insert', index, string) + + def scan_mark(self, x): + """Remember the current X, Y coordinates.""" + self.tk.call(self._w, 'scan', 'mark', x) + + def scan_dragto(self, x): + """Adjust the view of the canvas to 10 times the + difference between X and Y and the coordinates given in + scan_mark.""" + self.tk.call(self._w, 'scan', 'dragto', x) + + def selection_adjust(self, index): + """Adjust the end of the selection near the cursor to INDEX.""" + self.tk.call(self._w, 'selection', 'adjust', index) + + select_adjust = selection_adjust + + def selection_clear(self): + """Clear the selection if it is in this widget.""" + self.tk.call(self._w, 'selection', 'clear') + + select_clear = selection_clear + + def selection_from(self, index): + """Set the fixed end of a selection to INDEX.""" + self.tk.call(self._w, 'selection', 'from', index) + + select_from = selection_from + + def selection_present(self): + """Return True if there are characters selected in the entry, False + otherwise.""" + return self.tk.getboolean( + self.tk.call(self._w, 'selection', 'present')) + + select_present = selection_present + + def selection_range(self, start, end): + """Set the selection from START to END (not included).""" + self.tk.call(self._w, 'selection', 'range', start, end) + + select_range = selection_range + + def selection_to(self, index): + """Set the variable end of a selection to INDEX.""" + self.tk.call(self._w, 'selection', 'to', index) + + select_to = selection_to + + +class Frame(Widget): + """Frame widget which may contain other widgets and can have a 3D border.""" + + def __init__(self, master=None, cnf={}, **kw): + """Construct a frame widget with the parent MASTER. + + Valid resource names: background, bd, bg, borderwidth, class, + colormap, container, cursor, height, highlightbackground, + highlightcolor, highlightthickness, relief, takefocus, visual, width.""" + cnf = _cnfmerge((cnf, kw)) + extra = () + if 'class_' in cnf: + extra = ('-class', cnf['class_']) + del cnf['class_'] + elif 'class' in cnf: + extra = ('-class', cnf['class']) + del cnf['class'] + Widget.__init__(self, master, 'frame', cnf, {}, extra) + + +class Label(Widget): + """Label widget which can display text and bitmaps.""" + + def __init__(self, master=None, cnf={}, **kw): + """Construct a label widget with the parent MASTER. + + STANDARD OPTIONS + + activebackground, activeforeground, anchor, + background, bitmap, borderwidth, cursor, + disabledforeground, font, foreground, + highlightbackground, highlightcolor, + highlightthickness, image, justify, + padx, pady, relief, takefocus, text, + textvariable, underline, wraplength + + WIDGET-SPECIFIC OPTIONS + + height, state, width + + """ + Widget.__init__(self, master, 'label', cnf, kw) + + +class Listbox(Widget, XView, YView): + """Listbox widget which can display a list of strings.""" + + def __init__(self, master=None, cnf={}, **kw): + """Construct a listbox widget with the parent MASTER. + + Valid resource names: background, bd, bg, borderwidth, cursor, + exportselection, fg, font, foreground, height, highlightbackground, + highlightcolor, highlightthickness, relief, selectbackground, + selectborderwidth, selectforeground, selectmode, setgrid, takefocus, + width, xscrollcommand, yscrollcommand, listvariable.""" + Widget.__init__(self, master, 'listbox', cnf, kw) + + def activate(self, index): + """Activate item identified by INDEX.""" + self.tk.call(self._w, 'activate', index) + + def bbox(self, index): + """Return a tuple of X1,Y1,X2,Y2 coordinates for a rectangle + which encloses the item identified by the given index.""" + return self._getints(self.tk.call(self._w, 'bbox', index)) or None + + def curselection(self): + """Return the indices of currently selected item.""" + return self._getints(self.tk.call(self._w, 'curselection')) or () + + def delete(self, first, last=None): + """Delete items from FIRST to LAST (included).""" + self.tk.call(self._w, 'delete', first, last) + + def get(self, first, last=None): + """Get list of items from FIRST to LAST (included).""" + if last is not None: + return self.tk.splitlist(self.tk.call( + self._w, 'get', first, last)) + else: + return self.tk.call(self._w, 'get', first) + + def index(self, index): + """Return index of item identified with INDEX.""" + i = self.tk.call(self._w, 'index', index) + if i == 'none': return None + return self.tk.getint(i) + + def insert(self, index, *elements): + """Insert ELEMENTS at INDEX.""" + self.tk.call((self._w, 'insert', index) + elements) + + def nearest(self, y): + """Get index of item which is nearest to y coordinate Y.""" + return self.tk.getint(self.tk.call( + self._w, 'nearest', y)) + + def scan_mark(self, x, y): + """Remember the current X, Y coordinates.""" + self.tk.call(self._w, 'scan', 'mark', x, y) + + def scan_dragto(self, x, y): + """Adjust the view of the listbox to 10 times the + difference between X and Y and the coordinates given in + scan_mark.""" + self.tk.call(self._w, 'scan', 'dragto', x, y) + + def see(self, index): + """Scroll such that INDEX is visible.""" + self.tk.call(self._w, 'see', index) + + def selection_anchor(self, index): + """Set the fixed end oft the selection to INDEX.""" + self.tk.call(self._w, 'selection', 'anchor', index) + + select_anchor = selection_anchor + + def selection_clear(self, first, last=None): + """Clear the selection from FIRST to LAST (included).""" + self.tk.call(self._w, + 'selection', 'clear', first, last) + + select_clear = selection_clear + + def selection_includes(self, index): + """Return True if INDEX is part of the selection.""" + return self.tk.getboolean(self.tk.call( + self._w, 'selection', 'includes', index)) + + select_includes = selection_includes + + def selection_set(self, first, last=None): + """Set the selection from FIRST to LAST (included) without + changing the currently selected elements.""" + self.tk.call(self._w, 'selection', 'set', first, last) + + select_set = selection_set + + def size(self): + """Return the number of elements in the listbox.""" + return self.tk.getint(self.tk.call(self._w, 'size')) + + def itemcget(self, index, option): + """Return the resource value for an ITEM and an OPTION.""" + return self.tk.call( + (self._w, 'itemcget') + (index, '-'+option)) + + def itemconfigure(self, index, cnf=None, **kw): + """Configure resources of an ITEM. + + The values for resources are specified as keyword arguments. + To get an overview about the allowed keyword arguments + call the method without arguments. + Valid resource names: background, bg, foreground, fg, + selectbackground, selectforeground.""" + return self._configure(('itemconfigure', index), cnf, kw) + + itemconfig = itemconfigure + + +class Menu(Widget): + """Menu widget which allows displaying menu bars, pull-down menus and pop-up menus.""" + + def __init__(self, master=None, cnf={}, **kw): + """Construct menu widget with the parent MASTER. + + Valid resource names: activebackground, activeborderwidth, + activeforeground, background, bd, bg, borderwidth, cursor, + disabledforeground, fg, font, foreground, postcommand, relief, + selectcolor, takefocus, tearoff, tearoffcommand, title, type.""" + Widget.__init__(self, master, 'menu', cnf, kw) + + def tk_popup(self, x, y, entry=""): + """Post the menu at position X,Y with entry ENTRY.""" + self.tk.call('tk_popup', self._w, x, y, entry) + + def activate(self, index): + """Activate entry at INDEX.""" + self.tk.call(self._w, 'activate', index) + + def add(self, itemType, cnf={}, **kw): + """Internal function.""" + self.tk.call((self._w, 'add', itemType) + + self._options(cnf, kw)) + + def add_cascade(self, cnf={}, **kw): + """Add hierarchical menu item.""" + self.add('cascade', cnf or kw) + + def add_checkbutton(self, cnf={}, **kw): + """Add checkbutton menu item.""" + self.add('checkbutton', cnf or kw) + + def add_command(self, cnf={}, **kw): + """Add command menu item.""" + self.add('command', cnf or kw) + + def add_radiobutton(self, cnf={}, **kw): + """Add radio menu item.""" + self.add('radiobutton', cnf or kw) + + def add_separator(self, cnf={}, **kw): + """Add separator.""" + self.add('separator', cnf or kw) + + def insert(self, index, itemType, cnf={}, **kw): + """Internal function.""" + self.tk.call((self._w, 'insert', index, itemType) + + self._options(cnf, kw)) + + def insert_cascade(self, index, cnf={}, **kw): + """Add hierarchical menu item at INDEX.""" + self.insert(index, 'cascade', cnf or kw) + + def insert_checkbutton(self, index, cnf={}, **kw): + """Add checkbutton menu item at INDEX.""" + self.insert(index, 'checkbutton', cnf or kw) + + def insert_command(self, index, cnf={}, **kw): + """Add command menu item at INDEX.""" + self.insert(index, 'command', cnf or kw) + + def insert_radiobutton(self, index, cnf={}, **kw): + """Add radio menu item at INDEX.""" + self.insert(index, 'radiobutton', cnf or kw) + + def insert_separator(self, index, cnf={}, **kw): + """Add separator at INDEX.""" + self.insert(index, 'separator', cnf or kw) + + def delete(self, index1, index2=None): + """Delete menu items between INDEX1 and INDEX2 (included).""" + if index2 is None: + index2 = index1 + + num_index1, num_index2 = self.index(index1), self.index(index2) + if (num_index1 is None) or (num_index2 is None): + num_index1, num_index2 = 0, -1 + + for i in range(num_index1, num_index2 + 1): + if 'command' in self.entryconfig(i): + c = str(self.entrycget(i, 'command')) + if c: + self.deletecommand(c) + self.tk.call(self._w, 'delete', index1, index2) + + def entrycget(self, index, option): + """Return the resource value of a menu item for OPTION at INDEX.""" + return self.tk.call(self._w, 'entrycget', index, '-' + option) + + def entryconfigure(self, index, cnf=None, **kw): + """Configure a menu item at INDEX.""" + return self._configure(('entryconfigure', index), cnf, kw) + + entryconfig = entryconfigure + + def index(self, index): + """Return the index of a menu item identified by INDEX.""" + i = self.tk.call(self._w, 'index', index) + return None if i in ('', 'none') else self.tk.getint(i) # GH-103685. + + def invoke(self, index): + """Invoke a menu item identified by INDEX and execute + the associated command.""" + return self.tk.call(self._w, 'invoke', index) + + def post(self, x, y): + """Display a menu at position X,Y.""" + self.tk.call(self._w, 'post', x, y) + + def type(self, index): + """Return the type of the menu item at INDEX.""" + return self.tk.call(self._w, 'type', index) + + def unpost(self): + """Unmap a menu.""" + self.tk.call(self._w, 'unpost') + + def xposition(self, index): # new in Tk 8.5 + """Return the x-position of the leftmost pixel of the menu item + at INDEX.""" + return self.tk.getint(self.tk.call(self._w, 'xposition', index)) + + def yposition(self, index): + """Return the y-position of the topmost pixel of the menu item at INDEX.""" + return self.tk.getint(self.tk.call( + self._w, 'yposition', index)) + + +class Menubutton(Widget): + """Menubutton widget, obsolete since Tk8.0.""" + + def __init__(self, master=None, cnf={}, **kw): + Widget.__init__(self, master, 'menubutton', cnf, kw) + + +class Message(Widget): + """Message widget to display multiline text. Obsolete since Label does it too.""" + + def __init__(self, master=None, cnf={}, **kw): + Widget.__init__(self, master, 'message', cnf, kw) + + +class Radiobutton(Widget): + """Radiobutton widget which shows only one of several buttons in on-state.""" + + def __init__(self, master=None, cnf={}, **kw): + """Construct a radiobutton widget with the parent MASTER. + + Valid resource names: activebackground, activeforeground, anchor, + background, bd, bg, bitmap, borderwidth, command, cursor, + disabledforeground, fg, font, foreground, height, + highlightbackground, highlightcolor, highlightthickness, image, + indicatoron, justify, padx, pady, relief, selectcolor, selectimage, + state, takefocus, text, textvariable, underline, value, variable, + width, wraplength.""" + Widget.__init__(self, master, 'radiobutton', cnf, kw) + + def deselect(self): + """Put the button in off-state.""" + + self.tk.call(self._w, 'deselect') + + def flash(self): + """Flash the button.""" + self.tk.call(self._w, 'flash') + + def invoke(self): + """Toggle the button and invoke a command if given as resource.""" + return self.tk.call(self._w, 'invoke') + + def select(self): + """Put the button in on-state.""" + self.tk.call(self._w, 'select') + + +class Scale(Widget): + """Scale widget which can display a numerical scale.""" + + def __init__(self, master=None, cnf={}, **kw): + """Construct a scale widget with the parent MASTER. + + Valid resource names: activebackground, background, bigincrement, bd, + bg, borderwidth, command, cursor, digits, fg, font, foreground, from, + highlightbackground, highlightcolor, highlightthickness, label, + length, orient, relief, repeatdelay, repeatinterval, resolution, + showvalue, sliderlength, sliderrelief, state, takefocus, + tickinterval, to, troughcolor, variable, width.""" + Widget.__init__(self, master, 'scale', cnf, kw) + + def get(self): + """Get the current value as integer or float.""" + value = self.tk.call(self._w, 'get') + try: + return self.tk.getint(value) + except (ValueError, TypeError, TclError): + return self.tk.getdouble(value) + + def set(self, value): + """Set the value to VALUE.""" + self.tk.call(self._w, 'set', value) + + def coords(self, value=None): + """Return a tuple (X,Y) of the point along the centerline of the + trough that corresponds to VALUE or the current value if None is + given.""" + + return self._getints(self.tk.call(self._w, 'coords', value)) + + def identify(self, x, y): + """Return where the point X,Y lies. Valid return values are "slider", + "though1" and "though2".""" + return self.tk.call(self._w, 'identify', x, y) + + +class Scrollbar(Widget): + """Scrollbar widget which displays a slider at a certain position.""" + + def __init__(self, master=None, cnf={}, **kw): + """Construct a scrollbar widget with the parent MASTER. + + Valid resource names: activebackground, activerelief, + background, bd, bg, borderwidth, command, cursor, + elementborderwidth, highlightbackground, + highlightcolor, highlightthickness, jump, orient, + relief, repeatdelay, repeatinterval, takefocus, + troughcolor, width.""" + Widget.__init__(self, master, 'scrollbar', cnf, kw) + + def activate(self, index=None): + """Marks the element indicated by index as active. + The only index values understood by this method are "arrow1", + "slider", or "arrow2". If any other value is specified then no + element of the scrollbar will be active. If index is not specified, + the method returns the name of the element that is currently active, + or None if no element is active.""" + return self.tk.call(self._w, 'activate', index) or None + + def delta(self, deltax, deltay): + """Return the fractional change of the scrollbar setting if it + would be moved by DELTAX or DELTAY pixels.""" + return self.tk.getdouble( + self.tk.call(self._w, 'delta', deltax, deltay)) + + def fraction(self, x, y): + """Return the fractional value which corresponds to a slider + position of X,Y.""" + return self.tk.getdouble(self.tk.call(self._w, 'fraction', x, y)) + + def identify(self, x, y): + """Return the element under position X,Y as one of + "arrow1","slider","arrow2" or "".""" + return self.tk.call(self._w, 'identify', x, y) + + def get(self): + """Return the current fractional values (upper and lower end) + of the slider position.""" + return self._getdoubles(self.tk.call(self._w, 'get')) + + def set(self, first, last): + """Set the fractional values of the slider position (upper and + lower ends as value between 0 and 1).""" + self.tk.call(self._w, 'set', first, last) + + +class Text(Widget, XView, YView): + """Text widget which can display text in various forms.""" + + def __init__(self, master=None, cnf={}, **kw): + """Construct a text widget with the parent MASTER. + + STANDARD OPTIONS + + background, borderwidth, cursor, + exportselection, font, foreground, + highlightbackground, highlightcolor, + highlightthickness, insertbackground, + insertborderwidth, insertofftime, + insertontime, insertwidth, padx, pady, + relief, selectbackground, + selectborderwidth, selectforeground, + setgrid, takefocus, + xscrollcommand, yscrollcommand, + + WIDGET-SPECIFIC OPTIONS + + autoseparators, height, maxundo, + spacing1, spacing2, spacing3, + state, tabs, undo, width, wrap, + + """ + Widget.__init__(self, master, 'text', cnf, kw) + + def bbox(self, index): + """Return a tuple of (x,y,width,height) which gives the bounding + box of the visible part of the character at the given index.""" + return self._getints( + self.tk.call(self._w, 'bbox', index)) or None + + def compare(self, index1, op, index2): + """Return whether between index INDEX1 and index INDEX2 the + relation OP is satisfied. OP is one of <, <=, ==, >=, >, or !=.""" + return self.tk.getboolean(self.tk.call( + self._w, 'compare', index1, op, index2)) + + def count(self, index1, index2, *options, return_ints=False): # new in Tk 8.5 + """Counts the number of relevant things between the two indices. + + If INDEX1 is after INDEX2, the result will be a negative number + (and this holds for each of the possible options). + + The actual items which are counted depends on the options given. + The result is a tuple of integers, one for the result of each + counting option given, if more than one option is specified or + return_ints is false (default), otherwise it is an integer. + Valid counting options are "chars", "displaychars", + "displayindices", "displaylines", "indices", "lines", "xpixels" + and "ypixels". The default value, if no option is specified, is + "indices". There is an additional possible option "update", + which if given then all subsequent options ensure that any + possible out of date information is recalculated. + """ + options = ['-%s' % arg for arg in options] + res = self.tk.call(self._w, 'count', *options, index1, index2) + if not isinstance(res, int): + res = self._getints(res) + if len(res) == 1: + res, = res + if not return_ints: + if not res: + res = None + elif len(options) <= 1: + res = (res,) + return res + + def debug(self, boolean=None): + """Turn on the internal consistency checks of the B-Tree inside the text + widget according to BOOLEAN.""" + if boolean is None: + return self.tk.getboolean(self.tk.call(self._w, 'debug')) + self.tk.call(self._w, 'debug', boolean) + + def delete(self, index1, index2=None): + """Delete the characters between INDEX1 and INDEX2 (not included).""" + self.tk.call(self._w, 'delete', index1, index2) + + def dlineinfo(self, index): + """Return tuple (x,y,width,height,baseline) giving the bounding box + and baseline position of the visible part of the line containing + the character at INDEX.""" + return self._getints(self.tk.call(self._w, 'dlineinfo', index)) + + def dump(self, index1, index2=None, command=None, **kw): + """Return the contents of the widget between index1 and index2. + + The type of contents returned in filtered based on the keyword + parameters; if 'all', 'image', 'mark', 'tag', 'text', or 'window' are + given and true, then the corresponding items are returned. The result + is a list of triples of the form (key, value, index). If none of the + keywords are true then 'all' is used by default. + + If the 'command' argument is given, it is called once for each element + of the list of triples, with the values of each triple serving as the + arguments to the function. In this case the list is not returned.""" + args = [] + func_name = None + result = None + if not command: + # Never call the dump command without the -command flag, since the + # output could involve Tcl quoting and would be a pain to parse + # right. Instead just set the command to build a list of triples + # as if we had done the parsing. + result = [] + def append_triple(key, value, index, result=result): + result.append((key, value, index)) + command = append_triple + try: + if not isinstance(command, str): + func_name = command = self._register(command) + args += ["-command", command] + for key in kw: + if kw[key]: args.append("-" + key) + args.append(index1) + if index2: + args.append(index2) + self.tk.call(self._w, "dump", *args) + return result + finally: + if func_name: + self.deletecommand(func_name) + + ## new in tk8.4 + def edit(self, *args): + """Internal method + + This method controls the undo mechanism and + the modified flag. The exact behavior of the + command depends on the option argument that + follows the edit argument. The following forms + of the command are currently supported: + + edit_modified, edit_redo, edit_reset, edit_separator + and edit_undo + + """ + return self.tk.call(self._w, 'edit', *args) + + def edit_modified(self, arg=None): + """Get or Set the modified flag + + If arg is not specified, returns the modified + flag of the widget. The insert, delete, edit undo and + edit redo commands or the user can set or clear the + modified flag. If boolean is specified, sets the + modified flag of the widget to arg. + """ + return self.edit("modified", arg) + + def edit_redo(self): + """Redo the last undone edit + + When the undo option is true, reapplies the last + undone edits provided no other edits were done since + then. Generates an error when the redo stack is empty. + Does nothing when the undo option is false. + """ + return self.edit("redo") + + def edit_reset(self): + """Clears the undo and redo stacks + """ + return self.edit("reset") + + def edit_separator(self): + """Inserts a separator (boundary) on the undo stack. + + Does nothing when the undo option is false + """ + return self.edit("separator") + + def edit_undo(self): + """Undoes the last edit action + + If the undo option is true. An edit action is defined + as all the insert and delete commands that are recorded + on the undo stack in between two separators. Generates + an error when the undo stack is empty. Does nothing + when the undo option is false + """ + return self.edit("undo") + + def get(self, index1, index2=None): + """Return the text from INDEX1 to INDEX2 (not included).""" + return self.tk.call(self._w, 'get', index1, index2) + # (Image commands are new in 8.0) + + def image_cget(self, index, option): + """Return the value of OPTION of an embedded image at INDEX.""" + if option[:1] != "-": + option = "-" + option + if option[-1:] == "_": + option = option[:-1] + return self.tk.call(self._w, "image", "cget", index, option) + + def image_configure(self, index, cnf=None, **kw): + """Configure an embedded image at INDEX.""" + return self._configure(('image', 'configure', index), cnf, kw) + + def image_create(self, index, cnf={}, **kw): + """Create an embedded image at INDEX.""" + return self.tk.call( + self._w, "image", "create", index, + *self._options(cnf, kw)) + + def image_names(self): + """Return all names of embedded images in this widget.""" + return self.tk.call(self._w, "image", "names") + + def index(self, index): + """Return the index in the form line.char for INDEX.""" + return str(self.tk.call(self._w, 'index', index)) + + def insert(self, index, chars, *args): + """Insert CHARS before the characters at INDEX. An additional + tag can be given in ARGS. Additional CHARS and tags can follow in ARGS.""" + self.tk.call((self._w, 'insert', index, chars) + args) + + def mark_gravity(self, markName, direction=None): + """Change the gravity of a mark MARKNAME to DIRECTION (LEFT or RIGHT). + Return the current value if None is given for DIRECTION.""" + return self.tk.call( + (self._w, 'mark', 'gravity', markName, direction)) + + def mark_names(self): + """Return all mark names.""" + return self.tk.splitlist(self.tk.call( + self._w, 'mark', 'names')) + + def mark_set(self, markName, index): + """Set mark MARKNAME before the character at INDEX.""" + self.tk.call(self._w, 'mark', 'set', markName, index) + + def mark_unset(self, *markNames): + """Delete all marks in MARKNAMES.""" + self.tk.call((self._w, 'mark', 'unset') + markNames) + + def mark_next(self, index): + """Return the name of the next mark after INDEX.""" + return self.tk.call(self._w, 'mark', 'next', index) or None + + def mark_previous(self, index): + """Return the name of the previous mark before INDEX.""" + return self.tk.call(self._w, 'mark', 'previous', index) or None + + def peer_create(self, newPathName, cnf={}, **kw): # new in Tk 8.5 + """Creates a peer text widget with the given newPathName, and any + optional standard configuration options. By default the peer will + have the same start and end line as the parent widget, but + these can be overridden with the standard configuration options.""" + self.tk.call(self._w, 'peer', 'create', newPathName, + *self._options(cnf, kw)) + + def peer_names(self): # new in Tk 8.5 + """Returns a list of peers of this widget (this does not include + the widget itself).""" + return self.tk.splitlist(self.tk.call(self._w, 'peer', 'names')) + + def replace(self, index1, index2, chars, *args): # new in Tk 8.5 + """Replaces the range of characters between index1 and index2 with + the given characters and tags specified by args. + + See the method insert for some more information about args, and the + method delete for information about the indices.""" + self.tk.call(self._w, 'replace', index1, index2, chars, *args) + + def scan_mark(self, x, y): + """Remember the current X, Y coordinates.""" + self.tk.call(self._w, 'scan', 'mark', x, y) + + def scan_dragto(self, x, y): + """Adjust the view of the text to 10 times the + difference between X and Y and the coordinates given in + scan_mark.""" + self.tk.call(self._w, 'scan', 'dragto', x, y) + + def search(self, pattern, index, stopindex=None, + forwards=None, backwards=None, exact=None, + regexp=None, nocase=None, count=None, elide=None): + """Search PATTERN beginning from INDEX until STOPINDEX. + Return the index of the first character of a match or an + empty string.""" + args = [self._w, 'search'] + if forwards: args.append('-forwards') + if backwards: args.append('-backwards') + if exact: args.append('-exact') + if regexp: args.append('-regexp') + if nocase: args.append('-nocase') + if elide: args.append('-elide') + if count: args.append('-count'); args.append(count) + if pattern and pattern[0] == '-': args.append('--') + args.append(pattern) + args.append(index) + if stopindex: args.append(stopindex) + return str(self.tk.call(tuple(args))) + + def see(self, index): + """Scroll such that the character at INDEX is visible.""" + self.tk.call(self._w, 'see', index) + + def tag_add(self, tagName, index1, *args): + """Add tag TAGNAME to all characters between INDEX1 and index2 in ARGS. + Additional pairs of indices may follow in ARGS.""" + self.tk.call( + (self._w, 'tag', 'add', tagName, index1) + args) + + def tag_unbind(self, tagName, sequence, funcid=None): + """Unbind for all characters with TAGNAME for event SEQUENCE the + function identified with FUNCID.""" + return self._unbind((self._w, 'tag', 'bind', tagName, sequence), funcid) + + def tag_bind(self, tagName, sequence, func, add=None): + """Bind to all characters with TAGNAME at event SEQUENCE a call to function FUNC. + + An additional boolean parameter ADD specifies whether FUNC will be + called additionally to the other bound function or whether it will + replace the previous function. See bind for the return value.""" + return self._bind((self._w, 'tag', 'bind', tagName), + sequence, func, add) + + def _tag_bind(self, tagName, sequence=None, func=None, add=None): + # For tests only + return self._bind((self._w, 'tag', 'bind', tagName), + sequence, func, add) + + def tag_cget(self, tagName, option): + """Return the value of OPTION for tag TAGNAME.""" + if option[:1] != '-': + option = '-' + option + if option[-1:] == '_': + option = option[:-1] + return self.tk.call(self._w, 'tag', 'cget', tagName, option) + + def tag_configure(self, tagName, cnf=None, **kw): + """Configure a tag TAGNAME.""" + return self._configure(('tag', 'configure', tagName), cnf, kw) + + tag_config = tag_configure + + def tag_delete(self, *tagNames): + """Delete all tags in TAGNAMES.""" + self.tk.call((self._w, 'tag', 'delete') + tagNames) + + def tag_lower(self, tagName, belowThis=None): + """Change the priority of tag TAGNAME such that it is lower + than the priority of BELOWTHIS.""" + self.tk.call(self._w, 'tag', 'lower', tagName, belowThis) + + def tag_names(self, index=None): + """Return a list of all tag names.""" + return self.tk.splitlist( + self.tk.call(self._w, 'tag', 'names', index)) + + def tag_nextrange(self, tagName, index1, index2=None): + """Return a list of start and end index for the first sequence of + characters between INDEX1 and INDEX2 which all have tag TAGNAME. + The text is searched forward from INDEX1.""" + return self.tk.splitlist(self.tk.call( + self._w, 'tag', 'nextrange', tagName, index1, index2)) + + def tag_prevrange(self, tagName, index1, index2=None): + """Return a list of start and end index for the first sequence of + characters between INDEX1 and INDEX2 which all have tag TAGNAME. + The text is searched backwards from INDEX1.""" + return self.tk.splitlist(self.tk.call( + self._w, 'tag', 'prevrange', tagName, index1, index2)) + + def tag_raise(self, tagName, aboveThis=None): + """Change the priority of tag TAGNAME such that it is higher + than the priority of ABOVETHIS.""" + self.tk.call( + self._w, 'tag', 'raise', tagName, aboveThis) + + def tag_ranges(self, tagName): + """Return a list of ranges of text which have tag TAGNAME.""" + return self.tk.splitlist(self.tk.call( + self._w, 'tag', 'ranges', tagName)) + + def tag_remove(self, tagName, index1, index2=None): + """Remove tag TAGNAME from all characters between INDEX1 and INDEX2.""" + self.tk.call( + self._w, 'tag', 'remove', tagName, index1, index2) + + def window_cget(self, index, option): + """Return the value of OPTION of an embedded window at INDEX.""" + if option[:1] != '-': + option = '-' + option + if option[-1:] == '_': + option = option[:-1] + return self.tk.call(self._w, 'window', 'cget', index, option) + + def window_configure(self, index, cnf=None, **kw): + """Configure an embedded window at INDEX.""" + return self._configure(('window', 'configure', index), cnf, kw) + + window_config = window_configure + + def window_create(self, index, cnf={}, **kw): + """Create a window at INDEX.""" + self.tk.call( + (self._w, 'window', 'create', index) + + self._options(cnf, kw)) + + def window_names(self): + """Return all names of embedded windows in this widget.""" + return self.tk.splitlist( + self.tk.call(self._w, 'window', 'names')) + + def yview_pickplace(self, *what): + """Obsolete function, use see.""" + self.tk.call((self._w, 'yview', '-pickplace') + what) + + +class _setit: + """Internal class. It wraps the command in the widget OptionMenu.""" + + def __init__(self, var, value, callback=None): + self.__value = value + self.__var = var + self.__callback = callback + + def __call__(self, *args): + self.__var.set(self.__value) + if self.__callback is not None: + self.__callback(self.__value, *args) + + +class OptionMenu(Menubutton): + """OptionMenu which allows the user to select a value from a menu.""" + + def __init__(self, master, variable, value, *values, **kwargs): + """Construct an optionmenu widget with the parent MASTER, with + the resource textvariable set to VARIABLE, the initially selected + value VALUE, the other menu values VALUES and an additional + keyword argument command.""" + kw = {"borderwidth": 2, "textvariable": variable, + "indicatoron": 1, "relief": RAISED, "anchor": "c", + "highlightthickness": 2} + Widget.__init__(self, master, "menubutton", kw) + self.widgetName = 'tk_optionMenu' + menu = self.__menu = Menu(self, name="menu", tearoff=0) + self.menuname = menu._w + # 'command' is the only supported keyword + callback = kwargs.get('command') + if 'command' in kwargs: + del kwargs['command'] + if kwargs: + raise TclError('unknown option -'+next(iter(kwargs))) + menu.add_command(label=value, + command=_setit(variable, value, callback)) + for v in values: + menu.add_command(label=v, + command=_setit(variable, v, callback)) + self["menu"] = menu + + def __getitem__(self, name): + if name == 'menu': + return self.__menu + return Widget.__getitem__(self, name) + + def destroy(self): + """Destroy this widget and the associated menu.""" + Menubutton.destroy(self) + self.__menu = None + + +class Image: + """Base class for images.""" + _last_id = 0 + + def __init__(self, imgtype, name=None, cnf={}, master=None, **kw): + self.name = None + if master is None: + master = _get_default_root('create image') + self.tk = getattr(master, 'tk', master) + if not name: + Image._last_id += 1 + name = "pyimage%r" % (Image._last_id,) # tk itself would use image + if kw and cnf: cnf = _cnfmerge((cnf, kw)) + elif kw: cnf = kw + options = () + for k, v in cnf.items(): + options = options + ('-'+k, v) + self.tk.call(('image', 'create', imgtype, name,) + options) + self.name = name + + def __str__(self): return self.name + + def __del__(self): + if self.name: + try: + self.tk.call('image', 'delete', self.name) + except TclError: + # May happen if the root was destroyed + pass + + def __setitem__(self, key, value): + self.tk.call(self.name, 'configure', '-'+key, value) + + def __getitem__(self, key): + return self.tk.call(self.name, 'configure', '-'+key) + + def configure(self, **kw): + """Configure the image.""" + res = () + for k, v in _cnfmerge(kw).items(): + if v is not None: + if k[-1] == '_': k = k[:-1] + res = res + ('-'+k, v) + self.tk.call((self.name, 'config') + res) + + config = configure + + def height(self): + """Return the height of the image.""" + return self.tk.getint( + self.tk.call('image', 'height', self.name)) + + def type(self): + """Return the type of the image, e.g. "photo" or "bitmap".""" + return self.tk.call('image', 'type', self.name) + + def width(self): + """Return the width of the image.""" + return self.tk.getint( + self.tk.call('image', 'width', self.name)) + + +class PhotoImage(Image): + """Widget which can display images in PGM, PPM, GIF, PNG format.""" + + def __init__(self, name=None, cnf={}, master=None, **kw): + """Create an image with NAME. + + Valid resource names: data, format, file, gamma, height, palette, + width.""" + Image.__init__(self, 'photo', name, cnf, master, **kw) + + def blank(self): + """Display a transparent image.""" + self.tk.call(self.name, 'blank') + + def cget(self, option): + """Return the value of OPTION.""" + return self.tk.call(self.name, 'cget', '-' + option) + # XXX config + + def __getitem__(self, key): + return self.tk.call(self.name, 'cget', '-' + key) + + def copy(self, *, from_coords=None, zoom=None, subsample=None): + """Return a new PhotoImage with the same image as this widget. + + The FROM_COORDS option specifies a rectangular sub-region of the + source image to be copied. It must be a tuple or a list of 1 to 4 + integers (x1, y1, x2, y2). (x1, y1) and (x2, y2) specify diagonally + opposite corners of the rectangle. If x2 and y2 are not specified, + the default value is the bottom-right corner of the source image. + The pixels copied will include the left and top edges of the + specified rectangle but not the bottom or right edges. If the + FROM_COORDS option is not given, the default is the whole source + image. + + If SUBSAMPLE or ZOOM are specified, the image is transformed as in + the subsample() or zoom() methods. The value must be a single + integer or a pair of integers. + """ + destImage = PhotoImage(master=self.tk) + destImage.copy_replace(self, from_coords=from_coords, + zoom=zoom, subsample=subsample) + return destImage + + def zoom(self, x, y='', *, from_coords=None): + """Return a new PhotoImage with the same image as this widget + but zoom it with a factor of X in the X direction and Y in the Y + direction. If Y is not given, the default value is the same as X. + + The FROM_COORDS option specifies a rectangular sub-region of the + source image to be copied, as in the copy() method. + """ + if y=='': y=x + return self.copy(zoom=(x, y), from_coords=from_coords) + + def subsample(self, x, y='', *, from_coords=None): + """Return a new PhotoImage based on the same image as this widget + but use only every Xth or Yth pixel. If Y is not given, the + default value is the same as X. + + The FROM_COORDS option specifies a rectangular sub-region of the + source image to be copied, as in the copy() method. + """ + if y=='': y=x + return self.copy(subsample=(x, y), from_coords=from_coords) + + def copy_replace(self, sourceImage, *, from_coords=None, to=None, shrink=False, + zoom=None, subsample=None, compositingrule=None): + """Copy a region from the source image (which must be a PhotoImage) to + this image, possibly with pixel zooming and/or subsampling. If no + options are specified, this command copies the whole of the source + image into this image, starting at coordinates (0, 0). + + The FROM_COORDS option specifies a rectangular sub-region of the + source image to be copied. It must be a tuple or a list of 1 to 4 + integers (x1, y1, x2, y2). (x1, y1) and (x2, y2) specify diagonally + opposite corners of the rectangle. If x2 and y2 are not specified, + the default value is the bottom-right corner of the source image. + The pixels copied will include the left and top edges of the + specified rectangle but not the bottom or right edges. If the + FROM_COORDS option is not given, the default is the whole source + image. + + The TO option specifies a rectangular sub-region of the destination + image to be affected. It must be a tuple or a list of 1 to 4 + integers (x1, y1, x2, y2). (x1, y1) and (x2, y2) specify diagonally + opposite corners of the rectangle. If x2 and y2 are not specified, + the default value is (x1,y1) plus the size of the source region + (after subsampling and zooming, if specified). If x2 and y2 are + specified, the source region will be replicated if necessary to fill + the destination region in a tiled fashion. + + If SHRINK is true, the size of the destination image should be + reduced, if necessary, so that the region being copied into is at + the bottom-right corner of the image. + + If SUBSAMPLE or ZOOM are specified, the image is transformed as in + the subsample() or zoom() methods. The value must be a single + integer or a pair of integers. + + The COMPOSITINGRULE option specifies how transparent pixels in the + source image are combined with the destination image. When a + compositing rule of 'overlay' is set, the old contents of the + destination image are visible, as if the source image were printed + on a piece of transparent film and placed over the top of the + destination. When a compositing rule of 'set' is set, the old + contents of the destination image are discarded and the source image + is used as-is. The default compositing rule is 'overlay'. + """ + options = [] + if from_coords is not None: + options.extend(('-from', *from_coords)) + if to is not None: + options.extend(('-to', *to)) + if shrink: + options.append('-shrink') + if zoom is not None: + if not isinstance(zoom, (tuple, list)): + zoom = (zoom,) + options.extend(('-zoom', *zoom)) + if subsample is not None: + if not isinstance(subsample, (tuple, list)): + subsample = (subsample,) + options.extend(('-subsample', *subsample)) + if compositingrule: + options.extend(('-compositingrule', compositingrule)) + self.tk.call(self.name, 'copy', sourceImage, *options) + + def get(self, x, y): + """Return the color (red, green, blue) of the pixel at X,Y.""" + return self.tk.call(self.name, 'get', x, y) + + def put(self, data, to=None): + """Put row formatted colors to image starting from + position TO, e.g. image.put("{red green} {blue yellow}", to=(4,6))""" + args = (self.name, 'put', data) + if to: + if to[0] == '-to': + to = to[1:] + args = args + ('-to',) + tuple(to) + self.tk.call(args) + + def read(self, filename, format=None, *, from_coords=None, to=None, shrink=False): + """Reads image data from the file named FILENAME into the image. + + The FORMAT option specifies the format of the image data in the + file. + + The FROM_COORDS option specifies a rectangular sub-region of the image + file data to be copied to the destination image. It must be a tuple + or a list of 1 to 4 integers (x1, y1, x2, y2). (x1, y1) and + (x2, y2) specify diagonally opposite corners of the rectangle. If + x2 and y2 are not specified, the default value is the bottom-right + corner of the source image. The default, if this option is not + specified, is the whole of the image in the image file. + + The TO option specifies the coordinates of the top-left corner of + the region of the image into which data from filename are to be + read. The default is (0, 0). + + If SHRINK is true, the size of the destination image will be + reduced, if necessary, so that the region into which the image file + data are read is at the bottom-right corner of the image. + """ + options = () + if format is not None: + options += ('-format', format) + if from_coords is not None: + options += ('-from', *from_coords) + if shrink: + options += ('-shrink',) + if to is not None: + options += ('-to', *to) + self.tk.call(self.name, 'read', filename, *options) + + def write(self, filename, format=None, from_coords=None, *, + background=None, grayscale=False): + """Writes image data from the image to a file named FILENAME. + + The FORMAT option specifies the name of the image file format + handler to be used to write the data to the file. If this option + is not given, the format is guessed from the file extension. + + The FROM_COORDS option specifies a rectangular region of the image + to be written to the image file. It must be a tuple or a list of 1 + to 4 integers (x1, y1, x2, y2). If only x1 and y1 are specified, + the region extends from (x1,y1) to the bottom-right corner of the + image. If all four coordinates are given, they specify diagonally + opposite corners of the rectangular region. The default, if this + option is not given, is the whole image. + + If BACKGROUND is specified, the data will not contain any + transparency information. In all transparent pixels the color will + be replaced by the specified color. + + If GRAYSCALE is true, the data will not contain color information. + All pixel data will be transformed into grayscale. + """ + options = () + if format is not None: + options += ('-format', format) + if from_coords is not None: + options += ('-from', *from_coords) + if grayscale: + options += ('-grayscale',) + if background is not None: + options += ('-background', background) + self.tk.call(self.name, 'write', filename, *options) + + def data(self, format=None, *, from_coords=None, + background=None, grayscale=False): + """Returns image data. + + The FORMAT option specifies the name of the image file format + handler to be used. If this option is not given, this method uses + a format that consists of a tuple (one element per row) of strings + containings space separated (one element per pixel/column) colors + in “#RRGGBB” format (where RR is a pair of hexadecimal digits for + the red channel, GG for green, and BB for blue). + + The FROM_COORDS option specifies a rectangular region of the image + to be returned. It must be a tuple or a list of 1 to 4 integers + (x1, y1, x2, y2). If only x1 and y1 are specified, the region + extends from (x1,y1) to the bottom-right corner of the image. If + all four coordinates are given, they specify diagonally opposite + corners of the rectangular region, including (x1, y1) and excluding + (x2, y2). The default, if this option is not given, is the whole + image. + + If BACKGROUND is specified, the data will not contain any + transparency information. In all transparent pixels the color will + be replaced by the specified color. + + If GRAYSCALE is true, the data will not contain color information. + All pixel data will be transformed into grayscale. + """ + options = () + if format is not None: + options += ('-format', format) + if from_coords is not None: + options += ('-from', *from_coords) + if grayscale: + options += ('-grayscale',) + if background is not None: + options += ('-background', background) + data = self.tk.call(self.name, 'data', *options) + if isinstance(data, str): # For wantobjects = 0. + if format is None: + data = self.tk.splitlist(data) + else: + data = bytes(data, 'latin1') + return data + + def transparency_get(self, x, y): + """Return True if the pixel at x,y is transparent.""" + return self.tk.getboolean(self.tk.call( + self.name, 'transparency', 'get', x, y)) + + def transparency_set(self, x, y, boolean): + """Set the transparency of the pixel at x,y.""" + self.tk.call(self.name, 'transparency', 'set', x, y, boolean) + + +class BitmapImage(Image): + """Widget which can display images in XBM format.""" + + def __init__(self, name=None, cnf={}, master=None, **kw): + """Create a bitmap with NAME. + + Valid resource names: background, data, file, foreground, maskdata, maskfile.""" + Image.__init__(self, 'bitmap', name, cnf, master, **kw) + + +def image_names(): + tk = _get_default_root('use image_names()').tk + return tk.splitlist(tk.call('image', 'names')) + + +def image_types(): + tk = _get_default_root('use image_types()').tk + return tk.splitlist(tk.call('image', 'types')) + + +class Spinbox(Widget, XView): + """spinbox widget.""" + + def __init__(self, master=None, cnf={}, **kw): + """Construct a spinbox widget with the parent MASTER. + + STANDARD OPTIONS + + activebackground, background, borderwidth, + cursor, exportselection, font, foreground, + highlightbackground, highlightcolor, + highlightthickness, insertbackground, + insertborderwidth, insertofftime, + insertontime, insertwidth, justify, relief, + repeatdelay, repeatinterval, + selectbackground, selectborderwidth + selectforeground, takefocus, textvariable + xscrollcommand. + + WIDGET-SPECIFIC OPTIONS + + buttonbackground, buttoncursor, + buttondownrelief, buttonuprelief, + command, disabledbackground, + disabledforeground, format, from, + invalidcommand, increment, + readonlybackground, state, to, + validate, validatecommand values, + width, wrap, + """ + Widget.__init__(self, master, 'spinbox', cnf, kw) + + def bbox(self, index): + """Return a tuple of X1,Y1,X2,Y2 coordinates for a + rectangle which encloses the character given by index. + + The first two elements of the list give the x and y + coordinates of the upper-left corner of the screen + area covered by the character (in pixels relative + to the widget) and the last two elements give the + width and height of the character, in pixels. The + bounding box may refer to a region outside the + visible area of the window. + """ + return self._getints(self.tk.call(self._w, 'bbox', index)) or None + + def delete(self, first, last=None): + """Delete one or more elements of the spinbox. + + First is the index of the first character to delete, + and last is the index of the character just after + the last one to delete. If last isn't specified it + defaults to first+1, i.e. a single character is + deleted. This command returns an empty string. + """ + return self.tk.call(self._w, 'delete', first, last) + + def get(self): + """Returns the spinbox's string""" + return self.tk.call(self._w, 'get') + + def icursor(self, index): + """Alter the position of the insertion cursor. + + The insertion cursor will be displayed just before + the character given by index. Returns an empty string + """ + return self.tk.call(self._w, 'icursor', index) + + def identify(self, x, y): + """Returns the name of the widget at position x, y + + Return value is one of: none, buttondown, buttonup, entry + """ + return self.tk.call(self._w, 'identify', x, y) + + def index(self, index): + """Returns the numerical index corresponding to index + """ + return self.tk.call(self._w, 'index', index) + + def insert(self, index, s): + """Insert string s at index + + Returns an empty string. + """ + return self.tk.call(self._w, 'insert', index, s) + + def invoke(self, element): + """Causes the specified element to be invoked + + The element could be buttondown or buttonup + triggering the action associated with it. + """ + return self.tk.call(self._w, 'invoke', element) + + def scan(self, *args): + """Internal function.""" + return self._getints( + self.tk.call((self._w, 'scan') + args)) or () + + def scan_mark(self, x): + """Records x and the current view in the spinbox window; + + used in conjunction with later scan dragto commands. + Typically this command is associated with a mouse button + press in the widget. It returns an empty string. + """ + return self.scan("mark", x) + + def scan_dragto(self, x): + """Compute the difference between the given x argument + and the x argument to the last scan mark command + + It then adjusts the view left or right by 10 times the + difference in x-coordinates. This command is typically + associated with mouse motion events in the widget, to + produce the effect of dragging the spinbox at high speed + through the window. The return value is an empty string. + """ + return self.scan("dragto", x) + + def selection(self, *args): + """Internal function.""" + return self._getints( + self.tk.call((self._w, 'selection') + args)) or () + + def selection_adjust(self, index): + """Locate the end of the selection nearest to the character + given by index, + + Then adjust that end of the selection to be at index + (i.e including but not going beyond index). The other + end of the selection is made the anchor point for future + select to commands. If the selection isn't currently in + the spinbox, then a new selection is created to include + the characters between index and the most recent selection + anchor point, inclusive. + """ + return self.selection("adjust", index) + + def selection_clear(self): + """Clear the selection + + If the selection isn't in this widget then the + command has no effect. + """ + return self.selection("clear") + + def selection_element(self, element=None): + """Sets or gets the currently selected element. + + If a spinbutton element is specified, it will be + displayed depressed. + """ + return self.tk.call(self._w, 'selection', 'element', element) + + def selection_from(self, index): + """Set the fixed end of a selection to INDEX.""" + self.selection('from', index) + + def selection_present(self): + """Return True if there are characters selected in the spinbox, False + otherwise.""" + return self.tk.getboolean( + self.tk.call(self._w, 'selection', 'present')) + + def selection_range(self, start, end): + """Set the selection from START to END (not included).""" + self.selection('range', start, end) + + def selection_to(self, index): + """Set the variable end of a selection to INDEX.""" + self.selection('to', index) + +########################################################################### + + +class LabelFrame(Widget): + """labelframe widget.""" + + def __init__(self, master=None, cnf={}, **kw): + """Construct a labelframe widget with the parent MASTER. + + STANDARD OPTIONS + + borderwidth, cursor, font, foreground, + highlightbackground, highlightcolor, + highlightthickness, padx, pady, relief, + takefocus, text + + WIDGET-SPECIFIC OPTIONS + + background, class, colormap, container, + height, labelanchor, labelwidget, + visual, width + """ + Widget.__init__(self, master, 'labelframe', cnf, kw) + +######################################################################## + + +class PanedWindow(Widget): + """panedwindow widget.""" + + def __init__(self, master=None, cnf={}, **kw): + """Construct a panedwindow widget with the parent MASTER. + + STANDARD OPTIONS + + background, borderwidth, cursor, height, + orient, relief, width + + WIDGET-SPECIFIC OPTIONS + + handlepad, handlesize, opaqueresize, + sashcursor, sashpad, sashrelief, + sashwidth, showhandle, + """ + Widget.__init__(self, master, 'panedwindow', cnf, kw) + + def add(self, child, **kw): + """Add a child widget to the panedwindow in a new pane. + + The child argument is the name of the child widget + followed by pairs of arguments that specify how to + manage the windows. The possible options and values + are the ones accepted by the paneconfigure method. + """ + self.tk.call((self._w, 'add', child) + self._options(kw)) + + def remove(self, child): + """Remove the pane containing child from the panedwindow + + All geometry management options for child will be forgotten. + """ + self.tk.call(self._w, 'forget', child) + + forget = remove + + def identify(self, x, y): + """Identify the panedwindow component at point x, y + + If the point is over a sash or a sash handle, the result + is a two element list containing the index of the sash or + handle, and a word indicating whether it is over a sash + or a handle, such as {0 sash} or {2 handle}. If the point + is over any other part of the panedwindow, the result is + an empty list. + """ + return self.tk.call(self._w, 'identify', x, y) + + def proxy(self, *args): + """Internal function.""" + return self._getints( + self.tk.call((self._w, 'proxy') + args)) or () + + def proxy_coord(self): + """Return the x and y pair of the most recent proxy location + """ + return self.proxy("coord") + + def proxy_forget(self): + """Remove the proxy from the display. + """ + return self.proxy("forget") + + def proxy_place(self, x, y): + """Place the proxy at the given x and y coordinates. + """ + return self.proxy("place", x, y) + + def sash(self, *args): + """Internal function.""" + return self._getints( + self.tk.call((self._w, 'sash') + args)) or () + + def sash_coord(self, index): + """Return the current x and y pair for the sash given by index. + + Index must be an integer between 0 and 1 less than the + number of panes in the panedwindow. The coordinates given are + those of the top left corner of the region containing the sash. + pathName sash dragto index x y This command computes the + difference between the given coordinates and the coordinates + given to the last sash coord command for the given sash. It then + moves that sash the computed difference. The return value is the + empty string. + """ + return self.sash("coord", index) + + def sash_mark(self, index): + """Records x and y for the sash given by index; + + Used in conjunction with later dragto commands to move the sash. + """ + return self.sash("mark", index) + + def sash_place(self, index, x, y): + """Place the sash given by index at the given coordinates + """ + return self.sash("place", index, x, y) + + def panecget(self, child, option): + """Query a management option for window. + + Option may be any value allowed by the paneconfigure subcommand + """ + return self.tk.call( + (self._w, 'panecget') + (child, '-'+option)) + + def paneconfigure(self, tagOrId, cnf=None, **kw): + """Query or modify the management options for window. + + If no option is specified, returns a list describing all + of the available options for pathName. If option is + specified with no value, then the command returns a list + describing the one named option (this list will be identical + to the corresponding sublist of the value returned if no + option is specified). If one or more option-value pairs are + specified, then the command modifies the given widget + option(s) to have the given value(s); in this case the + command returns an empty string. The following options + are supported: + + after window + Insert the window after the window specified. window + should be the name of a window already managed by pathName. + before window + Insert the window before the window specified. window + should be the name of a window already managed by pathName. + height size + Specify a height for the window. The height will be the + outer dimension of the window including its border, if + any. If size is an empty string, or if -height is not + specified, then the height requested internally by the + window will be used initially; the height may later be + adjusted by the movement of sashes in the panedwindow. + Size may be any value accepted by Tk_GetPixels. + minsize n + Specifies that the size of the window cannot be made + less than n. This constraint only affects the size of + the widget in the paned dimension -- the x dimension + for horizontal panedwindows, the y dimension for + vertical panedwindows. May be any value accepted by + Tk_GetPixels. + padx n + Specifies a non-negative value indicating how much + extra space to leave on each side of the window in + the X-direction. The value may have any of the forms + accepted by Tk_GetPixels. + pady n + Specifies a non-negative value indicating how much + extra space to leave on each side of the window in + the Y-direction. The value may have any of the forms + accepted by Tk_GetPixels. + sticky style + If a window's pane is larger than the requested + dimensions of the window, this option may be used + to position (or stretch) the window within its pane. + Style is a string that contains zero or more of the + characters n, s, e or w. The string can optionally + contains spaces or commas, but they are ignored. Each + letter refers to a side (north, south, east, or west) + that the window will "stick" to. If both n and s + (or e and w) are specified, the window will be + stretched to fill the entire height (or width) of + its cavity. + width size + Specify a width for the window. The width will be + the outer dimension of the window including its + border, if any. If size is an empty string, or + if -width is not specified, then the width requested + internally by the window will be used initially; the + width may later be adjusted by the movement of sashes + in the panedwindow. Size may be any value accepted by + Tk_GetPixels. + + """ + if cnf is None and not kw: + return self._getconfigure(self._w, 'paneconfigure', tagOrId) + if isinstance(cnf, str) and not kw: + return self._getconfigure1( + self._w, 'paneconfigure', tagOrId, '-'+cnf) + self.tk.call((self._w, 'paneconfigure', tagOrId) + + self._options(cnf, kw)) + + paneconfig = paneconfigure + + def panes(self): + """Returns an ordered list of the child panes.""" + return self.tk.splitlist(self.tk.call(self._w, 'panes')) + +# Test: + + +def _test(): + root = Tk() + text = "This is Tcl/Tk %s" % root.globalgetvar('tk_patchLevel') + text += "\nThis should be a cedilla: \xe7" + label = Label(root, text=text) + label.pack() + test = Button(root, text="Click me!", + command=lambda root=root: root.test.configure( + text="[%s]" % root.test['text'])) + test.pack() + root.test = test + quit = Button(root, text="QUIT", command=root.destroy) + quit.pack() + # The following three commands are needed so the window pops + # up on top on Windows... + root.iconify() + root.update() + root.deiconify() + root.mainloop() + + +__all__ = [name for name, obj in globals().items() + if not name.startswith('_') and not isinstance(obj, types.ModuleType) + and name not in {'wantobjects'}] + +if __name__ == '__main__': + _test() diff --git a/Lib/tkinter/__main__.py b/Lib/tkinter/__main__.py new file mode 100644 index 0000000000..757880d439 --- /dev/null +++ b/Lib/tkinter/__main__.py @@ -0,0 +1,7 @@ +"""Main entry point""" + +import sys +if sys.argv[0].endswith("__main__.py"): + sys.argv[0] = "python -m tkinter" +from . import _test as main +main() diff --git a/Lib/tkinter/colorchooser.py b/Lib/tkinter/colorchooser.py new file mode 100644 index 0000000000..e2fb69dba9 --- /dev/null +++ b/Lib/tkinter/colorchooser.py @@ -0,0 +1,86 @@ +# tk common color chooser dialogue +# +# this module provides an interface to the native color dialogue +# available in Tk 4.2 and newer. +# +# written by Fredrik Lundh, May 1997 +# +# fixed initialcolor handling in August 1998 +# + + +from tkinter.commondialog import Dialog + +__all__ = ["Chooser", "askcolor"] + + +class Chooser(Dialog): + """Create a dialog for the tk_chooseColor command. + + Args: + master: The master widget for this dialog. If not provided, + defaults to options['parent'] (if defined). + options: Dictionary of options for the tk_chooseColor call. + initialcolor: Specifies the selected color when the + dialog is first displayed. This can be a tk color + string or a 3-tuple of ints in the range (0, 255) + for an RGB triplet. + parent: The parent window of the color dialog. The + color dialog is displayed on top of this. + title: A string for the title of the dialog box. + """ + + command = "tk_chooseColor" + + def _fixoptions(self): + """Ensure initialcolor is a tk color string. + + Convert initialcolor from a RGB triplet to a color string. + """ + try: + color = self.options["initialcolor"] + if isinstance(color, tuple): + # Assume an RGB triplet. + self.options["initialcolor"] = "#%02x%02x%02x" % color + except KeyError: + pass + + def _fixresult(self, widget, result): + """Adjust result returned from call to tk_chooseColor. + + Return both an RGB tuple of ints in the range (0, 255) and the + tk color string in the form #rrggbb. + """ + # Result can be many things: an empty tuple, an empty string, or + # a _tkinter.Tcl_Obj, so this somewhat weird check handles that. + if not result or not str(result): + return None, None # canceled + + # To simplify application code, the color chooser returns + # an RGB tuple together with the Tk color string. + r, g, b = widget.winfo_rgb(result) + return (r//256, g//256, b//256), str(result) + + +# +# convenience stuff + +def askcolor(color=None, **options): + """Display dialog window for selection of a color. + + Convenience wrapper for the Chooser class. Displays the color + chooser dialog with color as the initial value. + """ + + if color: + options = options.copy() + options["initialcolor"] = color + + return Chooser(**options).show() + + +# -------------------------------------------------------------------- +# test stuff + +if __name__ == "__main__": + print("color", askcolor()) diff --git a/Lib/tkinter/commondialog.py b/Lib/tkinter/commondialog.py new file mode 100644 index 0000000000..86f5387e00 --- /dev/null +++ b/Lib/tkinter/commondialog.py @@ -0,0 +1,53 @@ +# base class for tk common dialogues +# +# this module provides a base class for accessing the common +# dialogues available in Tk 4.2 and newer. use filedialog, +# colorchooser, and messagebox to access the individual +# dialogs. +# +# written by Fredrik Lundh, May 1997 +# + +__all__ = ["Dialog"] + +from tkinter import _get_temp_root, _destroy_temp_root + + +class Dialog: + + command = None + + def __init__(self, master=None, **options): + if master is None: + master = options.get('parent') + self.master = master + self.options = options + + def _fixoptions(self): + pass # hook + + def _fixresult(self, widget, result): + return result # hook + + def show(self, **options): + + # update instance options + for k, v in options.items(): + self.options[k] = v + + self._fixoptions() + + master = self.master + if master is None: + master = _get_temp_root() + try: + self._test_callback(master) # The function below is replaced for some tests. + s = master.tk.call(self.command, *master._options(self.options)) + s = self._fixresult(master, s) + finally: + _destroy_temp_root(master) + + return s + + def _test_callback(self, master): + pass diff --git a/Lib/tkinter/constants.py b/Lib/tkinter/constants.py new file mode 100644 index 0000000000..63eee33d24 --- /dev/null +++ b/Lib/tkinter/constants.py @@ -0,0 +1,110 @@ +# Symbolic constants for Tk + +# Booleans +NO=FALSE=OFF=0 +YES=TRUE=ON=1 + +# -anchor and -sticky +N='n' +S='s' +W='w' +E='e' +NW='nw' +SW='sw' +NE='ne' +SE='se' +NS='ns' +EW='ew' +NSEW='nsew' +CENTER='center' + +# -fill +NONE='none' +X='x' +Y='y' +BOTH='both' + +# -side +LEFT='left' +TOP='top' +RIGHT='right' +BOTTOM='bottom' + +# -relief +RAISED='raised' +SUNKEN='sunken' +FLAT='flat' +RIDGE='ridge' +GROOVE='groove' +SOLID = 'solid' + +# -orient +HORIZONTAL='horizontal' +VERTICAL='vertical' + +# -tabs +NUMERIC='numeric' + +# -wrap +CHAR='char' +WORD='word' + +# -align +BASELINE='baseline' + +# -bordermode +INSIDE='inside' +OUTSIDE='outside' + +# Special tags, marks and insert positions +SEL='sel' +SEL_FIRST='sel.first' +SEL_LAST='sel.last' +END='end' +INSERT='insert' +CURRENT='current' +ANCHOR='anchor' +ALL='all' # e.g. Canvas.delete(ALL) + +# Text widget and button states +NORMAL='normal' +DISABLED='disabled' +ACTIVE='active' +# Canvas state +HIDDEN='hidden' + +# Menu item types +CASCADE='cascade' +CHECKBUTTON='checkbutton' +COMMAND='command' +RADIOBUTTON='radiobutton' +SEPARATOR='separator' + +# Selection modes for list boxes +SINGLE='single' +BROWSE='browse' +MULTIPLE='multiple' +EXTENDED='extended' + +# Activestyle for list boxes +# NONE='none' is also valid +DOTBOX='dotbox' +UNDERLINE='underline' + +# Various canvas styles +PIESLICE='pieslice' +CHORD='chord' +ARC='arc' +FIRST='first' +LAST='last' +BUTT='butt' +PROJECTING='projecting' +ROUND='round' +BEVEL='bevel' +MITER='miter' + +# Arguments to xview/yview +MOVETO='moveto' +SCROLL='scroll' +UNITS='units' +PAGES='pages' diff --git a/Lib/tkinter/dialog.py b/Lib/tkinter/dialog.py new file mode 100644 index 0000000000..36ae6c277c --- /dev/null +++ b/Lib/tkinter/dialog.py @@ -0,0 +1,49 @@ +# dialog.py -- Tkinter interface to the tk_dialog script. + +from tkinter import _cnfmerge, Widget, TclError, Button, Pack + +__all__ = ["Dialog"] + +DIALOG_ICON = 'questhead' + + +class Dialog(Widget): + def __init__(self, master=None, cnf={}, **kw): + cnf = _cnfmerge((cnf, kw)) + self.widgetName = '__dialog__' + self._setup(master, cnf) + self.num = self.tk.getint( + self.tk.call( + 'tk_dialog', self._w, + cnf['title'], cnf['text'], + cnf['bitmap'], cnf['default'], + *cnf['strings'])) + try: Widget.destroy(self) + except TclError: pass + + def destroy(self): pass + + +def _test(): + d = Dialog(None, {'title': 'File Modified', + 'text': + 'File "Python.h" has been modified' + ' since the last time it was saved.' + ' Do you want to save it before' + ' exiting the application.', + 'bitmap': DIALOG_ICON, + 'default': 0, + 'strings': ('Save File', + 'Discard Changes', + 'Return to Editor')}) + print(d.num) + + +if __name__ == '__main__': + t = Button(None, {'text': 'Test', + 'command': _test, + Pack: {}}) + q = Button(None, {'text': 'Quit', + 'command': t.quit, + Pack: {}}) + t.mainloop() diff --git a/Lib/tkinter/dnd.py b/Lib/tkinter/dnd.py new file mode 100644 index 0000000000..acec61ba71 --- /dev/null +++ b/Lib/tkinter/dnd.py @@ -0,0 +1,324 @@ +"""Drag-and-drop support for Tkinter. + +This is very preliminary. I currently only support dnd *within* one +application, between different windows (or within the same window). + +I am trying to make this as generic as possible -- not dependent on +the use of a particular widget or icon type, etc. I also hope that +this will work with Pmw. + +To enable an object to be dragged, you must create an event binding +for it that starts the drag-and-drop process. Typically, you should +bind to a callback function that you write. The function +should call Tkdnd.dnd_start(source, event), where 'source' is the +object to be dragged, and 'event' is the event that invoked the call +(the argument to your callback function). Even though this is a class +instantiation, the returned instance should not be stored -- it will +be kept alive automatically for the duration of the drag-and-drop. + +When a drag-and-drop is already in process for the Tk interpreter, the +call is *ignored*; this normally averts starting multiple simultaneous +dnd processes, e.g. because different button callbacks all +dnd_start(). + +The object is *not* necessarily a widget -- it can be any +application-specific object that is meaningful to potential +drag-and-drop targets. + +Potential drag-and-drop targets are discovered as follows. Whenever +the mouse moves, and at the start and end of a drag-and-drop move, the +Tk widget directly under the mouse is inspected. This is the target +widget (not to be confused with the target object, yet to be +determined). If there is no target widget, there is no dnd target +object. If there is a target widget, and it has an attribute +dnd_accept, this should be a function (or any callable object). The +function is called as dnd_accept(source, event), where 'source' is the +object being dragged (the object passed to dnd_start() above), and +'event' is the most recent event object (generally a event; +it can also be or ). If the dnd_accept() +function returns something other than None, this is the new dnd target +object. If dnd_accept() returns None, or if the target widget has no +dnd_accept attribute, the target widget's parent is considered as the +target widget, and the search for a target object is repeated from +there. If necessary, the search is repeated all the way up to the +root widget. If none of the target widgets can produce a target +object, there is no target object (the target object is None). + +The target object thus produced, if any, is called the new target +object. It is compared with the old target object (or None, if there +was no old target widget). There are several cases ('source' is the +source object, and 'event' is the most recent event object): + +- Both the old and new target objects are None. Nothing happens. + +- The old and new target objects are the same object. Its method +dnd_motion(source, event) is called. + +- The old target object was None, and the new target object is not +None. The new target object's method dnd_enter(source, event) is +called. + +- The new target object is None, and the old target object is not +None. The old target object's method dnd_leave(source, event) is +called. + +- The old and new target objects differ and neither is None. The old +target object's method dnd_leave(source, event), and then the new +target object's method dnd_enter(source, event) is called. + +Once this is done, the new target object replaces the old one, and the +Tk mainloop proceeds. The return value of the methods mentioned above +is ignored; if they raise an exception, the normal exception handling +mechanisms take over. + +The drag-and-drop processes can end in two ways: a final target object +is selected, or no final target object is selected. When a final +target object is selected, it will always have been notified of the +potential drop by a call to its dnd_enter() method, as described +above, and possibly one or more calls to its dnd_motion() method; its +dnd_leave() method has not been called since the last call to +dnd_enter(). The target is notified of the drop by a call to its +method dnd_commit(source, event). + +If no final target object is selected, and there was an old target +object, its dnd_leave(source, event) method is called to complete the +dnd sequence. + +Finally, the source object is notified that the drag-and-drop process +is over, by a call to source.dnd_end(target, event), specifying either +the selected target object, or None if no target object was selected. +The source object can use this to implement the commit action; this is +sometimes simpler than to do it in the target's dnd_commit(). The +target's dnd_commit() method could then simply be aliased to +dnd_leave(). + +At any time during a dnd sequence, the application can cancel the +sequence by calling the cancel() method on the object returned by +dnd_start(). This will call dnd_leave() if a target is currently +active; it will never call dnd_commit(). + +""" + +import tkinter + +__all__ = ["dnd_start", "DndHandler"] + + +# The factory function + +def dnd_start(source, event): + h = DndHandler(source, event) + if h.root is not None: + return h + else: + return None + + +# The class that does the work + +class DndHandler: + + root = None + + def __init__(self, source, event): + if event.num > 5: + return + root = event.widget._root() + try: + root.__dnd + return # Don't start recursive dnd + except AttributeError: + root.__dnd = self + self.root = root + self.source = source + self.target = None + self.initial_button = button = event.num + self.initial_widget = widget = event.widget + self.release_pattern = "" % (button, button) + self.save_cursor = widget['cursor'] or "" + widget.bind(self.release_pattern, self.on_release) + widget.bind("", self.on_motion) + widget['cursor'] = "hand2" + + def __del__(self): + root = self.root + self.root = None + if root is not None: + try: + del root.__dnd + except AttributeError: + pass + + def on_motion(self, event): + x, y = event.x_root, event.y_root + target_widget = self.initial_widget.winfo_containing(x, y) + source = self.source + new_target = None + while target_widget is not None: + try: + attr = target_widget.dnd_accept + except AttributeError: + pass + else: + new_target = attr(source, event) + if new_target is not None: + break + target_widget = target_widget.master + old_target = self.target + if old_target is new_target: + if old_target is not None: + old_target.dnd_motion(source, event) + else: + if old_target is not None: + self.target = None + old_target.dnd_leave(source, event) + if new_target is not None: + new_target.dnd_enter(source, event) + self.target = new_target + + def on_release(self, event): + self.finish(event, 1) + + def cancel(self, event=None): + self.finish(event, 0) + + def finish(self, event, commit=0): + target = self.target + source = self.source + widget = self.initial_widget + root = self.root + try: + del root.__dnd + self.initial_widget.unbind(self.release_pattern) + self.initial_widget.unbind("") + widget['cursor'] = self.save_cursor + self.target = self.source = self.initial_widget = self.root = None + if target is not None: + if commit: + target.dnd_commit(source, event) + else: + target.dnd_leave(source, event) + finally: + source.dnd_end(target, event) + + +# ---------------------------------------------------------------------- +# The rest is here for testing and demonstration purposes only! + +class Icon: + + def __init__(self, name): + self.name = name + self.canvas = self.label = self.id = None + + def attach(self, canvas, x=10, y=10): + if canvas is self.canvas: + self.canvas.coords(self.id, x, y) + return + if self.canvas is not None: + self.detach() + if canvas is None: + return + label = tkinter.Label(canvas, text=self.name, + borderwidth=2, relief="raised") + id = canvas.create_window(x, y, window=label, anchor="nw") + self.canvas = canvas + self.label = label + self.id = id + label.bind("", self.press) + + def detach(self): + canvas = self.canvas + if canvas is None: + return + id = self.id + label = self.label + self.canvas = self.label = self.id = None + canvas.delete(id) + label.destroy() + + def press(self, event): + if dnd_start(self, event): + # where the pointer is relative to the label widget: + self.x_off = event.x + self.y_off = event.y + # where the widget is relative to the canvas: + self.x_orig, self.y_orig = self.canvas.coords(self.id) + + def move(self, event): + x, y = self.where(self.canvas, event) + self.canvas.coords(self.id, x, y) + + def putback(self): + self.canvas.coords(self.id, self.x_orig, self.y_orig) + + def where(self, canvas, event): + # where the corner of the canvas is relative to the screen: + x_org = canvas.winfo_rootx() + y_org = canvas.winfo_rooty() + # where the pointer is relative to the canvas widget: + x = event.x_root - x_org + y = event.y_root - y_org + # compensate for initial pointer offset + return x - self.x_off, y - self.y_off + + def dnd_end(self, target, event): + pass + + +class Tester: + + def __init__(self, root): + self.top = tkinter.Toplevel(root) + self.canvas = tkinter.Canvas(self.top, width=100, height=100) + self.canvas.pack(fill="both", expand=1) + self.canvas.dnd_accept = self.dnd_accept + + def dnd_accept(self, source, event): + return self + + def dnd_enter(self, source, event): + self.canvas.focus_set() # Show highlight border + x, y = source.where(self.canvas, event) + x1, y1, x2, y2 = source.canvas.bbox(source.id) + dx, dy = x2-x1, y2-y1 + self.dndid = self.canvas.create_rectangle(x, y, x+dx, y+dy) + self.dnd_motion(source, event) + + def dnd_motion(self, source, event): + x, y = source.where(self.canvas, event) + x1, y1, x2, y2 = self.canvas.bbox(self.dndid) + self.canvas.move(self.dndid, x-x1, y-y1) + + def dnd_leave(self, source, event): + self.top.focus_set() # Hide highlight border + self.canvas.delete(self.dndid) + self.dndid = None + + def dnd_commit(self, source, event): + self.dnd_leave(source, event) + x, y = source.where(self.canvas, event) + source.attach(self.canvas, x, y) + + +def test(): + root = tkinter.Tk() + root.geometry("+1+1") + tkinter.Button(command=root.quit, text="Quit").pack() + t1 = Tester(root) + t1.top.geometry("+1+60") + t2 = Tester(root) + t2.top.geometry("+120+60") + t3 = Tester(root) + t3.top.geometry("+240+60") + i1 = Icon("ICON1") + i2 = Icon("ICON2") + i3 = Icon("ICON3") + i1.attach(t1.canvas) + i2.attach(t2.canvas) + i3.attach(t3.canvas) + root.mainloop() + + +if __name__ == '__main__': + test() diff --git a/Lib/tkinter/filedialog.py b/Lib/tkinter/filedialog.py new file mode 100644 index 0000000000..e2eff98e60 --- /dev/null +++ b/Lib/tkinter/filedialog.py @@ -0,0 +1,492 @@ +"""File selection dialog classes. + +Classes: + +- FileDialog +- LoadFileDialog +- SaveFileDialog + +This module also presents tk common file dialogues, it provides interfaces +to the native file dialogues available in Tk 4.2 and newer, and the +directory dialogue available in Tk 8.3 and newer. +These interfaces were written by Fredrik Lundh, May 1997. +""" +__all__ = ["FileDialog", "LoadFileDialog", "SaveFileDialog", + "Open", "SaveAs", "Directory", + "askopenfilename", "asksaveasfilename", "askopenfilenames", + "askopenfile", "askopenfiles", "asksaveasfile", "askdirectory"] + +import fnmatch +import os +from tkinter import ( + Frame, LEFT, YES, BOTTOM, Entry, TOP, Button, Tk, X, + Toplevel, RIGHT, Y, END, Listbox, BOTH, Scrollbar, +) +from tkinter.dialog import Dialog +from tkinter import commondialog +from tkinter.simpledialog import _setup_dialog + + +dialogstates = {} + + +class FileDialog: + + """Standard file selection dialog -- no checks on selected file. + + Usage: + + d = FileDialog(master) + fname = d.go(dir_or_file, pattern, default, key) + if fname is None: ...canceled... + else: ...open file... + + All arguments to go() are optional. + + The 'key' argument specifies a key in the global dictionary + 'dialogstates', which keeps track of the values for the directory + and pattern arguments, overriding the values passed in (it does + not keep track of the default argument!). If no key is specified, + the dialog keeps no memory of previous state. Note that memory is + kept even when the dialog is canceled. (All this emulates the + behavior of the Macintosh file selection dialogs.) + + """ + + title = "File Selection Dialog" + + def __init__(self, master, title=None): + if title is None: title = self.title + self.master = master + self.directory = None + + self.top = Toplevel(master) + self.top.title(title) + self.top.iconname(title) + _setup_dialog(self.top) + + self.botframe = Frame(self.top) + self.botframe.pack(side=BOTTOM, fill=X) + + self.selection = Entry(self.top) + self.selection.pack(side=BOTTOM, fill=X) + self.selection.bind('', self.ok_event) + + self.filter = Entry(self.top) + self.filter.pack(side=TOP, fill=X) + self.filter.bind('', self.filter_command) + + self.midframe = Frame(self.top) + self.midframe.pack(expand=YES, fill=BOTH) + + self.filesbar = Scrollbar(self.midframe) + self.filesbar.pack(side=RIGHT, fill=Y) + self.files = Listbox(self.midframe, exportselection=0, + yscrollcommand=(self.filesbar, 'set')) + self.files.pack(side=RIGHT, expand=YES, fill=BOTH) + btags = self.files.bindtags() + self.files.bindtags(btags[1:] + btags[:1]) + self.files.bind('', self.files_select_event) + self.files.bind('', self.files_double_event) + self.filesbar.config(command=(self.files, 'yview')) + + self.dirsbar = Scrollbar(self.midframe) + self.dirsbar.pack(side=LEFT, fill=Y) + self.dirs = Listbox(self.midframe, exportselection=0, + yscrollcommand=(self.dirsbar, 'set')) + self.dirs.pack(side=LEFT, expand=YES, fill=BOTH) + self.dirsbar.config(command=(self.dirs, 'yview')) + btags = self.dirs.bindtags() + self.dirs.bindtags(btags[1:] + btags[:1]) + self.dirs.bind('', self.dirs_select_event) + self.dirs.bind('', self.dirs_double_event) + + self.ok_button = Button(self.botframe, + text="OK", + command=self.ok_command) + self.ok_button.pack(side=LEFT) + self.filter_button = Button(self.botframe, + text="Filter", + command=self.filter_command) + self.filter_button.pack(side=LEFT, expand=YES) + self.cancel_button = Button(self.botframe, + text="Cancel", + command=self.cancel_command) + self.cancel_button.pack(side=RIGHT) + + self.top.protocol('WM_DELETE_WINDOW', self.cancel_command) + # XXX Are the following okay for a general audience? + self.top.bind('', self.cancel_command) + self.top.bind('', self.cancel_command) + + def go(self, dir_or_file=os.curdir, pattern="*", default="", key=None): + if key and key in dialogstates: + self.directory, pattern = dialogstates[key] + else: + dir_or_file = os.path.expanduser(dir_or_file) + if os.path.isdir(dir_or_file): + self.directory = dir_or_file + else: + self.directory, default = os.path.split(dir_or_file) + self.set_filter(self.directory, pattern) + self.set_selection(default) + self.filter_command() + self.selection.focus_set() + self.top.wait_visibility() # window needs to be visible for the grab + self.top.grab_set() + self.how = None + self.master.mainloop() # Exited by self.quit(how) + if key: + directory, pattern = self.get_filter() + if self.how: + directory = os.path.dirname(self.how) + dialogstates[key] = directory, pattern + self.top.destroy() + return self.how + + def quit(self, how=None): + self.how = how + self.master.quit() # Exit mainloop() + + def dirs_double_event(self, event): + self.filter_command() + + def dirs_select_event(self, event): + dir, pat = self.get_filter() + subdir = self.dirs.get('active') + dir = os.path.normpath(os.path.join(self.directory, subdir)) + self.set_filter(dir, pat) + + def files_double_event(self, event): + self.ok_command() + + def files_select_event(self, event): + file = self.files.get('active') + self.set_selection(file) + + def ok_event(self, event): + self.ok_command() + + def ok_command(self): + self.quit(self.get_selection()) + + def filter_command(self, event=None): + dir, pat = self.get_filter() + try: + names = os.listdir(dir) + except OSError: + self.master.bell() + return + self.directory = dir + self.set_filter(dir, pat) + names.sort() + subdirs = [os.pardir] + matchingfiles = [] + for name in names: + fullname = os.path.join(dir, name) + if os.path.isdir(fullname): + subdirs.append(name) + elif fnmatch.fnmatch(name, pat): + matchingfiles.append(name) + self.dirs.delete(0, END) + for name in subdirs: + self.dirs.insert(END, name) + self.files.delete(0, END) + for name in matchingfiles: + self.files.insert(END, name) + head, tail = os.path.split(self.get_selection()) + if tail == os.curdir: tail = '' + self.set_selection(tail) + + def get_filter(self): + filter = self.filter.get() + filter = os.path.expanduser(filter) + if filter[-1:] == os.sep or os.path.isdir(filter): + filter = os.path.join(filter, "*") + return os.path.split(filter) + + def get_selection(self): + file = self.selection.get() + file = os.path.expanduser(file) + return file + + def cancel_command(self, event=None): + self.quit() + + def set_filter(self, dir, pat): + if not os.path.isabs(dir): + try: + pwd = os.getcwd() + except OSError: + pwd = None + if pwd: + dir = os.path.join(pwd, dir) + dir = os.path.normpath(dir) + self.filter.delete(0, END) + self.filter.insert(END, os.path.join(dir or os.curdir, pat or "*")) + + def set_selection(self, file): + self.selection.delete(0, END) + self.selection.insert(END, os.path.join(self.directory, file)) + + +class LoadFileDialog(FileDialog): + + """File selection dialog which checks that the file exists.""" + + title = "Load File Selection Dialog" + + def ok_command(self): + file = self.get_selection() + if not os.path.isfile(file): + self.master.bell() + else: + self.quit(file) + + +class SaveFileDialog(FileDialog): + + """File selection dialog which checks that the file may be created.""" + + title = "Save File Selection Dialog" + + def ok_command(self): + file = self.get_selection() + if os.path.exists(file): + if os.path.isdir(file): + self.master.bell() + return + d = Dialog(self.top, + title="Overwrite Existing File Question", + text="Overwrite existing file %r?" % (file,), + bitmap='questhead', + default=1, + strings=("Yes", "Cancel")) + if d.num != 0: + return + else: + head, tail = os.path.split(file) + if not os.path.isdir(head): + self.master.bell() + return + self.quit(file) + + +# For the following classes and modules: +# +# options (all have default values): +# +# - defaultextension: added to filename if not explicitly given +# +# - filetypes: sequence of (label, pattern) tuples. the same pattern +# may occur with several patterns. use "*" as pattern to indicate +# all files. +# +# - initialdir: initial directory. preserved by dialog instance. +# +# - initialfile: initial file (ignored by the open dialog). preserved +# by dialog instance. +# +# - parent: which window to place the dialog on top of +# +# - title: dialog title +# +# - multiple: if true user may select more than one file +# +# options for the directory chooser: +# +# - initialdir, parent, title: see above +# +# - mustexist: if true, user must pick an existing directory +# + + +class _Dialog(commondialog.Dialog): + + def _fixoptions(self): + try: + # make sure "filetypes" is a tuple + self.options["filetypes"] = tuple(self.options["filetypes"]) + except KeyError: + pass + + def _fixresult(self, widget, result): + if result: + # keep directory and filename until next time + # convert Tcl path objects to strings + try: + result = result.string + except AttributeError: + # it already is a string + pass + path, file = os.path.split(result) + self.options["initialdir"] = path + self.options["initialfile"] = file + self.filename = result # compatibility + return result + + +# +# file dialogs + +class Open(_Dialog): + "Ask for a filename to open" + + command = "tk_getOpenFile" + + def _fixresult(self, widget, result): + if isinstance(result, tuple): + # multiple results: + result = tuple([getattr(r, "string", r) for r in result]) + if result: + path, file = os.path.split(result[0]) + self.options["initialdir"] = path + # don't set initialfile or filename, as we have multiple of these + return result + if not widget.tk.wantobjects() and "multiple" in self.options: + # Need to split result explicitly + return self._fixresult(widget, widget.tk.splitlist(result)) + return _Dialog._fixresult(self, widget, result) + + +class SaveAs(_Dialog): + "Ask for a filename to save as" + + command = "tk_getSaveFile" + + +# the directory dialog has its own _fix routines. +class Directory(commondialog.Dialog): + "Ask for a directory" + + command = "tk_chooseDirectory" + + def _fixresult(self, widget, result): + if result: + # convert Tcl path objects to strings + try: + result = result.string + except AttributeError: + # it already is a string + pass + # keep directory until next time + self.options["initialdir"] = result + self.directory = result # compatibility + return result + +# +# convenience stuff + + +def askopenfilename(**options): + "Ask for a filename to open" + + return Open(**options).show() + + +def asksaveasfilename(**options): + "Ask for a filename to save as" + + return SaveAs(**options).show() + + +def askopenfilenames(**options): + """Ask for multiple filenames to open + + Returns a list of filenames or empty list if + cancel button selected + """ + options["multiple"]=1 + return Open(**options).show() + +# FIXME: are the following perhaps a bit too convenient? + + +def askopenfile(mode = "r", **options): + "Ask for a filename to open, and returned the opened file" + + filename = Open(**options).show() + if filename: + return open(filename, mode) + return None + + +def askopenfiles(mode = "r", **options): + """Ask for multiple filenames and return the open file + objects + + returns a list of open file objects or an empty list if + cancel selected + """ + + files = askopenfilenames(**options) + if files: + ofiles=[] + for filename in files: + ofiles.append(open(filename, mode)) + files=ofiles + return files + + +def asksaveasfile(mode = "w", **options): + "Ask for a filename to save as, and returned the opened file" + + filename = SaveAs(**options).show() + if filename: + return open(filename, mode) + return None + + +def askdirectory (**options): + "Ask for a directory, and return the file name" + return Directory(**options).show() + + +# -------------------------------------------------------------------- +# test stuff + +def test(): + """Simple test program.""" + root = Tk() + root.withdraw() + fd = LoadFileDialog(root) + loadfile = fd.go(key="test") + fd = SaveFileDialog(root) + savefile = fd.go(key="test") + print(loadfile, savefile) + + # Since the file name may contain non-ASCII characters, we need + # to find an encoding that likely supports the file name, and + # displays correctly on the terminal. + + # Start off with UTF-8 + enc = "utf-8" + + # See whether CODESET is defined + try: + import locale + locale.setlocale(locale.LC_ALL,'') + enc = locale.nl_langinfo(locale.CODESET) + except (ImportError, AttributeError): + pass + + # dialog for opening files + + openfilename=askopenfilename(filetypes=[("all files", "*")]) + try: + fp=open(openfilename,"r") + fp.close() + except BaseException as exc: + print("Could not open File: ") + print(exc) + + print("open", openfilename.encode(enc)) + + # dialog for saving files + + saveasfilename=asksaveasfilename() + print("saveas", saveasfilename.encode(enc)) + + +if __name__ == '__main__': + test() diff --git a/Lib/tkinter/font.py b/Lib/tkinter/font.py new file mode 100644 index 0000000000..3e24e28ef5 --- /dev/null +++ b/Lib/tkinter/font.py @@ -0,0 +1,239 @@ +# Tkinter font wrapper +# +# written by Fredrik Lundh, February 1998 +# + +import itertools +import tkinter + +__version__ = "0.9" +__all__ = ["NORMAL", "ROMAN", "BOLD", "ITALIC", + "nametofont", "Font", "families", "names"] + +# weight/slant +NORMAL = "normal" +ROMAN = "roman" +BOLD = "bold" +ITALIC = "italic" + + +def nametofont(name, root=None): + """Given the name of a tk named font, returns a Font representation. + """ + return Font(name=name, exists=True, root=root) + + +class Font: + """Represents a named font. + + Constructor options are: + + font -- font specifier (name, system font, or (family, size, style)-tuple) + name -- name to use for this font configuration (defaults to a unique name) + exists -- does a named font by this name already exist? + Creates a new named font if False, points to the existing font if True. + Raises _tkinter.TclError if the assertion is false. + + the following are ignored if font is specified: + + family -- font 'family', e.g. Courier, Times, Helvetica + size -- font size in points + weight -- font thickness: NORMAL, BOLD + slant -- font slant: ROMAN, ITALIC + underline -- font underlining: false (0), true (1) + overstrike -- font strikeout: false (0), true (1) + + """ + + counter = itertools.count(1) + + def _set(self, kw): + options = [] + for k, v in kw.items(): + options.append("-"+k) + options.append(str(v)) + return tuple(options) + + def _get(self, args): + options = [] + for k in args: + options.append("-"+k) + return tuple(options) + + def _mkdict(self, args): + options = {} + for i in range(0, len(args), 2): + options[args[i][1:]] = args[i+1] + return options + + def __init__(self, root=None, font=None, name=None, exists=False, + **options): + if root is None: + root = tkinter._get_default_root('use font') + tk = getattr(root, 'tk', root) + if font: + # get actual settings corresponding to the given font + font = tk.splitlist(tk.call("font", "actual", font)) + else: + font = self._set(options) + if not name: + name = "font" + str(next(self.counter)) + self.name = name + + if exists: + self.delete_font = False + # confirm font exists + if self.name not in tk.splitlist(tk.call("font", "names")): + raise tkinter._tkinter.TclError( + "named font %s does not already exist" % (self.name,)) + # if font config info supplied, apply it + if font: + tk.call("font", "configure", self.name, *font) + else: + # create new font (raises TclError if the font exists) + tk.call("font", "create", self.name, *font) + self.delete_font = True + self._tk = tk + self._split = tk.splitlist + self._call = tk.call + + def __str__(self): + return self.name + + def __repr__(self): + return f"<{self.__class__.__module__}.{self.__class__.__qualname__}" \ + f" object {self.name!r}>" + + def __eq__(self, other): + if not isinstance(other, Font): + return NotImplemented + return self.name == other.name and self._tk == other._tk + + def __getitem__(self, key): + return self.cget(key) + + def __setitem__(self, key, value): + self.configure(**{key: value}) + + def __del__(self): + try: + if self.delete_font: + self._call("font", "delete", self.name) + except Exception: + pass + + def copy(self): + "Return a distinct copy of the current font" + return Font(self._tk, **self.actual()) + + def actual(self, option=None, displayof=None): + "Return actual font attributes" + args = () + if displayof: + args = ('-displayof', displayof) + if option: + args = args + ('-' + option, ) + return self._call("font", "actual", self.name, *args) + else: + return self._mkdict( + self._split(self._call("font", "actual", self.name, *args))) + + def cget(self, option): + "Get font attribute" + return self._call("font", "config", self.name, "-"+option) + + def config(self, **options): + "Modify font attributes" + if options: + self._call("font", "config", self.name, + *self._set(options)) + else: + return self._mkdict( + self._split(self._call("font", "config", self.name))) + + configure = config + + def measure(self, text, displayof=None): + "Return text width" + args = (text,) + if displayof: + args = ('-displayof', displayof, text) + return self._tk.getint(self._call("font", "measure", self.name, *args)) + + def metrics(self, *options, **kw): + """Return font metrics. + + For best performance, create a dummy widget + using this font before calling this method.""" + args = () + displayof = kw.pop('displayof', None) + if displayof: + args = ('-displayof', displayof) + if options: + args = args + self._get(options) + return self._tk.getint( + self._call("font", "metrics", self.name, *args)) + else: + res = self._split(self._call("font", "metrics", self.name, *args)) + options = {} + for i in range(0, len(res), 2): + options[res[i][1:]] = self._tk.getint(res[i+1]) + return options + + +def families(root=None, displayof=None): + "Get font families (as a tuple)" + if root is None: + root = tkinter._get_default_root('use font.families()') + args = () + if displayof: + args = ('-displayof', displayof) + return root.tk.splitlist(root.tk.call("font", "families", *args)) + + +def names(root=None): + "Get names of defined fonts (as a tuple)" + if root is None: + root = tkinter._get_default_root('use font.names()') + return root.tk.splitlist(root.tk.call("font", "names")) + + +# -------------------------------------------------------------------- +# test stuff + +if __name__ == "__main__": + + root = tkinter.Tk() + + # create a font + f = Font(family="times", size=30, weight=NORMAL) + + print(f.actual()) + print(f.actual("family")) + print(f.actual("weight")) + + print(f.config()) + print(f.cget("family")) + print(f.cget("weight")) + + print(names()) + + print(f.measure("hello"), f.metrics("linespace")) + + print(f.metrics(displayof=root)) + + f = Font(font=("Courier", 20, "bold")) + print(f.measure("hello"), f.metrics("linespace", displayof=root)) + + w = tkinter.Label(root, text="Hello, world", font=f) + w.pack() + + w = tkinter.Button(root, text="Quit!", command=root.destroy) + w.pack() + + fb = Font(font=w["font"]).copy() + fb.config(weight=BOLD) + + w.config(font=fb) + + tkinter.mainloop() diff --git a/Lib/tkinter/messagebox.py b/Lib/tkinter/messagebox.py new file mode 100644 index 0000000000..5f0343b660 --- /dev/null +++ b/Lib/tkinter/messagebox.py @@ -0,0 +1,146 @@ +# tk common message boxes +# +# this module provides an interface to the native message boxes +# available in Tk 4.2 and newer. +# +# written by Fredrik Lundh, May 1997 +# + +# +# options (all have default values): +# +# - default: which button to make default (one of the reply codes) +# +# - icon: which icon to display (see below) +# +# - message: the message to display +# +# - parent: which window to place the dialog on top of +# +# - title: dialog title +# +# - type: dialog type; that is, which buttons to display (see below) +# + +from tkinter.commondialog import Dialog + +__all__ = ["showinfo", "showwarning", "showerror", + "askquestion", "askokcancel", "askyesno", + "askyesnocancel", "askretrycancel"] + +# +# constants + +# icons +ERROR = "error" +INFO = "info" +QUESTION = "question" +WARNING = "warning" + +# types +ABORTRETRYIGNORE = "abortretryignore" +OK = "ok" +OKCANCEL = "okcancel" +RETRYCANCEL = "retrycancel" +YESNO = "yesno" +YESNOCANCEL = "yesnocancel" + +# replies +ABORT = "abort" +RETRY = "retry" +IGNORE = "ignore" +OK = "ok" +CANCEL = "cancel" +YES = "yes" +NO = "no" + + +# +# message dialog class + +class Message(Dialog): + "A message box" + + command = "tk_messageBox" + + +# +# convenience stuff + +# Rename _icon and _type options to allow overriding them in options +def _show(title=None, message=None, _icon=None, _type=None, **options): + if _icon and "icon" not in options: options["icon"] = _icon + if _type and "type" not in options: options["type"] = _type + if title: options["title"] = title + if message: options["message"] = message + res = Message(**options).show() + # In some Tcl installations, yes/no is converted into a boolean. + if isinstance(res, bool): + if res: + return YES + return NO + # In others we get a Tcl_Obj. + return str(res) + + +def showinfo(title=None, message=None, **options): + "Show an info message" + return _show(title, message, INFO, OK, **options) + + +def showwarning(title=None, message=None, **options): + "Show a warning message" + return _show(title, message, WARNING, OK, **options) + + +def showerror(title=None, message=None, **options): + "Show an error message" + return _show(title, message, ERROR, OK, **options) + + +def askquestion(title=None, message=None, **options): + "Ask a question" + return _show(title, message, QUESTION, YESNO, **options) + + +def askokcancel(title=None, message=None, **options): + "Ask if operation should proceed; return true if the answer is ok" + s = _show(title, message, QUESTION, OKCANCEL, **options) + return s == OK + + +def askyesno(title=None, message=None, **options): + "Ask a question; return true if the answer is yes" + s = _show(title, message, QUESTION, YESNO, **options) + return s == YES + + +def askyesnocancel(title=None, message=None, **options): + "Ask a question; return true if the answer is yes, None if cancelled." + s = _show(title, message, QUESTION, YESNOCANCEL, **options) + # s might be a Tcl index object, so convert it to a string + s = str(s) + if s == CANCEL: + return None + return s == YES + + +def askretrycancel(title=None, message=None, **options): + "Ask if operation should be retried; return true if the answer is yes" + s = _show(title, message, WARNING, RETRYCANCEL, **options) + return s == RETRY + + +# -------------------------------------------------------------------- +# test stuff + +if __name__ == "__main__": + + print("info", showinfo("Spam", "Egg Information")) + print("warning", showwarning("Spam", "Egg Warning")) + print("error", showerror("Spam", "Egg Alert")) + print("question", askquestion("Spam", "Question?")) + print("proceed", askokcancel("Spam", "Proceed?")) + print("yes/no", askyesno("Spam", "Got it?")) + print("yes/no/cancel", askyesnocancel("Spam", "Want it?")) + print("try again", askretrycancel("Spam", "Try again?")) diff --git a/Lib/tkinter/scrolledtext.py b/Lib/tkinter/scrolledtext.py new file mode 100644 index 0000000000..4f9a8815b6 --- /dev/null +++ b/Lib/tkinter/scrolledtext.py @@ -0,0 +1,56 @@ +"""A ScrolledText widget feels like a text widget but also has a +vertical scroll bar on its right. (Later, options may be added to +add a horizontal bar as well, to make the bars disappear +automatically when not needed, to move them to the other side of the +window, etc.) + +Configuration options are passed to the Text widget. +A Frame widget is inserted between the master and the text, to hold +the Scrollbar widget. +Most methods calls are inherited from the Text widget; Pack, Grid and +Place methods are redirected to the Frame widget however. +""" + +from tkinter import Frame, Text, Scrollbar, Pack, Grid, Place +from tkinter.constants import RIGHT, LEFT, Y, BOTH + +__all__ = ['ScrolledText'] + + +class ScrolledText(Text): + def __init__(self, master=None, **kw): + self.frame = Frame(master) + self.vbar = Scrollbar(self.frame) + self.vbar.pack(side=RIGHT, fill=Y) + + kw.update({'yscrollcommand': self.vbar.set}) + Text.__init__(self, self.frame, **kw) + self.pack(side=LEFT, fill=BOTH, expand=True) + self.vbar['command'] = self.yview + + # Copy geometry methods of self.frame without overriding Text + # methods -- hack! + text_meths = vars(Text).keys() + methods = vars(Pack).keys() | vars(Grid).keys() | vars(Place).keys() + methods = methods.difference(text_meths) + + for m in methods: + if m[0] != '_' and m != 'config' and m != 'configure': + setattr(self, m, getattr(self.frame, m)) + + def __str__(self): + return str(self.frame) + + +def example(): + from tkinter.constants import END + + stext = ScrolledText(bg='white', height=10) + stext.insert(END, __doc__) + stext.pack(fill=BOTH, side=LEFT, expand=True) + stext.focus_set() + stext.mainloop() + + +if __name__ == "__main__": + example() diff --git a/Lib/tkinter/simpledialog.py b/Lib/tkinter/simpledialog.py new file mode 100644 index 0000000000..6e5b025a9f --- /dev/null +++ b/Lib/tkinter/simpledialog.py @@ -0,0 +1,441 @@ +# +# An Introduction to Tkinter +# +# Copyright (c) 1997 by Fredrik Lundh +# +# This copyright applies to Dialog, askinteger, askfloat and asktring +# +# fredrik@pythonware.com +# http://www.pythonware.com +# +"""This modules handles dialog boxes. + +It contains the following public symbols: + +SimpleDialog -- A simple but flexible modal dialog box + +Dialog -- a base class for dialogs + +askinteger -- get an integer from the user + +askfloat -- get a float from the user + +askstring -- get a string from the user +""" + +from tkinter import * +from tkinter import _get_temp_root, _destroy_temp_root +from tkinter import messagebox + + +class SimpleDialog: + + def __init__(self, master, + text='', buttons=[], default=None, cancel=None, + title=None, class_=None): + if class_: + self.root = Toplevel(master, class_=class_) + else: + self.root = Toplevel(master) + if title: + self.root.title(title) + self.root.iconname(title) + + _setup_dialog(self.root) + + self.message = Message(self.root, text=text, aspect=400) + self.message.pack(expand=1, fill=BOTH) + self.frame = Frame(self.root) + self.frame.pack() + self.num = default + self.cancel = cancel + self.default = default + self.root.bind('', self.return_event) + for num in range(len(buttons)): + s = buttons[num] + b = Button(self.frame, text=s, + command=(lambda self=self, num=num: self.done(num))) + if num == default: + b.config(relief=RIDGE, borderwidth=8) + b.pack(side=LEFT, fill=BOTH, expand=1) + self.root.protocol('WM_DELETE_WINDOW', self.wm_delete_window) + self.root.transient(master) + _place_window(self.root, master) + + def go(self): + self.root.wait_visibility() + self.root.grab_set() + self.root.mainloop() + self.root.destroy() + return self.num + + def return_event(self, event): + if self.default is None: + self.root.bell() + else: + self.done(self.default) + + def wm_delete_window(self): + if self.cancel is None: + self.root.bell() + else: + self.done(self.cancel) + + def done(self, num): + self.num = num + self.root.quit() + + +class Dialog(Toplevel): + + '''Class to open dialogs. + + This class is intended as a base class for custom dialogs + ''' + + def __init__(self, parent, title = None): + '''Initialize a dialog. + + Arguments: + + parent -- a parent window (the application window) + + title -- the dialog title + ''' + master = parent + if master is None: + master = _get_temp_root() + + Toplevel.__init__(self, master) + + self.withdraw() # remain invisible for now + # If the parent is not viewable, don't + # make the child transient, or else it + # would be opened withdrawn + if parent is not None and parent.winfo_viewable(): + self.transient(parent) + + if title: + self.title(title) + + _setup_dialog(self) + + self.parent = parent + + self.result = None + + body = Frame(self) + self.initial_focus = self.body(body) + body.pack(padx=5, pady=5) + + self.buttonbox() + + if self.initial_focus is None: + self.initial_focus = self + + self.protocol("WM_DELETE_WINDOW", self.cancel) + + _place_window(self, parent) + + self.initial_focus.focus_set() + + # wait for window to appear on screen before calling grab_set + self.wait_visibility() + self.grab_set() + self.wait_window(self) + + def destroy(self): + '''Destroy the window''' + self.initial_focus = None + Toplevel.destroy(self) + _destroy_temp_root(self.master) + + # + # construction hooks + + def body(self, master): + '''create dialog body. + + return widget that should have initial focus. + This method should be overridden, and is called + by the __init__ method. + ''' + pass + + def buttonbox(self): + '''add standard button box. + + override if you do not want the standard buttons + ''' + + box = Frame(self) + + w = Button(box, text="OK", width=10, command=self.ok, default=ACTIVE) + w.pack(side=LEFT, padx=5, pady=5) + w = Button(box, text="Cancel", width=10, command=self.cancel) + w.pack(side=LEFT, padx=5, pady=5) + + self.bind("", self.ok) + self.bind("", self.cancel) + + box.pack() + + # + # standard button semantics + + def ok(self, event=None): + + if not self.validate(): + self.initial_focus.focus_set() # put focus back + return + + self.withdraw() + self.update_idletasks() + + try: + self.apply() + finally: + self.cancel() + + def cancel(self, event=None): + + # put focus back to the parent window + if self.parent is not None: + self.parent.focus_set() + self.destroy() + + # + # command hooks + + def validate(self): + '''validate the data + + This method is called automatically to validate the data before the + dialog is destroyed. By default, it always validates OK. + ''' + + return 1 # override + + def apply(self): + '''process the data + + This method is called automatically to process the data, *after* + the dialog is destroyed. By default, it does nothing. + ''' + + pass # override + + +# Place a toplevel window at the center of parent or screen +# It is a Python implementation of ::tk::PlaceWindow. +def _place_window(w, parent=None): + w.wm_withdraw() # Remain invisible while we figure out the geometry + w.update_idletasks() # Actualize geometry information + + minwidth = w.winfo_reqwidth() + minheight = w.winfo_reqheight() + maxwidth = w.winfo_vrootwidth() + maxheight = w.winfo_vrootheight() + if parent is not None and parent.winfo_ismapped(): + x = parent.winfo_rootx() + (parent.winfo_width() - minwidth) // 2 + y = parent.winfo_rooty() + (parent.winfo_height() - minheight) // 2 + vrootx = w.winfo_vrootx() + vrooty = w.winfo_vrooty() + x = min(x, vrootx + maxwidth - minwidth) + x = max(x, vrootx) + y = min(y, vrooty + maxheight - minheight) + y = max(y, vrooty) + if w._windowingsystem == 'aqua': + # Avoid the native menu bar which sits on top of everything. + y = max(y, 22) + else: + x = (w.winfo_screenwidth() - minwidth) // 2 + y = (w.winfo_screenheight() - minheight) // 2 + + w.wm_maxsize(maxwidth, maxheight) + w.wm_geometry('+%d+%d' % (x, y)) + w.wm_deiconify() # Become visible at the desired location + + +def _setup_dialog(w): + if w._windowingsystem == "aqua": + w.tk.call("::tk::unsupported::MacWindowStyle", "style", + w, "moveableModal", "") + elif w._windowingsystem == "x11": + w.wm_attributes(type="dialog") + +# -------------------------------------------------------------------- +# convenience dialogues + +class _QueryDialog(Dialog): + + def __init__(self, title, prompt, + initialvalue=None, + minvalue = None, maxvalue = None, + parent = None): + + self.prompt = prompt + self.minvalue = minvalue + self.maxvalue = maxvalue + + self.initialvalue = initialvalue + + Dialog.__init__(self, parent, title) + + def destroy(self): + self.entry = None + Dialog.destroy(self) + + def body(self, master): + + w = Label(master, text=self.prompt, justify=LEFT) + w.grid(row=0, padx=5, sticky=W) + + self.entry = Entry(master, name="entry") + self.entry.grid(row=1, padx=5, sticky=W+E) + + if self.initialvalue is not None: + self.entry.insert(0, self.initialvalue) + self.entry.select_range(0, END) + + return self.entry + + def validate(self): + try: + result = self.getresult() + except ValueError: + messagebox.showwarning( + "Illegal value", + self.errormessage + "\nPlease try again", + parent = self + ) + return 0 + + if self.minvalue is not None and result < self.minvalue: + messagebox.showwarning( + "Too small", + "The allowed minimum value is %s. " + "Please try again." % self.minvalue, + parent = self + ) + return 0 + + if self.maxvalue is not None and result > self.maxvalue: + messagebox.showwarning( + "Too large", + "The allowed maximum value is %s. " + "Please try again." % self.maxvalue, + parent = self + ) + return 0 + + self.result = result + + return 1 + + +class _QueryInteger(_QueryDialog): + errormessage = "Not an integer." + + def getresult(self): + return self.getint(self.entry.get()) + + +def askinteger(title, prompt, **kw): + '''get an integer from the user + + Arguments: + + title -- the dialog title + prompt -- the label text + **kw -- see SimpleDialog class + + Return value is an integer + ''' + d = _QueryInteger(title, prompt, **kw) + return d.result + + +class _QueryFloat(_QueryDialog): + errormessage = "Not a floating-point value." + + def getresult(self): + return self.getdouble(self.entry.get()) + + +def askfloat(title, prompt, **kw): + '''get a float from the user + + Arguments: + + title -- the dialog title + prompt -- the label text + **kw -- see SimpleDialog class + + Return value is a float + ''' + d = _QueryFloat(title, prompt, **kw) + return d.result + + +class _QueryString(_QueryDialog): + def __init__(self, *args, **kw): + if "show" in kw: + self.__show = kw["show"] + del kw["show"] + else: + self.__show = None + _QueryDialog.__init__(self, *args, **kw) + + def body(self, master): + entry = _QueryDialog.body(self, master) + if self.__show is not None: + entry.configure(show=self.__show) + return entry + + def getresult(self): + return self.entry.get() + + +def askstring(title, prompt, **kw): + '''get a string from the user + + Arguments: + + title -- the dialog title + prompt -- the label text + **kw -- see SimpleDialog class + + Return value is a string + ''' + d = _QueryString(title, prompt, **kw) + return d.result + + +if __name__ == '__main__': + + def test(): + root = Tk() + def doit(root=root): + d = SimpleDialog(root, + text="This is a test dialog. " + "Would this have been an actual dialog, " + "the buttons below would have been glowing " + "in soft pink light.\n" + "Do you believe this?", + buttons=["Yes", "No", "Cancel"], + default=0, + cancel=2, + title="Test Dialog") + print(d.go()) + print(askinteger("Spam", "Egg count", initialvalue=12*12)) + print(askfloat("Spam", "Egg weight\n(in tons)", minvalue=1, + maxvalue=100)) + print(askstring("Spam", "Egg label")) + t = Button(root, text='Test', command=doit) + t.pack() + q = Button(root, text='Quit', command=t.quit) + q.pack() + t.mainloop() + + test() diff --git a/Lib/tkinter/ttk.py b/Lib/tkinter/ttk.py new file mode 100644 index 0000000000..073b3ae207 --- /dev/null +++ b/Lib/tkinter/ttk.py @@ -0,0 +1,1647 @@ +"""Ttk wrapper. + +This module provides classes to allow using Tk themed widget set. + +Ttk is based on a revised and enhanced version of +TIP #48 (http://tip.tcl.tk/48) specified style engine. + +Its basic idea is to separate, to the extent possible, the code +implementing a widget's behavior from the code implementing its +appearance. Widget class bindings are primarily responsible for +maintaining the widget state and invoking callbacks, all aspects +of the widgets appearance lies at Themes. +""" + +__version__ = "0.3.1" + +__author__ = "Guilherme Polo " + +__all__ = ["Button", "Checkbutton", "Combobox", "Entry", "Frame", "Label", + "Labelframe", "LabelFrame", "Menubutton", "Notebook", "Panedwindow", + "PanedWindow", "Progressbar", "Radiobutton", "Scale", "Scrollbar", + "Separator", "Sizegrip", "Spinbox", "Style", "Treeview", + # Extensions + "LabeledScale", "OptionMenu", + # functions + "tclobjs_to_py", "setup_master"] + +import tkinter +from tkinter import _flatten, _join, _stringify, _splitdict + + +def _format_optvalue(value, script=False): + """Internal function.""" + if script: + # if caller passes a Tcl script to tk.call, all the values need to + # be grouped into words (arguments to a command in Tcl dialect) + value = _stringify(value) + elif isinstance(value, (list, tuple)): + value = _join(value) + return value + +def _format_optdict(optdict, script=False, ignore=None): + """Formats optdict to a tuple to pass it to tk.call. + + E.g. (script=False): + {'foreground': 'blue', 'padding': [1, 2, 3, 4]} returns: + ('-foreground', 'blue', '-padding', '1 2 3 4')""" + + opts = [] + for opt, value in optdict.items(): + if not ignore or opt not in ignore: + opts.append("-%s" % opt) + if value is not None: + opts.append(_format_optvalue(value, script)) + + return _flatten(opts) + +def _mapdict_values(items): + # each value in mapdict is expected to be a sequence, where each item + # is another sequence containing a state (or several) and a value + # E.g. (script=False): + # [('active', 'selected', 'grey'), ('focus', [1, 2, 3, 4])] + # returns: + # ['active selected', 'grey', 'focus', [1, 2, 3, 4]] + opt_val = [] + for *state, val in items: + if len(state) == 1: + # if it is empty (something that evaluates to False), then + # format it to Tcl code to denote the "normal" state + state = state[0] or '' + else: + # group multiple states + state = ' '.join(state) # raise TypeError if not str + opt_val.append(state) + if val is not None: + opt_val.append(val) + return opt_val + +def _format_mapdict(mapdict, script=False): + """Formats mapdict to pass it to tk.call. + + E.g. (script=False): + {'expand': [('active', 'selected', 'grey'), ('focus', [1, 2, 3, 4])]} + + returns: + + ('-expand', '{active selected} grey focus {1, 2, 3, 4}')""" + + opts = [] + for opt, value in mapdict.items(): + opts.extend(("-%s" % opt, + _format_optvalue(_mapdict_values(value), script))) + + return _flatten(opts) + +def _format_elemcreate(etype, script=False, *args, **kw): + """Formats args and kw according to the given element factory etype.""" + specs = () + opts = () + if etype == "image": # define an element based on an image + # first arg should be the default image name + iname = args[0] + # next args, if any, are statespec/value pairs which is almost + # a mapdict, but we just need the value + imagespec = (iname, *_mapdict_values(args[1:])) + if script: + specs = (imagespec,) + else: + specs = (_join(imagespec),) + opts = _format_optdict(kw, script) + + if etype == "vsapi": + # define an element whose visual appearance is drawn using the + # Microsoft Visual Styles API which is responsible for the + # themed styles on Windows XP and Vista. + # Availability: Tk 8.6, Windows XP and Vista. + if len(args) < 3: + class_name, part_id = args + statemap = (((), 1),) + else: + class_name, part_id, statemap = args + specs = (class_name, part_id, tuple(_mapdict_values(statemap))) + opts = _format_optdict(kw, script) + + elif etype == "from": # clone an element + # it expects a themename and optionally an element to clone from, + # otherwise it will clone {} (empty element) + specs = (args[0],) # theme name + if len(args) > 1: # elementfrom specified + opts = (_format_optvalue(args[1], script),) + + if script: + specs = _join(specs) + opts = ' '.join(opts) + return specs, opts + else: + return *specs, opts + + +def _format_layoutlist(layout, indent=0, indent_size=2): + """Formats a layout list so we can pass the result to ttk::style + layout and ttk::style settings. Note that the layout doesn't have to + be a list necessarily. + + E.g.: + [("Menubutton.background", None), + ("Menubutton.button", {"children": + [("Menubutton.focus", {"children": + [("Menubutton.padding", {"children": + [("Menubutton.label", {"side": "left", "expand": 1})] + })] + })] + }), + ("Menubutton.indicator", {"side": "right"}) + ] + + returns: + + Menubutton.background + Menubutton.button -children { + Menubutton.focus -children { + Menubutton.padding -children { + Menubutton.label -side left -expand 1 + } + } + } + Menubutton.indicator -side right""" + script = [] + + for layout_elem in layout: + elem, opts = layout_elem + opts = opts or {} + fopts = ' '.join(_format_optdict(opts, True, ("children",))) + head = "%s%s%s" % (' ' * indent, elem, (" %s" % fopts) if fopts else '') + + if "children" in opts: + script.append(head + " -children {") + indent += indent_size + newscript, indent = _format_layoutlist(opts['children'], indent, + indent_size) + script.append(newscript) + indent -= indent_size + script.append('%s}' % (' ' * indent)) + else: + script.append(head) + + return '\n'.join(script), indent + +def _script_from_settings(settings): + """Returns an appropriate script, based on settings, according to + theme_settings definition to be used by theme_settings and + theme_create.""" + script = [] + # a script will be generated according to settings passed, which + # will then be evaluated by Tcl + for name, opts in settings.items(): + # will format specific keys according to Tcl code + if opts.get('configure'): # format 'configure' + s = ' '.join(_format_optdict(opts['configure'], True)) + script.append("ttk::style configure %s %s;" % (name, s)) + + if opts.get('map'): # format 'map' + s = ' '.join(_format_mapdict(opts['map'], True)) + script.append("ttk::style map %s %s;" % (name, s)) + + if 'layout' in opts: # format 'layout' which may be empty + if not opts['layout']: + s = 'null' # could be any other word, but this one makes sense + else: + s, _ = _format_layoutlist(opts['layout']) + script.append("ttk::style layout %s {\n%s\n}" % (name, s)) + + if opts.get('element create'): # format 'element create' + eopts = opts['element create'] + etype = eopts[0] + + # find where args end, and where kwargs start + argc = 1 # etype was the first one + while argc < len(eopts) and not hasattr(eopts[argc], 'items'): + argc += 1 + + elemargs = eopts[1:argc] + elemkw = eopts[argc] if argc < len(eopts) and eopts[argc] else {} + specs, eopts = _format_elemcreate(etype, True, *elemargs, **elemkw) + + script.append("ttk::style element create %s %s %s %s" % ( + name, etype, specs, eopts)) + + return '\n'.join(script) + +def _list_from_statespec(stuple): + """Construct a list from the given statespec tuple according to the + accepted statespec accepted by _format_mapdict.""" + if isinstance(stuple, str): + return stuple + result = [] + it = iter(stuple) + for state, val in zip(it, it): + if hasattr(state, 'typename'): # this is a Tcl object + state = str(state).split() + elif isinstance(state, str): + state = state.split() + elif not isinstance(state, (tuple, list)): + state = (state,) + if hasattr(val, 'typename'): + val = str(val) + result.append((*state, val)) + + return result + +def _list_from_layouttuple(tk, ltuple): + """Construct a list from the tuple returned by ttk::layout, this is + somewhat the reverse of _format_layoutlist.""" + ltuple = tk.splitlist(ltuple) + res = [] + + indx = 0 + while indx < len(ltuple): + name = ltuple[indx] + opts = {} + res.append((name, opts)) + indx += 1 + + while indx < len(ltuple): # grab name's options + opt, val = ltuple[indx:indx + 2] + if not opt.startswith('-'): # found next name + break + + opt = opt[1:] # remove the '-' from the option + indx += 2 + + if opt == 'children': + val = _list_from_layouttuple(tk, val) + + opts[opt] = val + + return res + +def _val_or_dict(tk, options, *args): + """Format options then call Tk command with args and options and return + the appropriate result. + + If no option is specified, a dict is returned. If an option is + specified with the None value, the value for that option is returned. + Otherwise, the function just sets the passed options and the caller + shouldn't be expecting a return value anyway.""" + options = _format_optdict(options) + res = tk.call(*(args + options)) + + if len(options) % 2: # option specified without a value, return its value + return res + + return _splitdict(tk, res, conv=_tclobj_to_py) + +def _convert_stringval(value): + """Converts a value to, hopefully, a more appropriate Python object.""" + value = str(value) + try: + value = int(value) + except (ValueError, TypeError): + pass + + return value + +def _to_number(x): + if isinstance(x, str): + if '.' in x: + x = float(x) + else: + x = int(x) + return x + +def _tclobj_to_py(val): + """Return value converted from Tcl object to Python object.""" + if val and hasattr(val, '__len__') and not isinstance(val, str): + if getattr(val[0], 'typename', None) == 'StateSpec': + val = _list_from_statespec(val) + else: + val = list(map(_convert_stringval, val)) + + elif hasattr(val, 'typename'): # some other (single) Tcl object + val = _convert_stringval(val) + + return val + +def tclobjs_to_py(adict): + """Returns adict with its values converted from Tcl objects to Python + objects.""" + for opt, val in adict.items(): + adict[opt] = _tclobj_to_py(val) + + return adict + +def setup_master(master=None): + """If master is not None, itself is returned. If master is None, + the default master is returned if there is one, otherwise a new + master is created and returned. + + If it is not allowed to use the default root and master is None, + RuntimeError is raised.""" + if master is None: + master = tkinter._get_default_root() + return master + + +class Style(object): + """Manipulate style database.""" + + _name = "ttk::style" + + def __init__(self, master=None): + master = setup_master(master) + self.master = master + self.tk = self.master.tk + + + def configure(self, style, query_opt=None, **kw): + """Query or sets the default value of the specified option(s) in + style. + + Each key in kw is an option and each value is either a string or + a sequence identifying the value for that option.""" + if query_opt is not None: + kw[query_opt] = None + result = _val_or_dict(self.tk, kw, self._name, "configure", style) + if result or query_opt: + return result + + + def map(self, style, query_opt=None, **kw): + """Query or sets dynamic values of the specified option(s) in + style. + + Each key in kw is an option and each value should be a list or a + tuple (usually) containing statespecs grouped in tuples, or list, + or something else of your preference. A statespec is compound of + one or more states and then a value.""" + if query_opt is not None: + result = self.tk.call(self._name, "map", style, '-%s' % query_opt) + return _list_from_statespec(self.tk.splitlist(result)) + + result = self.tk.call(self._name, "map", style, *_format_mapdict(kw)) + return {k: _list_from_statespec(self.tk.splitlist(v)) + for k, v in _splitdict(self.tk, result).items()} + + + def lookup(self, style, option, state=None, default=None): + """Returns the value specified for option in style. + + If state is specified it is expected to be a sequence of one + or more states. If the default argument is set, it is used as + a fallback value in case no specification for option is found.""" + state = ' '.join(state) if state else '' + + return self.tk.call(self._name, "lookup", style, '-%s' % option, + state, default) + + + def layout(self, style, layoutspec=None): + """Define the widget layout for given style. If layoutspec is + omitted, return the layout specification for given style. + + layoutspec is expected to be a list or an object different than + None that evaluates to False if you want to "turn off" that style. + If it is a list (or tuple, or something else), each item should be + a tuple where the first item is the layout name and the second item + should have the format described below: + + LAYOUTS + + A layout can contain the value None, if takes no options, or + a dict of options specifying how to arrange the element. + The layout mechanism uses a simplified version of the pack + geometry manager: given an initial cavity, each element is + allocated a parcel. Valid options/values are: + + side: whichside + Specifies which side of the cavity to place the + element; one of top, right, bottom or left. If + omitted, the element occupies the entire cavity. + + sticky: nswe + Specifies where the element is placed inside its + allocated parcel. + + children: [sublayout... ] + Specifies a list of elements to place inside the + element. Each element is a tuple (or other sequence) + where the first item is the layout name, and the other + is a LAYOUT.""" + lspec = None + if layoutspec: + lspec = _format_layoutlist(layoutspec)[0] + elif layoutspec is not None: # will disable the layout ({}, '', etc) + lspec = "null" # could be any other word, but this may make sense + # when calling layout(style) later + + return _list_from_layouttuple(self.tk, + self.tk.call(self._name, "layout", style, lspec)) + + + def element_create(self, elementname, etype, *args, **kw): + """Create a new element in the current theme of given etype.""" + *specs, opts = _format_elemcreate(etype, False, *args, **kw) + self.tk.call(self._name, "element", "create", elementname, etype, + *specs, *opts) + + + def element_names(self): + """Returns the list of elements defined in the current theme.""" + return tuple(n.lstrip('-') for n in self.tk.splitlist( + self.tk.call(self._name, "element", "names"))) + + + def element_options(self, elementname): + """Return the list of elementname's options.""" + return tuple(o.lstrip('-') for o in self.tk.splitlist( + self.tk.call(self._name, "element", "options", elementname))) + + + def theme_create(self, themename, parent=None, settings=None): + """Creates a new theme. + + It is an error if themename already exists. If parent is + specified, the new theme will inherit styles, elements and + layouts from the specified parent theme. If settings are present, + they are expected to have the same syntax used for theme_settings.""" + script = _script_from_settings(settings) if settings else '' + + if parent: + self.tk.call(self._name, "theme", "create", themename, + "-parent", parent, "-settings", script) + else: + self.tk.call(self._name, "theme", "create", themename, + "-settings", script) + + + def theme_settings(self, themename, settings): + """Temporarily sets the current theme to themename, apply specified + settings and then restore the previous theme. + + Each key in settings is a style and each value may contain the + keys 'configure', 'map', 'layout' and 'element create' and they + are expected to have the same format as specified by the methods + configure, map, layout and element_create respectively.""" + script = _script_from_settings(settings) + self.tk.call(self._name, "theme", "settings", themename, script) + + + def theme_names(self): + """Returns a list of all known themes.""" + return self.tk.splitlist(self.tk.call(self._name, "theme", "names")) + + + def theme_use(self, themename=None): + """If themename is None, returns the theme in use, otherwise, set + the current theme to themename, refreshes all widgets and emits + a <> event.""" + if themename is None: + # Starting on Tk 8.6, checking this global is no longer needed + # since it allows doing self.tk.call(self._name, "theme", "use") + return self.tk.eval("return $ttk::currentTheme") + + # using "ttk::setTheme" instead of "ttk::style theme use" causes + # the variable currentTheme to be updated, also, ttk::setTheme calls + # "ttk::style theme use" in order to change theme. + self.tk.call("ttk::setTheme", themename) + + +class Widget(tkinter.Widget): + """Base class for Tk themed widgets.""" + + def __init__(self, master, widgetname, kw=None): + """Constructs a Ttk Widget with the parent master. + + STANDARD OPTIONS + + class, cursor, takefocus, style + + SCROLLABLE WIDGET OPTIONS + + xscrollcommand, yscrollcommand + + LABEL WIDGET OPTIONS + + text, textvariable, underline, image, compound, width + + WIDGET STATES + + active, disabled, focus, pressed, selected, background, + readonly, alternate, invalid + """ + master = setup_master(master) + tkinter.Widget.__init__(self, master, widgetname, kw=kw) + + + def identify(self, x, y): + """Returns the name of the element at position x, y, or the empty + string if the point does not lie within any element. + + x and y are pixel coordinates relative to the widget.""" + return self.tk.call(self._w, "identify", x, y) + + + def instate(self, statespec, callback=None, *args, **kw): + """Test the widget's state. + + If callback is not specified, returns True if the widget state + matches statespec and False otherwise. If callback is specified, + then it will be invoked with *args, **kw if the widget state + matches statespec. statespec is expected to be a sequence.""" + ret = self.tk.getboolean( + self.tk.call(self._w, "instate", ' '.join(statespec))) + if ret and callback is not None: + return callback(*args, **kw) + + return ret + + + def state(self, statespec=None): + """Modify or inquire widget state. + + Widget state is returned if statespec is None, otherwise it is + set according to the statespec flags and then a new state spec + is returned indicating which flags were changed. statespec is + expected to be a sequence.""" + if statespec is not None: + statespec = ' '.join(statespec) + + return self.tk.splitlist(str(self.tk.call(self._w, "state", statespec))) + + +class Button(Widget): + """Ttk Button widget, displays a textual label and/or image, and + evaluates a command when pressed.""" + + def __init__(self, master=None, **kw): + """Construct a Ttk Button widget with the parent master. + + STANDARD OPTIONS + + class, compound, cursor, image, state, style, takefocus, + text, textvariable, underline, width + + WIDGET-SPECIFIC OPTIONS + + command, default, width + """ + Widget.__init__(self, master, "ttk::button", kw) + + + def invoke(self): + """Invokes the command associated with the button.""" + return self.tk.call(self._w, "invoke") + + +class Checkbutton(Widget): + """Ttk Checkbutton widget which is either in on- or off-state.""" + + def __init__(self, master=None, **kw): + """Construct a Ttk Checkbutton widget with the parent master. + + STANDARD OPTIONS + + class, compound, cursor, image, state, style, takefocus, + text, textvariable, underline, width + + WIDGET-SPECIFIC OPTIONS + + command, offvalue, onvalue, variable + """ + Widget.__init__(self, master, "ttk::checkbutton", kw) + + + def invoke(self): + """Toggles between the selected and deselected states and + invokes the associated command. If the widget is currently + selected, sets the option variable to the offvalue option + and deselects the widget; otherwise, sets the option variable + to the option onvalue. + + Returns the result of the associated command.""" + return self.tk.call(self._w, "invoke") + + +class Entry(Widget, tkinter.Entry): + """Ttk Entry widget displays a one-line text string and allows that + string to be edited by the user.""" + + def __init__(self, master=None, widget=None, **kw): + """Constructs a Ttk Entry widget with the parent master. + + STANDARD OPTIONS + + class, cursor, style, takefocus, xscrollcommand + + WIDGET-SPECIFIC OPTIONS + + exportselection, invalidcommand, justify, show, state, + textvariable, validate, validatecommand, width + + VALIDATION MODES + + none, key, focus, focusin, focusout, all + """ + Widget.__init__(self, master, widget or "ttk::entry", kw) + + + def bbox(self, index): + """Return a tuple of (x, y, width, height) which describes the + bounding box of the character given by index.""" + return self._getints(self.tk.call(self._w, "bbox", index)) + + + def identify(self, x, y): + """Returns the name of the element at position x, y, or the + empty string if the coordinates are outside the window.""" + return self.tk.call(self._w, "identify", x, y) + + + def validate(self): + """Force revalidation, independent of the conditions specified + by the validate option. Returns False if validation fails, True + if it succeeds. Sets or clears the invalid state accordingly.""" + return self.tk.getboolean(self.tk.call(self._w, "validate")) + + +class Combobox(Entry): + """Ttk Combobox widget combines a text field with a pop-down list of + values.""" + + def __init__(self, master=None, **kw): + """Construct a Ttk Combobox widget with the parent master. + + STANDARD OPTIONS + + class, cursor, style, takefocus + + WIDGET-SPECIFIC OPTIONS + + exportselection, justify, height, postcommand, state, + textvariable, values, width + """ + Entry.__init__(self, master, "ttk::combobox", **kw) + + + def current(self, newindex=None): + """If newindex is supplied, sets the combobox value to the + element at position newindex in the list of values. Otherwise, + returns the index of the current value in the list of values + or -1 if the current value does not appear in the list.""" + if newindex is None: + res = self.tk.call(self._w, "current") + if res == '': + return -1 + return self.tk.getint(res) + return self.tk.call(self._w, "current", newindex) + + + def set(self, value): + """Sets the value of the combobox to value.""" + self.tk.call(self._w, "set", value) + + +class Frame(Widget): + """Ttk Frame widget is a container, used to group other widgets + together.""" + + def __init__(self, master=None, **kw): + """Construct a Ttk Frame with parent master. + + STANDARD OPTIONS + + class, cursor, style, takefocus + + WIDGET-SPECIFIC OPTIONS + + borderwidth, relief, padding, width, height + """ + Widget.__init__(self, master, "ttk::frame", kw) + + +class Label(Widget): + """Ttk Label widget displays a textual label and/or image.""" + + def __init__(self, master=None, **kw): + """Construct a Ttk Label with parent master. + + STANDARD OPTIONS + + class, compound, cursor, image, style, takefocus, text, + textvariable, underline, width + + WIDGET-SPECIFIC OPTIONS + + anchor, background, font, foreground, justify, padding, + relief, text, wraplength + """ + Widget.__init__(self, master, "ttk::label", kw) + + +class Labelframe(Widget): + """Ttk Labelframe widget is a container used to group other widgets + together. It has an optional label, which may be a plain text string + or another widget.""" + + def __init__(self, master=None, **kw): + """Construct a Ttk Labelframe with parent master. + + STANDARD OPTIONS + + class, cursor, style, takefocus + + WIDGET-SPECIFIC OPTIONS + labelanchor, text, underline, padding, labelwidget, width, + height + """ + Widget.__init__(self, master, "ttk::labelframe", kw) + +LabelFrame = Labelframe # tkinter name compatibility + + +class Menubutton(Widget): + """Ttk Menubutton widget displays a textual label and/or image, and + displays a menu when pressed.""" + + def __init__(self, master=None, **kw): + """Construct a Ttk Menubutton with parent master. + + STANDARD OPTIONS + + class, compound, cursor, image, state, style, takefocus, + text, textvariable, underline, width + + WIDGET-SPECIFIC OPTIONS + + direction, menu + """ + Widget.__init__(self, master, "ttk::menubutton", kw) + + +class Notebook(Widget): + """Ttk Notebook widget manages a collection of windows and displays + a single one at a time. Each child window is associated with a tab, + which the user may select to change the currently-displayed window.""" + + def __init__(self, master=None, **kw): + """Construct a Ttk Notebook with parent master. + + STANDARD OPTIONS + + class, cursor, style, takefocus + + WIDGET-SPECIFIC OPTIONS + + height, padding, width + + TAB OPTIONS + + state, sticky, padding, text, image, compound, underline + + TAB IDENTIFIERS (tab_id) + + The tab_id argument found in several methods may take any of + the following forms: + + * An integer between zero and the number of tabs + * The name of a child window + * A positional specification of the form "@x,y", which + defines the tab + * The string "current", which identifies the + currently-selected tab + * The string "end", which returns the number of tabs (only + valid for method index) + """ + Widget.__init__(self, master, "ttk::notebook", kw) + + + def add(self, child, **kw): + """Adds a new tab to the notebook. + + If window is currently managed by the notebook but hidden, it is + restored to its previous position.""" + self.tk.call(self._w, "add", child, *(_format_optdict(kw))) + + + def forget(self, tab_id): + """Removes the tab specified by tab_id, unmaps and unmanages the + associated window.""" + self.tk.call(self._w, "forget", tab_id) + + + def hide(self, tab_id): + """Hides the tab specified by tab_id. + + The tab will not be displayed, but the associated window remains + managed by the notebook and its configuration remembered. Hidden + tabs may be restored with the add command.""" + self.tk.call(self._w, "hide", tab_id) + + + def identify(self, x, y): + """Returns the name of the tab element at position x, y, or the + empty string if none.""" + return self.tk.call(self._w, "identify", x, y) + + + def index(self, tab_id): + """Returns the numeric index of the tab specified by tab_id, or + the total number of tabs if tab_id is the string "end".""" + return self.tk.getint(self.tk.call(self._w, "index", tab_id)) + + + def insert(self, pos, child, **kw): + """Inserts a pane at the specified position. + + pos is either the string end, an integer index, or the name of + a managed child. If child is already managed by the notebook, + moves it to the specified position.""" + self.tk.call(self._w, "insert", pos, child, *(_format_optdict(kw))) + + + def select(self, tab_id=None): + """Selects the specified tab. + + The associated child window will be displayed, and the + previously-selected window (if different) is unmapped. If tab_id + is omitted, returns the widget name of the currently selected + pane.""" + return self.tk.call(self._w, "select", tab_id) + + + def tab(self, tab_id, option=None, **kw): + """Query or modify the options of the specific tab_id. + + If kw is not given, returns a dict of the tab option values. If option + is specified, returns the value of that option. Otherwise, sets the + options to the corresponding values.""" + if option is not None: + kw[option] = None + return _val_or_dict(self.tk, kw, self._w, "tab", tab_id) + + + def tabs(self): + """Returns a list of windows managed by the notebook.""" + return self.tk.splitlist(self.tk.call(self._w, "tabs") or ()) + + + def enable_traversal(self): + """Enable keyboard traversal for a toplevel window containing + this notebook. + + This will extend the bindings for the toplevel window containing + this notebook as follows: + + Control-Tab: selects the tab following the currently selected + one + + Shift-Control-Tab: selects the tab preceding the currently + selected one + + Alt-K: where K is the mnemonic (underlined) character of any + tab, will select that tab. + + Multiple notebooks in a single toplevel may be enabled for + traversal, including nested notebooks. However, notebook traversal + only works properly if all panes are direct children of the + notebook.""" + # The only, and good, difference I see is about mnemonics, which works + # after calling this method. Control-Tab and Shift-Control-Tab always + # works (here at least). + self.tk.call("ttk::notebook::enableTraversal", self._w) + + +class Panedwindow(Widget, tkinter.PanedWindow): + """Ttk Panedwindow widget displays a number of subwindows, stacked + either vertically or horizontally.""" + + def __init__(self, master=None, **kw): + """Construct a Ttk Panedwindow with parent master. + + STANDARD OPTIONS + + class, cursor, style, takefocus + + WIDGET-SPECIFIC OPTIONS + + orient, width, height + + PANE OPTIONS + + weight + """ + Widget.__init__(self, master, "ttk::panedwindow", kw) + + + forget = tkinter.PanedWindow.forget # overrides Pack.forget + + + def insert(self, pos, child, **kw): + """Inserts a pane at the specified positions. + + pos is either the string end, and integer index, or the name + of a child. If child is already managed by the paned window, + moves it to the specified position.""" + self.tk.call(self._w, "insert", pos, child, *(_format_optdict(kw))) + + + def pane(self, pane, option=None, **kw): + """Query or modify the options of the specified pane. + + pane is either an integer index or the name of a managed subwindow. + If kw is not given, returns a dict of the pane option values. If + option is specified then the value for that option is returned. + Otherwise, sets the options to the corresponding values.""" + if option is not None: + kw[option] = None + return _val_or_dict(self.tk, kw, self._w, "pane", pane) + + + def sashpos(self, index, newpos=None): + """If newpos is specified, sets the position of sash number index. + + May adjust the positions of adjacent sashes to ensure that + positions are monotonically increasing. Sash positions are further + constrained to be between 0 and the total size of the widget. + + Returns the new position of sash number index.""" + return self.tk.getint(self.tk.call(self._w, "sashpos", index, newpos)) + +PanedWindow = Panedwindow # tkinter name compatibility + + +class Progressbar(Widget): + """Ttk Progressbar widget shows the status of a long-running + operation. They can operate in two modes: determinate mode shows the + amount completed relative to the total amount of work to be done, and + indeterminate mode provides an animated display to let the user know + that something is happening.""" + + def __init__(self, master=None, **kw): + """Construct a Ttk Progressbar with parent master. + + STANDARD OPTIONS + + class, cursor, style, takefocus + + WIDGET-SPECIFIC OPTIONS + + orient, length, mode, maximum, value, variable, phase + """ + Widget.__init__(self, master, "ttk::progressbar", kw) + + + def start(self, interval=None): + """Begin autoincrement mode: schedules a recurring timer event + that calls method step every interval milliseconds. + + interval defaults to 50 milliseconds (20 steps/second) if omitted.""" + self.tk.call(self._w, "start", interval) + + + def step(self, amount=None): + """Increments the value option by amount. + + amount defaults to 1.0 if omitted.""" + self.tk.call(self._w, "step", amount) + + + def stop(self): + """Stop autoincrement mode: cancels any recurring timer event + initiated by start.""" + self.tk.call(self._w, "stop") + + +class Radiobutton(Widget): + """Ttk Radiobutton widgets are used in groups to show or change a + set of mutually-exclusive options.""" + + def __init__(self, master=None, **kw): + """Construct a Ttk Radiobutton with parent master. + + STANDARD OPTIONS + + class, compound, cursor, image, state, style, takefocus, + text, textvariable, underline, width + + WIDGET-SPECIFIC OPTIONS + + command, value, variable + """ + Widget.__init__(self, master, "ttk::radiobutton", kw) + + + def invoke(self): + """Sets the option variable to the option value, selects the + widget, and invokes the associated command. + + Returns the result of the command, or an empty string if + no command is specified.""" + return self.tk.call(self._w, "invoke") + + +class Scale(Widget, tkinter.Scale): + """Ttk Scale widget is typically used to control the numeric value of + a linked variable that varies uniformly over some range.""" + + def __init__(self, master=None, **kw): + """Construct a Ttk Scale with parent master. + + STANDARD OPTIONS + + class, cursor, style, takefocus + + WIDGET-SPECIFIC OPTIONS + + command, from, length, orient, to, value, variable + """ + Widget.__init__(self, master, "ttk::scale", kw) + + + def configure(self, cnf=None, **kw): + """Modify or query scale options. + + Setting a value for any of the "from", "from_" or "to" options + generates a <> event.""" + retval = Widget.configure(self, cnf, **kw) + if not isinstance(cnf, (type(None), str)): + kw.update(cnf) + if any(['from' in kw, 'from_' in kw, 'to' in kw]): + self.event_generate('<>') + return retval + + + def get(self, x=None, y=None): + """Get the current value of the value option, or the value + corresponding to the coordinates x, y if they are specified. + + x and y are pixel coordinates relative to the scale widget + origin.""" + return self.tk.call(self._w, 'get', x, y) + + +class Scrollbar(Widget, tkinter.Scrollbar): + """Ttk Scrollbar controls the viewport of a scrollable widget.""" + + def __init__(self, master=None, **kw): + """Construct a Ttk Scrollbar with parent master. + + STANDARD OPTIONS + + class, cursor, style, takefocus + + WIDGET-SPECIFIC OPTIONS + + command, orient + """ + Widget.__init__(self, master, "ttk::scrollbar", kw) + + +class Separator(Widget): + """Ttk Separator widget displays a horizontal or vertical separator + bar.""" + + def __init__(self, master=None, **kw): + """Construct a Ttk Separator with parent master. + + STANDARD OPTIONS + + class, cursor, style, takefocus + + WIDGET-SPECIFIC OPTIONS + + orient + """ + Widget.__init__(self, master, "ttk::separator", kw) + + +class Sizegrip(Widget): + """Ttk Sizegrip allows the user to resize the containing toplevel + window by pressing and dragging the grip.""" + + def __init__(self, master=None, **kw): + """Construct a Ttk Sizegrip with parent master. + + STANDARD OPTIONS + + class, cursor, state, style, takefocus + """ + Widget.__init__(self, master, "ttk::sizegrip", kw) + + +class Spinbox(Entry): + """Ttk Spinbox is an Entry with increment and decrement arrows + + It is commonly used for number entry or to select from a list of + string values. + """ + + def __init__(self, master=None, **kw): + """Construct a Ttk Spinbox widget with the parent master. + + STANDARD OPTIONS + + class, cursor, style, takefocus, validate, + validatecommand, xscrollcommand, invalidcommand + + WIDGET-SPECIFIC OPTIONS + + to, from_, increment, values, wrap, format, command + """ + Entry.__init__(self, master, "ttk::spinbox", **kw) + + + def set(self, value): + """Sets the value of the Spinbox to value.""" + self.tk.call(self._w, "set", value) + + +class Treeview(Widget, tkinter.XView, tkinter.YView): + """Ttk Treeview widget displays a hierarchical collection of items. + + Each item has a textual label, an optional image, and an optional list + of data values. The data values are displayed in successive columns + after the tree label.""" + + def __init__(self, master=None, **kw): + """Construct a Ttk Treeview with parent master. + + STANDARD OPTIONS + + class, cursor, style, takefocus, xscrollcommand, + yscrollcommand + + WIDGET-SPECIFIC OPTIONS + + columns, displaycolumns, height, padding, selectmode, show + + ITEM OPTIONS + + text, image, values, open, tags + + TAG OPTIONS + + foreground, background, font, image + """ + Widget.__init__(self, master, "ttk::treeview", kw) + + + def bbox(self, item, column=None): + """Returns the bounding box (relative to the treeview widget's + window) of the specified item in the form x y width height. + + If column is specified, returns the bounding box of that cell. + If the item is not visible (i.e., if it is a descendant of a + closed item or is scrolled offscreen), returns an empty string.""" + return self._getints(self.tk.call(self._w, "bbox", item, column)) or '' + + + def get_children(self, item=None): + """Returns a tuple of children belonging to item. + + If item is not specified, returns root children.""" + return self.tk.splitlist( + self.tk.call(self._w, "children", item or '') or ()) + + + def set_children(self, item, *newchildren): + """Replaces item's child with newchildren. + + Children present in item that are not present in newchildren + are detached from tree. No items in newchildren may be an + ancestor of item.""" + self.tk.call(self._w, "children", item, newchildren) + + + def column(self, column, option=None, **kw): + """Query or modify the options for the specified column. + + If kw is not given, returns a dict of the column option values. If + option is specified then the value for that option is returned. + Otherwise, sets the options to the corresponding values.""" + if option is not None: + kw[option] = None + return _val_or_dict(self.tk, kw, self._w, "column", column) + + + def delete(self, *items): + """Delete all specified items and all their descendants. The root + item may not be deleted.""" + self.tk.call(self._w, "delete", items) + + + def detach(self, *items): + """Unlinks all of the specified items from the tree. + + The items and all of their descendants are still present, and may + be reinserted at another point in the tree, but will not be + displayed. The root item may not be detached.""" + self.tk.call(self._w, "detach", items) + + + def exists(self, item): + """Returns True if the specified item is present in the tree, + False otherwise.""" + return self.tk.getboolean(self.tk.call(self._w, "exists", item)) + + + def focus(self, item=None): + """If item is specified, sets the focus item to item. Otherwise, + returns the current focus item, or '' if there is none.""" + return self.tk.call(self._w, "focus", item) + + + def heading(self, column, option=None, **kw): + """Query or modify the heading options for the specified column. + + If kw is not given, returns a dict of the heading option values. If + option is specified then the value for that option is returned. + Otherwise, sets the options to the corresponding values. + + Valid options/values are: + text: text + The text to display in the column heading + image: image_name + Specifies an image to display to the right of the column + heading + anchor: anchor + Specifies how the heading text should be aligned. One of + the standard Tk anchor values + command: callback + A callback to be invoked when the heading label is + pressed. + + To configure the tree column heading, call this with column = "#0" """ + cmd = kw.get('command') + if cmd and not isinstance(cmd, str): + # callback not registered yet, do it now + kw['command'] = self.master.register(cmd, self._substitute) + + if option is not None: + kw[option] = None + + return _val_or_dict(self.tk, kw, self._w, 'heading', column) + + + def identify(self, component, x, y): + """Returns a description of the specified component under the + point given by x and y, or the empty string if no such component + is present at that position.""" + return self.tk.call(self._w, "identify", component, x, y) + + + def identify_row(self, y): + """Returns the item ID of the item at position y.""" + return self.identify("row", 0, y) + + + def identify_column(self, x): + """Returns the data column identifier of the cell at position x. + + The tree column has ID #0.""" + return self.identify("column", x, 0) + + + def identify_region(self, x, y): + """Returns one of: + + heading: Tree heading area. + separator: Space between two columns headings; + tree: The tree area. + cell: A data cell. + + * Availability: Tk 8.6""" + return self.identify("region", x, y) + + + def identify_element(self, x, y): + """Returns the element at position x, y. + + * Availability: Tk 8.6""" + return self.identify("element", x, y) + + + def index(self, item): + """Returns the integer index of item within its parent's list + of children.""" + return self.tk.getint(self.tk.call(self._w, "index", item)) + + + def insert(self, parent, index, iid=None, **kw): + """Creates a new item and return the item identifier of the newly + created item. + + parent is the item ID of the parent item, or the empty string + to create a new top-level item. index is an integer, or the value + end, specifying where in the list of parent's children to insert + the new item. If index is less than or equal to zero, the new node + is inserted at the beginning, if index is greater than or equal to + the current number of children, it is inserted at the end. If iid + is specified, it is used as the item identifier, iid must not + already exist in the tree. Otherwise, a new unique identifier + is generated.""" + opts = _format_optdict(kw) + if iid is not None: + res = self.tk.call(self._w, "insert", parent, index, + "-id", iid, *opts) + else: + res = self.tk.call(self._w, "insert", parent, index, *opts) + + return res + + + def item(self, item, option=None, **kw): + """Query or modify the options for the specified item. + + If no options are given, a dict with options/values for the item + is returned. If option is specified then the value for that option + is returned. Otherwise, sets the options to the corresponding + values as given by kw.""" + if option is not None: + kw[option] = None + return _val_or_dict(self.tk, kw, self._w, "item", item) + + + def move(self, item, parent, index): + """Moves item to position index in parent's list of children. + + It is illegal to move an item under one of its descendants. If + index is less than or equal to zero, item is moved to the + beginning, if greater than or equal to the number of children, + it is moved to the end. If item was detached it is reattached.""" + self.tk.call(self._w, "move", item, parent, index) + + reattach = move # A sensible method name for reattaching detached items + + + def next(self, item): + """Returns the identifier of item's next sibling, or '' if item + is the last child of its parent.""" + return self.tk.call(self._w, "next", item) + + + def parent(self, item): + """Returns the ID of the parent of item, or '' if item is at the + top level of the hierarchy.""" + return self.tk.call(self._w, "parent", item) + + + def prev(self, item): + """Returns the identifier of item's previous sibling, or '' if + item is the first child of its parent.""" + return self.tk.call(self._w, "prev", item) + + + def see(self, item): + """Ensure that item is visible. + + Sets all of item's ancestors open option to True, and scrolls + the widget if necessary so that item is within the visible + portion of the tree.""" + self.tk.call(self._w, "see", item) + + + def selection(self): + """Returns the tuple of selected items.""" + return self.tk.splitlist(self.tk.call(self._w, "selection")) + + + def _selection(self, selop, items): + if len(items) == 1 and isinstance(items[0], (tuple, list)): + items = items[0] + + self.tk.call(self._w, "selection", selop, items) + + + def selection_set(self, *items): + """The specified items becomes the new selection.""" + self._selection("set", items) + + + def selection_add(self, *items): + """Add all of the specified items to the selection.""" + self._selection("add", items) + + + def selection_remove(self, *items): + """Remove all of the specified items from the selection.""" + self._selection("remove", items) + + + def selection_toggle(self, *items): + """Toggle the selection state of each specified item.""" + self._selection("toggle", items) + + + def set(self, item, column=None, value=None): + """Query or set the value of given item. + + With one argument, return a dictionary of column/value pairs + for the specified item. With two arguments, return the current + value of the specified column. With three arguments, set the + value of given column in given item to the specified value.""" + res = self.tk.call(self._w, "set", item, column, value) + if column is None and value is None: + return _splitdict(self.tk, res, + cut_minus=False, conv=_tclobj_to_py) + else: + return res + + + def tag_bind(self, tagname, sequence=None, callback=None): + """Bind a callback for the given event sequence to the tag tagname. + When an event is delivered to an item, the callbacks for each + of the item's tags option are called.""" + self._bind((self._w, "tag", "bind", tagname), sequence, callback, add=0) + + + def tag_configure(self, tagname, option=None, **kw): + """Query or modify the options for the specified tagname. + + If kw is not given, returns a dict of the option settings for tagname. + If option is specified, returns the value for that option for the + specified tagname. Otherwise, sets the options to the corresponding + values for the given tagname.""" + if option is not None: + kw[option] = None + return _val_or_dict(self.tk, kw, self._w, "tag", "configure", + tagname) + + + def tag_has(self, tagname, item=None): + """If item is specified, returns 1 or 0 depending on whether the + specified item has the given tagname. Otherwise, returns a list of + all items which have the specified tag. + + * Availability: Tk 8.6""" + if item is None: + return self.tk.splitlist( + self.tk.call(self._w, "tag", "has", tagname)) + else: + return self.tk.getboolean( + self.tk.call(self._w, "tag", "has", tagname, item)) + + +# Extensions + +class LabeledScale(Frame): + """A Ttk Scale widget with a Ttk Label widget indicating its + current value. + + The Ttk Scale can be accessed through instance.scale, and Ttk Label + can be accessed through instance.label""" + + def __init__(self, master=None, variable=None, from_=0, to=10, **kw): + """Construct a horizontal LabeledScale with parent master, a + variable to be associated with the Ttk Scale widget and its range. + If variable is not specified, a tkinter.IntVar is created. + + WIDGET-SPECIFIC OPTIONS + + compound: 'top' or 'bottom' + Specifies how to display the label relative to the scale. + Defaults to 'top'. + """ + self._label_top = kw.pop('compound', 'top') == 'top' + + Frame.__init__(self, master, **kw) + self._variable = variable or tkinter.IntVar(master) + self._variable.set(from_) + self._last_valid = from_ + + self.label = Label(self) + self.scale = Scale(self, variable=self._variable, from_=from_, to=to) + self.scale.bind('<>', self._adjust) + + # position scale and label according to the compound option + scale_side = 'bottom' if self._label_top else 'top' + label_side = 'top' if scale_side == 'bottom' else 'bottom' + self.scale.pack(side=scale_side, fill='x') + # Dummy required to make frame correct height + dummy = Label(self) + dummy.pack(side=label_side) + dummy.lower() + self.label.place(anchor='n' if label_side == 'top' else 's') + + # update the label as scale or variable changes + self.__tracecb = self._variable.trace_add('write', self._adjust) + self.bind('', self._adjust) + self.bind('', self._adjust) + + + def destroy(self): + """Destroy this widget and possibly its associated variable.""" + try: + self._variable.trace_remove('write', self.__tracecb) + except AttributeError: + pass + else: + del self._variable + super().destroy() + self.label = None + self.scale = None + + + def _adjust(self, *args): + """Adjust the label position according to the scale.""" + def adjust_label(): + self.update_idletasks() # "force" scale redraw + + x, y = self.scale.coords() + if self._label_top: + y = self.scale.winfo_y() - self.label.winfo_reqheight() + else: + y = self.scale.winfo_reqheight() + self.label.winfo_reqheight() + + self.label.place_configure(x=x, y=y) + + from_ = _to_number(self.scale['from']) + to = _to_number(self.scale['to']) + if to < from_: + from_, to = to, from_ + newval = self._variable.get() + if not from_ <= newval <= to: + # value outside range, set value back to the last valid one + self.value = self._last_valid + return + + self._last_valid = newval + self.label['text'] = newval + self.after_idle(adjust_label) + + @property + def value(self): + """Return current scale value.""" + return self._variable.get() + + @value.setter + def value(self, val): + """Set new scale value.""" + self._variable.set(val) + + +class OptionMenu(Menubutton): + """Themed OptionMenu, based after tkinter's OptionMenu, which allows + the user to select a value from a menu.""" + + def __init__(self, master, variable, default=None, *values, **kwargs): + """Construct a themed OptionMenu widget with master as the parent, + the resource textvariable set to variable, the initially selected + value specified by the default parameter, the menu values given by + *values and additional keywords. + + WIDGET-SPECIFIC OPTIONS + + style: stylename + Menubutton style. + direction: 'above', 'below', 'left', 'right', or 'flush' + Menubutton direction. + command: callback + A callback that will be invoked after selecting an item. + """ + kw = {'textvariable': variable, 'style': kwargs.pop('style', None), + 'direction': kwargs.pop('direction', None)} + Menubutton.__init__(self, master, **kw) + self['menu'] = tkinter.Menu(self, tearoff=False) + + self._variable = variable + self._callback = kwargs.pop('command', None) + if kwargs: + raise tkinter.TclError('unknown option -%s' % ( + next(iter(kwargs.keys())))) + + self.set_menu(default, *values) + + + def __getitem__(self, item): + if item == 'menu': + return self.nametowidget(Menubutton.__getitem__(self, item)) + + return Menubutton.__getitem__(self, item) + + + def set_menu(self, default=None, *values): + """Build a new menu of radiobuttons with *values and optionally + a default value.""" + menu = self['menu'] + menu.delete(0, 'end') + for val in values: + menu.add_radiobutton(label=val, + command=( + None if self._callback is None + else lambda val=val: self._callback(val) + ), + variable=self._variable) + + if default: + self._variable.set(default) + + + def destroy(self): + """Destroy this widget and its associated variable.""" + try: + del self._variable + except AttributeError: + pass + super().destroy() diff --git a/stdlib/Cargo.toml b/stdlib/Cargo.toml index d7552844be..b46fb16b8f 100644 --- a/stdlib/Cargo.toml +++ b/stdlib/Cargo.toml @@ -18,6 +18,7 @@ bz2 = ["bzip2"] sqlite = ["dep:libsqlite3-sys"] ssl = ["openssl", "openssl-sys", "foreign-types-shared", "openssl-probe"] ssl-vendor = ["ssl", "openssl/vendored"] +tkinter = ["dep:tk", "dep:tcl"] [dependencies] # rustpython crates @@ -83,6 +84,10 @@ flate2 = { version = "1.1", default-features = false, features = ["zlib-rs"] } libz-sys = { package = "libz-rs-sys", version = "0.4" } bzip2 = { version = "0.4", optional = true } +# tkinter +tk = { version = "0.1.10", optional = true } +tcl = { version = "0.1.9", optional = true } + # uuid [target.'cfg(not(any(target_os = "ios", target_os = "android", target_os = "windows", target_arch = "wasm32", target_os = "redox")))'.dependencies] mac_address = "1.1.3" diff --git a/stdlib/src/lib.rs b/stdlib/src/lib.rs index 33eb114f0a..9ee8e3e81d 100644 --- a/stdlib/src/lib.rs +++ b/stdlib/src/lib.rs @@ -80,6 +80,9 @@ mod termios; )))] mod uuid; +#[cfg(feature = "tkinter")] +mod tkinter; + use rustpython_common as common; use rustpython_vm as vm; @@ -192,5 +195,9 @@ pub fn get_module_inits() -> impl Iterator, StdlibInit { "_locale" => locale::make_module, } + #[cfg(feature = "tkinter")] + { + "_tkinter" => tkinter::make_module, + } } } diff --git a/stdlib/src/tkinter.rs b/stdlib/src/tkinter.rs new file mode 100644 index 0000000000..1d14c9f38c --- /dev/null +++ b/stdlib/src/tkinter.rs @@ -0,0 +1,84 @@ +pub(crate) use self::_tkinter::make_module; + +#[pymodule] +mod _tkinter { + use crate::builtins::PyTypeRef; + use rustpython_vm::function::{Either, FuncArgs}; + use rustpython_vm::{PyResult, VirtualMachine, function::OptionalArg}; + + use crate::common::lock::PyRwLock; + use std::sync::Arc; + use tk::cmd::*; + use tk::*; + + #[pyattr] + const TK_VERSION: &str = "8.6"; + #[pyattr] + const TCL_VERSION: &str = "8.6"; + #[pyattr] + const READABLE: i32 = 2; + #[pyattr] + const WRITABLE: i32 = 4; + #[pyattr] + const EXCEPTION: i32 = 8; + + fn demo() -> tk::TkResult<()> { + let tk = make_tk!()?; + let root = tk.root(); + root.add_label(-text("constructs widgets and layout step by step"))? + .pack(())?; + let f = root.add_frame(())?.pack(())?; + let _btn = f + .add_button("btn" - text("quit") - command("destroy ."))? + .pack(())?; + Ok(main_loop()) + } + + #[pyattr(once, name = "TclError")] + fn tcl_error(vm: &VirtualMachine) -> PyTypeRef { + vm.ctx.new_exception_type( + "zlib", + "TclError", + Some(vec![vm.ctx.exceptions.exception_type.to_owned()]), + ) + } + + #[pyfunction] + fn create(args: FuncArgs, _vm: &VirtualMachine) -> PyResult { + // TODO: handle arguements + // TODO: this means creating 2 tk instances is not possible. + let tk = Tk::new(()).unwrap(); + Ok(TkApp { + tk: Arc::new(PyRwLock::new(tk)), + }) + } + + #[pyattr] + #[pyclass(name = "tkapp")] + #[derive(PyPayload)] + struct TkApp { + tk: Arc>>, + } + + unsafe impl Send for TkApp {} + + unsafe impl Sync for TkApp {} + + impl std::fmt::Debug for TkApp { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TkApp").finish() + } + } + + #[pyclass] + impl TkApp { + #[pymethod] + fn getvar(&self, name: &str) -> PyResult { + let tk = self.tk.read().unwrap(); + Ok(tk.getvar(name).unwrap()) + } + + #[pymethod] + fn createcommand(&self, name: String, callback: PyObjectRef) {} + } +} From 01d470ff77540298821c75d3f98bc79b3536b4d8 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Mon, 24 Mar 2025 17:09:29 -0700 Subject: [PATCH 103/295] _ctypes pt. 4 (#5582) * correct error type when symbol is not found * restype get/set * base of PyCSimple for PyCArray * PyCSimple::to_arg * par down ctypes * nt._LOAD_LIBRARY_SEARCH_DEFAULT_DIRS * arguments for ctypes function * force failure to import ctypes --- Lib/ctypes/__init__.py | 23 +++-- extra_tests/snippets/builtins_ctypes.py | 63 +++++++++++++- vm/src/stdlib/ctypes.rs | 68 ++++++++++++++- vm/src/stdlib/ctypes/array.rs | 110 +++++++++++++++++++++++- vm/src/stdlib/ctypes/base.rs | 60 +++++++++++++ vm/src/stdlib/ctypes/function.rs | 70 ++++++++++++--- vm/src/stdlib/nt.rs | 3 + 7 files changed, 368 insertions(+), 29 deletions(-) diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index 2e9d4c5e72..b8b005061f 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -36,6 +36,9 @@ FUNCFLAG_USE_ERRNO as _FUNCFLAG_USE_ERRNO, \ FUNCFLAG_USE_LASTERROR as _FUNCFLAG_USE_LASTERROR +# TODO: RUSTPYTHON remove this +from _ctypes import _non_existing_function + # WINOLEAPI -> HRESULT # WINOLEAPI_(type) # @@ -296,7 +299,9 @@ def create_unicode_buffer(init, size=None): return buf elif isinstance(init, int): _sys.audit("ctypes.create_unicode_buffer", None, init) - buftype = c_wchar * init + # XXX: RUSTPYTHON + # buftype = c_wchar * init + buftype = c_wchar.__mul__(init) buf = buftype() return buf raise TypeError(init) @@ -495,14 +500,15 @@ def WinError(code=None, descr=None): c_ssize_t = c_longlong # functions - from _ctypes import _memmove_addr, _memset_addr, _string_at_addr, _cast_addr ## void *memmove(void *, const void *, size_t); -memmove = CFUNCTYPE(c_void_p, c_void_p, c_void_p, c_size_t)(_memmove_addr) +# XXX: RUSTPYTHON +# memmove = CFUNCTYPE(c_void_p, c_void_p, c_void_p, c_size_t)(_memmove_addr) ## void *memset(void *, int, size_t) -memset = CFUNCTYPE(c_void_p, c_void_p, c_int, c_size_t)(_memset_addr) +# XXX: RUSTPYTHON +# memset = CFUNCTYPE(c_void_p, c_void_p, c_int, c_size_t)(_memset_addr) def PYFUNCTYPE(restype, *argtypes): class CFunctionType(_CFuncPtr): @@ -511,11 +517,13 @@ class CFunctionType(_CFuncPtr): _flags_ = _FUNCFLAG_CDECL | _FUNCFLAG_PYTHONAPI return CFunctionType -_cast = PYFUNCTYPE(py_object, c_void_p, py_object, py_object)(_cast_addr) +# XXX: RUSTPYTHON +# _cast = PYFUNCTYPE(py_object, c_void_p, py_object, py_object)(_cast_addr) def cast(obj, typ): return _cast(obj, obj, typ) -_string_at = PYFUNCTYPE(py_object, c_void_p, c_int)(_string_at_addr) +# XXX: RUSTPYTHON +# _string_at = PYFUNCTYPE(py_object, c_void_p, c_int)(_string_at_addr) def string_at(ptr, size=-1): """string_at(addr[, size]) -> string @@ -527,7 +535,8 @@ def string_at(ptr, size=-1): except ImportError: pass else: - _wstring_at = PYFUNCTYPE(py_object, c_void_p, c_int)(_wstring_at_addr) + # XXX: RUSTPYTHON + # _wstring_at = PYFUNCTYPE(py_object, c_void_p, c_int)(_wstring_at_addr) def wstring_at(ptr, size=-1): """wstring_at(addr[, size]) -> string diff --git a/extra_tests/snippets/builtins_ctypes.py b/extra_tests/snippets/builtins_ctypes.py index c5c563a48e..2108ce41a4 100644 --- a/extra_tests/snippets/builtins_ctypes.py +++ b/extra_tests/snippets/builtins_ctypes.py @@ -33,13 +33,16 @@ def create_string_buffer(init, size=None): if size is None: size = len(init)+1 _sys.audit("ctypes.create_string_buffer", init, size) - buftype = c_char * size + buftype = c_char.__mul__(size) + print(type(c_char.__mul__(size))) + # buftype = c_char * size buf = buftype() buf.value = init return buf elif isinstance(init, int): _sys.audit("ctypes.create_string_buffer", None, init) - buftype = c_char * init + buftype = c_char.__mul__(init) + # buftype = c_char * init buf = buftype() return buf raise TypeError(init) @@ -260,8 +263,62 @@ def LoadLibrary(self, name): cdll = LibraryLoader(CDLL) +test_byte_array = create_string_buffer(b"Hello, World!\n") +assert test_byte_array._length_ == 15 + if _os.name == "posix" or _sys.platform == "darwin": pass else: + import os + libc = cdll.msvcrt - print("rand", libc.rand()) + libc.rand() + i = c_int(1) + print("start srand") + print(libc.srand(i)) + print(test_byte_array) + print(test_byte_array._type_) + # print("start printf") + # libc.printf(test_byte_array) + + # windows pip support + + def get_win_folder_via_ctypes(csidl_name: str) -> str: + """Get folder with ctypes.""" + # There is no 'CSIDL_DOWNLOADS'. + # Use 'CSIDL_PROFILE' (40) and append the default folder 'Downloads' instead. + # https://learn.microsoft.com/en-us/windows/win32/shell/knownfolderid + + import ctypes # noqa: PLC0415 + + csidl_const = { + "CSIDL_APPDATA": 26, + "CSIDL_COMMON_APPDATA": 35, + "CSIDL_LOCAL_APPDATA": 28, + "CSIDL_PERSONAL": 5, + "CSIDL_MYPICTURES": 39, + "CSIDL_MYVIDEO": 14, + "CSIDL_MYMUSIC": 13, + "CSIDL_DOWNLOADS": 40, + "CSIDL_DESKTOPDIRECTORY": 16, + }.get(csidl_name) + if csidl_const is None: + msg = f"Unknown CSIDL name: {csidl_name}" + raise ValueError(msg) + + buf = ctypes.create_unicode_buffer(1024) + windll = getattr(ctypes, "windll") # noqa: B009 # using getattr to avoid false positive with mypy type checker + windll.shell32.SHGetFolderPathW(None, csidl_const, None, 0, buf) + + # Downgrade to short path name if it has high-bit chars. + if any(ord(c) > 255 for c in buf): # noqa: PLR2004 + buf2 = ctypes.create_unicode_buffer(1024) + if windll.kernel32.GetShortPathNameW(buf.value, buf2, 1024): + buf = buf2 + + if csidl_name == "CSIDL_DOWNLOADS": + return os.path.join(buf.value, "Downloads") # noqa: PTH118 + + return buf.value + + # print(get_win_folder_via_ctypes("CSIDL_DOWNLOADS")) diff --git a/vm/src/stdlib/ctypes.rs b/vm/src/stdlib/ctypes.rs index 99866bae70..b6fd150889 100644 --- a/vm/src/stdlib/ctypes.rs +++ b/vm/src/stdlib/ctypes.rs @@ -17,6 +17,7 @@ pub fn extend_module_nodes(vm: &VirtualMachine, module: &Py) { extend_module!(vm, module, { "_CData" => PyCData::make_class(ctx), "_SimpleCData" => PyCSimple::make_class(ctx), + "ArrayType" => array::PyCArrayType::make_class(ctx), "Array" => array::PyCArray::make_class(ctx), "CFuncPtr" => function::PyCFuncPtr::make_class(ctx), "_Pointer" => pointer::PyCPointer::make_class(ctx), @@ -37,7 +38,7 @@ pub(crate) mod _ctypes { use super::base::PyCSimple; use crate::builtins::PyTypeRef; use crate::class::StaticType; - use crate::function::{Either, OptionalArg}; + use crate::function::{Either, FuncArgs, OptionalArg}; use crate::stdlib::ctypes::library; use crate::{AsObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine}; use crossbeam_utils::atomic::AtomicCell; @@ -124,11 +125,12 @@ pub(crate) mod _ctypes { "d" | "g" => mem::size_of::(), "?" | "B" => mem::size_of::(), "P" | "z" | "Z" => mem::size_of::(), + "O" => mem::size_of::(), _ => unreachable!(), } } - const SIMPLE_TYPE_CHARS: &str = "cbBhHiIlLdfguzZPqQ?"; + const SIMPLE_TYPE_CHARS: &str = "cbBhHiIlLdfguzZPqQ?O"; pub fn new_simple_type( cls: Either<&PyObjectRef, &PyTypeRef>, @@ -218,9 +220,14 @@ pub(crate) mod _ctypes { #[pyfunction(name = "POINTER")] pub fn pointer(_cls: PyTypeRef) {} - #[pyfunction] + #[pyfunction(name = "pointer")] pub fn pointer_fn(_inst: PyObjectRef) {} + #[pyfunction] + fn _pointer_type_cache() -> PyObjectRef { + todo!() + } + #[cfg(target_os = "windows")] #[pyfunction(name = "_check_HRESULT")] pub fn check_hresult(_self: PyObjectRef, hr: i32, _vm: &VirtualMachine) -> PyResult { @@ -243,6 +250,24 @@ pub(crate) mod _ctypes { } } + #[pyfunction] + fn byref(_args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + // TODO: RUSTPYTHON + Err(vm.new_value_error("not implemented".to_string())) + } + + #[pyfunction] + fn alignment(_args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + // TODO: RUSTPYTHON + Err(vm.new_value_error("not implemented".to_string())) + } + + #[pyfunction] + fn resize(_args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + // TODO: RUSTPYTHON + Err(vm.new_value_error("not implemented".to_string())) + } + #[pyfunction] fn get_errno() -> i32 { errno::errno().0 @@ -252,4 +277,41 @@ pub(crate) mod _ctypes { fn set_errno(value: i32) { errno::set_errno(errno::Errno(value)); } + + #[cfg(windows)] + #[pyfunction] + fn get_last_error() -> PyResult { + Ok(unsafe { windows_sys::Win32::Foundation::GetLastError() }) + } + + #[cfg(windows)] + #[pyfunction] + fn set_last_error(value: u32) -> PyResult<()> { + unsafe { windows_sys::Win32::Foundation::SetLastError(value) }; + Ok(()) + } + + #[pyattr] + fn _memmove_addr(_vm: &VirtualMachine) -> usize { + let f = libc::memmove; + f as usize + } + + #[pyattr] + fn _memset_addr(_vm: &VirtualMachine) -> usize { + let f = libc::memset; + f as usize + } + + #[pyattr] + fn _string_at_addr(_vm: &VirtualMachine) -> usize { + let f = libc::strnlen; + f as usize + } + + #[pyattr] + fn _cast_addr(_vm: &VirtualMachine) -> usize { + // TODO: RUSTPYTHON + 0 + } } diff --git a/vm/src/stdlib/ctypes/array.rs b/vm/src/stdlib/ctypes/array.rs index 8b023582c9..44d237cd7e 100644 --- a/vm/src/stdlib/ctypes/array.rs +++ b/vm/src/stdlib/ctypes/array.rs @@ -1,5 +1,107 @@ -#[pyclass(name = "Array", module = "_ctypes")] -pub struct PyCArray {} +use crate::builtins::PyBytes; +use crate::types::Callable; +use crate::{Py, PyObjectRef, PyPayload}; +use crate::{PyResult, VirtualMachine, builtins::PyTypeRef, types::Constructor}; +use crossbeam_utils::atomic::AtomicCell; +use rustpython_common::lock::PyRwLock; +use rustpython_vm::stdlib::ctypes::base::PyCSimple; -#[pyclass(flags(BASETYPE, IMMUTABLETYPE))] -impl PyCArray {} +// TODO: make it metaclass +#[pyclass(name = "ArrayType", module = "_ctypes")] +#[derive(PyPayload)] +pub struct PyCArrayType { + pub(super) inner: PyCArray, +} + +impl std::fmt::Debug for PyCArrayType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PyCArrayType") + .field("inner", &self.inner) + .finish() + } +} + +impl Callable for PyCArrayType { + type Args = (); + fn call(zelf: &Py, _args: Self::Args, vm: &VirtualMachine) -> PyResult { + Ok(PyCArray { + typ: PyRwLock::new(zelf.inner.typ.read().clone()), + length: AtomicCell::new(zelf.inner.length.load()), + value: PyRwLock::new(zelf.inner.value.read().clone()), + } + .into_pyobject(vm)) + } +} + +impl Constructor for PyCArrayType { + type Args = PyObjectRef; + + fn py_new(_cls: PyTypeRef, _args: Self::Args, _vm: &VirtualMachine) -> PyResult { + unreachable!() + } +} + +#[pyclass(flags(IMMUTABLETYPE), with(Callable, Constructor))] +impl PyCArrayType {} + +#[pyclass(name = "Array", base = "PyCSimple", module = "_ctypes")] +#[derive(PyPayload)] +pub struct PyCArray { + pub(super) typ: PyRwLock, + pub(super) length: AtomicCell, + pub(super) value: PyRwLock, +} + +impl std::fmt::Debug for PyCArray { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PyCArray") + .field("typ", &self.typ) + .field("length", &self.length) + .finish() + } +} + +impl Constructor for PyCArray { + type Args = (PyTypeRef, usize); + + fn py_new(_cls: PyTypeRef, args: Self::Args, vm: &VirtualMachine) -> PyResult { + Ok(Self { + typ: PyRwLock::new(args.0), + length: AtomicCell::new(args.1), + value: PyRwLock::new(vm.ctx.none()), + } + .into_pyobject(vm)) + } +} + +#[pyclass(flags(BASETYPE, IMMUTABLETYPE), with(Constructor))] +impl PyCArray { + #[pygetset(name = "_type_")] + fn typ(&self) -> PyTypeRef { + self.typ.read().clone() + } + + #[pygetset(name = "_length_")] + fn length(&self) -> usize { + self.length.load() + } + + #[pygetset] + fn value(&self) -> PyObjectRef { + self.value.read().clone() + } + + #[pygetset(setter)] + fn set_value(&self, value: PyObjectRef) { + *self.value.write() = value; + } +} + +impl PyCArray { + pub fn to_arg(&self, _vm: &VirtualMachine) -> PyResult { + let value = self.value.read(); + let py_bytes = value.payload::().unwrap(); + let bytes = py_bytes.as_ref().to_vec(); + Ok(libffi::middle::Arg::new(&bytes)) + } +} diff --git a/vm/src/stdlib/ctypes/base.rs b/vm/src/stdlib/ctypes/base.rs index 5c5396be29..d07ccba30b 100644 --- a/vm/src/stdlib/ctypes/base.rs +++ b/vm/src/stdlib/ctypes/base.rs @@ -1,3 +1,4 @@ +use super::array::{PyCArray, PyCArrayType}; use crate::builtins::PyType; use crate::builtins::{PyBytes, PyFloat, PyInt, PyNone, PyStr, PyTypeRef}; use crate::convert::ToPyObject; @@ -246,4 +247,63 @@ impl PyCSimple { zelf.value.store(content); Ok(()) } + + #[pyclassmethod] + fn repeat(cls: PyTypeRef, n: isize, vm: &VirtualMachine) -> PyResult { + if n < 0 { + return Err(vm.new_value_error(format!("Array length must be >= 0, not {}", n))); + } + Ok(PyCArrayType { + inner: PyCArray { + typ: PyRwLock::new(cls), + length: AtomicCell::new(n as usize), + value: PyRwLock::new(vm.ctx.none()), + }, + } + .to_pyobject(vm)) + } + + #[pyclassmethod(magic)] + fn mul(cls: PyTypeRef, n: isize, vm: &VirtualMachine) -> PyResult { + PyCSimple::repeat(cls, n, vm) + } +} + +impl PyCSimple { + pub fn to_arg( + &self, + ty: libffi::middle::Type, + vm: &VirtualMachine, + ) -> Option { + let value = unsafe { (*self.value.as_ptr()).clone() }; + if let Ok(i) = value.try_int(vm) { + let i = i.as_bigint(); + if ty.as_raw_ptr() == libffi::middle::Type::u8().as_raw_ptr() { + return i.to_u8().map(|r: u8| libffi::middle::Arg::new(&r)); + } else if ty.as_raw_ptr() == libffi::middle::Type::i8().as_raw_ptr() { + return i.to_i8().map(|r: i8| libffi::middle::Arg::new(&r)); + } else if ty.as_raw_ptr() == libffi::middle::Type::u16().as_raw_ptr() { + return i.to_u16().map(|r: u16| libffi::middle::Arg::new(&r)); + } else if ty.as_raw_ptr() == libffi::middle::Type::i16().as_raw_ptr() { + return i.to_i16().map(|r: i16| libffi::middle::Arg::new(&r)); + } else if ty.as_raw_ptr() == libffi::middle::Type::u32().as_raw_ptr() { + return i.to_u32().map(|r: u32| libffi::middle::Arg::new(&r)); + } else if ty.as_raw_ptr() == libffi::middle::Type::i32().as_raw_ptr() { + return i.to_i32().map(|r: i32| libffi::middle::Arg::new(&r)); + } else if ty.as_raw_ptr() == libffi::middle::Type::u64().as_raw_ptr() { + return i.to_u64().map(|r: u64| libffi::middle::Arg::new(&r)); + } else if ty.as_raw_ptr() == libffi::middle::Type::i64().as_raw_ptr() { + return i.to_i64().map(|r: i64| libffi::middle::Arg::new(&r)); + } else { + return None; + } + } + if let Ok(_f) = value.try_float(vm) { + todo!(); + } + if let Ok(_b) = value.try_to_bool(vm) { + todo!(); + } + None + } } diff --git a/vm/src/stdlib/ctypes/function.rs b/vm/src/stdlib/ctypes/function.rs index 7d8dc0386a..c1c1230e03 100644 --- a/vm/src/stdlib/ctypes/function.rs +++ b/vm/src/stdlib/ctypes/function.rs @@ -2,6 +2,7 @@ use crate::builtins::{PyStr, PyTupleRef, PyTypeRef}; use crate::convert::ToPyObject; use crate::function::FuncArgs; use crate::stdlib::ctypes::PyCData; +use crate::stdlib::ctypes::array::PyCArray; use crate::stdlib::ctypes::base::{PyCSimple, ffi_type_from_str}; use crate::types::{Callable, Constructor}; use crate::{Py, PyObjectRef, PyResult, VirtualMachine}; @@ -16,6 +17,7 @@ use std::fmt::Debug; #[derive(Debug)] pub struct Function { + args: Vec, // TODO: no protection from use-after-free pointer: CodePtr, cif: Cif, @@ -38,8 +40,36 @@ impl Function { let args = args .iter() .map(|arg| { - if let Some(data) = arg.downcast_ref::() { - Ok(ffi_type_from_str(&data._type_).unwrap()) + if let Some(arg) = arg.payload_if_subclass::(vm) { + let converted = ffi_type_from_str(&arg._type_); + return match converted { + Some(t) => Ok(t), + None => Err(vm.new_type_error("Invalid type".to_string())), // TODO: add type name + }; + } + if let Some(arg) = arg.payload_if_subclass::(vm) { + let t = arg.typ.read(); + let ty_attributes = t.attributes.read(); + let ty_pystr = + ty_attributes + .get(vm.ctx.intern_str("_type_")) + .ok_or_else(|| { + vm.new_type_error("Expected a ctypes simple type".to_string()) + })?; + let ty_str = ty_pystr + .downcast_ref::() + .ok_or_else(|| { + vm.new_type_error("Expected a ctypes simple type".to_string()) + })? + .to_string(); + let converted = ffi_type_from_str(&ty_str); + match converted { + Some(_t) => { + // TODO: Use + Ok(Type::void()) + } + None => Err(vm.new_type_error("Invalid type".to_string())), // TODO: add type name + } } else { Err(vm.new_type_error("Expected a ctypes simple type".to_string())) } @@ -50,7 +80,7 @@ impl Function { library .get(terminated.as_bytes()) .map_err(|err| err.to_string()) - .map_err(|err| vm.new_value_error(err))? + .map_err(|err| vm.new_attribute_error(err))? }; let code_ptr = CodePtr(*pointer as *mut _); let return_type = match ret_type { @@ -60,8 +90,9 @@ impl Function { } None => Type::c_int(), }; - let cif = Cif::new(args, return_type); + let cif = Cif::new(args.clone(), return_type); Ok(Function { + args, cif, pointer: code_ptr, }) @@ -74,16 +105,19 @@ impl Function { ) -> PyResult { let args = args .into_iter() - .map(|arg| { - if let Some(data) = arg.downcast_ref::() { - dbg!(&data); - todo!("HANDLE ARGUMENTS") - } else { - Err(vm.new_type_error("Expected a ctypes simple type".to_string())) + .enumerate() + .map(|(count, arg)| { + // none type check + if let Some(d) = arg.payload_if_subclass::(vm) { + return Ok(d.to_arg(self.args[count].clone(), vm).unwrap()); + } + if let Some(d) = arg.payload_if_subclass::(vm) { + return Ok(d.to_arg(vm).unwrap()); } + Err(vm.new_type_error("Expected a ctypes simple type".to_string())) }) .collect::>>()?; - // TODO: FIX return type + // TODO: FIX return let result: i32 = unsafe { self.cif.call(self.pointer, &args) }; Ok(vm.ctx.new_int(result).into()) } @@ -151,7 +185,9 @@ impl Callable for PyCFuncPtr { let name = zelf.name.read(); let res_type = zelf._restype_.read(); let func = Function::load( - inner_lib.as_ref().unwrap(), + inner_lib + .as_ref() + .ok_or_else(|| vm.new_value_error("Library not found".to_string()))?, &name, &args.args, &res_type, @@ -173,4 +209,14 @@ impl PyCFuncPtr { fn set_name(&self, name: String) { *self.name.write() = name; } + + #[pygetset(name = "_restype_")] + fn restype(&self) -> Option { + self._restype_.read().as_ref().cloned() + } + + #[pygetset(name = "_restype_", setter)] + fn set_restype(&self, restype: PyTypeRef) { + *self._restype_.write() = Some(restype); + } } diff --git a/vm/src/stdlib/nt.rs b/vm/src/stdlib/nt.rs index ecc63e0aa6..48f1ab668b 100644 --- a/vm/src/stdlib/nt.rs +++ b/vm/src/stdlib/nt.rs @@ -34,6 +34,9 @@ pub(crate) mod module { #[pyattr] use libc::{O_BINARY, O_TEMPORARY}; + #[pyattr] + const _LOAD_LIBRARY_SEARCH_DEFAULT_DIRS: i32 = 4096; + #[pyfunction] pub(super) fn access(path: OsPath, mode: u8, vm: &VirtualMachine) -> PyResult { let attr = unsafe { FileSystem::GetFileAttributesW(path.to_widecstring(vm)?.as_ptr()) }; From e3a1031081ffa2b2e3c4a09c531d726b1b0e9d28 Mon Sep 17 00:00:00 2001 From: Noa Date: Tue, 25 Mar 2025 11:37:51 -0500 Subject: [PATCH 104/295] Refactor cformat in prep for wtf8 --- common/src/cformat.rs | 570 +++++++++++++++++++++++------------------ common/src/format.rs | 9 +- vm/src/anystr.rs | 7 - vm/src/builtins/str.rs | 8 +- vm/src/bytesinner.rs | 4 - vm/src/cformat.rs | 159 ++++++------ vm/src/dictdatatype.rs | 71 ++++- 7 files changed, 463 insertions(+), 365 deletions(-) diff --git a/common/src/cformat.rs b/common/src/cformat.rs index cd282eb4cb..94762aceab 100644 --- a/common/src/cformat.rs +++ b/common/src/cformat.rs @@ -1,6 +1,7 @@ //! Implementation of Printf-Style string formatting //! as per the [Python Docs](https://docs.python.org/3/library/stdtypes.html#printf-style-string-formatting). use bitflags::bitflags; +use itertools::Itertools; use malachite_bigint::{BigInt, Sign}; use num_traits::Signed; use rustpython_literal::{float, format::Case}; @@ -48,29 +49,64 @@ impl fmt::Display for CFormatError { pub type CFormatConversion = super::format::FormatConversion; -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Clone, Copy)] +#[repr(u8)] pub enum CNumberType { - Decimal, - Octal, - Hex(Case), + DecimalD = b'd', + DecimalI = b'i', + DecimalU = b'u', + Octal = b'o', + HexLower = b'x', + HexUpper = b'X', } -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Clone, Copy)] +#[repr(u8)] pub enum CFloatType { - Exponent(Case), - PointDecimal(Case), - General(Case), + ExponentLower = b'e', + ExponentUpper = b'E', + PointDecimalLower = b'f', + PointDecimalUpper = b'F', + GeneralLower = b'g', + GeneralUpper = b'G', } -#[derive(Debug, PartialEq)] +impl CFloatType { + fn case(self) -> Case { + use CFloatType::*; + match self { + ExponentLower | PointDecimalLower | GeneralLower => Case::Lower, + ExponentUpper | PointDecimalUpper | GeneralUpper => Case::Upper, + } + } +} + +#[derive(Debug, PartialEq, Clone, Copy)] +#[repr(u8)] +pub enum CCharacterType { + Character = b'c', +} + +#[derive(Debug, PartialEq, Clone, Copy)] pub enum CFormatType { Number(CNumberType), Float(CFloatType), - Character, + Character(CCharacterType), String(CFormatConversion), } -#[derive(Debug, PartialEq)] +impl CFormatType { + pub fn to_char(self) -> char { + match self { + CFormatType::Number(x) => x as u8 as char, + CFormatType::Float(x) => x as u8 as char, + CFormatType::Character(x) => x as u8 as char, + CFormatType::String(x) => x as u8 as char, + } + } +} + +#[derive(Debug, PartialEq, Clone, Copy)] pub enum CFormatPrecision { Quantity(CFormatQuantity), Dot, @@ -106,73 +142,151 @@ impl CConversionFlags { } } -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Clone, Copy)] pub enum CFormatQuantity { Amount(usize), FromValuesTuple, } -#[derive(Debug, PartialEq)] +pub trait FormatBuf: + Extend + Default + FromIterator + From +{ + type Char: FormatChar; + fn chars(&self) -> impl Iterator; + fn len(&self) -> usize; + fn is_empty(&self) -> bool { + self.len() == 0 + } + fn concat(self, other: Self) -> Self; +} + +pub trait FormatChar: Copy + Into + From { + fn to_char_lossy(self) -> char; + fn eq_char(self, c: char) -> bool; +} + +impl FormatBuf for String { + type Char = char; + fn chars(&self) -> impl Iterator { + (**self).chars() + } + fn len(&self) -> usize { + self.len() + } + fn concat(mut self, other: Self) -> Self { + self.extend([other]); + self + } +} + +impl FormatChar for char { + fn to_char_lossy(self) -> char { + self + } + fn eq_char(self, c: char) -> bool { + self == c + } +} + +impl FormatBuf for Vec { + type Char = u8; + fn chars(&self) -> impl Iterator { + self.iter().copied() + } + fn len(&self) -> usize { + self.len() + } + fn concat(mut self, other: Self) -> Self { + self.extend(other); + self + } +} + +impl FormatChar for u8 { + fn to_char_lossy(self) -> char { + self.into() + } + fn eq_char(self, c: char) -> bool { + char::from(self) == c + } +} + +#[derive(Debug, PartialEq, Clone, Copy)] pub struct CFormatSpec { - pub mapping_key: Option, pub flags: CConversionFlags, pub min_field_width: Option, pub precision: Option, pub format_type: CFormatType, - pub format_char: char, // chars_consumed: usize, } +#[derive(Debug, PartialEq)] +pub struct CFormatSpecKeyed { + pub mapping_key: Option, + pub spec: CFormatSpec, +} + +#[cfg(test)] impl FromStr for CFormatSpec { type Err = ParsingError; + fn from_str(text: &str) -> Result { + text.parse::>() + .map(|CFormatSpecKeyed { mapping_key, spec }| { + assert!(mapping_key.is_none()); + spec + }) + } +} + +impl FromStr for CFormatSpecKeyed { + type Err = ParsingError; + fn from_str(text: &str) -> Result { let mut chars = text.chars().enumerate().peekable(); if chars.next().map(|x| x.1) != Some('%') { return Err((CFormatErrorType::MissingModuloSign, 1)); } - CFormatSpec::parse(&mut chars) + Self::parse(&mut chars) } } pub type ParseIter = Peekable>; -impl CFormatSpec { - pub fn parse(iter: &mut ParseIter) -> Result +impl CFormatSpecKeyed { + pub fn parse(iter: &mut ParseIter) -> Result where - T: Into + Copy, - I: Iterator, + I: Iterator, { let mapping_key = parse_spec_mapping_key(iter)?; let flags = parse_flags(iter); let min_field_width = parse_quantity(iter)?; let precision = parse_precision(iter)?; consume_length(iter); - let (format_type, format_char) = parse_format_type(iter)?; + let format_type = parse_format_type(iter)?; - Ok(CFormatSpec { - mapping_key, + let spec = CFormatSpec { flags, min_field_width, precision, format_type, - format_char, - }) + }; + Ok(Self { mapping_key, spec }) } +} - fn compute_fill_string(fill_char: char, fill_chars_needed: usize) -> String { - (0..fill_chars_needed) - .map(|_| fill_char) - .collect::() +impl CFormatSpec { + fn compute_fill_string(fill_char: T::Char, fill_chars_needed: usize) -> T { + (0..fill_chars_needed).map(|_| fill_char).collect() } - fn fill_string( + fn fill_string( &self, - string: String, - fill_char: char, + string: T, + fill_char: T::Char, num_prefix_chars: Option, - ) -> String { + ) -> T { let mut num_chars = string.chars().count(); if let Some(num_prefix_chars) = num_prefix_chars { num_chars += num_prefix_chars; @@ -184,20 +298,20 @@ impl CFormatSpec { _ => &num_chars, }; let fill_chars_needed = width.saturating_sub(num_chars); - let fill_string = CFormatSpec::compute_fill_string(fill_char, fill_chars_needed); + let fill_string: T = CFormatSpec::compute_fill_string(fill_char, fill_chars_needed); if !fill_string.is_empty() { if self.flags.contains(CConversionFlags::LEFT_ADJUST) { - format!("{string}{fill_string}") + string.concat(fill_string) } else { - format!("{fill_string}{string}") + fill_string.concat(string) } } else { string } } - fn fill_string_with_precision(&self, string: String, fill_char: char) -> String { + fn fill_string_with_precision(&self, string: T, fill_char: T::Char) -> T { let num_chars = string.chars().count(); let width = match &self.precision { @@ -207,52 +321,51 @@ impl CFormatSpec { _ => &num_chars, }; let fill_chars_needed = width.saturating_sub(num_chars); - let fill_string = CFormatSpec::compute_fill_string(fill_char, fill_chars_needed); + let fill_string: T = CFormatSpec::compute_fill_string(fill_char, fill_chars_needed); if !fill_string.is_empty() { // Don't left-adjust if precision-filling: that will always be prepending 0s to %d // arguments, the LEFT_ADJUST flag will be used by a later call to fill_string with // the 0-filled string as the string param. - format!("{fill_string}{string}") + fill_string.concat(string) } else { string } } - fn format_string_with_precision( + fn format_string_with_precision( &self, - string: String, + string: T, precision: Option<&CFormatPrecision>, - ) -> String { + ) -> T { // truncate if needed let string = match precision { Some(CFormatPrecision::Quantity(CFormatQuantity::Amount(precision))) if string.chars().count() > *precision => { - string.chars().take(*precision).collect::() + string.chars().take(*precision).collect::() } Some(CFormatPrecision::Dot) => { // truncate to 0 - String::new() + T::default() } _ => string, }; - self.fill_string(string, ' ', None) + self.fill_string(string, b' '.into(), None) } #[inline] - pub fn format_string(&self, string: String) -> String { + pub fn format_string(&self, string: T) -> T { self.format_string_with_precision(string, self.precision.as_ref()) } #[inline] - pub fn format_char(&self, ch: char) -> String { + pub fn format_char(&self, ch: T::Char) -> T { self.format_string_with_precision( - ch.to_string(), + T::from_iter([ch]), Some(&(CFormatQuantity::Amount(1).into())), ) } - pub fn format_bytes(&self, bytes: &[u8]) -> Vec { let bytes = if let Some(CFormatPrecision::Quantity(CFormatQuantity::Amount(precision))) = self.precision @@ -279,28 +392,30 @@ impl CFormatSpec { pub fn format_number(&self, num: &BigInt) -> String { use CNumberType::*; + let CFormatType::Number(format_type) = self.format_type else { + unreachable!() + }; let magnitude = num.abs(); let prefix = if self.flags.contains(CConversionFlags::ALTERNATE_FORM) { - match self.format_type { - CFormatType::Number(Octal) => "0o", - CFormatType::Number(Hex(Case::Lower)) => "0x", - CFormatType::Number(Hex(Case::Upper)) => "0X", + match format_type { + Octal => "0o", + HexLower => "0x", + HexUpper => "0X", _ => "", } } else { "" }; - let magnitude_string: String = match self.format_type { - CFormatType::Number(Decimal) => magnitude.to_str_radix(10), - CFormatType::Number(Octal) => magnitude.to_str_radix(8), - CFormatType::Number(Hex(Case::Lower)) => magnitude.to_str_radix(16), - CFormatType::Number(Hex(Case::Upper)) => { + let magnitude_string: String = match format_type { + DecimalD | DecimalI | DecimalU => magnitude.to_str_radix(10), + Octal => magnitude.to_str_radix(8), + HexLower => magnitude.to_str_radix(16), + HexUpper => { let mut result = magnitude.to_str_radix(16); result.make_ascii_uppercase(); result } - _ => unreachable!(), // Should not happen because caller has to make sure that this is a number }; let sign_string = match num.sign() { @@ -351,37 +466,36 @@ impl CFormatSpec { None => 6, }; - let magnitude_string = match &self.format_type { - CFormatType::Float(CFloatType::PointDecimal(case)) => { - let magnitude = num.abs(); - float::format_fixed( - precision, - magnitude, - *case, - self.flags.contains(CConversionFlags::ALTERNATE_FORM), - ) - } - CFormatType::Float(CFloatType::Exponent(case)) => { - let magnitude = num.abs(); - float::format_exponent( - precision, - magnitude, - *case, - self.flags.contains(CConversionFlags::ALTERNATE_FORM), - ) - } - CFormatType::Float(CFloatType::General(case)) => { + let CFormatType::Float(format_type) = self.format_type else { + unreachable!() + }; + + let magnitude = num.abs(); + let case = format_type.case(); + + let magnitude_string = match format_type { + CFloatType::PointDecimalLower | CFloatType::PointDecimalUpper => float::format_fixed( + precision, + magnitude, + case, + self.flags.contains(CConversionFlags::ALTERNATE_FORM), + ), + CFloatType::ExponentLower | CFloatType::ExponentUpper => float::format_exponent( + precision, + magnitude, + case, + self.flags.contains(CConversionFlags::ALTERNATE_FORM), + ), + CFloatType::GeneralLower | CFloatType::GeneralUpper => { let precision = if precision == 0 { 1 } else { precision }; - let magnitude = num.abs(); float::format_general( precision, magnitude, - *case, + case, self.flags.contains(CConversionFlags::ALTERNATE_FORM), false, ) } - _ => unreachable!(), }; if self.flags.contains(CConversionFlags::ZERO_PAD) { @@ -405,110 +519,101 @@ impl CFormatSpec { } } -fn parse_spec_mapping_key(iter: &mut ParseIter) -> Result, ParsingError> +fn parse_spec_mapping_key(iter: &mut ParseIter) -> Result, ParsingError> where - T: Into + Copy, - I: Iterator, + T: FormatBuf, + I: Iterator, { - if let Some(&(index, c)) = iter.peek() { - if c.into() == '(' { - iter.next().unwrap(); - return match parse_text_inside_parentheses(iter) { - Some(key) => Ok(Some(key)), - None => Err((CFormatErrorType::UnmatchedKeyParentheses, index)), - }; - } + if let Some((index, _)) = iter.next_if(|(_, c)| c.eq_char('(')) { + return match parse_text_inside_parentheses(iter) { + Some(key) => Ok(Some(key)), + None => Err((CFormatErrorType::UnmatchedKeyParentheses, index)), + }; } Ok(None) } -fn parse_flags(iter: &mut ParseIter) -> CConversionFlags +fn parse_flags(iter: &mut ParseIter) -> CConversionFlags where - T: Into + Copy, - I: Iterator, + C: FormatChar, + I: Iterator, { let mut flags = CConversionFlags::empty(); - while let Some(&(_, c)) = iter.peek() { - let flag = match c.into() { + iter.peeking_take_while(|(_, c)| { + let flag = match c.to_char_lossy() { '#' => CConversionFlags::ALTERNATE_FORM, '0' => CConversionFlags::ZERO_PAD, '-' => CConversionFlags::LEFT_ADJUST, ' ' => CConversionFlags::BLANK_SIGN, '+' => CConversionFlags::SIGN_CHAR, - _ => break, + _ => return false, }; - iter.next().unwrap(); flags |= flag; - } + true + }) + .for_each(drop); flags } -fn consume_length(iter: &mut ParseIter) +fn consume_length(iter: &mut ParseIter) where - T: Into + Copy, - I: Iterator, + C: FormatChar, + I: Iterator, { - if let Some(&(_, c)) = iter.peek() { - let c = c.into(); - if c == 'h' || c == 'l' || c == 'L' { - iter.next().unwrap(); - } - } + iter.next_if(|(_, c)| matches!(c.to_char_lossy(), 'h' | 'l' | 'L')); } -fn parse_format_type(iter: &mut ParseIter) -> Result<(CFormatType, char), ParsingError> +fn parse_format_type(iter: &mut ParseIter) -> Result where - T: Into, - I: Iterator, + C: FormatChar, + I: Iterator, { use CFloatType::*; use CNumberType::*; - let (index, c) = match iter.next() { - Some((index, c)) => (index, c.into()), - None => { - return Err(( - CFormatErrorType::IncompleteFormat, - iter.peek().map(|x| x.0).unwrap_or(0), - )); - } - }; - let format_type = match c { - 'd' | 'i' | 'u' => CFormatType::Number(Decimal), + let (index, c) = iter.next().ok_or_else(|| { + ( + CFormatErrorType::IncompleteFormat, + iter.peek().map(|x| x.0).unwrap_or(0), + ) + })?; + let format_type = match c.to_char_lossy() { + 'd' => CFormatType::Number(DecimalD), + 'i' => CFormatType::Number(DecimalI), + 'u' => CFormatType::Number(DecimalU), 'o' => CFormatType::Number(Octal), - 'x' => CFormatType::Number(Hex(Case::Lower)), - 'X' => CFormatType::Number(Hex(Case::Upper)), - 'e' => CFormatType::Float(Exponent(Case::Lower)), - 'E' => CFormatType::Float(Exponent(Case::Upper)), - 'f' => CFormatType::Float(PointDecimal(Case::Lower)), - 'F' => CFormatType::Float(PointDecimal(Case::Upper)), - 'g' => CFormatType::Float(General(Case::Lower)), - 'G' => CFormatType::Float(General(Case::Upper)), - 'c' => CFormatType::Character, + 'x' => CFormatType::Number(HexLower), + 'X' => CFormatType::Number(HexUpper), + 'e' => CFormatType::Float(ExponentLower), + 'E' => CFormatType::Float(ExponentUpper), + 'f' => CFormatType::Float(PointDecimalLower), + 'F' => CFormatType::Float(PointDecimalUpper), + 'g' => CFormatType::Float(GeneralLower), + 'G' => CFormatType::Float(GeneralUpper), + 'c' => CFormatType::Character(CCharacterType::Character), 'r' => CFormatType::String(CFormatConversion::Repr), 's' => CFormatType::String(CFormatConversion::Str), 'b' => CFormatType::String(CFormatConversion::Bytes), 'a' => CFormatType::String(CFormatConversion::Ascii), - _ => return Err((CFormatErrorType::UnsupportedFormatChar(c), index)), + _ => return Err((CFormatErrorType::UnsupportedFormatChar(c.into()), index)), }; - Ok((format_type, c)) + Ok(format_type) } -fn parse_quantity(iter: &mut ParseIter) -> Result, ParsingError> +fn parse_quantity(iter: &mut ParseIter) -> Result, ParsingError> where - T: Into + Copy, - I: Iterator, + C: FormatChar, + I: Iterator, { if let Some(&(_, c)) = iter.peek() { - let c: char = c.into(); - if c == '*' { + if c.eq_char('*') { iter.next().unwrap(); return Ok(Some(CFormatQuantity::FromValuesTuple)); } - if let Some(i) = c.to_digit(10) { + if let Some(i) = c.to_char_lossy().to_digit(10) { let mut num = i as i32; iter.next().unwrap(); while let Some(&(index, c)) = iter.peek() { - if let Some(i) = c.into().to_digit(10) { + if let Some(i) = c.to_char_lossy().to_digit(10) { num = num .checked_mul(10) .and_then(|num| num.checked_add(i as i32)) @@ -524,44 +629,40 @@ where Ok(None) } -fn parse_precision(iter: &mut ParseIter) -> Result, ParsingError> +fn parse_precision(iter: &mut ParseIter) -> Result, ParsingError> where - T: Into + Copy, - I: Iterator, + C: FormatChar, + I: Iterator, { - if let Some(&(_, c)) = iter.peek() { - if c.into() == '.' { - iter.next().unwrap(); - let quantity = parse_quantity(iter)?; - let precision = quantity.map_or(CFormatPrecision::Dot, CFormatPrecision::Quantity); - return Ok(Some(precision)); - } + if iter.next_if(|(_, c)| c.eq_char('.')).is_some() { + let quantity = parse_quantity(iter)?; + let precision = quantity.map_or(CFormatPrecision::Dot, CFormatPrecision::Quantity); + return Ok(Some(precision)); } Ok(None) } -fn parse_text_inside_parentheses(iter: &mut ParseIter) -> Option +fn parse_text_inside_parentheses(iter: &mut ParseIter) -> Option where - T: Into, - I: Iterator, + T: FormatBuf, + I: Iterator, { let mut counter: i32 = 1; - let mut contained_text = String::new(); + let mut contained_text = T::default(); loop { let (_, c) = iter.next()?; - let c = c.into(); - match c { - _ if c == '(' => { + match c.to_char_lossy() { + '(' => { counter += 1; } - _ if c == ')' => { + ')' => { counter -= 1; } _ => (), } if counter > 0 { - contained_text.push(c); + contained_text.extend([c]); } else { break; } @@ -573,13 +674,13 @@ where #[derive(Debug, PartialEq)] pub enum CFormatPart { Literal(T), - Spec(CFormatSpec), + Spec(CFormatSpecKeyed), } impl CFormatPart { #[inline] pub fn is_specifier(&self) -> bool { - matches!(self, CFormatPart::Spec(_)) + matches!(self, CFormatPart::Spec { .. }) } #[inline] @@ -623,21 +724,21 @@ impl CFormatStrOrBytes { pub fn iter_mut(&mut self) -> impl Iterator)> { self.parts.iter_mut() } -} - -pub type CFormatBytes = CFormatStrOrBytes>; -impl CFormatBytes { - pub fn parse>(iter: &mut ParseIter) -> Result { + pub fn parse(iter: &mut ParseIter) -> Result + where + S: FormatBuf, + I: Iterator, + { let mut parts = vec![]; - let mut literal = vec![]; + let mut literal = S::default(); let mut part_index = 0; while let Some((index, c)) = iter.next() { - if c == b'%' { + if c.eq_char('%') { if let Some(&(_, second)) = iter.peek() { - if second == b'%' { + if second.eq_char('%') { iter.next().unwrap(); - literal.push(b'%'); + literal.extend([second]); continue; } else { if !literal.is_empty() { @@ -646,7 +747,7 @@ impl CFormatBytes { CFormatPart::Literal(std::mem::take(&mut literal)), )); } - let spec = CFormatSpec::parse(iter).map_err(|err| CFormatError { + let spec = CFormatSpecKeyed::parse(iter).map_err(|err| CFormatError { typ: err.0, index: err.1, })?; @@ -662,7 +763,7 @@ impl CFormatBytes { }); } } else { - literal.push(c); + literal.extend([c]); } } if !literal.is_empty() { @@ -670,7 +771,19 @@ impl CFormatBytes { } Ok(Self { parts }) } +} + +impl IntoIterator for CFormatStrOrBytes { + type Item = (usize, CFormatPart); + type IntoIter = std::vec::IntoIter; + fn into_iter(self) -> Self::IntoIter { + self.parts.into_iter() + } +} + +pub type CFormatBytes = CFormatStrOrBytes>; +impl CFormatBytes { pub fn parse_from_bytes(bytes: &[u8]) -> Result { let mut iter = bytes.iter().cloned().enumerate().peekable(); Self::parse(&mut iter) @@ -688,53 +801,6 @@ impl FromStr for CFormatString { } } -impl CFormatString { - pub(crate) fn parse>( - iter: &mut ParseIter, - ) -> Result { - let mut parts = vec![]; - let mut literal = String::new(); - let mut part_index = 0; - while let Some((index, c)) = iter.next() { - if c == '%' { - if let Some(&(_, second)) = iter.peek() { - if second == '%' { - iter.next().unwrap(); - literal.push('%'); - continue; - } else { - if !literal.is_empty() { - parts.push(( - part_index, - CFormatPart::Literal(std::mem::take(&mut literal)), - )); - } - let spec = CFormatSpec::parse(iter).map_err(|err| CFormatError { - typ: err.0, - index: err.1, - })?; - parts.push((index, CFormatPart::Spec(spec))); - if let Some(&(index, _)) = iter.peek() { - part_index = index; - } - } - } else { - return Err(CFormatError { - typ: CFormatErrorType::IncompleteFormat, - index: index + 1, - }); - } - } else { - literal.push(c); - } - } - if !literal.is_empty() { - parts.push((part_index, CFormatPart::Literal(literal))); - } - Ok(Self { parts }) - } -} - #[cfg(test)] mod tests { use super::*; @@ -773,26 +839,28 @@ mod tests { #[test] fn test_parse_key() { - let expected = Ok(CFormatSpec { + let expected = Ok(CFormatSpecKeyed { mapping_key: Some("amount".to_owned()), - format_type: CFormatType::Number(CNumberType::Decimal), - format_char: 'd', - min_field_width: None, - precision: None, - flags: CConversionFlags::empty(), + spec: CFormatSpec { + format_type: CFormatType::Number(CNumberType::DecimalD), + min_field_width: None, + precision: None, + flags: CConversionFlags::empty(), + }, }); - assert_eq!("%(amount)d".parse::(), expected); + assert_eq!("%(amount)d".parse::>(), expected); - let expected = Ok(CFormatSpec { + let expected = Ok(CFormatSpecKeyed { mapping_key: Some("m((u(((l((((ti))))p)))l))e".to_owned()), - format_type: CFormatType::Number(CNumberType::Decimal), - format_char: 'd', - min_field_width: None, - precision: None, - flags: CConversionFlags::empty(), + spec: CFormatSpec { + format_type: CFormatType::Number(CNumberType::DecimalD), + min_field_width: None, + precision: None, + flags: CConversionFlags::empty(), + }, }); assert_eq!( - "%(m((u(((l((((ti))))p)))l))e)d".parse::(), + "%(m((u(((l((((ti))))p)))l))e)d".parse::>(), expected ); } @@ -813,7 +881,7 @@ mod tests { assert_eq!( "Hello %n".parse::(), Err(CFormatError { - typ: CFormatErrorType::UnsupportedFormatChar('n'), + typ: CFormatErrorType::UnsupportedFormatChar('n'.into()), index: 7 }) ); @@ -833,11 +901,9 @@ mod tests { #[test] fn test_parse_flags() { let expected = Ok(CFormatSpec { - format_type: CFormatType::Number(CNumberType::Decimal), - format_char: 'd', + format_type: CFormatType::Number(CNumberType::DecimalD), min_field_width: Some(CFormatQuantity::Amount(10)), precision: None, - mapping_key: None, flags: CConversionFlags::all(), }); let parsed = "% 0 -+++###10d".parse::(); @@ -1011,25 +1077,27 @@ mod tests { (0, CFormatPart::Literal("Hello, my name is ".to_owned())), ( 18, - CFormatPart::Spec(CFormatSpec { - format_type: CFormatType::String(CFormatConversion::Str), - format_char: 's', + CFormatPart::Spec(CFormatSpecKeyed { mapping_key: None, - min_field_width: None, - precision: None, - flags: CConversionFlags::empty(), + spec: CFormatSpec { + format_type: CFormatType::String(CFormatConversion::Str), + min_field_width: None, + precision: None, + flags: CConversionFlags::empty(), + }, }), ), (20, CFormatPart::Literal(" and I'm ".to_owned())), ( 29, - CFormatPart::Spec(CFormatSpec { - format_type: CFormatType::Number(CNumberType::Decimal), - format_char: 'd', + CFormatPart::Spec(CFormatSpecKeyed { mapping_key: None, - min_field_width: None, - precision: None, - flags: CConversionFlags::empty(), + spec: CFormatSpec { + format_type: CFormatType::Number(CNumberType::DecimalD), + min_field_width: None, + precision: None, + flags: CConversionFlags::empty(), + }, }), ), (31, CFormatPart::Literal(" years old".to_owned())), diff --git a/common/src/format.rs b/common/src/format.rs index 1c41df38c9..d9f821658b 100644 --- a/common/src/format.rs +++ b/common/src/format.rs @@ -14,11 +14,12 @@ trait FormatParse { } #[derive(Debug, Copy, Clone, PartialEq)] +#[repr(u8)] pub enum FormatConversion { - Str, - Repr, - Ascii, - Bytes, + Str = b's', + Repr = b'r', + Ascii = b'b', + Bytes = b'a', } impl FormatParse for FormatConversion { diff --git a/vm/src/anystr.rs b/vm/src/anystr.rs index 0fd1d8f2f6..d01136b0fb 100644 --- a/vm/src/anystr.rs +++ b/vm/src/anystr.rs @@ -1,7 +1,6 @@ use crate::{ Py, PyObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine, builtins::{PyIntRef, PyTuple}, - cformat::cformat_string, convert::TryFromBorrowedObject, function::OptionalOption, }; @@ -155,7 +154,6 @@ pub trait AnyStr { fn to_container(&self) -> Self::Container; fn as_bytes(&self) -> &[u8]; - fn as_utf8_str(&self) -> Result<&str, std::str::Utf8Error>; fn chars(&self) -> impl Iterator; fn elements(&self) -> impl Iterator; fn get_bytes(&self, range: std::ops::Range) -> &Self; @@ -425,11 +423,6 @@ pub trait AnyStr { } cased } - - fn py_cformat(&self, values: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let format_string = self.as_utf8_str().unwrap(); - cformat_string(vm, format_string, values) - } } /// Tests that the predicate is True on a single value, or if the value is a tuple a tuple, then diff --git a/vm/src/builtins/str.rs b/vm/src/builtins/str.rs index 554fa1de7f..76cdca81ed 100644 --- a/vm/src/builtins/str.rs +++ b/vm/src/builtins/str.rs @@ -8,6 +8,7 @@ use crate::{ TryFromBorrowedObject, VirtualMachine, anystr::{self, AnyStr, AnyStrContainer, AnyStrWrapper, adjust_indices}, atomic_func, + cformat::cformat_string, class::PyClassImpl, common::str::{BorrowedStr, PyStrKind, PyStrKindData}, convert::{IntoPyException, ToPyException, ToPyObject, ToPyResult}, @@ -729,8 +730,7 @@ impl PyStr { #[pymethod(name = "__mod__")] fn modulo(&self, values: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let formatted = self.as_str().py_cformat(values, vm)?; - Ok(formatted) + cformat_string(vm, self.as_str(), values) } #[pymethod(magic)] @@ -1598,10 +1598,6 @@ impl AnyStr for str { self.as_bytes() } - fn as_utf8_str(&self) -> Result<&str, std::str::Utf8Error> { - Ok(self) - } - fn chars(&self) -> impl Iterator { str::chars(self) } diff --git a/vm/src/bytesinner.rs b/vm/src/bytesinner.rs index 3754b75ee1..bcbbd4fce6 100644 --- a/vm/src/bytesinner.rs +++ b/vm/src/bytesinner.rs @@ -1037,10 +1037,6 @@ impl AnyStr for [u8] { self } - fn as_utf8_str(&self) -> Result<&str, std::str::Utf8Error> { - std::str::from_utf8(self) - } - fn chars(&self) -> impl Iterator { bstr::ByteSlice::chars(self) } diff --git a/vm/src/cformat.rs b/vm/src/cformat.rs index a11c06a252..af78fde021 100644 --- a/vm/src/cformat.rs +++ b/vm/src/cformat.rs @@ -13,7 +13,6 @@ use crate::{ }; use itertools::Itertools; use num_traits::cast::ToPrimitive; -use std::str::FromStr; fn spec_format_bytes( vm: &VirtualMachine, @@ -48,36 +47,38 @@ fn spec_format_bytes( } }, CFormatType::Number(number_type) => match number_type { - CNumberType::Decimal => match_class!(match &obj { - ref i @ PyInt => { - Ok(spec.format_number(i.as_bigint()).into_bytes()) - } - ref f @ PyFloat => { - Ok(spec - .format_number(&try_f64_to_bigint(f.to_f64(), vm)?) - .into_bytes()) - } - obj => { - if let Some(method) = vm.get_method(obj.clone(), identifier!(vm, __int__)) { - let result = method?.call((), vm)?; - if let Some(i) = result.payload::() { - return Ok(spec.format_number(i.as_bigint()).into_bytes()); + CNumberType::DecimalD | CNumberType::DecimalI | CNumberType::DecimalU => { + match_class!(match &obj { + ref i @ PyInt => { + Ok(spec.format_number(i.as_bigint()).into_bytes()) + } + ref f @ PyFloat => { + Ok(spec + .format_number(&try_f64_to_bigint(f.to_f64(), vm)?) + .into_bytes()) + } + obj => { + if let Some(method) = vm.get_method(obj.clone(), identifier!(vm, __int__)) { + let result = method?.call((), vm)?; + if let Some(i) = result.payload::() { + return Ok(spec.format_number(i.as_bigint()).into_bytes()); + } } + Err(vm.new_type_error(format!( + "%{} format: a number is required, not {}", + spec.format_type.to_char(), + obj.class().name() + ))) } - Err(vm.new_type_error(format!( - "%{} format: a number is required, not {}", - spec.format_char, - obj.class().name() - ))) - } - }), + }) + } _ => { if let Some(i) = obj.payload::() { Ok(spec.format_number(i.as_bigint()).into_bytes()) } else { Err(vm.new_type_error(format!( "%{} format: an integer is required, not {}", - spec.format_char, + spec.format_type.to_char(), obj.class().name() ))) } @@ -96,22 +97,21 @@ fn spec_format_bytes( })?; Ok(spec.format_float(value.into()).into_bytes()) } - CFormatType::Character => { + CFormatType::Character(CCharacterType::Character) => { if let Some(i) = obj.payload::() { let ch = i .try_to_primitive::(vm) - .map_err(|_| vm.new_overflow_error("%c arg not in range(256)".to_owned()))? - as char; - return Ok(spec.format_char(ch).into_bytes()); + .map_err(|_| vm.new_overflow_error("%c arg not in range(256)".to_owned()))?; + return Ok(spec.format_char(ch)); } if let Some(b) = obj.payload::() { if b.len() == 1 { - return Ok(spec.format_char(b.as_bytes()[0] as char).into_bytes()); + return Ok(spec.format_char(b.as_bytes()[0])); } } else if let Some(ba) = obj.payload::() { let buf = ba.borrow_buf(); if buf.len() == 1 { - return Ok(spec.format_char(buf[0] as char).into_bytes()); + return Ok(spec.format_char(buf[0])); } } Err(vm @@ -124,7 +124,7 @@ fn spec_format_string( vm: &VirtualMachine, spec: &CFormatSpec, obj: PyObjectRef, - idx: &usize, + idx: usize, ) -> PyResult { match &spec.format_type { CFormatType::String(conversion) => { @@ -143,34 +143,36 @@ fn spec_format_string( Ok(spec.format_string(result)) } CFormatType::Number(number_type) => match number_type { - CNumberType::Decimal => match_class!(match &obj { - ref i @ PyInt => { - Ok(spec.format_number(i.as_bigint())) - } - ref f @ PyFloat => { - Ok(spec.format_number(&try_f64_to_bigint(f.to_f64(), vm)?)) - } - obj => { - if let Some(method) = vm.get_method(obj.clone(), identifier!(vm, __int__)) { - let result = method?.call((), vm)?; - if let Some(i) = result.payload::() { - return Ok(spec.format_number(i.as_bigint())); + CNumberType::DecimalD | CNumberType::DecimalI | CNumberType::DecimalU => { + match_class!(match &obj { + ref i @ PyInt => { + Ok(spec.format_number(i.as_bigint())) + } + ref f @ PyFloat => { + Ok(spec.format_number(&try_f64_to_bigint(f.to_f64(), vm)?)) + } + obj => { + if let Some(method) = vm.get_method(obj.clone(), identifier!(vm, __int__)) { + let result = method?.call((), vm)?; + if let Some(i) = result.payload::() { + return Ok(spec.format_number(i.as_bigint())); + } } + Err(vm.new_type_error(format!( + "%{} format: a number is required, not {}", + spec.format_type.to_char(), + obj.class().name() + ))) } - Err(vm.new_type_error(format!( - "%{} format: a number is required, not {}", - spec.format_char, - obj.class().name() - ))) - } - }), + }) + } _ => { if let Some(i) = obj.payload::() { Ok(spec.format_number(i.as_bigint())) } else { Err(vm.new_type_error(format!( "%{} format: an integer is required, not {}", - spec.format_char, + spec.format_type.to_char(), obj.class().name() ))) } @@ -180,12 +182,12 @@ fn spec_format_string( let value = ArgIntoFloat::try_from_object(vm, obj)?; Ok(spec.format_float(value.into())) } - CFormatType::Character => { + CFormatType::Character(CCharacterType::Character) => { if let Some(i) = obj.payload::() { let ch = i .as_bigint() .to_u32() - .and_then(std::char::from_u32) + .and_then(char::from_u32) .ok_or_else(|| { vm.new_overflow_error("%c arg not in range(0x110000)".to_owned()) })?; @@ -313,19 +315,14 @@ pub(crate) fn cformat_bytes( if mapping_required { // dict return if is_mapping { - for (_, part) in format.iter_mut() { + for (_, part) in format { match part { - CFormatPart::Literal(literal) => result.append(literal), - CFormatPart::Spec(spec) => { - let value = match &spec.mapping_key { - Some(key) => { - let k = vm.ctx.new_bytes(key.as_str().as_bytes().to_vec()); - values_obj.get_item(k.as_object(), vm)? - } - None => unreachable!(), - }; - let mut part_result = spec_format_bytes(vm, spec, value)?; - result.append(&mut part_result); + CFormatPart::Literal(literal) => result.extend(literal), + CFormatPart::Spec(CFormatSpecKeyed { mapping_key, spec }) => { + let key = mapping_key.unwrap(); + let value = values_obj.get_item(&key, vm)?; + let part_result = spec_format_bytes(vm, &spec, value)?; + result.extend(part_result); } } } @@ -343,10 +340,10 @@ pub(crate) fn cformat_bytes( }; let mut value_iter = values.iter(); - for (_, part) in format.iter_mut() { + for (_, part) in format { match part { - CFormatPart::Literal(literal) => result.append(literal), - CFormatPart::Spec(spec) => { + CFormatPart::Literal(literal) => result.extend(literal), + CFormatPart::Spec(CFormatSpecKeyed { mut spec, .. }) => { try_update_quantity_from_tuple( vm, &mut value_iter, @@ -361,8 +358,8 @@ pub(crate) fn cformat_bytes( Err(vm.new_type_error("not enough arguments for format string".to_owned())) } }?; - let mut part_result = spec_format_bytes(vm, spec, value)?; - result.append(&mut part_result); + let part_result = spec_format_bytes(vm, &spec, value)?; + result.extend(part_result); } } } @@ -380,7 +377,8 @@ pub(crate) fn cformat_string( format_string: &str, values_obj: PyObjectRef, ) -> PyResult { - let mut format = CFormatString::from_str(format_string) + let format = format_string + .parse::() .map_err(|err| vm.new_value_error(err.to_string()))?; let (num_specifiers, mapping_required) = format .check_specifiers() @@ -415,15 +413,12 @@ pub(crate) fn cformat_string( if mapping_required { // dict return if is_mapping { - for (idx, part) in format.iter() { + for (idx, part) in format { match part { - CFormatPart::Literal(literal) => result.push_str(literal), - CFormatPart::Spec(spec) => { - let value = match &spec.mapping_key { - Some(key) => values_obj.get_item(key.as_str(), vm)?, - None => unreachable!(), - }; - let part_result = spec_format_string(vm, spec, value, idx)?; + CFormatPart::Literal(literal) => result.push_str(&literal), + CFormatPart::Spec(CFormatSpecKeyed { mapping_key, spec }) => { + let value = values_obj.get_item(&mapping_key.unwrap(), vm)?; + let part_result = spec_format_string(vm, &spec, value, idx)?; result.push_str(&part_result); } } @@ -442,10 +437,10 @@ pub(crate) fn cformat_string( }; let mut value_iter = values.iter(); - for (idx, part) in format.iter_mut() { + for (idx, part) in format { match part { - CFormatPart::Literal(literal) => result.push_str(literal), - CFormatPart::Spec(spec) => { + CFormatPart::Literal(literal) => result.push_str(&literal), + CFormatPart::Spec(CFormatSpecKeyed { mut spec, .. }) => { try_update_quantity_from_tuple( vm, &mut value_iter, @@ -460,7 +455,7 @@ pub(crate) fn cformat_string( Err(vm.new_type_error("not enough arguments for format string".to_owned())) } }?; - let part_result = spec_format_string(vm, spec, value, idx)?; + let part_result = spec_format_string(vm, &spec, value, idx)?; result.push_str(&part_result); } } diff --git a/vm/src/dictdatatype.rs b/vm/src/dictdatatype.rs index 912aa8e776..b36485e86a 100644 --- a/vm/src/dictdatatype.rs +++ b/vm/src/dictdatatype.rs @@ -5,7 +5,7 @@ use crate::{ AsObject, Py, PyExact, PyObject, PyObjectRef, PyRefExact, PyResult, VirtualMachine, - builtins::{PyInt, PyStr, PyStrInterned, PyStrRef}, + builtins::{PyBytes, PyInt, PyStr, PyStrInterned, PyStrRef}, convert::ToPyObject, }; use crate::{ @@ -749,7 +749,7 @@ impl DictKey for Py { fn key_eq(&self, vm: &VirtualMachine, other_key: &PyObject) -> PyResult { if self.is(other_key) { Ok(true) - } else if let Some(pystr) = str_exact(other_key, vm) { + } else if let Some(pystr) = other_key.payload_if_exact::(vm) { Ok(pystr.as_str() == self.as_str()) } else { vm.bool_eq(self.as_object(), other_key) @@ -833,7 +833,7 @@ impl DictKey for str { } fn key_eq(&self, vm: &VirtualMachine, other_key: &PyObject) -> PyResult { - if let Some(pystr) = str_exact(other_key, vm) { + if let Some(pystr) = other_key.payload_if_exact::(vm) { Ok(pystr.as_str() == self) } else { // Fall back to PyObjectRef implementation. @@ -871,6 +871,63 @@ impl DictKey for String { } } +impl DictKey for [u8] { + type Owned = Vec; + #[inline(always)] + fn _to_owned(&self, _vm: &VirtualMachine) -> Self::Owned { + self.to_owned() + } + #[inline] + fn key_hash(&self, vm: &VirtualMachine) -> PyResult { + // follow a similar route as the hashing of PyStrRef + Ok(vm.state.hash_secret.hash_bytes(self)) + } + #[inline(always)] + fn key_is(&self, _other: &PyObject) -> bool { + // No matter who the other pyobject is, we are never the same thing, since + // we are a str, not a pyobject. + false + } + + fn key_eq(&self, vm: &VirtualMachine, other_key: &PyObject) -> PyResult { + if let Some(pystr) = other_key.payload_if_exact::(vm) { + Ok(pystr.as_bytes() == self) + } else { + // Fall back to PyObjectRef implementation. + let s = vm.ctx.new_bytes(self.to_vec()); + s.key_eq(vm, other_key) + } + } + + fn key_as_isize(&self, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("'str' object cannot be interpreted as an integer".to_owned())) + } +} + +impl DictKey for Vec { + type Owned = Vec; + #[inline] + fn _to_owned(&self, _vm: &VirtualMachine) -> Self::Owned { + self.clone() + } + + fn key_hash(&self, vm: &VirtualMachine) -> PyResult { + self.as_slice().key_hash(vm) + } + + fn key_is(&self, other: &PyObject) -> bool { + self.as_slice().key_is(other) + } + + fn key_eq(&self, vm: &VirtualMachine, other_key: &PyObject) -> PyResult { + self.as_slice().key_eq(vm, other_key) + } + + fn key_as_isize(&self, vm: &VirtualMachine) -> PyResult { + self.as_slice().key_as_isize(vm) + } +} + impl DictKey for usize { type Owned = usize; #[inline] @@ -904,14 +961,6 @@ impl DictKey for usize { } } -fn str_exact<'a>(obj: &'a PyObject, vm: &VirtualMachine) -> Option<&'a PyStr> { - if obj.class().is(vm.ctx.types.str_type) { - obj.payload::() - } else { - None - } -} - #[cfg(test)] mod tests { use super::*; From cace112b1ada0a51a8d2a191c656332a2d425c76 Mon Sep 17 00:00:00 2001 From: Noa Date: Thu, 6 Mar 2025 19:20:33 -0600 Subject: [PATCH 105/295] Allow surrogates in str --- Cargo.lock | 2 + common/Cargo.toml | 2 + common/src/encodings.rs | 22 +- common/src/lib.rs | 1 + common/src/str.rs | 370 +++++++-- common/src/wtf8/core_char.rs | 113 +++ common/src/wtf8/core_str.rs | 113 +++ common/src/wtf8/mod.rs | 1347 +++++++++++++++++++++++++++++++ stdlib/src/pystruct.rs | 31 +- stdlib/src/unicodedata.rs | 49 +- vm/src/anystr.rs | 124 +-- vm/src/builtins/genericalias.rs | 33 +- vm/src/builtins/object.rs | 2 +- vm/src/builtins/set.rs | 4 +- vm/src/builtins/str.rs | 894 +++++++++++++++----- vm/src/builtins/union.rs | 2 +- vm/src/bytesinner.rs | 38 +- vm/src/codecs.rs | 23 +- vm/src/frame.rs | 7 +- vm/src/import.rs | 10 +- vm/src/stdlib/builtins.rs | 28 +- vm/src/stdlib/codecs.rs | 4 +- vm/src/vm/interpreter.rs | 2 +- vm/src/vm/mod.rs | 2 +- 24 files changed, 2747 insertions(+), 476 deletions(-) create mode 100644 common/src/wtf8/core_char.rs create mode 100644 common/src/wtf8/core_str.rs create mode 100644 common/src/wtf8/mod.rs diff --git a/Cargo.lock b/Cargo.lock index b214780cb4..94b0732b8a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2338,6 +2338,7 @@ version = "0.4.0" dependencies = [ "ascii", "bitflags 2.8.0", + "bstr", "cfg-if", "itertools 0.14.0", "libc", @@ -2345,6 +2346,7 @@ dependencies = [ "malachite-base", "malachite-bigint", "malachite-q", + "memchr", "num-complex", "num-traits", "once_cell", diff --git a/common/Cargo.toml b/common/Cargo.toml index 589170064c..e9aeba7459 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -16,12 +16,14 @@ rustpython-literal = { workspace = true } ascii = { workspace = true } bitflags = { workspace = true } +bstr = { workspace = true } cfg-if = { workspace = true } itertools = { workspace = true } libc = { workspace = true } malachite-bigint = { workspace = true } malachite-q = { workspace = true } malachite-base = { workspace = true } +memchr = { workspace = true } num-complex = { workspace = true } num-traits = { workspace = true } once_cell = { workspace = true } diff --git a/common/src/encodings.rs b/common/src/encodings.rs index 858d3b8c6b..3aaca68c3d 100644 --- a/common/src/encodings.rs +++ b/common/src/encodings.rs @@ -1,10 +1,12 @@ use std::ops::Range; +use crate::wtf8::{Wtf8, Wtf8Buf}; + pub type EncodeErrorResult = Result<(EncodeReplace, usize), E>; pub type DecodeErrorResult = Result<(S, Option, usize), E>; -pub trait StrBuffer: AsRef { +pub trait StrBuffer: AsRef { fn is_ascii(&self) -> bool { self.as_ref().is_ascii() } @@ -63,19 +65,19 @@ fn decode_utf8_compatible( errors: &E, decode: DecodeF, handle_error: ErrF, -) -> Result<(String, usize), E::Error> +) -> Result<(Wtf8Buf, usize), E::Error> where DecodeF: Fn(&[u8]) -> Result<&str, DecodeError<'_>>, ErrF: Fn(&[u8], Option) -> HandleResult<'_>, { if data.is_empty() { - return Ok((String::new(), 0)); + return Ok((Wtf8Buf::new(), 0)); } // we need to coerce the lifetime to that of the function body rather than the // anonymous input lifetime, so that we can assign it data borrowed from data_from_err let mut data = data; let mut data_from_err: E::BytesBuf; - let mut out = String::with_capacity(data.len()); + let mut out = Wtf8Buf::with_capacity(data.len()); let mut remaining_index = 0; let mut remaining_data = data; loop { @@ -98,7 +100,7 @@ where err_idx..err_len.map_or_else(|| data.len(), |len| err_idx + len); let (replace, new_data, restart) = errors.handle_decode_error(data, err_range, reason)?; - out.push_str(replace.as_ref()); + out.push_wtf8(replace.as_ref()); if let Some(new_data) = new_data { data_from_err = new_data; data = data_from_err.as_ref(); @@ -130,7 +132,7 @@ pub mod utf8 { data: &[u8], errors: &E, final_decode: bool, - ) -> Result<(String, usize), E::Error> { + ) -> Result<(Wtf8Buf, usize), E::Error> { decode_utf8_compatible( data, errors, @@ -218,7 +220,7 @@ pub mod latin_1 { )?; match replace { EncodeReplace::Str(s) => { - if s.as_ref().chars().any(|c| (c as u32) > 255) { + if s.as_ref().code_points().any(|c| c.to_u32() > 255) { return Err( errors.error_encoding(full_data, char_range, ERR_REASON) ); @@ -240,10 +242,10 @@ pub mod latin_1 { Ok(out) } - pub fn decode(data: &[u8], _errors: &E) -> Result<(String, usize), E::Error> { + pub fn decode(data: &[u8], _errors: &E) -> Result<(Wtf8Buf, usize), E::Error> { let out: String = data.iter().map(|c| *c as char).collect(); let out_len = out.len(); - Ok((out, out_len)) + Ok((out.into(), out_len)) } } @@ -303,7 +305,7 @@ pub mod ascii { Ok(out) } - pub fn decode(data: &[u8], errors: &E) -> Result<(String, usize), E::Error> { + pub fn decode(data: &[u8], errors: &E) -> Result<(Wtf8Buf, usize), E::Error> { decode_utf8_compatible( data, errors, diff --git a/common/src/lib.rs b/common/src/lib.rs index 760ba8c55f..e83a9af43a 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -29,6 +29,7 @@ pub mod static_cell; pub mod str; #[cfg(windows)] pub mod windows; +pub mod wtf8; pub mod vendored { pub use ascii; diff --git a/common/src/str.rs b/common/src/str.rs index 89d2381d3e..dbe4e22234 100644 --- a/common/src/str.rs +++ b/common/src/str.rs @@ -1,9 +1,9 @@ -use crate::{ - atomic::{PyAtomic, Radium}, - format::CharLen, - hash::PyHash, -}; -use ascii::AsciiString; +use crate::atomic::{PyAtomic, Radium}; +use crate::format::CharLen; +use crate::wtf8::{CodePoint, Wtf8, Wtf8Buf}; +use ascii::{AsciiChar, AsciiStr, AsciiString}; +use core::fmt; +use core::sync::atomic::Ordering::Relaxed; use std::ops::{Bound, RangeBounds}; #[cfg(not(target_arch = "wasm32"))] @@ -15,132 +15,313 @@ pub type wchar_t = u32; /// Utf8 + state.ascii (+ PyUnicode_Kind in future) #[derive(Debug, Copy, Clone, PartialEq)] -pub enum PyStrKind { +pub enum StrKind { Ascii, Utf8, + Wtf8, } -impl std::ops::BitOr for PyStrKind { +impl std::ops::BitOr for StrKind { type Output = Self; fn bitor(self, other: Self) -> Self { + use StrKind::*; match (self, other) { - (Self::Ascii, Self::Ascii) => Self::Ascii, - _ => Self::Utf8, + (Wtf8, _) | (_, Wtf8) => Wtf8, + (Utf8, _) | (_, Utf8) => Utf8, + (Ascii, Ascii) => Ascii, } } } -impl PyStrKind { - #[inline] - pub fn new_data(self) -> PyStrKindData { - match self { - PyStrKind::Ascii => PyStrKindData::Ascii, - PyStrKind::Utf8 => PyStrKindData::Utf8(Radium::new(usize::MAX)), +impl StrKind { + pub fn is_ascii(&self) -> bool { + matches!(self, Self::Ascii) + } + + pub fn is_utf8(&self) -> bool { + matches!(self, Self::Ascii | Self::Utf8) + } +} + +pub trait DeduceStrKind { + fn str_kind(&self) -> StrKind; +} + +impl DeduceStrKind for str { + fn str_kind(&self) -> StrKind { + if self.is_ascii() { + StrKind::Ascii + } else { + StrKind::Utf8 + } + } +} + +impl DeduceStrKind for Wtf8 { + fn str_kind(&self) -> StrKind { + if self.is_ascii() { + StrKind::Ascii + } else if self.is_utf8() { + StrKind::Utf8 + } else { + StrKind::Wtf8 } } } +impl DeduceStrKind for String { + fn str_kind(&self) -> StrKind { + (**self).str_kind() + } +} + +impl DeduceStrKind for Wtf8Buf { + fn str_kind(&self) -> StrKind { + (**self).str_kind() + } +} + +impl DeduceStrKind for &T { + fn str_kind(&self) -> StrKind { + (**self).str_kind() + } +} + +impl DeduceStrKind for Box { + fn str_kind(&self) -> StrKind { + (**self).str_kind() + } +} + #[derive(Debug)] -pub enum PyStrKindData { - Ascii, - // uses usize::MAX as a sentinel for "uncomputed" - Utf8(PyAtomic), +pub enum PyKindStr<'a> { + Ascii(&'a AsciiStr), + Utf8(&'a str), + Wtf8(&'a Wtf8), } -impl PyStrKindData { - #[inline] - pub fn kind(&self) -> PyStrKind { - match self { - PyStrKindData::Ascii => PyStrKind::Ascii, - PyStrKindData::Utf8(_) => PyStrKind::Utf8, +#[derive(Debug, Clone)] +pub struct StrData { + data: Box, + kind: StrKind, + len: StrLen, +} + +struct StrLen(PyAtomic); + +impl From for StrLen { + #[inline(always)] + fn from(value: usize) -> Self { + Self(Radium::new(value)) + } +} + +impl fmt::Debug for StrLen { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let len = self.0.load(Relaxed); + if len == usize::MAX { + f.write_str("") + } else { + len.fmt(f) } } } -pub struct BorrowedStr<'a> { - bytes: &'a [u8], - kind: PyStrKindData, - #[allow(dead_code)] - hash: PyAtomic, +impl StrLen { + #[inline(always)] + fn zero() -> Self { + 0usize.into() + } + #[inline(always)] + fn uncomputed() -> Self { + usize::MAX.into() + } } -impl<'a> BorrowedStr<'a> { - /// # Safety - /// `s` have to be an ascii string - #[inline] - pub unsafe fn from_ascii_unchecked(s: &'a [u8]) -> Self { - debug_assert!(s.is_ascii()); +impl Clone for StrLen { + fn clone(&self) -> Self { + Self(self.0.load(Relaxed).into()) + } +} + +impl Default for StrData { + fn default() -> Self { Self { - bytes: s, - kind: PyStrKind::Ascii.new_data(), - hash: PyAtomic::::new(0), + data: >::default(), + kind: StrKind::Ascii, + len: StrLen::zero(), } } +} + +impl From> for StrData { + fn from(value: Box) -> Self { + // doing the check is ~10x faster for ascii, and is actually only 2% slower worst case for + // non-ascii; see https://github.com/RustPython/RustPython/pull/2586#issuecomment-844611532 + let kind = value.str_kind(); + unsafe { Self::new_str_unchecked(value, kind) } + } +} +impl From> for StrData { #[inline] - pub fn from_bytes(s: &'a [u8]) -> Self { - let k = if s.is_ascii() { - PyStrKind::Ascii.new_data() + fn from(value: Box) -> Self { + // doing the check is ~10x faster for ascii, and is actually only 2% slower worst case for + // non-ascii; see https://github.com/RustPython/RustPython/pull/2586#issuecomment-844611532 + let kind = value.str_kind(); + unsafe { Self::new_str_unchecked(value.into(), kind) } + } +} + +impl From> for StrData { + #[inline] + fn from(value: Box) -> Self { + Self { + len: value.len().into(), + data: value.into(), + kind: StrKind::Ascii, + } + } +} + +impl From for StrData { + fn from(ch: AsciiChar) -> Self { + AsciiString::from(ch).into_boxed_ascii_str().into() + } +} + +impl From for StrData { + fn from(ch: char) -> Self { + if let Ok(ch) = ascii::AsciiChar::from_ascii(ch) { + ch.into() + } else { + Self { + data: ch.to_string().into(), + kind: StrKind::Utf8, + len: 1.into(), + } + } + } +} + +impl From for StrData { + fn from(ch: CodePoint) -> Self { + if let Some(ch) = ch.to_char() { + ch.into() } else { - PyStrKind::Utf8.new_data() + Self { + data: Wtf8Buf::from(ch).into(), + kind: StrKind::Wtf8, + len: 1.into(), + } + } + } +} + +impl StrData { + /// # Safety + /// + /// Given `bytes` must be valid data for given `kind` + pub unsafe fn new_str_unchecked(data: Box, kind: StrKind) -> Self { + let len = match kind { + StrKind::Ascii => data.len().into(), + _ => StrLen::uncomputed(), }; + Self { data, kind, len } + } + + /// # Safety + /// + /// `char_len` must be accurate. + pub unsafe fn new_with_char_len(data: Box, kind: StrKind, char_len: usize) -> Self { Self { - bytes: s, - kind: k, - hash: PyAtomic::::new(0), + data, + kind, + len: char_len.into(), } } #[inline] - pub fn as_str(&self) -> &str { - unsafe { - // SAFETY: Both PyStrKind::{Ascii, Utf8} are valid utf8 string - std::str::from_utf8_unchecked(self.bytes) + pub fn as_wtf8(&self) -> &Wtf8 { + &self.data + } + + #[inline] + pub fn as_str(&self) -> Option<&str> { + self.kind + .is_utf8() + .then(|| unsafe { std::str::from_utf8_unchecked(self.data.as_bytes()) }) + } + + pub fn as_ascii(&self) -> Option<&AsciiStr> { + self.kind + .is_ascii() + .then(|| unsafe { AsciiStr::from_ascii_unchecked(self.data.as_bytes()) }) + } + + pub fn kind(&self) -> StrKind { + self.kind + } + + #[inline] + pub fn as_str_kind(&self) -> PyKindStr<'_> { + match self.kind { + StrKind::Ascii => { + PyKindStr::Ascii(unsafe { AsciiStr::from_ascii_unchecked(self.data.as_bytes()) }) + } + StrKind::Utf8 => { + PyKindStr::Utf8(unsafe { std::str::from_utf8_unchecked(self.data.as_bytes()) }) + } + StrKind::Wtf8 => PyKindStr::Wtf8(&self.data), } } + #[inline] + pub fn len(&self) -> usize { + self.data.len() + } + + pub fn is_empty(&self) -> bool { + self.data.is_empty() + } + #[inline] pub fn char_len(&self) -> usize { - match self.kind { - PyStrKindData::Ascii => self.bytes.len(), - PyStrKindData::Utf8(ref len) => match len.load(core::sync::atomic::Ordering::Relaxed) { - usize::MAX => self._compute_char_len(), - len => len, - }, + match self.len.0.load(Relaxed) { + usize::MAX => self._compute_char_len(), + len => len, } } #[cold] fn _compute_char_len(&self) -> usize { - match self.kind { - PyStrKindData::Utf8(ref char_len) => { - let len = self.as_str().chars().count(); - // len cannot be usize::MAX, since vec.capacity() < sys.maxsize - char_len.store(len, core::sync::atomic::Ordering::Relaxed); - len - } - _ => unsafe { - debug_assert!(false); // invalid for non-utf8 strings - std::hint::unreachable_unchecked() - }, - } + let len = if let Some(s) = self.as_str() { + // utf8 chars().count() is optimized + s.chars().count() + } else { + self.data.code_points().count() + }; + // len cannot be usize::MAX, since vec.capacity() < sys.maxsize + self.len.0.store(len, Relaxed); + len } -} -impl std::ops::Deref for BorrowedStr<'_> { - type Target = str; - fn deref(&self) -> &str { - self.as_str() + pub fn nth_char(&self, index: usize) -> CodePoint { + match self.as_str_kind() { + PyKindStr::Ascii(s) => s[index].into(), + PyKindStr::Utf8(s) => s.chars().nth(index).unwrap().into(), + PyKindStr::Wtf8(w) => w.code_points().nth(index).unwrap(), + } } } -impl std::fmt::Display for BorrowedStr<'_> { +impl std::fmt::Display for StrData { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - self.as_str().fmt(f) + self.data.fmt(f) } } -impl CharLen for BorrowedStr<'_> { +impl CharLen for StrData { fn char_len(&self) -> usize { self.char_len() } @@ -181,6 +362,41 @@ pub fn char_range_end(s: &str, nchars: usize) -> Option { Some(i) } +pub fn try_get_codepoints(w: &Wtf8, range: impl RangeBounds) -> Option<&Wtf8> { + let mut chars = w.code_points(); + let start = match range.start_bound() { + Bound::Included(&i) => i, + Bound::Excluded(&i) => i + 1, + Bound::Unbounded => 0, + }; + for _ in 0..start { + chars.next()?; + } + let s = chars.as_wtf8(); + let range_len = match range.end_bound() { + Bound::Included(&i) => i + 1 - start, + Bound::Excluded(&i) => i - start, + Bound::Unbounded => return Some(s), + }; + codepoint_range_end(s, range_len).map(|end| &s[..end]) +} + +pub fn get_codepoints(w: &Wtf8, range: impl RangeBounds) -> &Wtf8 { + try_get_codepoints(w, range).unwrap() +} + +#[inline] +pub fn codepoint_range_end(s: &Wtf8, nchars: usize) -> Option { + let i = match nchars.checked_sub(1) { + Some(last_char_index) => { + let (index, c) = s.code_point_indices().nth(last_char_index)?; + index + c.len_wtf8() + } + None => 0, + }; + Some(i) +} + pub fn zfill(bytes: &[u8], width: usize) -> Vec { if width <= bytes.len() { bytes.to_vec() diff --git a/common/src/wtf8/core_char.rs b/common/src/wtf8/core_char.rs new file mode 100644 index 0000000000..1444e8e130 --- /dev/null +++ b/common/src/wtf8/core_char.rs @@ -0,0 +1,113 @@ +//! Unstable functions from [`core::char`] + +use core::slice; + +pub const MAX_LEN_UTF8: usize = 4; +pub const MAX_LEN_UTF16: usize = 2; + +// UTF-8 ranges and tags for encoding characters +const TAG_CONT: u8 = 0b1000_0000; +const TAG_TWO_B: u8 = 0b1100_0000; +const TAG_THREE_B: u8 = 0b1110_0000; +const TAG_FOUR_B: u8 = 0b1111_0000; +const MAX_ONE_B: u32 = 0x80; +const MAX_TWO_B: u32 = 0x800; +const MAX_THREE_B: u32 = 0x10000; + +#[inline] +#[must_use] +pub const fn len_utf8(code: u32) -> usize { + match code { + ..MAX_ONE_B => 1, + ..MAX_TWO_B => 2, + ..MAX_THREE_B => 3, + _ => 4, + } +} + +#[inline] +#[must_use] +const fn len_utf16(code: u32) -> usize { + if (code & 0xFFFF) == code { 1 } else { 2 } +} + +/// Encodes a raw `u32` value as UTF-8 into the provided byte buffer, +/// and then returns the subslice of the buffer that contains the encoded character. +/// +/// Unlike `char::encode_utf8`, this method also handles codepoints in the surrogate range. +/// (Creating a `char` in the surrogate range is UB.) +/// The result is valid [generalized UTF-8] but not valid UTF-8. +/// +/// [generalized UTF-8]: https://simonsapin.github.io/wtf-8/#generalized-utf8 +/// +/// # Panics +/// +/// Panics if the buffer is not large enough. +/// A buffer of length four is large enough to encode any `char`. +#[doc(hidden)] +#[inline] +pub fn encode_utf8_raw(code: u32, dst: &mut [u8]) -> &mut [u8] { + let len = len_utf8(code); + match (len, &mut *dst) { + (1, [a, ..]) => { + *a = code as u8; + } + (2, [a, b, ..]) => { + *a = (code >> 6 & 0x1F) as u8 | TAG_TWO_B; + *b = (code & 0x3F) as u8 | TAG_CONT; + } + (3, [a, b, c, ..]) => { + *a = (code >> 12 & 0x0F) as u8 | TAG_THREE_B; + *b = (code >> 6 & 0x3F) as u8 | TAG_CONT; + *c = (code & 0x3F) as u8 | TAG_CONT; + } + (4, [a, b, c, d, ..]) => { + *a = (code >> 18 & 0x07) as u8 | TAG_FOUR_B; + *b = (code >> 12 & 0x3F) as u8 | TAG_CONT; + *c = (code >> 6 & 0x3F) as u8 | TAG_CONT; + *d = (code & 0x3F) as u8 | TAG_CONT; + } + _ => { + panic!( + "encode_utf8: need {len} bytes to encode U+{code:04X} but buffer has just {dst_len}", + dst_len = dst.len(), + ) + } + }; + // SAFETY: `<&mut [u8]>::as_mut_ptr` is guaranteed to return a valid pointer and `len` has been tested to be within bounds. + unsafe { slice::from_raw_parts_mut(dst.as_mut_ptr(), len) } +} + +/// Encodes a raw `u32` value as UTF-16 into the provided `u16` buffer, +/// and then returns the subslice of the buffer that contains the encoded character. +/// +/// Unlike `char::encode_utf16`, this method also handles codepoints in the surrogate range. +/// (Creating a `char` in the surrogate range is UB.) +/// +/// # Panics +/// +/// Panics if the buffer is not large enough. +/// A buffer of length 2 is large enough to encode any `char`. +#[doc(hidden)] +#[inline] +pub fn encode_utf16_raw(mut code: u32, dst: &mut [u16]) -> &mut [u16] { + let len = len_utf16(code); + match (len, &mut *dst) { + (1, [a, ..]) => { + *a = code as u16; + } + (2, [a, b, ..]) => { + code -= 0x1_0000; + *a = (code >> 10) as u16 | 0xD800; + *b = (code & 0x3FF) as u16 | 0xDC00; + } + _ => { + panic!( + "encode_utf16: need {len} bytes to encode U+{code:04X} but buffer has just {dst_len}", + dst_len = dst.len(), + ) + } + }; + // SAFETY: `<&mut [u16]>::as_mut_ptr` is guaranteed to return a valid pointer and `len` has been tested to be within bounds. + unsafe { slice::from_raw_parts_mut(dst.as_mut_ptr(), len) } +} diff --git a/common/src/wtf8/core_str.rs b/common/src/wtf8/core_str.rs new file mode 100644 index 0000000000..56f715cf75 --- /dev/null +++ b/common/src/wtf8/core_str.rs @@ -0,0 +1,113 @@ +//! Operations related to UTF-8 validation. +//! +//! Copied from `core::str::validations` + +/// Returns the initial codepoint accumulator for the first byte. +/// The first byte is special, only want bottom 5 bits for width 2, 4 bits +/// for width 3, and 3 bits for width 4. +#[inline] +const fn utf8_first_byte(byte: u8, width: u32) -> u32 { + (byte & (0x7F >> width)) as u32 +} + +/// Returns the value of `ch` updated with continuation byte `byte`. +#[inline] +const fn utf8_acc_cont_byte(ch: u32, byte: u8) -> u32 { + (ch << 6) | (byte & CONT_MASK) as u32 +} + +/// Checks whether the byte is a UTF-8 continuation byte (i.e., starts with the +/// bits `10`). +#[inline] +pub(super) const fn utf8_is_cont_byte(byte: u8) -> bool { + (byte as i8) < -64 +} + +/// Reads the next code point out of a byte iterator (assuming a +/// UTF-8-like encoding). +/// +/// # Safety +/// +/// `bytes` must produce a valid UTF-8-like (UTF-8 or WTF-8) string +#[inline] +pub unsafe fn next_code_point<'a, I: Iterator>(bytes: &mut I) -> Option { + // Decode UTF-8 + let x = *bytes.next()?; + if x < 128 { + return Some(x as u32); + } + + // Multibyte case follows + // Decode from a byte combination out of: [[[x y] z] w] + // NOTE: Performance is sensitive to the exact formulation here + let init = utf8_first_byte(x, 2); + // SAFETY: `bytes` produces an UTF-8-like string, + // so the iterator must produce a value here. + let y = unsafe { *bytes.next().unwrap_unchecked() }; + let mut ch = utf8_acc_cont_byte(init, y); + if x >= 0xE0 { + // [[x y z] w] case + // 5th bit in 0xE0 .. 0xEF is always clear, so `init` is still valid + // SAFETY: `bytes` produces an UTF-8-like string, + // so the iterator must produce a value here. + let z = unsafe { *bytes.next().unwrap_unchecked() }; + let y_z = utf8_acc_cont_byte((y & CONT_MASK) as u32, z); + ch = init << 12 | y_z; + if x >= 0xF0 { + // [x y z w] case + // use only the lower 3 bits of `init` + // SAFETY: `bytes` produces an UTF-8-like string, + // so the iterator must produce a value here. + let w = unsafe { *bytes.next().unwrap_unchecked() }; + ch = (init & 7) << 18 | utf8_acc_cont_byte(y_z, w); + } + } + + Some(ch) +} + +/// Reads the last code point out of a byte iterator (assuming a +/// UTF-8-like encoding). +/// +/// # Safety +/// +/// `bytes` must produce a valid UTF-8-like (UTF-8 or WTF-8) string +#[inline] +pub unsafe fn next_code_point_reverse<'a, I>(bytes: &mut I) -> Option +where + I: DoubleEndedIterator, +{ + // Decode UTF-8 + let w = match *bytes.next_back()? { + next_byte if next_byte < 128 => return Some(next_byte as u32), + back_byte => back_byte, + }; + + // Multibyte case follows + // Decode from a byte combination out of: [x [y [z w]]] + let mut ch; + // SAFETY: `bytes` produces an UTF-8-like string, + // so the iterator must produce a value here. + let z = unsafe { *bytes.next_back().unwrap_unchecked() }; + ch = utf8_first_byte(z, 2); + if utf8_is_cont_byte(z) { + // SAFETY: `bytes` produces an UTF-8-like string, + // so the iterator must produce a value here. + let y = unsafe { *bytes.next_back().unwrap_unchecked() }; + ch = utf8_first_byte(y, 3); + if utf8_is_cont_byte(y) { + // SAFETY: `bytes` produces an UTF-8-like string, + // so the iterator must produce a value here. + let x = unsafe { *bytes.next_back().unwrap_unchecked() }; + ch = utf8_first_byte(x, 4); + ch = utf8_acc_cont_byte(ch, y); + } + ch = utf8_acc_cont_byte(ch, z); + } + ch = utf8_acc_cont_byte(ch, w); + + Some(ch) +} + +/// Mask of the value bits of a continuation byte. +const CONT_MASK: u8 = 0b0011_1111; diff --git a/common/src/wtf8/mod.rs b/common/src/wtf8/mod.rs new file mode 100644 index 0000000000..8ec3ae0afe --- /dev/null +++ b/common/src/wtf8/mod.rs @@ -0,0 +1,1347 @@ +//! An implementation of [WTF-8], a utf8-compatible encoding that allows for +//! unpaired surrogate codepoints. This implementation additionally allows for +//! paired surrogates that are nonetheless treated as two separate codepoints. +//! +//! +//! RustPython uses this because CPython internally uses a variant of UCS-1/2/4 +//! as its string storage, which treats each `u8`/`u16`/`u32` value (depending +//! on the highest codepoint value in the string) as simply integers, unlike +//! UTF-8 or UTF-16 where some characters are encoded using multi-byte +//! sequences. CPython additionally doesn't disallow the use of surrogates in +//! `str`s (which in UTF-16 pair together to represent codepoints with a value +//! higher than `u16::MAX`) and in fact takes quite extensive advantage of the +//! fact that they're allowed. The `surrogateescape` codec-error handler uses +//! them to represent byte sequences which are invalid in the given codec (e.g. +//! bytes with their high bit set in ASCII or UTF-8) by mapping them into the +//! surrogate range. `surrogateescape` is the default error handler in Python +//! for interacting with the filesystem, and thus if RustPython is to properly +//! support `surrogateescape`, its `str`s must be able to represent surrogates. +//! +//! We use WTF-8 over something more similar to CPython's string implementation +//! because of its compatibility with UTF-8, meaning that in the case where a +//! string has no surrogates, it can be viewed as a UTF-8 Rust [`str`] without +//! needing any copies or re-encoding. +//! +//! This implementation is mostly copied from the WTF-8 implentation in the +//! Rust standard library, which is used as the backing for [`OsStr`] on +//! Windows targets. As previously mentioned, however, it is modified to not +//! join two surrogates into one codepoint when concatenating strings, in order +//! to match CPython's behavior. +//! +//! [WTF-8]: https://simonsapin.github.io/wtf-8 +//! [`OsStr`]: std::ffi::OsStr + +#![allow(clippy::precedence, clippy::match_overlapping_arm)] + +use core::fmt; +use core::hash::{Hash, Hasher}; +use core::iter::FusedIterator; +use core::mem; +use core::ops; +use core::slice; +use core::str; +use core_char::MAX_LEN_UTF8; +use core_char::{MAX_LEN_UTF16, encode_utf8_raw, encode_utf16_raw, len_utf8}; +use core_str::{next_code_point, next_code_point_reverse}; +use itertools::{Either, Itertools}; +use std::borrow::{Borrow, Cow}; +use std::collections::TryReserveError; +use std::string::String; +use std::vec::Vec; + +use bstr::ByteSlice; + +mod core_char; +mod core_str; + +const UTF8_REPLACEMENT_CHARACTER: &str = "\u{FFFD}"; + +/// A Unicode code point: from U+0000 to U+10FFFF. +/// +/// Compares with the `char` type, +/// which represents a Unicode scalar value: +/// a code point that is not a surrogate (U+D800 to U+DFFF). +#[derive(Eq, PartialEq, Ord, PartialOrd, Clone, Copy)] +pub struct CodePoint { + value: u32, +} + +/// Format the code point as `U+` followed by four to six hexadecimal digits. +/// Example: `U+1F4A9` +impl fmt::Debug for CodePoint { + #[inline] + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(formatter, "U+{:04X}", self.value) + } +} + +impl CodePoint { + /// Unsafely creates a new `CodePoint` without checking the value. + /// + /// # Safety + /// + /// `value` must be less than or equal to 0x10FFFF. + #[inline] + pub unsafe fn from_u32_unchecked(value: u32) -> CodePoint { + CodePoint { value } + } + + /// Creates a new `CodePoint` if the value is a valid code point. + /// + /// Returns `None` if `value` is above 0x10FFFF. + #[inline] + pub fn from_u32(value: u32) -> Option { + match value { + 0..=0x10FFFF => Some(CodePoint { value }), + _ => None, + } + } + + /// Creates a new `CodePoint` from a `char`. + /// + /// Since all Unicode scalar values are code points, this always succeeds. + #[inline] + pub fn from_char(value: char) -> CodePoint { + CodePoint { + value: value as u32, + } + } + + /// Returns the numeric value of the code point. + #[inline] + pub fn to_u32(&self) -> u32 { + self.value + } + + /// Returns the numeric value of the code point if it is a leading surrogate. + #[inline] + pub fn to_lead_surrogate(&self) -> Option { + match self.value { + lead @ 0xD800..=0xDBFF => Some(lead as u16), + _ => None, + } + } + + /// Returns the numeric value of the code point if it is a trailing surrogate. + #[inline] + pub fn to_trail_surrogate(&self) -> Option { + match self.value { + trail @ 0xDC00..=0xDFFF => Some(trail as u16), + _ => None, + } + } + + /// Optionally returns a Unicode scalar value for the code point. + /// + /// Returns `None` if the code point is a surrogate (from U+D800 to U+DFFF). + #[inline] + pub fn to_char(&self) -> Option { + match self.value { + 0xD800..=0xDFFF => None, + _ => Some(unsafe { char::from_u32_unchecked(self.value) }), + } + } + + /// Returns a Unicode scalar value for the code point. + /// + /// Returns `'\u{FFFD}'` (the replacement character “�”) + /// if the code point is a surrogate (from U+D800 to U+DFFF). + #[inline] + pub fn to_char_lossy(&self) -> char { + self.to_char().unwrap_or('\u{FFFD}') + } + + pub fn is_char_and(self, f: impl FnOnce(char) -> bool) -> bool { + self.to_char().is_some_and(f) + } + + pub fn encode_wtf8(self, dst: &mut [u8]) -> &mut Wtf8 { + unsafe { Wtf8::from_mut_bytes_unchecked(encode_utf8_raw(self.value, dst)) } + } + + pub fn len_wtf8(&self) -> usize { + len_utf8(self.value) + } +} + +impl From for CodePoint { + fn from(value: u16) -> Self { + unsafe { Self::from_u32_unchecked(value.into()) } + } +} + +impl From for CodePoint { + fn from(value: char) -> Self { + Self::from_char(value) + } +} + +impl From for CodePoint { + fn from(value: ascii::AsciiChar) -> Self { + Self::from_char(value.into()) + } +} + +impl From for Wtf8Buf { + fn from(ch: CodePoint) -> Self { + ch.encode_wtf8(&mut [0; MAX_LEN_UTF8]).to_owned() + } +} + +impl PartialEq for CodePoint { + fn eq(&self, other: &char) -> bool { + self.to_u32() == *other as u32 + } +} +impl PartialEq for char { + fn eq(&self, other: &CodePoint) -> bool { + *self as u32 == other.to_u32() + } +} + +/// An owned, growable string of well-formed WTF-8 data. +/// +/// Similar to `String`, but can additionally contain surrogate code points +/// if they’re not in a surrogate pair. +#[derive(Eq, PartialEq, Ord, PartialOrd, Clone, Default)] +pub struct Wtf8Buf { + bytes: Vec, +} + +impl ops::Deref for Wtf8Buf { + type Target = Wtf8; + + fn deref(&self) -> &Wtf8 { + self.as_slice() + } +} + +impl ops::DerefMut for Wtf8Buf { + fn deref_mut(&mut self) -> &mut Wtf8 { + self.as_mut_slice() + } +} + +impl Borrow for Wtf8Buf { + fn borrow(&self) -> &Wtf8 { + self + } +} + +/// Formats the string in double quotes, with characters escaped according to +/// [`char::escape_debug`] and unpaired surrogates represented as `\u{xxxx}`, +/// where each `x` is a hexadecimal digit. +/// +/// For example, the code units [U+0061, U+D800, U+000A] are formatted as +/// `"a\u{D800}\n"`. +impl fmt::Debug for Wtf8Buf { + #[inline] + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&**self, formatter) + } +} + +/// Formats the string with unpaired surrogates substituted with the replacement +/// character, U+FFFD. +impl fmt::Display for Wtf8Buf { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&**self, formatter) + } +} + +impl Wtf8Buf { + /// Creates a new, empty WTF-8 string. + #[inline] + pub fn new() -> Wtf8Buf { + Wtf8Buf::default() + } + + /// Creates a new, empty WTF-8 string with pre-allocated capacity for `capacity` bytes. + #[inline] + pub fn with_capacity(capacity: usize) -> Wtf8Buf { + Wtf8Buf { + bytes: Vec::with_capacity(capacity), + } + } + + /// Creates a WTF-8 string from a WTF-8 byte vec. + /// + /// # Safety + /// + /// `value` must contain valid WTF-8. + #[inline] + pub unsafe fn from_bytes_unchecked(value: Vec) -> Wtf8Buf { + Wtf8Buf { bytes: value } + } + + /// Creates a WTF-8 string from a UTF-8 `String`. + /// + /// This takes ownership of the `String` and does not copy. + /// + /// Since WTF-8 is a superset of UTF-8, this always succeeds. + #[inline] + pub fn from_string(string: String) -> Wtf8Buf { + Wtf8Buf { + bytes: string.into_bytes(), + } + } + + pub fn clear(&mut self) { + self.bytes.clear(); + } + + /// Creates a WTF-8 string from a potentially ill-formed UTF-16 slice of 16-bit code units. + /// + /// This is lossless: calling `.encode_wide()` on the resulting string + /// will always return the original code units. + pub fn from_wide(v: &[u16]) -> Wtf8Buf { + let mut string = Wtf8Buf::with_capacity(v.len()); + for item in char::decode_utf16(v.iter().cloned()) { + match item { + Ok(ch) => string.push_char(ch), + Err(surrogate) => { + let surrogate = surrogate.unpaired_surrogate(); + // Surrogates are known to be in the code point range. + let code_point = CodePoint::from(surrogate); + // Skip the WTF-8 concatenation check, + // surrogate pairs are already decoded by decode_utf16 + string.push(code_point); + } + } + } + string + } + + #[inline] + pub fn as_slice(&self) -> &Wtf8 { + unsafe { Wtf8::from_bytes_unchecked(&self.bytes) } + } + + #[inline] + pub fn as_mut_slice(&mut self) -> &mut Wtf8 { + // Safety: `Wtf8` doesn't expose any way to mutate the bytes that would + // cause them to change from well-formed UTF-8 to ill-formed UTF-8, + // which would break the assumptions of the `is_known_utf8` field. + unsafe { Wtf8::from_mut_bytes_unchecked(&mut self.bytes) } + } + + /// Reserves capacity for at least `additional` more bytes to be inserted + /// in the given `Wtf8Buf`. + /// The collection may reserve more space to avoid frequent reallocations. + /// + /// # Panics + /// + /// Panics if the new capacity exceeds `isize::MAX` bytes. + #[inline] + pub fn reserve(&mut self, additional: usize) { + self.bytes.reserve(additional) + } + + /// Tries to reserve capacity for at least `additional` more bytes to be + /// inserted in the given `Wtf8Buf`. The `Wtf8Buf` may reserve more space to + /// avoid frequent reallocations. After calling `try_reserve`, capacity will + /// be greater than or equal to `self.len() + additional`. Does nothing if + /// capacity is already sufficient. This method preserves the contents even + /// if an error occurs. + /// + /// # Errors + /// + /// If the capacity overflows, or the allocator reports a failure, then an error + /// is returned. + #[inline] + pub fn try_reserve(&mut self, additional: usize) -> Result<(), TryReserveError> { + self.bytes.try_reserve(additional) + } + + #[inline] + pub fn reserve_exact(&mut self, additional: usize) { + self.bytes.reserve_exact(additional) + } + + /// Tries to reserve the minimum capacity for exactly `additional` more + /// bytes to be inserted in the given `Wtf8Buf`. After calling + /// `try_reserve_exact`, capacity will be greater than or equal to + /// `self.len() + additional` if it returns `Ok(())`. + /// Does nothing if the capacity is already sufficient. + /// + /// Note that the allocator may give the `Wtf8Buf` more space than it + /// requests. Therefore, capacity can not be relied upon to be precisely + /// minimal. Prefer [`try_reserve`] if future insertions are expected. + /// + /// [`try_reserve`]: Wtf8Buf::try_reserve + /// + /// # Errors + /// + /// If the capacity overflows, or the allocator reports a failure, then an error + /// is returned. + #[inline] + pub fn try_reserve_exact(&mut self, additional: usize) -> Result<(), TryReserveError> { + self.bytes.try_reserve_exact(additional) + } + + #[inline] + pub fn shrink_to_fit(&mut self) { + self.bytes.shrink_to_fit() + } + + #[inline] + pub fn shrink_to(&mut self, min_capacity: usize) { + self.bytes.shrink_to(min_capacity) + } + + #[inline] + pub fn leak<'a>(self) -> &'a mut Wtf8 { + unsafe { Wtf8::from_mut_bytes_unchecked(self.bytes.leak()) } + } + + /// Returns the number of bytes that this string buffer can hold without reallocating. + #[inline] + pub fn capacity(&self) -> usize { + self.bytes.capacity() + } + + /// Append a UTF-8 slice at the end of the string. + #[inline] + pub fn push_str(&mut self, other: &str) { + self.bytes.extend_from_slice(other.as_bytes()) + } + + /// Append a WTF-8 slice at the end of the string. + #[inline] + pub fn push_wtf8(&mut self, other: &Wtf8) { + self.bytes.extend_from_slice(&other.bytes); + } + + /// Append a Unicode scalar value at the end of the string. + #[inline] + pub fn push_char(&mut self, c: char) { + self.push(CodePoint::from_char(c)) + } + + /// Append a code point at the end of the string. + #[inline] + pub fn push(&mut self, code_point: CodePoint) { + self.push_wtf8(code_point.encode_wtf8(&mut [0; MAX_LEN_UTF8])) + } + + /// Shortens a string to the specified length. + /// + /// # Panics + /// + /// Panics if `new_len` > current length, + /// or if `new_len` is not a code point boundary. + #[inline] + pub fn truncate(&mut self, new_len: usize) { + assert!(is_code_point_boundary(self, new_len)); + self.bytes.truncate(new_len) + } + + /// Consumes the WTF-8 string and tries to convert it to a vec of bytes. + #[inline] + pub fn into_bytes(self) -> Vec { + self.bytes + } + + /// Consumes the WTF-8 string and tries to convert it to UTF-8. + /// + /// This does not copy the data. + /// + /// If the contents are not well-formed UTF-8 + /// (that is, if the string contains surrogates), + /// the original WTF-8 string is returned instead. + pub fn into_string(self) -> Result { + if self.is_utf8() { + Ok(unsafe { String::from_utf8_unchecked(self.bytes) }) + } else { + Err(self) + } + } + + /// Consumes the WTF-8 string and converts it lossily to UTF-8. + /// + /// This does not copy the data (but may overwrite parts of it in place). + /// + /// Surrogates are replaced with `"\u{FFFD}"` (the replacement character “�”) + pub fn into_string_lossy(mut self) -> String { + let mut pos = 0; + while let Some((surrogate_pos, _)) = self.next_surrogate(pos) { + pos = surrogate_pos + 3; + // Surrogates and the replacement character are all 3 bytes, so + // they can substituted in-place. + self.bytes[surrogate_pos..pos].copy_from_slice(UTF8_REPLACEMENT_CHARACTER.as_bytes()); + } + unsafe { String::from_utf8_unchecked(self.bytes) } + } + + /// Converts this `Wtf8Buf` into a boxed `Wtf8`. + #[inline] + pub fn into_box(self) -> Box { + // SAFETY: relies on `Wtf8` being `repr(transparent)`. + unsafe { mem::transmute(self.bytes.into_boxed_slice()) } + } + + /// Converts a `Box` into a `Wtf8Buf`. + pub fn from_box(boxed: Box) -> Wtf8Buf { + let bytes: Box<[u8]> = unsafe { mem::transmute(boxed) }; + Wtf8Buf { + bytes: bytes.into_vec(), + } + } +} + +/// Creates a new WTF-8 string from an iterator of code points. +/// +/// This replaces surrogate code point pairs with supplementary code points, +/// like concatenating ill-formed UTF-16 strings effectively would. +impl FromIterator for Wtf8Buf { + fn from_iter>(iter: T) -> Wtf8Buf { + let mut string = Wtf8Buf::new(); + string.extend(iter); + string + } +} + +/// Append code points from an iterator to the string. +/// +/// This replaces surrogate code point pairs with supplementary code points, +/// like concatenating ill-formed UTF-16 strings effectively would. +impl Extend for Wtf8Buf { + fn extend>(&mut self, iter: T) { + let iterator = iter.into_iter(); + let (low, _high) = iterator.size_hint(); + // Lower bound of one byte per code point (ASCII only) + self.bytes.reserve(low); + iterator.for_each(move |code_point| self.push(code_point)); + } +} + +impl> FromIterator for Wtf8Buf { + fn from_iter>(iter: T) -> Self { + let mut buf = Wtf8Buf::new(); + iter.into_iter().for_each(|w| buf.push_wtf8(w.as_ref())); + buf + } +} + +impl AsRef for Wtf8Buf { + fn as_ref(&self) -> &Wtf8 { + self + } +} + +impl From for Wtf8Buf { + fn from(s: String) -> Self { + Wtf8Buf::from_string(s) + } +} + +impl From<&str> for Wtf8Buf { + fn from(s: &str) -> Self { + Wtf8Buf::from_string(s.to_owned()) + } +} + +/// A borrowed slice of well-formed WTF-8 data. +/// +/// Similar to `&str`, but can additionally contain surrogate code points +/// if they’re not in a surrogate pair. +#[derive(PartialEq, Eq, PartialOrd, Ord)] +pub struct Wtf8 { + bytes: [u8], +} + +impl AsRef for Wtf8 { + fn as_ref(&self) -> &Wtf8 { + self + } +} + +impl ToOwned for Wtf8 { + type Owned = Wtf8Buf; + fn to_owned(&self) -> Self::Owned { + self.to_wtf8_buf() + } +} + +/// Formats the string in double quotes, with characters escaped according to +/// [`char::escape_debug`] and unpaired surrogates represented as `\u{xxxx}`, +/// where each `x` is a hexadecimal digit. +impl fmt::Debug for Wtf8 { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + fn write_str_escaped(f: &mut fmt::Formatter<'_>, s: &str) -> fmt::Result { + use std::fmt::Write; + for c in s.chars().flat_map(|c| c.escape_debug()) { + f.write_char(c)? + } + Ok(()) + } + + formatter.write_str("\"")?; + let mut pos = 0; + while let Some((surrogate_pos, surrogate)) = self.next_surrogate(pos) { + write_str_escaped(formatter, unsafe { + str::from_utf8_unchecked(&self.bytes[pos..surrogate_pos]) + })?; + write!(formatter, "\\u{{{:x}}}", surrogate)?; + pos = surrogate_pos + 3; + } + write_str_escaped(formatter, unsafe { + str::from_utf8_unchecked(&self.bytes[pos..]) + })?; + formatter.write_str("\"") + } +} + +/// Formats the string with unpaired surrogates substituted with the replacement +/// character, U+FFFD. +impl fmt::Display for Wtf8 { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + let wtf8_bytes = &self.bytes; + let mut pos = 0; + loop { + match self.next_surrogate(pos) { + Some((surrogate_pos, _)) => { + formatter.write_str(unsafe { + str::from_utf8_unchecked(&wtf8_bytes[pos..surrogate_pos]) + })?; + formatter.write_str(UTF8_REPLACEMENT_CHARACTER)?; + pos = surrogate_pos + 3; + } + None => { + let s = unsafe { str::from_utf8_unchecked(&wtf8_bytes[pos..]) }; + if pos == 0 { + return s.fmt(formatter); + } else { + return formatter.write_str(s); + } + } + } + } + } +} + +impl Default for &Wtf8 { + fn default() -> Self { + unsafe { Wtf8::from_bytes_unchecked(&[]) } + } +} + +impl Wtf8 { + /// Creates a WTF-8 slice from a UTF-8 `&str` slice. + /// + /// Since WTF-8 is a superset of UTF-8, this always succeeds. + #[inline] + pub fn new + ?Sized>(value: &S) -> &Wtf8 { + value.as_ref() + } + + /// Creates a WTF-8 slice from a WTF-8 byte slice. + /// + /// # Safety + /// + /// `value` must contain valid WTF-8. + #[inline] + pub unsafe fn from_bytes_unchecked(value: &[u8]) -> &Wtf8 { + // SAFETY: start with &[u8], end with fancy &[u8] + unsafe { &*(value as *const [u8] as *const Wtf8) } + } + + /// Creates a mutable WTF-8 slice from a mutable WTF-8 byte slice. + /// + /// Since the byte slice is not checked for valid WTF-8, this functions is + /// marked unsafe. + #[inline] + unsafe fn from_mut_bytes_unchecked(value: &mut [u8]) -> &mut Wtf8 { + // SAFETY: start with &mut [u8], end with fancy &mut [u8] + unsafe { &mut *(value as *mut [u8] as *mut Wtf8) } + } + + /// Returns the length, in WTF-8 bytes. + #[inline] + pub fn len(&self) -> usize { + self.bytes.len() + } + + #[inline] + pub fn is_empty(&self) -> bool { + self.bytes.is_empty() + } + + /// Returns the code point at `position` if it is in the ASCII range, + /// or `b'\xFF'` otherwise. + /// + /// # Panics + /// + /// Panics if `position` is beyond the end of the string. + #[inline] + pub fn ascii_byte_at(&self, position: usize) -> u8 { + match self.bytes[position] { + ascii_byte @ 0x00..=0x7F => ascii_byte, + _ => 0xFF, + } + } + + /// Returns an iterator for the string’s code points. + #[inline] + pub fn code_points(&self) -> Wtf8CodePoints<'_> { + Wtf8CodePoints { + bytes: self.bytes.iter(), + } + } + + /// Returns an iterator for the string’s code points and their indices. + #[inline] + pub fn code_point_indices(&self) -> Wtf8CodePointIndices<'_> { + Wtf8CodePointIndices { + front_offset: 0, + iter: self.code_points(), + } + } + + /// Access raw bytes of WTF-8 data + #[inline] + pub fn as_bytes(&self) -> &[u8] { + &self.bytes + } + + /// Tries to convert the string to UTF-8 and return a `&str` slice. + /// + /// Returns `None` if the string contains surrogates. + /// + /// This does not copy the data. + #[inline] + pub fn as_str(&self) -> Result<&str, str::Utf8Error> { + str::from_utf8(&self.bytes) + } + + /// Creates an owned `Wtf8Buf` from a borrowed `Wtf8`. + pub fn to_wtf8_buf(&self) -> Wtf8Buf { + Wtf8Buf { + bytes: self.bytes.to_vec(), + } + } + + /// Lossily converts the string to UTF-8. + /// Returns a UTF-8 `&str` slice if the contents are well-formed in UTF-8. + /// + /// Surrogates are replaced with `"\u{FFFD}"` (the replacement character “�”). + /// + /// This only copies the data if necessary (if it contains any surrogate). + pub fn to_string_lossy(&self) -> Cow<'_, str> { + let Some((surrogate_pos, _)) = self.next_surrogate(0) else { + return Cow::Borrowed(unsafe { str::from_utf8_unchecked(&self.bytes) }); + }; + let wtf8_bytes = &self.bytes; + let mut utf8_bytes = Vec::with_capacity(self.len()); + utf8_bytes.extend_from_slice(&wtf8_bytes[..surrogate_pos]); + utf8_bytes.extend_from_slice(UTF8_REPLACEMENT_CHARACTER.as_bytes()); + let mut pos = surrogate_pos + 3; + loop { + match self.next_surrogate(pos) { + Some((surrogate_pos, _)) => { + utf8_bytes.extend_from_slice(&wtf8_bytes[pos..surrogate_pos]); + utf8_bytes.extend_from_slice(UTF8_REPLACEMENT_CHARACTER.as_bytes()); + pos = surrogate_pos + 3; + } + None => { + utf8_bytes.extend_from_slice(&wtf8_bytes[pos..]); + return Cow::Owned(unsafe { String::from_utf8_unchecked(utf8_bytes) }); + } + } + } + } + + /// Converts the WTF-8 string to potentially ill-formed UTF-16 + /// and return an iterator of 16-bit code units. + /// + /// This is lossless: + /// calling `Wtf8Buf::from_ill_formed_utf16` on the resulting code units + /// would always return the original WTF-8 string. + #[inline] + pub fn encode_wide(&self) -> EncodeWide<'_> { + EncodeWide { + code_points: self.code_points(), + extra: 0, + } + } + + pub fn chunks(&self) -> Wtf8Chunks<'_> { + Wtf8Chunks { wtf8: self } + } + + pub fn map_utf8<'a, I>(&'a self, f: impl Fn(&'a str) -> I) -> impl Iterator + where + I: Iterator, + { + self.chunks().flat_map(move |chunk| match chunk { + Wtf8Chunk::Utf8(s) => Either::Left(f(s).map_into()), + Wtf8Chunk::Surrogate(c) => Either::Right(std::iter::once(c)), + }) + } + + #[inline] + fn next_surrogate(&self, mut pos: usize) -> Option<(usize, u16)> { + let mut iter = self.bytes[pos..].iter(); + loop { + let b = *iter.next()?; + if b < 0x80 { + pos += 1; + } else if b < 0xE0 { + iter.next(); + pos += 2; + } else if b == 0xED { + match (iter.next(), iter.next()) { + (Some(&b2), Some(&b3)) if b2 >= 0xA0 => { + return Some((pos, decode_surrogate(b2, b3))); + } + _ => pos += 3, + } + } else if b < 0xF0 { + iter.next(); + iter.next(); + pos += 3; + } else { + iter.next(); + iter.next(); + iter.next(); + pos += 4; + } + } + } + + pub fn clone_into(&self, buf: &mut Wtf8Buf) { + self.bytes.clone_into(&mut buf.bytes); + } + + /// Boxes this `Wtf8`. + #[inline] + pub fn into_box(&self) -> Box { + let boxed: Box<[u8]> = self.bytes.into(); + unsafe { mem::transmute(boxed) } + } + + /// Creates a boxed, empty `Wtf8`. + pub fn empty_box() -> Box { + let boxed: Box<[u8]> = Default::default(); + unsafe { mem::transmute(boxed) } + } + + #[inline] + pub fn make_ascii_lowercase(&mut self) { + self.bytes.make_ascii_lowercase() + } + + #[inline] + pub fn make_ascii_uppercase(&mut self) { + self.bytes.make_ascii_uppercase() + } + + #[inline] + pub fn to_ascii_lowercase(&self) -> Wtf8Buf { + Wtf8Buf { + bytes: self.bytes.to_ascii_lowercase(), + } + } + + #[inline] + pub fn to_ascii_uppercase(&self) -> Wtf8Buf { + Wtf8Buf { + bytes: self.bytes.to_ascii_uppercase(), + } + } + + #[inline] + pub fn is_ascii(&self) -> bool { + self.bytes.is_ascii() + } + + #[inline] + pub fn is_utf8(&self) -> bool { + self.next_surrogate(0).is_none() + } + + #[inline] + pub fn eq_ignore_ascii_case(&self, other: &Self) -> bool { + self.bytes.eq_ignore_ascii_case(&other.bytes) + } + + pub fn split(&self, pat: &Wtf8) -> impl Iterator { + self.as_bytes() + .split_str(pat) + .map(|w| unsafe { Wtf8::from_bytes_unchecked(w) }) + } + + pub fn splitn(&self, n: usize, pat: &Wtf8) -> impl Iterator { + self.as_bytes() + .splitn_str(n, pat) + .map(|w| unsafe { Wtf8::from_bytes_unchecked(w) }) + } + + pub fn rsplit(&self, pat: &Wtf8) -> impl Iterator { + self.as_bytes() + .rsplit_str(pat) + .map(|w| unsafe { Wtf8::from_bytes_unchecked(w) }) + } + + pub fn rsplitn(&self, n: usize, pat: &Wtf8) -> impl Iterator { + self.as_bytes() + .rsplitn_str(n, pat) + .map(|w| unsafe { Wtf8::from_bytes_unchecked(w) }) + } + + pub fn trim_start_matches(&self, f: impl Fn(CodePoint) -> bool) -> &Self { + let mut iter = self.code_points(); + loop { + let old = iter.clone(); + match iter.next().map(&f) { + Some(true) => continue, + Some(false) => { + iter = old; + break; + } + None => return iter.as_wtf8(), + } + } + iter.as_wtf8() + } + + pub fn trim_end_matches(&self, f: impl Fn(CodePoint) -> bool) -> &Self { + let mut iter = self.code_points(); + loop { + let old = iter.clone(); + match iter.next_back().map(&f) { + Some(true) => continue, + Some(false) => { + iter = old; + break; + } + None => return iter.as_wtf8(), + } + } + iter.as_wtf8() + } + + pub fn trim_matches(&self, f: impl Fn(CodePoint) -> bool) -> &Self { + self.trim_start_matches(&f).trim_end_matches(&f) + } + + pub fn find(&self, pat: &Wtf8) -> Option { + memchr::memmem::find(self.as_bytes(), pat.as_bytes()) + } + + pub fn rfind(&self, pat: &Wtf8) -> Option { + memchr::memmem::rfind(self.as_bytes(), pat.as_bytes()) + } + + pub fn get(&self, range: impl ops::RangeBounds) -> Option<&Self> { + let start = match range.start_bound() { + ops::Bound::Included(&i) => i, + ops::Bound::Excluded(&i) => i.saturating_add(1), + ops::Bound::Unbounded => 0, + }; + let end = match range.end_bound() { + ops::Bound::Included(&i) => i.saturating_add(1), + ops::Bound::Excluded(&i) => i, + ops::Bound::Unbounded => self.len(), + }; + // is_code_point_boundary checks that the index is in [0, .len()] + if start <= end && is_code_point_boundary(self, start) && is_code_point_boundary(self, end) + { + Some(unsafe { slice_unchecked(self, start, end) }) + } else { + None + } + } +} + +impl AsRef for str { + fn as_ref(&self) -> &Wtf8 { + unsafe { Wtf8::from_bytes_unchecked(self.as_bytes()) } + } +} + +impl AsRef<[u8]> for Wtf8 { + fn as_ref(&self) -> &[u8] { + self.as_bytes() + } +} + +/// Returns a slice of the given string for the byte range \[`begin`..`end`). +/// +/// # Panics +/// +/// Panics when `begin` and `end` do not point to code point boundaries, +/// or point beyond the end of the string. +impl ops::Index> for Wtf8 { + type Output = Wtf8; + + #[inline] + fn index(&self, range: ops::Range) -> &Wtf8 { + // is_code_point_boundary checks that the index is in [0, .len()] + if range.start <= range.end + && is_code_point_boundary(self, range.start) + && is_code_point_boundary(self, range.end) + { + unsafe { slice_unchecked(self, range.start, range.end) } + } else { + slice_error_fail(self, range.start, range.end) + } + } +} + +/// Returns a slice of the given string from byte `begin` to its end. +/// +/// # Panics +/// +/// Panics when `begin` is not at a code point boundary, +/// or is beyond the end of the string. +impl ops::Index> for Wtf8 { + type Output = Wtf8; + + #[inline] + fn index(&self, range: ops::RangeFrom) -> &Wtf8 { + // is_code_point_boundary checks that the index is in [0, .len()] + if is_code_point_boundary(self, range.start) { + unsafe { slice_unchecked(self, range.start, self.len()) } + } else { + slice_error_fail(self, range.start, self.len()) + } + } +} + +/// Returns a slice of the given string from its beginning to byte `end`. +/// +/// # Panics +/// +/// Panics when `end` is not at a code point boundary, +/// or is beyond the end of the string. +impl ops::Index> for Wtf8 { + type Output = Wtf8; + + #[inline] + fn index(&self, range: ops::RangeTo) -> &Wtf8 { + // is_code_point_boundary checks that the index is in [0, .len()] + if is_code_point_boundary(self, range.end) { + unsafe { slice_unchecked(self, 0, range.end) } + } else { + slice_error_fail(self, 0, range.end) + } + } +} + +impl ops::Index for Wtf8 { + type Output = Wtf8; + + #[inline] + fn index(&self, _range: ops::RangeFull) -> &Wtf8 { + self + } +} + +#[inline] +fn decode_surrogate(second_byte: u8, third_byte: u8) -> u16 { + // The first byte is assumed to be 0xED + 0xD800 | (second_byte as u16 & 0x3F) << 6 | third_byte as u16 & 0x3F +} + +/// Copied from str::is_char_boundary +#[inline] +pub fn is_code_point_boundary(slice: &Wtf8, index: usize) -> bool { + if index == 0 { + return true; + } + match slice.bytes.get(index) { + None => index == slice.len(), + Some(&b) => (b as i8) >= -0x40, + } +} + +/// Verify that `index` is at the edge of either a valid UTF-8 codepoint +/// (i.e. a codepoint that's not a surrogate) or of the whole string. +/// +/// These are the cases currently permitted by `OsStr::slice_encoded_bytes`. +/// Splitting between surrogates is valid as far as WTF-8 is concerned, but +/// we do not permit it in the public API because WTF-8 is considered an +/// implementation detail. +#[track_caller] +#[inline] +pub fn check_utf8_boundary(slice: &Wtf8, index: usize) { + if index == 0 { + return; + } + match slice.bytes.get(index) { + Some(0xED) => (), // Might be a surrogate + Some(&b) if (b as i8) >= -0x40 => return, + Some(_) => panic!("byte index {index} is not a codepoint boundary"), + None if index == slice.len() => return, + None => panic!("byte index {index} is out of bounds"), + } + if slice.bytes[index + 1] >= 0xA0 { + // There's a surrogate after index. Now check before index. + if index >= 3 && slice.bytes[index - 3] == 0xED && slice.bytes[index - 2] >= 0xA0 { + panic!("byte index {index} lies between surrogate codepoints"); + } + } +} + +/// Copied from core::str::raw::slice_unchecked +/// +/// # Safety +/// +/// `begin` and `end` must be within bounds and on codepoint boundaries. +#[inline] +pub unsafe fn slice_unchecked(s: &Wtf8, begin: usize, end: usize) -> &Wtf8 { + // SAFETY: memory layout of a &[u8] and &Wtf8 are the same + unsafe { + let len = end - begin; + let start = s.as_bytes().as_ptr().add(begin); + Wtf8::from_bytes_unchecked(slice::from_raw_parts(start, len)) + } +} + +/// Copied from core::str::raw::slice_error_fail +#[inline(never)] +pub fn slice_error_fail(s: &Wtf8, begin: usize, end: usize) -> ! { + assert!(begin <= end); + panic!("index {begin} and/or {end} in `{s:?}` do not lie on character boundary"); +} + +/// Iterator for the code points of a WTF-8 string. +/// +/// Created with the method `.code_points()`. +#[derive(Clone)] +pub struct Wtf8CodePoints<'a> { + bytes: slice::Iter<'a, u8>, +} + +impl Iterator for Wtf8CodePoints<'_> { + type Item = CodePoint; + + #[inline] + fn next(&mut self) -> Option { + // SAFETY: `self.bytes` has been created from a WTF-8 string + unsafe { next_code_point(&mut self.bytes).map(|c| CodePoint { value: c }) } + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + let len = self.bytes.len(); + (len.saturating_add(3) / 4, Some(len)) + } + + fn last(mut self) -> Option { + self.next_back() + } +} + +impl DoubleEndedIterator for Wtf8CodePoints<'_> { + #[inline] + fn next_back(&mut self) -> Option { + // SAFETY: `str` invariant says `self.iter` is a valid WTF-8 string and + // the resulting `ch` is a valid Unicode Code Point. + unsafe { + next_code_point_reverse(&mut self.bytes).map(|ch| CodePoint::from_u32_unchecked(ch)) + } + } +} + +impl<'a> Wtf8CodePoints<'a> { + pub fn as_wtf8(&self) -> &'a Wtf8 { + unsafe { Wtf8::from_bytes_unchecked(self.bytes.as_slice()) } + } +} + +#[derive(Clone)] +pub struct Wtf8CodePointIndices<'a> { + pub(super) front_offset: usize, + pub(super) iter: Wtf8CodePoints<'a>, +} + +impl Iterator for Wtf8CodePointIndices<'_> { + type Item = (usize, CodePoint); + + #[inline] + fn next(&mut self) -> Option<(usize, CodePoint)> { + let pre_len = self.iter.bytes.len(); + match self.iter.next() { + None => None, + Some(ch) => { + let index = self.front_offset; + let len = self.iter.bytes.len(); + self.front_offset += pre_len - len; + Some((index, ch)) + } + } + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + self.iter.size_hint() + } + + #[inline] + fn last(mut self) -> Option<(usize, CodePoint)> { + // No need to go through the entire string. + self.next_back() + } +} + +impl DoubleEndedIterator for Wtf8CodePointIndices<'_> { + #[inline] + fn next_back(&mut self) -> Option<(usize, CodePoint)> { + self.iter.next_back().map(|ch| { + let index = self.front_offset + self.iter.bytes.len(); + (index, ch) + }) + } +} + +impl FusedIterator for Wtf8CodePointIndices<'_> {} + +/// Generates a wide character sequence for potentially ill-formed UTF-16. +#[derive(Clone)] +pub struct EncodeWide<'a> { + code_points: Wtf8CodePoints<'a>, + extra: u16, +} + +// Copied from libunicode/u_str.rs +impl Iterator for EncodeWide<'_> { + type Item = u16; + + #[inline] + fn next(&mut self) -> Option { + if self.extra != 0 { + let tmp = self.extra; + self.extra = 0; + return Some(tmp); + } + + let mut buf = [0; MAX_LEN_UTF16]; + self.code_points.next().map(|code_point| { + let n = encode_utf16_raw(code_point.value, &mut buf).len(); + if n == 2 { + self.extra = buf[1]; + } + buf[0] + }) + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + let (low, high) = self.code_points.size_hint(); + let ext = (self.extra != 0) as usize; + // every code point gets either one u16 or two u16, + // so this iterator is between 1 or 2 times as + // long as the underlying iterator. + ( + low + ext, + high.and_then(|n| n.checked_mul(2)) + .and_then(|n| n.checked_add(ext)), + ) + } +} + +impl FusedIterator for EncodeWide<'_> {} + +pub struct Wtf8Chunks<'a> { + wtf8: &'a Wtf8, +} + +impl<'a> Iterator for Wtf8Chunks<'a> { + type Item = Wtf8Chunk<'a>; + + fn next(&mut self) -> Option { + match self.wtf8.next_surrogate(0) { + Some((0, surrogate)) => { + self.wtf8 = &self.wtf8[3..]; + Some(Wtf8Chunk::Surrogate(surrogate.into())) + } + Some((n, _)) => { + let s = unsafe { str::from_utf8_unchecked(&self.wtf8.as_bytes()[..n]) }; + self.wtf8 = &self.wtf8[n..]; + Some(Wtf8Chunk::Utf8(s)) + } + None => { + let s = + unsafe { str::from_utf8_unchecked(std::mem::take(&mut self.wtf8).as_bytes()) }; + (!s.is_empty()).then_some(Wtf8Chunk::Utf8(s)) + } + } + } +} + +pub enum Wtf8Chunk<'a> { + Utf8(&'a str), + Surrogate(CodePoint), +} + +impl Hash for CodePoint { + #[inline] + fn hash(&self, state: &mut H) { + self.value.hash(state) + } +} + +// == BOX IMPLS == + +/// # Safety +/// +/// `value` must be valid WTF-8. +pub unsafe fn from_boxed_wtf8_unchecked(value: Box<[u8]>) -> Box { + unsafe { Box::from_raw(Box::into_raw(value) as *mut Wtf8) } +} + +impl Clone for Box { + fn clone(&self) -> Self { + (&**self).into() + } +} + +impl Default for Box { + fn default() -> Self { + unsafe { from_boxed_wtf8_unchecked(Box::default()) } + } +} + +impl From<&Wtf8> for Box { + fn from(w: &Wtf8) -> Self { + w.into_box() + } +} + +impl From<&str> for Box { + fn from(s: &str) -> Self { + Box::::from(s).into() + } +} + +impl From> for Box { + fn from(s: Box) -> Self { + unsafe { from_boxed_wtf8_unchecked(s.into_boxed_bytes()) } + } +} + +impl From> for Box { + fn from(s: Box) -> Self { + >::from(s).into() + } +} + +impl From> for Box<[u8]> { + fn from(w: Box) -> Self { + unsafe { Box::from_raw(Box::into_raw(w) as *mut [u8]) } + } +} + +impl From for Box { + fn from(w: Wtf8Buf) -> Self { + w.into_box() + } +} + +impl From for Box { + fn from(s: String) -> Self { + s.into_boxed_str().into() + } +} diff --git a/stdlib/src/pystruct.rs b/stdlib/src/pystruct.rs index f8d41414f7..702e5b6681 100644 --- a/stdlib/src/pystruct.rs +++ b/stdlib/src/pystruct.rs @@ -27,23 +27,20 @@ pub(crate) mod _struct { fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { // CPython turns str to bytes but we do reversed way here // The only performance difference is this transition cost - let fmt = match_class! { - match obj { - s @ PyStr => if s.is_ascii() { - Some(s) - } else { - None - }, - b @ PyBytes => if b.is_ascii() { - Some(unsafe { - PyStr::new_ascii_unchecked(b.as_bytes().to_vec()) - }.into_ref(&vm.ctx)) - } else { - None - }, - other => return Err(vm.new_type_error(format!("Struct() argument 1 must be a str or bytes object, not {}", other.class().name()))), - } - }.ok_or_else(|| vm.new_unicode_decode_error("Struct format must be a ascii string".to_owned()))?; + let fmt = match_class!(match obj { + s @ PyStr => s.is_ascii().then_some(s), + b @ PyBytes => ascii::AsciiStr::from_ascii(&b) + .ok() + .map(|s| vm.ctx.new_str(s)), + other => + return Err(vm.new_type_error(format!( + "Struct() argument 1 must be a str or bytes object, not {}", + other.class().name() + ))), + }) + .ok_or_else(|| { + vm.new_unicode_decode_error("Struct format must be a ascii string".to_owned()) + })?; Ok(IntoStructFormatBytes(fmt)) } } diff --git a/stdlib/src/unicodedata.rs b/stdlib/src/unicodedata.rs index 49f3ef6250..9af921d360 100644 --- a/stdlib/src/unicodedata.rs +++ b/stdlib/src/unicodedata.rs @@ -65,6 +65,7 @@ mod unicodedata { function::OptionalArg, }; use itertools::Itertools; + use rustpython_common::wtf8::{CodePoint, Wtf8Buf}; use ucd::{Codepoint, EastAsianWidth}; use unic_char_property::EnumeratedCharProperty; use unic_normal::StrNormalForm; @@ -84,14 +85,23 @@ mod unicodedata { Self { unic_version } } - fn check_age(&self, c: char) -> bool { - Age::of(c).is_some_and(|age| age.actual() <= self.unic_version) + fn check_age(&self, c: CodePoint) -> bool { + c.to_char() + .is_none_or(|c| Age::of(c).is_some_and(|age| age.actual() <= self.unic_version)) } - fn extract_char(&self, character: PyStrRef, vm: &VirtualMachine) -> PyResult> { - let c = character.as_str().chars().exactly_one().map_err(|_| { - vm.new_type_error("argument must be an unicode character, not str".to_owned()) - })?; + fn extract_char( + &self, + character: PyStrRef, + vm: &VirtualMachine, + ) -> PyResult> { + let c = character + .as_wtf8() + .code_points() + .exactly_one() + .map_err(|_| { + vm.new_type_error("argument must be an unicode character, not str".to_owned()) + })?; Ok(self.check_age(c).then_some(c)) } @@ -103,7 +113,10 @@ mod unicodedata { fn category(&self, character: PyStrRef, vm: &VirtualMachine) -> PyResult { Ok(self .extract_char(character, vm)? - .map_or(GeneralCategory::Unassigned, GeneralCategory::of) + .map_or(GeneralCategory::Unassigned, |c| { + c.to_char() + .map_or(GeneralCategory::Surrogate, GeneralCategory::of) + }) .abbr_name() .to_owned()) } @@ -111,7 +124,7 @@ mod unicodedata { #[pymethod] fn lookup(&self, name: PyStrRef, vm: &VirtualMachine) -> PyResult { if let Some(character) = unicode_names2::character(name.as_str()) { - if self.check_age(character) { + if self.check_age(character.into()) { return Ok(character.to_string()); } } @@ -129,7 +142,7 @@ mod unicodedata { if let Some(c) = c { if self.check_age(c) { - if let Some(name) = unicode_names2::name(c) { + if let Some(name) = c.to_char().and_then(unicode_names2::name) { return Ok(vm.ctx.new_str(name.to_string()).into()); } } @@ -144,7 +157,10 @@ mod unicodedata { vm: &VirtualMachine, ) -> PyResult<&'static str> { let bidi = match self.extract_char(character, vm)? { - Some(c) => BidiClass::of(c).abbr_name(), + Some(c) => c + .to_char() + .map_or(BidiClass::LeftToRight, BidiClass::of) + .abbr_name(), None => "", }; Ok(bidi) @@ -159,19 +175,20 @@ mod unicodedata { ) -> PyResult<&'static str> { Ok(self .extract_char(character, vm)? + .and_then(|c| c.to_char()) .map_or(EastAsianWidth::Neutral, |c| c.east_asian_width()) .abbr_name()) } #[pymethod] - fn normalize(&self, form: super::NormalizeForm, unistr: PyStrRef) -> PyResult { + fn normalize(&self, form: super::NormalizeForm, unistr: PyStrRef) -> PyResult { use super::NormalizeForm::*; - let text = unistr.as_str(); + let text = unistr.as_wtf8(); let normalized_text = match form { - Nfc => text.nfc().collect::(), - Nfkc => text.nfkc().collect::(), - Nfd => text.nfd().collect::(), - Nfkd => text.nfkd().collect::(), + Nfc => text.map_utf8(|s| s.nfc()).collect(), + Nfkc => text.map_utf8(|s| s.nfkc()).collect(), + Nfd => text.map_utf8(|s| s.nfd()).collect(), + Nfkd => text.map_utf8(|s| s.nfkd()).collect(), }; Ok(normalized_text) } diff --git a/vm/src/anystr.rs b/vm/src/anystr.rs index d01136b0fb..0a884fc732 100644 --- a/vm/src/anystr.rs +++ b/vm/src/anystr.rs @@ -7,28 +7,13 @@ use crate::{ use num_traits::{cast::ToPrimitive, sign::Signed}; #[derive(FromArgs)] -pub struct SplitArgs { +pub struct SplitArgs { #[pyarg(any, default)] sep: Option, #[pyarg(any, default = "-1")] maxsplit: isize, } -impl SplitArgs { - pub fn get_value(self, vm: &VirtualMachine) -> PyResult<(Option, isize)> { - let sep = if let Some(s) = self.sep { - let sep = s.as_ref(); - if sep.is_empty() { - return Err(vm.new_value_error("empty separator".to_owned())); - } - Some(s) - } else { - None - }; - Ok((sep, self.maxsplit)) - } -} - #[derive(FromArgs)] pub struct SplitLinesArgs { #[pyarg(any, default = "false")] @@ -132,9 +117,9 @@ impl StringRange for std::ops::Range { } } -pub trait AnyStrWrapper { - type Str: ?Sized + AnyStr; - fn as_ref(&self) -> &Self::Str; +pub trait AnyStrWrapper { + fn as_ref(&self) -> Option<&S>; + fn is_empty(&self) -> bool; } pub trait AnyStrContainer @@ -146,15 +131,18 @@ where fn push_str(&mut self, s: &S); } +pub trait AnyChar: Copy { + fn is_lowercase(self) -> bool; + fn is_uppercase(self) -> bool; + fn bytes_len(self) -> usize; +} + pub trait AnyStr { - type Char: Copy; + type Char: AnyChar; type Container: AnyStrContainer + Extend; - fn element_bytes_len(c: Self::Char) -> usize; - fn to_container(&self) -> Self::Container; fn as_bytes(&self) -> &[u8]; - fn chars(&self) -> impl Iterator; fn elements(&self) -> impl Iterator; fn get_bytes(&self, range: std::ops::Range) -> &Self; // FIXME: get_chars is expensive for str @@ -172,29 +160,35 @@ pub trait AnyStr { new } - fn py_split( + fn py_split( &self, args: SplitArgs, vm: &VirtualMachine, + full_obj: impl FnOnce() -> PyObjectRef, split: SP, splitn: SN, splitw: SW, - ) -> PyResult> + ) -> PyResult> where - T: TryFromObject + AnyStrWrapper, - SP: Fn(&Self, &Self, &VirtualMachine) -> Vec, - SN: Fn(&Self, &Self, usize, &VirtualMachine) -> Vec, - SW: Fn(&Self, isize, &VirtualMachine) -> Vec, + T: TryFromObject + AnyStrWrapper, + SP: Fn(&Self, &Self, &VirtualMachine) -> Vec, + SN: Fn(&Self, &Self, usize, &VirtualMachine) -> Vec, + SW: Fn(&Self, isize, &VirtualMachine) -> Vec, { - let (sep, maxsplit) = args.get_value(vm)?; - let splits = if let Some(pattern) = sep { - if maxsplit < 0 { - split(self, pattern.as_ref(), vm) + if args.sep.as_ref().is_some_and(|sep| sep.is_empty()) { + return Err(vm.new_value_error("empty separator".to_owned())); + } + let splits = if let Some(pattern) = args.sep { + let Some(pattern) = pattern.as_ref() else { + return Ok(vec![full_obj()]); + }; + if args.maxsplit < 0 { + split(self, pattern, vm) } else { - splitn(self, pattern.as_ref(), (maxsplit + 1) as usize, vm) + splitn(self, pattern, (args.maxsplit + 1) as usize, vm) } } else { - splitw(self, maxsplit, vm) + splitw(self, args.maxsplit, vm) }; Ok(splits) } @@ -242,13 +236,19 @@ pub trait AnyStr { func_default: FD, ) -> &'a Self where - S: AnyStrWrapper, + S: AnyStrWrapper, FC: Fn(&'a Self, &Self) -> &'a Self, FD: Fn(&'a Self) -> &'a Self, { let chars = chars.flatten(); match chars { - Some(chars) => func_chars(self, chars.as_ref()), + Some(chars) => { + if let Some(chars) = chars.as_ref() { + func_chars(self, chars) + } else { + self + } + } None => func_default(self), } } @@ -281,7 +281,7 @@ pub trait AnyStr { fn py_pad(&self, left: usize, right: usize, fillchar: Self::Char) -> Self::Container { let mut u = Self::Container::with_capacity( - (left + right) * Self::element_bytes_len(fillchar) + self.bytes_len(), + (left + right) * fillchar.bytes_len() + self.bytes_len(), ); u.extend(std::iter::repeat(fillchar).take(left)); u.push_str(self); @@ -305,19 +305,17 @@ pub trait AnyStr { fn py_join( &self, - mut iter: impl std::iter::Iterator< - Item = PyResult + TryFromObject>, - >, + mut iter: impl std::iter::Iterator + TryFromObject>>, ) -> PyResult { let mut joined = if let Some(elem) = iter.next() { - elem?.as_ref().to_container() + elem?.as_ref().unwrap().to_container() } else { return Ok(Self::Container::new()); }; for elem in iter { let elem = elem?; joined.push_str(self); - joined.push_str(elem.as_ref()); + joined.push_str(elem.as_ref().unwrap()); } Ok(joined) } @@ -403,25 +401,33 @@ pub trait AnyStr { rustpython_common::str::zfill(self.as_bytes(), width) } - fn py_iscase(&self, is_case: F, is_opposite: G) -> bool - where - F: Fn(char) -> bool, - G: Fn(char) -> bool, - { - // Unified form of CPython functions: - // _Py_bytes_islower - // Py_bytes_isupper - // unicode_islower_impl - // unicode_isupper_impl - let mut cased = false; - for c in self.chars() { - if is_opposite(c) { + // Unified form of CPython functions: + // _Py_bytes_islower + // Py_bytes_isupper + // unicode_islower_impl + // unicode_isupper_impl + fn py_islower(&self) -> bool { + let mut lower = false; + for c in self.elements() { + if c.is_uppercase() { + return false; + } else if !lower && c.is_lowercase() { + lower = true + } + } + lower + } + + fn py_isupper(&self) -> bool { + let mut upper = false; + for c in self.elements() { + if c.is_lowercase() { return false; - } else if !cased && is_case(c) { - cased = true + } else if !upper && c.is_uppercase() { + upper = true } } - cased + upper } } diff --git a/vm/src/builtins/genericalias.rs b/vm/src/builtins/genericalias.rs index c03e3145b2..0e0a34227b 100644 --- a/vm/src/builtins/genericalias.rs +++ b/vm/src/builtins/genericalias.rs @@ -124,7 +124,7 @@ impl PyGenericAlias { Ok(format!( "{}[{}]", repr_item(self.origin.clone().into(), vm)?, - if self.args.len() == 0 { + if self.args.is_empty() { "()".to_owned() } else { self.args @@ -261,23 +261,20 @@ fn subs_tvars( .and_then(|sub_params| { PyTupleRef::try_from_object(vm, sub_params) .ok() - .and_then(|sub_params| { - if sub_params.len() > 0 { - let sub_args = sub_params - .iter() - .map(|arg| { - if let Some(idx) = tuple_index(params, arg) { - argitems[idx].clone() - } else { - arg.clone() - } - }) - .collect::>(); - let sub_args: PyObjectRef = PyTuple::new_ref(sub_args, &vm.ctx).into(); - Some(obj.get_item(&*sub_args, vm)) - } else { - None - } + .filter(|sub_params| !sub_params.is_empty()) + .map(|sub_params| { + let sub_args = sub_params + .iter() + .map(|arg| { + if let Some(idx) = tuple_index(params, arg) { + argitems[idx].clone() + } else { + arg.clone() + } + }) + .collect::>(); + let sub_args: PyObjectRef = PyTuple::new_ref(sub_args, &vm.ctx).into(); + obj.get_item(&*sub_args, vm) }) }) .unwrap_or(Ok(obj)) diff --git a/vm/src/builtins/object.rs b/vm/src/builtins/object.rs index cce1422d56..be14327542 100644 --- a/vm/src/builtins/object.rs +++ b/vm/src/builtins/object.rs @@ -159,7 +159,7 @@ fn object_getstate_default(obj: &PyObject, required: bool, vm: &VirtualMachine) slots.set_item(name.as_str(), value, vm).unwrap(); } - if slots.len() > 0 { + if !slots.is_empty() { return (state, slots).to_pyresult(vm); } } diff --git a/vm/src/builtins/set.rs b/vm/src/builtins/set.rs index a135af1bde..40823aa37b 100644 --- a/vm/src/builtins/set.rs +++ b/vm/src/builtins/set.rs @@ -784,9 +784,7 @@ impl Initializer for PySet { type Args = OptionalArg; fn init(zelf: PyRef, iterable: Self::Args, vm: &VirtualMachine) -> PyResult<()> { - if zelf.len() > 0 { - zelf.clear(); - } + zelf.clear(); if let OptionalArg::Present(it) = iterable { zelf.update(PosArgs::new(vec![it]), vm)?; } diff --git a/vm/src/builtins/str.rs b/vm/src/builtins/str.rs index 76cdca81ed..6ff079c644 100644 --- a/vm/src/builtins/str.rs +++ b/vm/src/builtins/str.rs @@ -10,7 +10,7 @@ use crate::{ atomic_func, cformat::cformat_string, class::PyClassImpl, - common::str::{BorrowedStr, PyStrKind, PyStrKindData}, + common::str::{PyKindStr, StrData, StrKind}, convert::{IntoPyException, ToPyException, ToPyObject, ToPyResult}, format::{format, format_map}, function::{ArgIterable, ArgSize, FuncArgs, OptionalArg, OptionalOption, PyComparisonValue}, @@ -24,7 +24,7 @@ use crate::{ PyComparisonOp, Representable, SelfIter, Unconstructible, }, }; -use ascii::{AsciiStr, AsciiString}; +use ascii::{AsciiChar, AsciiStr, AsciiString}; use bstr::ByteSlice; use itertools::Itertools; use num_traits::ToPrimitive; @@ -35,8 +35,10 @@ use rustpython_common::{ format::{FormatSpec, FormatString, FromTemplate}, hash, lock::PyMutex, + str::DeduceStrKind, + wtf8::{CodePoint, Wtf8, Wtf8Buf, Wtf8Chunk}, }; -use std::{char, fmt, ops::Range, string::ToString}; +use std::{char, fmt, ops::Range}; use unic_ucd_bidi::BidiClass; use unic_ucd_category::GeneralCategory; use unic_ucd_ident::{is_xid_continue, is_xid_start}; @@ -57,39 +59,59 @@ impl<'a> TryFromBorrowedObject<'a> for &'a str { #[pyclass(module = false, name = "str")] pub struct PyStr { - bytes: Box<[u8]>, - kind: PyStrKindData, + data: StrData, hash: PyAtomic, } impl fmt::Debug for PyStr { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("PyStr") - .field("value", &self.as_str()) - .field("kind", &self.kind) + .field("value", &self.as_wtf8()) + .field("kind", &self.data.kind()) .field("hash", &self.hash) .finish() } } impl AsRef for PyStr { + #[track_caller] // <- can remove this once it doesn't panic fn as_ref(&self) -> &str { self.as_str() } } impl AsRef for Py { + #[track_caller] // <- can remove this once it doesn't panic fn as_ref(&self) -> &str { self.as_str() } } impl AsRef for PyStrRef { + #[track_caller] // <- can remove this once it doesn't panic fn as_ref(&self) -> &str { self.as_str() } } +impl AsRef for PyStr { + fn as_ref(&self) -> &Wtf8 { + self.as_wtf8() + } +} + +impl AsRef for Py { + fn as_ref(&self) -> &Wtf8 { + self.as_wtf8() + } +} + +impl AsRef for PyStrRef { + fn as_ref(&self) -> &Wtf8 { + self.as_wtf8() + } +} + impl<'a> From<&'a AsciiStr> for PyStr { fn from(s: &'a AsciiStr) -> Self { s.to_owned().into() @@ -98,7 +120,19 @@ impl<'a> From<&'a AsciiStr> for PyStr { impl From for PyStr { fn from(s: AsciiString) -> Self { - unsafe { Self::new_ascii_unchecked(s.into()) } + s.into_boxed_ascii_str().into() + } +} + +impl From> for PyStr { + fn from(s: Box) -> Self { + StrData::from(s).into() + } +} + +impl From for PyStr { + fn from(ch: AsciiChar) -> Self { + AsciiString::from(ch).into() } } @@ -108,12 +142,45 @@ impl<'a> From<&'a str> for PyStr { } } +impl<'a> From<&'a Wtf8> for PyStr { + fn from(s: &'a Wtf8) -> Self { + s.to_owned().into() + } +} + impl From for PyStr { fn from(s: String) -> Self { s.into_boxed_str().into() } } +impl From for PyStr { + fn from(w: Wtf8Buf) -> Self { + w.into_box().into() + } +} + +impl From for PyStr { + fn from(ch: char) -> Self { + StrData::from(ch).into() + } +} + +impl From for PyStr { + fn from(ch: CodePoint) -> Self { + StrData::from(ch).into() + } +} + +impl From for PyStr { + fn from(data: StrData) -> Self { + PyStr { + data, + hash: Radium::new(hash::SENTINEL), + } + } +} + impl<'a> From> for PyStr { fn from(s: std::borrow::Cow<'a, str>) -> Self { s.into_owned().into() @@ -123,19 +190,21 @@ impl<'a> From> for PyStr { impl From> for PyStr { #[inline] fn from(value: Box) -> Self { - // doing the check is ~10x faster for ascii, and is actually only 2% slower worst case for - // non-ascii; see https://github.com/RustPython/RustPython/pull/2586#issuecomment-844611532 - let is_ascii = value.is_ascii(); - let bytes = value.into_boxed_bytes(); - let kind = if is_ascii { - PyStrKind::Ascii - } else { - PyStrKind::Utf8 - } - .new_data(); + StrData::from(value).into() + } +} + +impl From> for PyStr { + #[inline] + fn from(value: Box) -> Self { + StrData::from(value).into() + } +} + +impl Default for PyStr { + fn default() -> Self { Self { - bytes, - kind, + data: StrData::default(), hash: Radium::new(hash::SENTINEL), } } @@ -146,7 +215,7 @@ pub type PyStrRef = PyRef; impl fmt::Display for PyStr { #[inline] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Display::fmt(self.as_str(), f) + self.as_wtf8().fmt(f) } } @@ -237,18 +306,18 @@ impl IterNext for PyStrIterator { let mut internal = zelf.internal.lock(); if let IterStatus::Active(s) = &internal.0.status { - let value = s.as_str(); + let value = s.as_wtf8(); if internal.1 == usize::MAX { - if let Some((offset, ch)) = value.char_indices().nth(internal.0.position) { + if let Some((offset, ch)) = value.code_point_indices().nth(internal.0.position) { internal.0.position += 1; - internal.1 = offset + ch.len_utf8(); + internal.1 = offset + ch.len_wtf8(); return Ok(PyIterReturn::Return(ch.to_pyobject(vm))); } } else if let Some(value) = value.get(internal.1..) { - if let Some(ch) = value.chars().next() { + if let Some(ch) = value.code_points().next() { internal.0.position += 1; - internal.1 += ch.len_utf8(); + internal.1 += ch.len_wtf8(); return Ok(PyIterReturn::Return(ch.to_pyobject(vm))); } } @@ -292,7 +361,7 @@ impl Constructor for PyStr { if string.class().is(&cls) { Ok(string.into()) } else { - PyStr::from(string.as_str()) + PyStr::from(string.as_wtf8()) .into_ref_with_type(vm, cls) .map(Into::into) } @@ -301,20 +370,19 @@ impl Constructor for PyStr { impl PyStr { /// # Safety: Given `bytes` must be valid data for given `kind` - pub(crate) unsafe fn new_str_unchecked(bytes: Vec, kind: PyStrKind) -> Self { - let s = Self { - bytes: bytes.into_boxed_slice(), - kind: kind.new_data(), - hash: Radium::new(hash::SENTINEL), - }; - debug_assert!(matches!(s.kind, PyStrKindData::Ascii) || !s.as_str().is_ascii()); - s + unsafe fn new_str_unchecked(data: Box, kind: StrKind) -> Self { + unsafe { StrData::new_str_unchecked(data, kind) }.into() + } + + unsafe fn new_with_char_len>>(s: T, char_len: usize) -> Self { + let kind = s.str_kind(); + unsafe { StrData::new_with_char_len(s.into(), kind, char_len) }.into() } /// # Safety /// Given `bytes` must be ascii pub unsafe fn new_ascii_unchecked(bytes: Vec) -> Self { - unsafe { Self::new_str_unchecked(bytes, PyStrKind::Ascii) } + unsafe { AsciiString::from_ascii_unchecked(bytes) }.into() } pub fn new_ref(zelf: impl Into, ctx: &Context) -> PyRef { @@ -322,40 +390,56 @@ impl PyStr { PyRef::new_ref(zelf, ctx.types.str_type.to_owned(), None) } - fn new_substr(&self, s: String) -> Self { - let kind = if self.kind.kind() == PyStrKind::Ascii || s.is_ascii() { - PyStrKind::Ascii + fn new_substr(&self, s: Wtf8Buf) -> Self { + let kind = if self.kind().is_ascii() || s.is_ascii() { + StrKind::Ascii + } else if self.kind().is_utf8() || s.is_utf8() { + StrKind::Utf8 } else { - PyStrKind::Utf8 + StrKind::Wtf8 }; unsafe { // SAFETY: kind is properly decided for substring - Self::new_str_unchecked(s.into_bytes(), kind) + Self::new_str_unchecked(s.into(), kind) } } #[inline] + pub fn as_wtf8(&self) -> &Wtf8 { + self.data.as_wtf8() + } + + pub fn as_bytes(&self) -> &[u8] { + self.data.as_wtf8().as_bytes() + } + + // FIXME: make this return an Option + #[inline] + #[track_caller] // <- can remove this once it doesn't panic pub fn as_str(&self) -> &str { - unsafe { - // SAFETY: Both PyStrKind::{Ascii, Utf8} are valid utf8 string - std::str::from_utf8_unchecked(&self.bytes) - } + self.data.as_str().expect("str has surrogates") + } + + pub fn kind(&self) -> StrKind { + self.data.kind() + } + + #[inline] + pub fn as_str_kind(&self) -> PyKindStr<'_> { + self.data.as_str_kind() } fn char_all(&self, test: F) -> bool where F: Fn(char) -> bool, { - match self.kind.kind() { - PyStrKind::Ascii => self.bytes.iter().all(|&x| test(char::from(x))), - PyStrKind::Utf8 => self.as_str().chars().all(test), + match self.as_str_kind() { + PyKindStr::Ascii(s) => s.chars().all(|ch| test(ch.into())), + PyKindStr::Utf8(s) => s.chars().all(test), + PyKindStr::Wtf8(w) => w.code_points().all(|ch| ch.is_char_and(&test)), } } - fn borrow(&self) -> &BorrowedStr<'_> { - unsafe { std::mem::transmute(self) } - } - fn repeat(zelf: PyRef, value: isize, vm: &VirtualMachine) -> PyResult> { if value == 0 && zelf.class().is(vm.ctx.types.str_type) { // Special case: when some `str` is multiplied by `0`, @@ -369,10 +453,10 @@ impl PyStr { // This only works for `str` itself, not its subclasses. return Ok(zelf); } - zelf.as_str() + zelf.as_wtf8() .as_bytes() .mul(vm, value) - .map(|x| Self::from(unsafe { String::from_utf8_unchecked(x) }).into_ref(&vm.ctx)) + .map(|x| Self::from(unsafe { Wtf8Buf::from_bytes_unchecked(x) }).into_ref(&vm.ctx)) } } @@ -394,11 +478,11 @@ impl PyStr { #[pymethod(magic)] fn add(zelf: PyRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if let Some(other) = other.payload::() { - let bytes = zelf.as_str().py_add(other.as_ref()); + let bytes = zelf.as_wtf8().py_add(other.as_wtf8()); Ok(unsafe { // SAFETY: `kind` is safely decided - let kind = zelf.kind.kind() | other.kind.kind(); - Self::new_str_unchecked(bytes.into_bytes(), kind) + let kind = zelf.kind() | other.kind(); + Self::new_str_unchecked(bytes.into(), kind) } .to_pyobject(vm)) } else if let Some(radd) = vm.get_method(other.clone(), identifier!(vm, __radd__)) { @@ -414,7 +498,7 @@ impl PyStr { fn _contains(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult { if let Some(needle) = needle.payload::() { - Ok(self.as_str().contains(needle.as_str())) + Ok(memchr::memmem::find(self.as_bytes(), needle.as_bytes()).is_some()) } else { Err(vm.new_type_error(format!( "'in ' requires string as left operand, not {}", @@ -429,11 +513,11 @@ impl PyStr { } fn _getitem(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult { - match SequenceIndex::try_from_borrowed_object(vm, needle, "str")? { - SequenceIndex::Int(i) => self.getitem_by_index(vm, i).map(|x| x.to_string()), - SequenceIndex::Slice(slice) => self.getitem_by_slice(vm, slice), - } - .map(|x| self.new_substr(x).into_ref(&vm.ctx).into()) + let item = match SequenceIndex::try_from_borrowed_object(vm, needle, "str")? { + SequenceIndex::Int(i) => self.getitem_by_index(vm, i)?.to_pyobject(vm), + SequenceIndex::Slice(slice) => self.getitem_by_slice(vm, slice)?.to_pyobject(vm), + }; + Ok(item) } #[pymethod(magic)] @@ -450,7 +534,7 @@ impl PyStr { } #[cold] fn _compute_hash(&self, vm: &VirtualMachine) -> hash::PyHash { - let hash_val = vm.state.hash_secret.hash_str(self.as_str()); + let hash_val = vm.state.hash_secret.hash_bytes(self.as_bytes()); debug_assert_ne!(hash_val, hash::SENTINEL); // like with char_len, we don't need a cmpxchg loop, since it'll always be the same value self.hash.store(hash_val, atomic::Ordering::Relaxed); @@ -459,26 +543,23 @@ impl PyStr { #[inline] pub fn byte_len(&self) -> usize { - self.bytes.len() + self.data.len() } #[inline] pub fn is_empty(&self) -> bool { - self.bytes.is_empty() + self.data.is_empty() } #[pymethod(name = "__len__")] #[inline] pub fn char_len(&self) -> usize { - self.borrow().char_len() + self.data.char_len() } #[pymethod(name = "isascii")] #[inline(always)] pub fn is_ascii(&self) -> bool { - match self.kind { - PyStrKindData::Ascii => true, - PyStrKindData::Utf8(_) => false, - } + matches!(self.kind(), StrKind::Ascii) } #[pymethod(magic)] @@ -495,6 +576,9 @@ impl PyStr { #[inline] pub(crate) fn repr(&self, vm: &VirtualMachine) -> PyResult { use crate::literal::escape::UnicodeEscape; + if !self.kind().is_utf8() { + return Ok(format!("{:?}", self.as_wtf8())); + } let escape = UnicodeEscape::new_repr(self.as_str()); escape .str_repr() @@ -503,10 +587,18 @@ impl PyStr { } #[pymethod] - fn lower(&self) -> String { - match self.kind.kind() { - PyStrKind::Ascii => self.as_str().to_ascii_lowercase(), - PyStrKind::Utf8 => self.as_str().to_lowercase(), + fn lower(&self) -> PyStr { + match self.as_str_kind() { + PyKindStr::Ascii(s) => s.to_ascii_lowercase().into(), + PyKindStr::Utf8(s) => s.to_lowercase().into(), + PyKindStr::Wtf8(w) => w + .chunks() + .map(|c| match c { + Wtf8Chunk::Utf8(s) => s.to_lowercase().into(), + Wtf8Chunk::Surrogate(c) => Wtf8Buf::from(c), + }) + .collect::() + .into(), } } @@ -517,10 +609,18 @@ impl PyStr { } #[pymethod] - fn upper(&self) -> String { - match self.kind.kind() { - PyStrKind::Ascii => self.as_str().to_ascii_uppercase(), - PyStrKind::Utf8 => self.as_str().to_uppercase(), + fn upper(&self) -> PyStr { + match self.as_str_kind() { + PyKindStr::Ascii(s) => s.to_ascii_uppercase().into(), + PyKindStr::Utf8(s) => s.to_uppercase().into(), + PyKindStr::Wtf8(w) => w + .chunks() + .map(|c| match c { + Wtf8Chunk::Utf8(s) => s.to_uppercase().into(), + Wtf8Chunk::Surrogate(c) => Wtf8Buf::from(c), + }) + .collect::() + .into(), } } @@ -539,36 +639,42 @@ impl PyStr { } #[pymethod] - fn split(&self, args: SplitArgs, vm: &VirtualMachine) -> PyResult> { - let elements = match self.kind.kind() { - PyStrKind::Ascii => self.as_str().py_split( + fn split(zelf: &Py, args: SplitArgs, vm: &VirtualMachine) -> PyResult> { + let elements = match zelf.as_str_kind() { + PyKindStr::Ascii(s) => s.py_split( args, vm, + || zelf.as_object().to_owned(), |v, s, vm| { v.as_bytes() .split_str(s) - .map(|s| { - unsafe { PyStr::new_ascii_unchecked(s.to_owned()) }.to_pyobject(vm) - }) + .map(|s| unsafe { AsciiStr::from_ascii_unchecked(s) }.to_pyobject(vm)) .collect() }, |v, s, n, vm| { v.as_bytes() .splitn_str(n, s) - .map(|s| { - unsafe { PyStr::new_ascii_unchecked(s.to_owned()) }.to_pyobject(vm) - }) + .map(|s| unsafe { AsciiStr::from_ascii_unchecked(s) }.to_pyobject(vm)) .collect() }, |v, n, vm| { v.as_bytes().py_split_whitespace(n, |s| { - unsafe { PyStr::new_ascii_unchecked(s.to_owned()) }.to_pyobject(vm) + unsafe { AsciiStr::from_ascii_unchecked(s) }.to_pyobject(vm) }) }, ), - PyStrKind::Utf8 => self.as_str().py_split( + PyKindStr::Utf8(s) => s.py_split( args, vm, + || zelf.as_object().to_owned(), + |v, s, vm| v.split(s).map(|s| vm.ctx.new_str(s).into()).collect(), + |v, s, n, vm| v.splitn(n, s).map(|s| vm.ctx.new_str(s).into()).collect(), + |v, n, vm| v.py_split_whitespace(n, |s| vm.ctx.new_str(s).into()), + ), + PyKindStr::Wtf8(w) => w.py_split( + args, + vm, + || zelf.as_object().to_owned(), |v, s, vm| v.split(s).map(|s| vm.ctx.new_str(s).into()).collect(), |v, s, n, vm| v.splitn(n, s).map(|s| vm.ctx.new_str(s).into()).collect(), |v, n, vm| v.py_split_whitespace(n, |s| vm.ctx.new_str(s).into()), @@ -578,10 +684,11 @@ impl PyStr { } #[pymethod] - fn rsplit(&self, args: SplitArgs, vm: &VirtualMachine) -> PyResult> { - let mut elements = self.as_str().py_split( + fn rsplit(zelf: &Py, args: SplitArgs, vm: &VirtualMachine) -> PyResult> { + let mut elements = zelf.as_wtf8().py_split( args, vm, + || zelf.as_object().to_owned(), |v, s, vm| v.rsplit(s).map(|s| vm.ctx.new_str(s).into()).collect(), |v, s, n, vm| v.rsplitn(n, s).map(|s| vm.ctx.new_str(s).into()).collect(), |v, n, vm| v.py_rsplit_whitespace(n, |s| vm.ctx.new_str(s).into()), @@ -593,14 +700,35 @@ impl PyStr { } #[pymethod] - fn strip(&self, chars: OptionalOption) -> String { - self.as_str() - .py_strip( - chars, - |s, chars| s.trim_matches(|c| chars.contains(c)), - |s| s.trim(), - ) - .to_owned() + fn strip(&self, chars: OptionalOption) -> PyStr { + match self.as_str_kind() { + PyKindStr::Ascii(s) => s + .py_strip( + chars, + |s, chars| { + let s = s + .as_str() + .trim_matches(|c| memchr::memchr(c as _, chars.as_bytes()).is_some()); + unsafe { AsciiStr::from_ascii_unchecked(s.as_bytes()) } + }, + |s| s.trim(), + ) + .into(), + PyKindStr::Utf8(s) => s + .py_strip( + chars, + |s, chars| s.trim_matches(|c| chars.contains(c)), + |s| s.trim(), + ) + .into(), + PyKindStr::Wtf8(w) => w + .py_strip( + chars, + |s, chars| s.trim_matches(|c| chars.code_points().contains(&c)), + |s| s.trim_matches(|c| c.is_char_and(char::is_whitespace)), + ) + .into(), + } } #[pymethod] @@ -701,12 +829,12 @@ impl PyStr { #[pymethod] fn isalnum(&self) -> bool { - !self.bytes.is_empty() && self.char_all(char::is_alphanumeric) + !self.data.is_empty() && self.char_all(char::is_alphanumeric) } #[pymethod] fn isnumeric(&self) -> bool { - !self.bytes.is_empty() && self.char_all(char::is_numeric) + !self.data.is_empty() && self.char_all(char::is_numeric) } #[pymethod] @@ -724,7 +852,7 @@ impl PyStr { #[pymethod] fn isdecimal(&self) -> bool { - !self.bytes.is_empty() + !self.data.is_empty() && self.char_all(|c| GeneralCategory::of(c) == GeneralCategory::DecimalNumber) } @@ -767,7 +895,9 @@ impl PyStr { } let s = FormatSpec::parse(spec) - .and_then(|format_spec| format_spec.format_string(zelf.borrow())) + .and_then(|format_spec| { + format_spec.format_string(&CharLenStr(zelf.as_str(), zelf.char_len())) + }) .map_err(|err| err.into_pyexception(vm))?; Ok(vm.ctx.new_str(s)) } @@ -776,7 +906,7 @@ impl PyStr { /// uppercase character and the remaining characters are lowercase. #[pymethod] fn title(&self) -> String { - let mut title = String::with_capacity(self.bytes.len()); + let mut title = String::with_capacity(self.data.len()); let mut previous_is_cased = false; for c in self.as_str().chars() { if c.is_lowercase() { @@ -803,7 +933,7 @@ impl PyStr { #[pymethod] fn swapcase(&self) -> String { - let mut swapped_str = String::with_capacity(self.bytes.len()); + let mut swapped_str = String::with_capacity(self.data.len()); for c in self.as_str().chars() { // to_uppercase returns an iterator, to_ascii_uppercase returns the char if c.is_lowercase() { @@ -819,7 +949,7 @@ impl PyStr { #[pymethod] fn isalpha(&self) -> bool { - !self.bytes.is_empty() && self.char_all(char::is_alphabetic) + !self.data.is_empty() && self.char_all(char::is_alphabetic) } #[pymethod] @@ -863,7 +993,7 @@ impl PyStr { #[pymethod] fn isspace(&self) -> bool { use unic_ucd_bidi::bidi_class::abbr_names::*; - !self.bytes.is_empty() + !self.data.is_empty() && self.char_all(|c| { GeneralCategory::of(c) == GeneralCategory::SpaceSeparator || matches!(BidiClass::of(c), WS | B | S) @@ -873,41 +1003,39 @@ impl PyStr { // Return true if all cased characters in the string are lowercase and there is at least one cased character, false otherwise. #[pymethod] fn islower(&self) -> bool { - match self.kind.kind() { - PyStrKind::Ascii => self.bytes.py_iscase(char::is_lowercase, char::is_uppercase), - PyStrKind::Utf8 => self - .as_str() - .py_iscase(char::is_lowercase, char::is_uppercase), + match self.as_str_kind() { + PyKindStr::Ascii(s) => s.py_islower(), + PyKindStr::Utf8(s) => s.py_islower(), + PyKindStr::Wtf8(w) => w.py_islower(), } } // Return true if all cased characters in the string are uppercase and there is at least one cased character, false otherwise. #[pymethod] fn isupper(&self) -> bool { - match self.kind.kind() { - PyStrKind::Ascii => self.bytes.py_iscase(char::is_uppercase, char::is_lowercase), - PyStrKind::Utf8 => self - .as_str() - .py_iscase(char::is_uppercase, char::is_lowercase), + match self.as_str_kind() { + PyKindStr::Ascii(s) => s.py_isupper(), + PyKindStr::Utf8(s) => s.py_isupper(), + PyKindStr::Wtf8(w) => w.py_isupper(), } } #[pymethod] fn splitlines(&self, args: anystr::SplitLinesArgs, vm: &VirtualMachine) -> Vec { - let into_wrapper = |s: &str| self.new_substr(s.to_owned()).to_pyobject(vm); + let into_wrapper = |s: &Wtf8| self.new_substr(s.to_owned()).to_pyobject(vm); let mut elements = Vec::new(); let mut last_i = 0; - let self_str = self.as_str(); - let mut enumerated = self_str.char_indices().peekable(); + let self_str = self.as_wtf8(); + let mut enumerated = self_str.code_point_indices().peekable(); while let Some((i, ch)) = enumerated.next() { - let end_len = match ch { + let end_len = match ch.to_char_lossy() { '\n' => 1, '\r' => { let is_rn = enumerated.next_if(|(_, ch)| *ch == '\n').is_some(); if is_rn { 2 } else { 1 } } '\x0b' | '\x0c' | '\x1c' | '\x1d' | '\x1e' | '\u{0085}' | '\u{2028}' - | '\u{2029}' => ch.len_utf8(), + | '\u{2029}' => ch.len_wtf8(), _ => continue, }; let range = if args.keepends { @@ -937,27 +1065,27 @@ impl PyStr { if first.as_object().class().is(vm.ctx.types.str_type) { return Ok(first); } else { - first.as_str().to_owned() + first.as_wtf8().to_owned() } } - Err(iter) => zelf.as_str().py_join(iter)?, + Err(iter) => zelf.as_wtf8().py_join(iter)?, }; Ok(vm.ctx.new_str(joined)) } // FIXME: two traversals of str is expensive #[inline] - fn _to_char_idx(r: &str, byte_idx: usize) -> usize { - r[..byte_idx].chars().count() + fn _to_char_idx(r: &Wtf8, byte_idx: usize) -> usize { + r[..byte_idx].code_points().count() } #[inline] fn _find(&self, args: FindArgs, find: F) -> Option where - F: Fn(&str, &str) -> Option, + F: Fn(&Wtf8, &Wtf8) -> Option, { let (sub, range) = args.get_value(self.len()); - self.as_str().py_find(sub.as_str(), range, find) + self.as_wtf8().py_find(sub.as_wtf8(), range, find) } #[pymethod] @@ -986,9 +1114,9 @@ impl PyStr { #[pymethod] fn partition(&self, sep: PyStrRef, vm: &VirtualMachine) -> PyResult { - let (front, has_mid, back) = self.as_str().py_partition( - sep.as_str(), - || self.as_str().splitn(2, sep.as_str()), + let (front, has_mid, back) = self.as_wtf8().py_partition( + sep.as_wtf8(), + || self.as_wtf8().splitn(2, sep.as_wtf8()), vm, )?; let partition = ( @@ -1005,9 +1133,9 @@ impl PyStr { #[pymethod] fn rpartition(&self, sep: PyStrRef, vm: &VirtualMachine) -> PyResult { - let (back, has_mid, front) = self.as_str().py_partition( - sep.as_str(), - || self.as_str().rsplitn(2, sep.as_str()), + let (back, has_mid, front) = self.as_wtf8().py_partition( + sep.as_wtf8(), + || self.as_wtf8().rsplitn(2, sep.as_wtf8()), vm, )?; Ok(( @@ -1026,7 +1154,7 @@ impl PyStr { /// empty, `false` otherwise. #[pymethod] fn istitle(&self) -> bool { - if self.bytes.is_empty() { + if self.data.is_empty() { return false; } @@ -1247,18 +1375,34 @@ impl PyStr { } } +struct CharLenStr<'a>(&'a str, usize); +impl std::ops::Deref for CharLenStr<'_> { + type Target = str; + fn deref(&self) -> &Self::Target { + self.0 + } +} +impl crate::common::format::CharLen for CharLenStr<'_> { + fn char_len(&self) -> usize { + self.1 + } +} + #[pyclass] impl PyRef { #[pymethod(magic)] fn str(self, vm: &VirtualMachine) -> PyRefExact { - self.into_exact_or(&vm.ctx, |zelf| unsafe { - // Creating a copy with same kind is safe - PyStr::new_str_unchecked(zelf.bytes.to_vec(), zelf.kind.kind()).into_exact_ref(&vm.ctx) + self.into_exact_or(&vm.ctx, |zelf| { + PyStr::from(zelf.data.clone()).into_exact_ref(&vm.ctx) }) } } impl PyStrRef { + pub fn is_empty(&self) -> bool { + (**self).is_empty() + } + pub fn concat_in_place(&mut self, other: &str, vm: &VirtualMachine) { // TODO: call [A]Rc::get_mut on the str to try to mutate the data in place if other.is_empty() { @@ -1296,7 +1440,7 @@ impl Comparable for PyStr { return Ok(res.into()); } let other = class_or_notimplemented!(Self, other); - Ok(op.eval_ord(zelf.as_str().cmp(other.as_str())).into()) + Ok(op.eval_ord(zelf.as_wtf8().cmp(other.as_wtf8())).into()) } } @@ -1352,8 +1496,7 @@ impl AsSequence for PyStr { }), item: atomic_func!(|seq, i, vm| { let zelf = PyStr::sequence_downcast(seq); - zelf.getitem_by_index(vm, i) - .map(|x| zelf.new_substr(x.to_string()).into_ref(&vm.ctx).into()) + zelf.getitem_by_index(vm, i).to_pyresult(vm) }), contains: atomic_func!( |seq, needle, vm| PyStr::sequence_downcast(seq)._contains(needle, vm) @@ -1396,9 +1539,21 @@ impl ToPyObject for String { } } +impl ToPyObject for Wtf8Buf { + fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.new_str(self).into() + } +} + impl ToPyObject for char { fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { - vm.ctx.new_str(self.to_string()).into() + vm.ctx.new_str(self).into() + } +} + +impl ToPyObject for CodePoint { + fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.new_str(self).into() } } @@ -1426,6 +1581,12 @@ impl ToPyObject for AsciiString { } } +impl ToPyObject for AsciiChar { + fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.new_str(self).into() + } +} + type SplitArgs = anystr::SplitArgs; #[derive(FromArgs)] @@ -1452,96 +1613,137 @@ pub fn init(ctx: &Context) { } impl SliceableSequenceOp for PyStr { - type Item = char; - type Sliced = String; + type Item = CodePoint; + type Sliced = PyStr; fn do_get(&self, index: usize) -> Self::Item { - if self.is_ascii() { - self.bytes[index] as char - } else { - self.as_str().chars().nth(index).unwrap() - } + self.data.nth_char(index) } fn do_slice(&self, range: Range) -> Self::Sliced { - let value = self.as_str(); - if self.is_ascii() { - value[range].to_owned() - } else { - rustpython_common::str::get_chars(value, range).to_owned() + match self.as_str_kind() { + PyKindStr::Ascii(s) => s[range].into(), + PyKindStr::Utf8(s) => { + let char_len = range.len(); + let out = rustpython_common::str::get_chars(s, range); + // SAFETY: char_len is accurate + unsafe { PyStr::new_with_char_len(out, char_len) } + } + PyKindStr::Wtf8(w) => { + let char_len = range.len(); + let out = rustpython_common::str::get_codepoints(w, range); + // SAFETY: char_len is accurate + unsafe { PyStr::new_with_char_len(out, char_len) } + } } } fn do_slice_reverse(&self, range: Range) -> Self::Sliced { - if self.is_ascii() { - // this is an ascii string - let mut v = self.bytes[range].to_vec(); - v.reverse(); - unsafe { - // SAFETY: an ascii string is always utf8 - String::from_utf8_unchecked(v) + match self.as_str_kind() { + PyKindStr::Ascii(s) => { + let mut out = s[range].to_owned(); + out.as_mut_slice().reverse(); + out.into() + } + PyKindStr::Utf8(s) => { + let char_len = range.len(); + let mut out = String::with_capacity(2 * char_len); + out.extend( + s.chars() + .rev() + .skip(self.char_len() - range.end) + .take(range.len()), + ); + // SAFETY: char_len is accurate + unsafe { PyStr::new_with_char_len(out, range.len()) } + } + PyKindStr::Wtf8(w) => { + let char_len = range.len(); + let mut out = Wtf8Buf::with_capacity(2 * char_len); + out.extend( + w.code_points() + .rev() + .skip(self.char_len() - range.end) + .take(range.len()), + ); + // SAFETY: char_len is accurate + unsafe { PyStr::new_with_char_len(out, char_len) } } - } else { - let mut s = String::with_capacity(self.bytes.len()); - s.extend( - self.as_str() - .chars() - .rev() - .skip(self.char_len() - range.end) - .take(range.end - range.start), - ); - s } } fn do_stepped_slice(&self, range: Range, step: usize) -> Self::Sliced { - if self.is_ascii() { - let v = self.bytes[range].iter().copied().step_by(step).collect(); - unsafe { - // SAFETY: Any subset of ascii string is a valid utf8 string - String::from_utf8_unchecked(v) + match self.as_str_kind() { + PyKindStr::Ascii(s) => s[range] + .as_slice() + .iter() + .copied() + .step_by(step) + .collect::() + .into(), + PyKindStr::Utf8(s) => { + let char_len = (range.len() / step) + 1; + let mut out = String::with_capacity(2 * char_len); + out.extend(s.chars().skip(range.start).take(range.len()).step_by(step)); + // SAFETY: char_len is accurate + unsafe { PyStr::new_with_char_len(out, char_len) } + } + PyKindStr::Wtf8(w) => { + let char_len = (range.len() / step) + 1; + let mut out = Wtf8Buf::with_capacity(2 * char_len); + out.extend( + w.code_points() + .skip(range.start) + .take(range.len()) + .step_by(step), + ); + // SAFETY: char_len is accurate + unsafe { PyStr::new_with_char_len(out, char_len) } } - } else { - let mut s = String::with_capacity(2 * ((range.len() / step) + 1)); - s.extend( - self.as_str() - .chars() - .skip(range.start) - .take(range.end - range.start) - .step_by(step), - ); - s } } fn do_stepped_slice_reverse(&self, range: Range, step: usize) -> Self::Sliced { - if self.is_ascii() { - // this is an ascii string - let v: Vec = self.bytes[range] - .iter() + match self.as_str_kind() { + PyKindStr::Ascii(s) => s[range] + .chars() .rev() - .copied() .step_by(step) - .collect(); - // TODO: from_utf8_unchecked? - String::from_utf8(v).unwrap() - } else { - // not ascii, so the codepoints have to be at least 2 bytes each - let mut s = String::with_capacity(2 * ((range.len() / step) + 1)); - s.extend( - self.as_str() - .chars() - .rev() - .skip(self.char_len() - range.end) - .take(range.end - range.start) - .step_by(step), - ); - s + .collect::() + .into(), + PyKindStr::Utf8(s) => { + let char_len = (range.len() / step) + 1; + // not ascii, so the codepoints have to be at least 2 bytes each + let mut out = String::with_capacity(2 * char_len); + out.extend( + s.chars() + .rev() + .skip(self.char_len() - range.end) + .take(range.len()) + .step_by(step), + ); + // SAFETY: char_len is accurate + unsafe { PyStr::new_with_char_len(out, char_len) } + } + PyKindStr::Wtf8(w) => { + let char_len = (range.len() / step) + 1; + // not ascii, so the codepoints have to be at least 2 bytes each + let mut out = Wtf8Buf::with_capacity(2 * char_len); + out.extend( + w.code_points() + .rev() + .skip(self.char_len() - range.end) + .take(range.len()) + .step_by(step), + ); + // SAFETY: char_len is accurate + unsafe { PyStr::new_with_char_len(out, char_len) } + } } } fn empty() -> Self::Sliced { - String::new() + PyStr::default() } fn len(&self) -> usize { @@ -1561,10 +1763,30 @@ impl AsRef for PyExact { } } -impl AnyStrWrapper for PyStrRef { - type Str = str; - fn as_ref(&self) -> &str { - self.as_str() +impl AnyStrWrapper for PyStrRef { + fn as_ref(&self) -> Option<&Wtf8> { + Some(self.as_wtf8()) + } + fn is_empty(&self) -> bool { + self.data.is_empty() + } +} + +impl AnyStrWrapper for PyStrRef { + fn as_ref(&self) -> Option<&str> { + self.data.as_str() + } + fn is_empty(&self) -> bool { + self.data.is_empty() + } +} + +impl AnyStrWrapper for PyStrRef { + fn as_ref(&self) -> Option<&AsciiStr> { + self.data.as_ascii() + } + fn is_empty(&self) -> bool { + self.data.is_empty() } } @@ -1582,14 +1804,22 @@ impl AnyStrContainer for String { } } +impl anystr::AnyChar for char { + fn is_lowercase(self) -> bool { + self.is_lowercase() + } + fn is_uppercase(self) -> bool { + self.is_uppercase() + } + fn bytes_len(self) -> usize { + self.len_utf8() + } +} + impl AnyStr for str { type Char = char; type Container = String; - fn element_bytes_len(c: char) -> usize { - c.len_utf8() - } - fn to_container(&self) -> Self::Container { self.to_owned() } @@ -1598,10 +1828,6 @@ impl AnyStr for str { self.as_bytes() } - fn chars(&self) -> impl Iterator { - str::chars(self) - } - fn elements(&self) -> impl Iterator { str::chars(self) } @@ -1675,6 +1901,232 @@ impl AnyStr for str { } } +impl AnyStrContainer for Wtf8Buf { + fn new() -> Self { + Wtf8Buf::new() + } + + fn with_capacity(capacity: usize) -> Self { + Wtf8Buf::with_capacity(capacity) + } + + fn push_str(&mut self, other: &Wtf8) { + self.push_wtf8(other) + } +} + +impl anystr::AnyChar for CodePoint { + fn is_lowercase(self) -> bool { + self.is_char_and(char::is_lowercase) + } + fn is_uppercase(self) -> bool { + self.is_char_and(char::is_uppercase) + } + fn bytes_len(self) -> usize { + self.len_wtf8() + } +} + +impl AnyStr for Wtf8 { + type Char = CodePoint; + type Container = Wtf8Buf; + + fn to_container(&self) -> Self::Container { + self.to_owned() + } + + fn as_bytes(&self) -> &[u8] { + self.as_bytes() + } + + fn elements(&self) -> impl Iterator { + self.code_points() + } + + fn get_bytes(&self, range: std::ops::Range) -> &Self { + &self[range] + } + + fn get_chars(&self, range: std::ops::Range) -> &Self { + rustpython_common::str::get_codepoints(self, range) + } + + fn bytes_len(&self) -> usize { + self.len() + } + + fn is_empty(&self) -> bool { + self.is_empty() + } + + fn py_split_whitespace(&self, maxsplit: isize, convert: F) -> Vec + where + F: Fn(&Self) -> PyObjectRef, + { + // CPython split_whitespace + let mut splits = Vec::new(); + let mut last_offset = 0; + let mut count = maxsplit; + for (offset, _) in self + .code_point_indices() + .filter(|(_, c)| c.is_char_and(|c| c.is_ascii_whitespace() || c == '\x0b')) + { + if last_offset == offset { + last_offset += 1; + continue; + } + if count == 0 { + break; + } + splits.push(convert(&self[last_offset..offset])); + last_offset = offset + 1; + count -= 1; + } + if last_offset != self.len() { + splits.push(convert(&self[last_offset..])); + } + splits + } + + fn py_rsplit_whitespace(&self, maxsplit: isize, convert: F) -> Vec + where + F: Fn(&Self) -> PyObjectRef, + { + // CPython rsplit_whitespace + let mut splits = Vec::new(); + let mut last_offset = self.len(); + let mut count = maxsplit; + for (offset, _) in self + .code_point_indices() + .rev() + .filter(|(_, c)| c.is_char_and(|c| c.is_ascii_whitespace() || c == '\x0b')) + { + if last_offset == offset + 1 { + last_offset -= 1; + continue; + } + if count == 0 { + break; + } + splits.push(convert(&self[offset + 1..last_offset])); + last_offset = offset; + count -= 1; + } + if last_offset != 0 { + splits.push(convert(&self[..last_offset])); + } + splits + } +} + +impl AnyStrContainer for AsciiString { + fn new() -> Self { + AsciiString::new() + } + + fn with_capacity(capacity: usize) -> Self { + AsciiString::with_capacity(capacity) + } + + fn push_str(&mut self, other: &AsciiStr) { + AsciiString::push_str(self, other) + } +} + +impl anystr::AnyChar for ascii::AsciiChar { + fn is_lowercase(self) -> bool { + self.is_lowercase() + } + fn is_uppercase(self) -> bool { + self.is_uppercase() + } + fn bytes_len(self) -> usize { + 1 + } +} + +const ASCII_WHITESPACES: [u8; 6] = [0x20, 0x09, 0x0a, 0x0c, 0x0d, 0x0b]; + +impl AnyStr for AsciiStr { + type Char = AsciiChar; + type Container = AsciiString; + + fn to_container(&self) -> Self::Container { + self.to_ascii_string() + } + + fn as_bytes(&self) -> &[u8] { + self.as_bytes() + } + + fn elements(&self) -> impl Iterator { + self.chars() + } + + fn get_bytes(&self, range: std::ops::Range) -> &Self { + &self[range] + } + + fn get_chars(&self, range: std::ops::Range) -> &Self { + &self[range] + } + + fn bytes_len(&self) -> usize { + self.len() + } + + fn is_empty(&self) -> bool { + self.is_empty() + } + + fn py_split_whitespace(&self, maxsplit: isize, convert: F) -> Vec + where + F: Fn(&Self) -> PyObjectRef, + { + let mut splits = Vec::new(); + let mut count = maxsplit; + let mut haystack = self; + while let Some(offset) = haystack.as_bytes().find_byteset(ASCII_WHITESPACES) { + if offset != 0 { + if count == 0 { + break; + } + splits.push(convert(&haystack[..offset])); + count -= 1; + } + haystack = &haystack[offset + 1..]; + } + if !haystack.is_empty() { + splits.push(convert(haystack)); + } + splits + } + + fn py_rsplit_whitespace(&self, maxsplit: isize, convert: F) -> Vec + where + F: Fn(&Self) -> PyObjectRef, + { + // CPython rsplit_whitespace + let mut splits = Vec::new(); + let mut count = maxsplit; + let mut haystack = self; + while let Some(offset) = haystack.as_bytes().rfind_byteset(ASCII_WHITESPACES) { + if offset + 1 != haystack.len() { + if count == 0 { + break; + } + splits.push(convert(&haystack[offset + 1..])); + count -= 1; + } + haystack = &haystack[..offset]; + } + if !haystack.is_empty() { + splits.push(convert(haystack)); + } + splits + } +} + /// The unique reference of interned PyStr /// Always intended to be used as a static reference pub type PyStrInterned = PyInterned; @@ -1688,7 +2140,7 @@ impl PyStrInterned { impl std::fmt::Display for PyStrInterned { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - std::fmt::Display::fmt(self.as_str(), f) + self.data.fmt(f) } } diff --git a/vm/src/builtins/union.rs b/vm/src/builtins/union.rs index 2798724ae5..165113e216 100644 --- a/vm/src/builtins/union.rs +++ b/vm/src/builtins/union.rs @@ -204,7 +204,7 @@ impl PyUnion { vm, )?; let mut res; - if new_args.len() == 0 { + if new_args.is_empty() { res = make_union(&new_args, vm); } else { res = new_args.fast_getitem(0); diff --git a/vm/src/bytesinner.rs b/vm/src/bytesinner.rs index bcbbd4fce6..63d5148e04 100644 --- a/vm/src/bytesinner.rs +++ b/vm/src/bytesinner.rs @@ -355,13 +355,11 @@ impl PyBytesInner { } pub fn islower(&self) -> bool { - self.elements - .py_iscase(char::is_lowercase, char::is_uppercase) + self.elements.py_islower() } pub fn isupper(&self) -> bool { - self.elements - .py_iscase(char::is_uppercase, char::is_lowercase) + self.elements.py_isupper() } pub fn isspace(&self) -> bool { @@ -654,6 +652,7 @@ impl PyBytesInner { let elements = self.elements.py_split( options, vm, + || convert(&self.elements, vm), |v, s, vm| v.split_str(s).map(|v| convert(v, vm)).collect(), |v, s, n, vm| v.splitn_str(n, s).map(|v| convert(v, vm)).collect(), |v, n, vm| v.py_split_whitespace(n, |v| convert(v, vm)), @@ -673,6 +672,7 @@ impl PyBytesInner { let mut elements = self.elements.py_split( options, vm, + || convert(&self.elements, vm), |v, s, vm| v.rsplit_str(s).map(|v| convert(v, vm)).collect(), |v, s, n, vm| v.rsplitn_str(n, s).map(|v| convert(v, vm)).collect(), |v, n, vm| v.py_rsplit_whitespace(n, |v| convert(v, vm)), @@ -998,10 +998,12 @@ pub trait ByteOr: ToPrimitive { impl ByteOr for BigInt {} -impl AnyStrWrapper for PyBytesInner { - type Str = [u8]; - fn as_ref(&self) -> &[u8] { - &self.elements +impl AnyStrWrapper<[u8]> for PyBytesInner { + fn as_ref(&self) -> Option<&[u8]> { + Some(&self.elements) + } + fn is_empty(&self) -> bool { + self.elements.is_empty() } } @@ -1021,14 +1023,22 @@ impl AnyStrContainer<[u8]> for Vec { const ASCII_WHITESPACES: [u8; 6] = [0x20, 0x09, 0x0a, 0x0c, 0x0d, 0x0b]; +impl anystr::AnyChar for u8 { + fn is_lowercase(self) -> bool { + self.is_ascii_lowercase() + } + fn is_uppercase(self) -> bool { + self.is_ascii_uppercase() + } + fn bytes_len(self) -> usize { + 1 + } +} + impl AnyStr for [u8] { type Char = u8; type Container = Vec; - fn element_bytes_len(_: u8) -> usize { - 1 - } - fn to_container(&self) -> Self::Container { self.to_vec() } @@ -1037,10 +1047,6 @@ impl AnyStr for [u8] { self } - fn chars(&self) -> impl Iterator { - bstr::ByteSlice::chars(self) - } - fn elements(&self) -> impl Iterator { self.iter().copied() } diff --git a/vm/src/codecs.rs b/vm/src/codecs.rs index e104097413..8a690fcb58 100644 --- a/vm/src/codecs.rs +++ b/vm/src/codecs.rs @@ -1,3 +1,5 @@ +use rustpython_common::wtf8::{CodePoint, Wtf8Buf}; + use crate::{ AsObject, Context, PyObject, PyObjectRef, PyPayload, PyResult, TryFromObject, VirtualMachine, builtins::{PyBaseExceptionRef, PyBytesRef, PyStr, PyStrRef, PyTuple, PyTupleRef}, @@ -570,10 +572,11 @@ fn surrogatepass_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<(PyOb return Err(err.downcast().unwrap()); } let s_after_start = - crate::common::str::try_get_chars(s.as_str(), range.start..).unwrap_or(""); + crate::common::str::try_get_codepoints(s.as_wtf8(), range.start..).unwrap_or_default(); let num_chars = range.len(); let mut out: Vec = Vec::with_capacity(num_chars * 4); - for c in s_after_start.chars().take(num_chars).map(|x| x as u32) { + for c in s_after_start.code_points().take(num_chars) { + let c = c.to_u32(); if !(0xd800..=0xdfff).contains(&c) { // Not a surrogate, fail with original exception return Err(err.downcast().unwrap()); @@ -671,7 +674,7 @@ fn surrogatepass_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<(PyOb } Ok(( - vm.new_pyobj(format!("\\x{c:x?}")), + vm.new_pyobj(CodePoint::from_u32(c).unwrap()), range.start + byte_length, )) } else { @@ -683,11 +686,11 @@ fn surrogateescape_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<(Py if err.fast_isinstance(vm.ctx.exceptions.unicode_encode_error) { let range = extract_unicode_error_range(&err, vm)?; let object = PyStrRef::try_from_object(vm, err.get_attr("object", vm)?)?; - let s_after_start = - crate::common::str::try_get_chars(object.as_str(), range.start..).unwrap_or(""); + let s_after_start = crate::common::str::try_get_codepoints(object.as_wtf8(), range.start..) + .unwrap_or_default(); let mut out: Vec = Vec::with_capacity(range.len()); - for ch in s_after_start.chars().take(range.len()) { - let ch = ch as u32; + for ch in s_after_start.code_points().take(range.len()) { + let ch = ch.to_u32(); if !(0xdc80..=0xdcff).contains(&ch) { // Not a UTF-8b surrogate, fail with original exception return Err(err.downcast().unwrap()); @@ -702,14 +705,14 @@ fn surrogateescape_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<(Py let object = PyBytesRef::try_from_object(vm, object)?; let p = &object.as_bytes()[range.clone()]; let mut consumed = 0; - let mut replace = String::with_capacity(4 * range.len()); + let mut replace = Wtf8Buf::with_capacity(4 * range.len()); while consumed < 4 && consumed < range.len() { - let c = p[consumed] as u32; + let c = p[consumed] as u16; // Refuse to escape ASCII bytes if c < 128 { break; } - write!(replace, "#{}", 0xdc00 + c).unwrap(); + replace.push(CodePoint::from(0xdc00 + c)); consumed += 1; } if consumed == 0 { diff --git a/vm/src/frame.rs b/vm/src/frame.rs index cf695cd87b..c4dc23c95f 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -21,6 +21,7 @@ use crate::{ }; use indexmap::IndexMap; use itertools::Itertools; +use rustpython_common::wtf8::Wtf8Buf; #[cfg(feature = "threading")] use std::sync::atomic; use std::{fmt, iter::zip}; @@ -697,8 +698,8 @@ impl ExecutingFrame<'_> { .pop_multiple(size.get(arg) as usize) .as_slice() .iter() - .map(|pyobj| pyobj.payload::().unwrap().as_ref()) - .collect::(); + .map(|pyobj| pyobj.payload::().unwrap()) + .collect::(); let str_obj = vm.ctx.new_str(s); self.push_value(str_obj.into()); Ok(None) @@ -1468,7 +1469,7 @@ impl ExecutingFrame<'_> { let kwarg_names = kwarg_names .as_slice() .iter() - .map(|pyobj| pyobj.payload::().unwrap().as_ref().to_owned()); + .map(|pyobj| pyobj.payload::().unwrap().as_str().to_owned()); FuncArgs::with_kwargs_names(args, kwarg_names) } diff --git a/vm/src/import.rs b/vm/src/import.rs index 860f0b8a16..2d86e47d08 100644 --- a/vm/src/import.rs +++ b/vm/src/import.rs @@ -204,15 +204,15 @@ fn remove_importlib_frames_inner( // TODO: This function should do nothing on verbose mode. // TODO: Fix this function after making PyTraceback.next mutable -pub fn remove_importlib_frames( - vm: &VirtualMachine, - exc: &PyBaseExceptionRef, -) -> PyBaseExceptionRef { +pub fn remove_importlib_frames(vm: &VirtualMachine, exc: &PyBaseExceptionRef) { + if vm.state.settings.verbose != 0 { + return; + } + let always_trim = exc.fast_isinstance(vm.ctx.exceptions.import_error); if let Some(tb) = exc.traceback() { let trimmed_tb = remove_importlib_frames_inner(vm, Some(tb), always_trim).0; exc.set_traceback(trimmed_tb); } - exc.clone() } diff --git a/vm/src/stdlib/builtins.rs b/vm/src/stdlib/builtins.rs index 9c2826a1e9..6cae1bb7d5 100644 --- a/vm/src/stdlib/builtins.rs +++ b/vm/src/stdlib/builtins.rs @@ -31,7 +31,9 @@ mod builtins { stdlib::sys, types::PyComparisonOp, }; + use itertools::Itertools; use num_traits::{Signed, ToPrimitive}; + use rustpython_common::wtf8::CodePoint; #[cfg(not(feature = "rustpython-compiler"))] const CODEGEN_NOT_SUPPORTED: &str = @@ -85,13 +87,13 @@ mod builtins { } #[pyfunction] - fn chr(i: PyIntRef, vm: &VirtualMachine) -> PyResult { + fn chr(i: PyIntRef, vm: &VirtualMachine) -> PyResult { let value = i .try_to_primitive::(vm)? .to_u32() - .and_then(char::from_u32) + .and_then(CodePoint::from_u32) .ok_or_else(|| vm.new_value_error("chr() arg not in range(0x110000)".to_owned()))?; - Ok(value.to_string()) + Ok(value) } #[derive(FromArgs)] @@ -618,21 +620,15 @@ mod builtins { } Ok(u32::from(bytes[0])) }), - Either::B(string) => { - let string = string.as_str(); - let string_len = string.chars().count(); - if string_len != 1 { - return Err(vm.new_type_error(format!( + Either::B(string) => match string.as_wtf8().code_points().exactly_one() { + Ok(character) => Ok(character.to_u32()), + Err(_) => { + let string_len = string.char_len(); + Err(vm.new_type_error(format!( "ord() expected a character, but string of length {string_len} found" - ))); + ))) } - match string.chars().next() { - Some(character) => Ok(character as u32), - None => Err(vm.new_type_error( - "ord() could not guess the integer representing this character".to_owned(), - )), - } - } + }, } } diff --git a/vm/src/stdlib/codecs.rs b/vm/src/stdlib/codecs.rs index 976545f64b..9f58c8693d 100644 --- a/vm/src/stdlib/codecs.rs +++ b/vm/src/stdlib/codecs.rs @@ -2,6 +2,8 @@ pub(crate) use _codecs::make_module; #[pymodule] mod _codecs { + use rustpython_common::wtf8::Wtf8Buf; + use crate::common::encodings; use crate::{ AsObject, PyObject, PyObjectRef, PyResult, TryFromBorrowedObject, VirtualMachine, @@ -257,7 +259,7 @@ mod _codecs { } } - type DecodeResult = PyResult<(String, usize)>; + type DecodeResult = PyResult<(Wtf8Buf, usize)>; #[derive(FromArgs)] struct DecodeArgs { diff --git a/vm/src/vm/interpreter.rs b/vm/src/vm/interpreter.rs index a375dbedc1..cc669e0661 100644 --- a/vm/src/vm/interpreter.rs +++ b/vm/src/vm/interpreter.rs @@ -163,7 +163,7 @@ mod tests { let b = vm.new_pyobj(4_i32); let res = vm._mul(&a, &b).unwrap(); let value = res.payload::().unwrap(); - assert_eq!(value.as_ref(), "Hello Hello Hello Hello ") + assert_eq!(value.as_str(), "Hello Hello Hello Hello ") }) } } diff --git a/vm/src/vm/mod.rs b/vm/src/vm/mod.rs index dd647e36f8..eb4e846dea 100644 --- a/vm/src/vm/mod.rs +++ b/vm/src/vm/mod.rs @@ -610,7 +610,7 @@ impl VirtualMachine { let from_list = from_list.to_pyobject(self); import_func .call((module.to_owned(), globals, locals, from_list, level), self) - .map_err(|exc| import::remove_importlib_frames(self, &exc)) + .inspect_err(|exc| import::remove_importlib_frames(self, exc)) } } } From 7f4582bb23bf19e0af4542e541dc7f33b5c787e8 Mon Sep 17 00:00:00 2001 From: Noa Date: Fri, 21 Mar 2025 16:06:08 -0500 Subject: [PATCH 106/295] Make cformat wtf8-compatible --- Lib/test/test_codecs.py | 2 -- common/src/cformat.rs | 42 ++++++++++++++++++++++++++-- common/src/wtf8/mod.rs | 35 +++++++++++++++++++---- vm/src/builtins/str.rs | 4 +-- vm/src/cformat.rs | 42 +++++++++++++++------------- vm/src/dictdatatype.rs | 62 +++++++++++++++++++++++++++++++++++++++-- 6 files changed, 153 insertions(+), 34 deletions(-) diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index f29e91e088..df04653c66 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -1698,8 +1698,6 @@ def test_decode_invalid(self): class NameprepTest(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_nameprep(self): from encodings.idna import nameprep for pos, (orig, prepped) in enumerate(nameprep_tests): diff --git a/common/src/cformat.rs b/common/src/cformat.rs index 94762aceab..e62ffca65e 100644 --- a/common/src/cformat.rs +++ b/common/src/cformat.rs @@ -11,11 +11,13 @@ use std::{ str::FromStr, }; +use crate::wtf8::{CodePoint, Wtf8, Wtf8Buf}; + #[derive(Debug, PartialEq)] pub enum CFormatErrorType { UnmatchedKeyParentheses, MissingModuloSign, - UnsupportedFormatChar(char), + UnsupportedFormatChar(CodePoint), IncompleteFormat, IntTooBig, // Unimplemented, @@ -39,7 +41,9 @@ impl fmt::Display for CFormatError { UnsupportedFormatChar(c) => write!( f, "unsupported format character '{}' ({:#x}) at index {}", - c, c as u32, self.index + c, + c.to_u32(), + self.index ), IntTooBig => write!(f, "width/precision too big"), _ => write!(f, "unexpected error parsing format string"), @@ -160,7 +164,7 @@ pub trait FormatBuf: fn concat(self, other: Self) -> Self; } -pub trait FormatChar: Copy + Into + From { +pub trait FormatChar: Copy + Into + From { fn to_char_lossy(self) -> char; fn eq_char(self, c: char) -> bool; } @@ -188,6 +192,29 @@ impl FormatChar for char { } } +impl FormatBuf for Wtf8Buf { + type Char = CodePoint; + fn chars(&self) -> impl Iterator { + self.code_points() + } + fn len(&self) -> usize { + (**self).len() + } + fn concat(mut self, other: Self) -> Self { + self.extend([other]); + self + } +} + +impl FormatChar for CodePoint { + fn to_char_lossy(self) -> char { + self.to_char_lossy() + } + fn eq_char(self, c: char) -> bool { + self == c + } +} + impl FormatBuf for Vec { type Char = u8; fn chars(&self) -> impl Iterator { @@ -801,6 +828,15 @@ impl FromStr for CFormatString { } } +pub type CFormatWtf8 = CFormatStrOrBytes; + +impl CFormatWtf8 { + pub fn parse_from_wtf8(s: &Wtf8) -> Result { + let mut iter = s.code_points().enumerate().peekable(); + Self::parse(&mut iter) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/common/src/wtf8/mod.rs b/common/src/wtf8/mod.rs index 8ec3ae0afe..14ff8b5ef2 100644 --- a/common/src/wtf8/mod.rs +++ b/common/src/wtf8/mod.rs @@ -75,6 +75,12 @@ impl fmt::Debug for CodePoint { } } +impl fmt::Display for CodePoint { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.to_char_lossy().fmt(f) + } +} + impl CodePoint { /// Unsafely creates a new `CodePoint` without checking the value. /// @@ -109,13 +115,13 @@ impl CodePoint { /// Returns the numeric value of the code point. #[inline] - pub fn to_u32(&self) -> u32 { + pub fn to_u32(self) -> u32 { self.value } /// Returns the numeric value of the code point if it is a leading surrogate. #[inline] - pub fn to_lead_surrogate(&self) -> Option { + pub fn to_lead_surrogate(self) -> Option { match self.value { lead @ 0xD800..=0xDBFF => Some(lead as u16), _ => None, @@ -124,7 +130,7 @@ impl CodePoint { /// Returns the numeric value of the code point if it is a trailing surrogate. #[inline] - pub fn to_trail_surrogate(&self) -> Option { + pub fn to_trail_surrogate(self) -> Option { match self.value { trail @ 0xDC00..=0xDFFF => Some(trail as u16), _ => None, @@ -135,7 +141,7 @@ impl CodePoint { /// /// Returns `None` if the code point is a surrogate (from U+D800 to U+DFFF). #[inline] - pub fn to_char(&self) -> Option { + pub fn to_char(self) -> Option { match self.value { 0xD800..=0xDFFF => None, _ => Some(unsafe { char::from_u32_unchecked(self.value) }), @@ -147,7 +153,7 @@ impl CodePoint { /// Returns `'\u{FFFD}'` (the replacement character “�”) /// if the code point is a surrogate (from U+D800 to U+DFFF). #[inline] - pub fn to_char_lossy(&self) -> char { + pub fn to_char_lossy(self) -> char { self.to_char().unwrap_or('\u{FFFD}') } @@ -170,6 +176,12 @@ impl From for CodePoint { } } +impl From for CodePoint { + fn from(value: u8) -> Self { + char::from(value).into() + } +} + impl From for CodePoint { fn from(value: char) -> Self { Self::from_char(value) @@ -515,6 +527,13 @@ impl Extend for Wtf8Buf { } } +impl> Extend for Wtf8Buf { + fn extend>(&mut self, iter: T) { + iter.into_iter() + .for_each(move |w| self.push_wtf8(w.as_ref())); + } +} + impl> FromIterator for Wtf8Buf { fn from_iter>(iter: T) -> Self { let mut buf = Wtf8Buf::new(); @@ -541,6 +560,12 @@ impl From<&str> for Wtf8Buf { } } +impl From for Wtf8Buf { + fn from(s: ascii::AsciiString) -> Self { + Wtf8Buf::from_string(s.into()) + } +} + /// A borrowed slice of well-formed WTF-8 data. /// /// Similar to `&str`, but can additionally contain surrogate code points diff --git a/vm/src/builtins/str.rs b/vm/src/builtins/str.rs index 6ff079c644..75921bac71 100644 --- a/vm/src/builtins/str.rs +++ b/vm/src/builtins/str.rs @@ -857,8 +857,8 @@ impl PyStr { } #[pymethod(name = "__mod__")] - fn modulo(&self, values: PyObjectRef, vm: &VirtualMachine) -> PyResult { - cformat_string(vm, self.as_str(), values) + fn modulo(&self, values: PyObjectRef, vm: &VirtualMachine) -> PyResult { + cformat_string(vm, self.as_wtf8(), values) } #[pymethod(magic)] diff --git a/vm/src/cformat.rs b/vm/src/cformat.rs index af78fde021..93c409172c 100644 --- a/vm/src/cformat.rs +++ b/vm/src/cformat.rs @@ -2,6 +2,7 @@ //! as per the [Python Docs](https://docs.python.org/3/library/stdtypes.html#printf-style-string-formatting). use crate::common::cformat::*; +use crate::common::wtf8::{CodePoint, Wtf8, Wtf8Buf}; use crate::{ AsObject, PyObjectRef, PyResult, TryFromBorrowedObject, TryFromObject, VirtualMachine, builtins::{ @@ -125,13 +126,13 @@ fn spec_format_string( spec: &CFormatSpec, obj: PyObjectRef, idx: usize, -) -> PyResult { +) -> PyResult { match &spec.format_type { CFormatType::String(conversion) => { let result = match conversion { CFormatConversion::Ascii => builtins::ascii(obj, vm)?.into(), - CFormatConversion::Str => obj.str(vm)?.as_str().to_owned(), - CFormatConversion::Repr => obj.repr(vm)?.as_str().to_owned(), + CFormatConversion::Str => obj.str(vm)?.as_wtf8().to_owned(), + CFormatConversion::Repr => obj.repr(vm)?.as_wtf8().to_owned(), CFormatConversion::Bytes => { // idx is the position of the %, we want the position of the b return Err(vm.new_value_error(format!( @@ -146,16 +147,18 @@ fn spec_format_string( CNumberType::DecimalD | CNumberType::DecimalI | CNumberType::DecimalU => { match_class!(match &obj { ref i @ PyInt => { - Ok(spec.format_number(i.as_bigint())) + Ok(spec.format_number(i.as_bigint()).into()) } ref f @ PyFloat => { - Ok(spec.format_number(&try_f64_to_bigint(f.to_f64(), vm)?)) + Ok(spec + .format_number(&try_f64_to_bigint(f.to_f64(), vm)?) + .into()) } obj => { if let Some(method) = vm.get_method(obj.clone(), identifier!(vm, __int__)) { let result = method?.call((), vm)?; if let Some(i) = result.payload::() { - return Ok(spec.format_number(i.as_bigint())); + return Ok(spec.format_number(i.as_bigint()).into()); } } Err(vm.new_type_error(format!( @@ -168,7 +171,7 @@ fn spec_format_string( } _ => { if let Some(i) = obj.payload::() { - Ok(spec.format_number(i.as_bigint())) + Ok(spec.format_number(i.as_bigint()).into()) } else { Err(vm.new_type_error(format!( "%{} format: an integer is required, not {}", @@ -180,21 +183,21 @@ fn spec_format_string( }, CFormatType::Float(_) => { let value = ArgIntoFloat::try_from_object(vm, obj)?; - Ok(spec.format_float(value.into())) + Ok(spec.format_float(value.into()).into()) } CFormatType::Character(CCharacterType::Character) => { if let Some(i) = obj.payload::() { let ch = i .as_bigint() .to_u32() - .and_then(char::from_u32) + .and_then(CodePoint::from_u32) .ok_or_else(|| { vm.new_overflow_error("%c arg not in range(0x110000)".to_owned()) })?; return Ok(spec.format_char(ch)); } if let Some(s) = obj.payload::() { - if let Ok(ch) = s.as_str().chars().exactly_one() { + if let Ok(ch) = s.as_wtf8().code_points().exactly_one() { return Ok(spec.format_char(ch)); } } @@ -374,17 +377,16 @@ pub(crate) fn cformat_bytes( pub(crate) fn cformat_string( vm: &VirtualMachine, - format_string: &str, + format_string: &Wtf8, values_obj: PyObjectRef, -) -> PyResult { - let format = format_string - .parse::() +) -> PyResult { + let format = CFormatWtf8::parse_from_wtf8(format_string) .map_err(|err| vm.new_value_error(err.to_string()))?; let (num_specifiers, mapping_required) = format .check_specifiers() .ok_or_else(|| specifier_error(vm))?; - let mut result = String::new(); + let mut result = Wtf8Buf::new(); let is_mapping = values_obj.class().has_attr(identifier!(vm, __getitem__)) && !values_obj.fast_isinstance(vm.ctx.types.tuple_type) @@ -399,7 +401,7 @@ pub(crate) fn cformat_string( { for (_, part) in format.iter() { match part { - CFormatPart::Literal(literal) => result.push_str(literal), + CFormatPart::Literal(literal) => result.push_wtf8(literal), CFormatPart::Spec(_) => unreachable!(), } } @@ -415,11 +417,11 @@ pub(crate) fn cformat_string( return if is_mapping { for (idx, part) in format { match part { - CFormatPart::Literal(literal) => result.push_str(&literal), + CFormatPart::Literal(literal) => result.push_wtf8(&literal), CFormatPart::Spec(CFormatSpecKeyed { mapping_key, spec }) => { let value = values_obj.get_item(&mapping_key.unwrap(), vm)?; let part_result = spec_format_string(vm, &spec, value, idx)?; - result.push_str(&part_result); + result.push_wtf8(&part_result); } } } @@ -439,7 +441,7 @@ pub(crate) fn cformat_string( for (idx, part) in format { match part { - CFormatPart::Literal(literal) => result.push_str(&literal), + CFormatPart::Literal(literal) => result.push_wtf8(&literal), CFormatPart::Spec(CFormatSpecKeyed { mut spec, .. }) => { try_update_quantity_from_tuple( vm, @@ -456,7 +458,7 @@ pub(crate) fn cformat_string( } }?; let part_result = spec_format_string(vm, &spec, value, idx)?; - result.push_str(&part_result); + result.push_wtf8(&part_result); } } } diff --git a/vm/src/dictdatatype.rs b/vm/src/dictdatatype.rs index b36485e86a..faf5152145 100644 --- a/vm/src/dictdatatype.rs +++ b/vm/src/dictdatatype.rs @@ -12,6 +12,7 @@ use crate::{ common::{ hash, lock::{PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard}, + wtf8::{Wtf8, Wtf8Buf}, }, object::{Traverse, TraverseFn}, }; @@ -750,7 +751,7 @@ impl DictKey for Py { if self.is(other_key) { Ok(true) } else if let Some(pystr) = other_key.payload_if_exact::(vm) { - Ok(pystr.as_str() == self.as_str()) + Ok(self.as_wtf8() == pystr.as_wtf8()) } else { vm.bool_eq(self.as_object(), other_key) } @@ -834,7 +835,7 @@ impl DictKey for str { fn key_eq(&self, vm: &VirtualMachine, other_key: &PyObject) -> PyResult { if let Some(pystr) = other_key.payload_if_exact::(vm) { - Ok(pystr.as_str() == self) + Ok(pystr.as_wtf8() == self.as_ref()) } else { // Fall back to PyObjectRef implementation. let s = vm.ctx.new_str(self); @@ -871,6 +872,63 @@ impl DictKey for String { } } +impl DictKey for Wtf8 { + type Owned = Wtf8Buf; + #[inline(always)] + fn _to_owned(&self, _vm: &VirtualMachine) -> Self::Owned { + self.to_owned() + } + #[inline] + fn key_hash(&self, vm: &VirtualMachine) -> PyResult { + // follow a similar route as the hashing of PyStrRef + Ok(vm.state.hash_secret.hash_bytes(self.as_bytes())) + } + #[inline(always)] + fn key_is(&self, _other: &PyObject) -> bool { + // No matter who the other pyobject is, we are never the same thing, since + // we are a str, not a pyobject. + false + } + + fn key_eq(&self, vm: &VirtualMachine, other_key: &PyObject) -> PyResult { + if let Some(pystr) = other_key.payload_if_exact::(vm) { + Ok(pystr.as_wtf8() == self) + } else { + // Fall back to PyObjectRef implementation. + let s = vm.ctx.new_str(self); + s.key_eq(vm, other_key) + } + } + + fn key_as_isize(&self, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("'str' object cannot be interpreted as an integer".to_owned())) + } +} + +impl DictKey for Wtf8Buf { + type Owned = Wtf8Buf; + #[inline] + fn _to_owned(&self, _vm: &VirtualMachine) -> Self::Owned { + self.clone() + } + + fn key_hash(&self, vm: &VirtualMachine) -> PyResult { + (**self).key_hash(vm) + } + + fn key_is(&self, other: &PyObject) -> bool { + (**self).key_is(other) + } + + fn key_eq(&self, vm: &VirtualMachine, other_key: &PyObject) -> PyResult { + (**self).key_eq(vm, other_key) + } + + fn key_as_isize(&self, vm: &VirtualMachine) -> PyResult { + (**self).key_as_isize(vm) + } +} + impl DictKey for [u8] { type Owned = Vec; #[inline(always)] From ba1b5811ee5f151d435d042836da99dcb816144e Mon Sep 17 00:00:00 2001 From: Noa Date: Fri, 21 Mar 2025 19:19:03 -0500 Subject: [PATCH 107/295] Update encoding to use wtf8 --- Lib/test/test_cmd_line_script.py | 2 + common/src/encodings.rs | 136 +++++++++++++++++-------------- common/src/str.rs | 11 ++- common/src/wtf8/mod.rs | 71 +++++++++++++++- stdlib/src/csv.rs | 10 +-- stdlib/src/pyexpat.rs | 2 +- stdlib/src/pystruct.rs | 2 +- stdlib/src/re.rs | 23 +++--- stdlib/src/sqlite.rs | 2 +- stdlib/src/ssl.rs | 2 +- vm/src/builtins/str.rs | 22 +++-- vm/src/builtins/type.rs | 4 +- vm/src/function/buffer.rs | 4 +- vm/src/function/fspath.rs | 10 ++- vm/src/stdlib/codecs.rs | 23 +++--- vm/src/stdlib/io.rs | 52 ++++++------ vm/src/stdlib/operator.rs | 2 +- vm/src/stdlib/os.rs | 2 +- 18 files changed, 242 insertions(+), 138 deletions(-) diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index e40069d780..d5fa1563ec 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -574,6 +574,8 @@ def test_pep_409_verbiage(self): self.assertTrue(text[1].startswith(' File ')) self.assertTrue(text[3].startswith('NameError')) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_non_ascii(self): # Mac OS X denies the creation of a file with an invalid UTF-8 name. # Windows allows creating a name with an arbitrary bytes name, but diff --git a/common/src/encodings.rs b/common/src/encodings.rs index 3aaca68c3d..402e0db77f 100644 --- a/common/src/encodings.rs +++ b/common/src/encodings.rs @@ -1,5 +1,8 @@ use std::ops::Range; +use num_traits::ToPrimitive; + +use crate::str::StrKind; use crate::wtf8::{Wtf8, Wtf8Buf}; pub type EncodeErrorResult = Result<(EncodeReplace, usize), E>; @@ -7,8 +10,13 @@ pub type EncodeErrorResult = Result<(EncodeReplace, usize), E>; pub type DecodeErrorResult = Result<(S, Option, usize), E>; pub trait StrBuffer: AsRef { - fn is_ascii(&self) -> bool { - self.as_ref().is_ascii() + fn is_compatible_with(&self, kind: StrKind) -> bool { + let s = self.as_ref(); + match kind { + StrKind::Ascii => s.is_ascii(), + StrKind::Utf8 => s.is_utf8(), + StrKind::Wtf8 => true, + } } } @@ -18,7 +26,7 @@ pub trait ErrorHandler { type BytesBuf: AsRef<[u8]>; fn handle_encode_error( &self, - data: &str, + data: &Wtf8, char_range: Range, reason: &str, ) -> EncodeErrorResult; @@ -29,7 +37,7 @@ pub trait ErrorHandler { reason: &str, ) -> DecodeErrorResult; fn error_oob_restart(&self, i: usize) -> Self::Error; - fn error_encoding(&self, data: &str, char_range: Range, reason: &str) -> Self::Error; + fn error_encoding(&self, data: &Wtf8, char_range: Range, reason: &str) -> Self::Error; } pub enum EncodeReplace { Str(S), @@ -118,14 +126,61 @@ where Ok((out, remaining_index)) } +#[inline] +fn encode_utf8_compatible( + s: &Wtf8, + errors: &E, + err_reason: &str, + target_kind: StrKind, +) -> Result, E::Error> { + let full_data = s; + let mut data = s; + let mut char_data_index = 0; + let mut out = Vec::::new(); + while let Some((char_i, (byte_i, _))) = data + .code_point_indices() + .enumerate() + .find(|(_, (_, c))| !target_kind.can_encode(*c)) + { + out.extend_from_slice(&data.as_bytes()[..byte_i]); + let char_start = char_data_index + char_i; + + // number of non-compatible chars between the first non-compatible char and the next compatible char + let non_compat_run_length = data[byte_i..] + .code_points() + .take_while(|c| !target_kind.can_encode(*c)) + .count(); + let char_range = char_start..char_start + non_compat_run_length; + let (replace, char_restart) = + errors.handle_encode_error(full_data, char_range.clone(), err_reason)?; + match replace { + EncodeReplace::Str(s) => { + if s.is_compatible_with(target_kind) { + out.extend_from_slice(s.as_ref().as_bytes()); + } else { + return Err(errors.error_encoding(full_data, char_range, err_reason)); + } + } + EncodeReplace::Bytes(b) => { + out.extend_from_slice(b.as_ref()); + } + } + data = crate::str::try_get_codepoints(full_data, char_restart..) + .ok_or_else(|| errors.error_oob_restart(char_restart))?; + char_data_index = char_restart; + } + out.extend_from_slice(data.as_bytes()); + Ok(out) +} + pub mod utf8 { use super::*; pub const ENCODING_NAME: &str = "utf-8"; #[inline] - pub fn encode(s: &str, _errors: &E) -> Result, E::Error> { - Ok(s.as_bytes().to_vec()) + pub fn encode(s: &Wtf8, errors: &E) -> Result, E::Error> { + encode_utf8_compatible(s, errors, "surrogates not allowed", StrKind::Utf8) } pub fn decode( @@ -175,6 +230,7 @@ pub mod utf8 { } pub mod latin_1 { + use super::*; pub const ENCODING_NAME: &str = "latin-1"; @@ -182,14 +238,14 @@ pub mod latin_1 { const ERR_REASON: &str = "ordinal not in range(256)"; #[inline] - pub fn encode(s: &str, errors: &E) -> Result, E::Error> { + pub fn encode(s: &Wtf8, errors: &E) -> Result, E::Error> { let full_data = s; let mut data = s; let mut char_data_index = 0; let mut out = Vec::::new(); loop { match data - .char_indices() + .code_point_indices() .enumerate() .find(|(_, (_, c))| !c.is_ascii()) { @@ -200,17 +256,16 @@ pub mod latin_1 { Some((char_i, (byte_i, ch))) => { out.extend_from_slice(&data.as_bytes()[..byte_i]); let char_start = char_data_index + char_i; - if (ch as u32) <= 255 { - out.push(ch as u8); - let char_restart = char_start + 1; - data = crate::str::try_get_chars(full_data, char_restart..) - .ok_or_else(|| errors.error_oob_restart(char_restart))?; - char_data_index = char_restart; + if let Some(byte) = ch.to_u32().to_u8() { + out.push(byte); + // if the codepoint is between 128..=255, it's utf8-length is 2 + data = &data[byte_i + 2..]; + char_data_index = char_start + 1; } else { // number of non-latin_1 chars between the first non-latin_1 char and the next latin_1 char let non_latin_1_run_length = data[byte_i..] - .chars() - .take_while(|c| (*c as u32) > 255) + .code_points() + .take_while(|c| c.to_u32() > 255) .count(); let char_range = char_start..char_start + non_latin_1_run_length; let (replace, char_restart) = errors.handle_encode_error( @@ -231,7 +286,7 @@ pub mod latin_1 { out.extend_from_slice(b.as_ref()); } } - data = crate::str::try_get_chars(full_data, char_restart..) + data = crate::str::try_get_codepoints(full_data, char_restart..) .ok_or_else(|| errors.error_oob_restart(char_restart))?; char_data_index = char_restart; } @@ -258,51 +313,8 @@ pub mod ascii { const ERR_REASON: &str = "ordinal not in range(128)"; #[inline] - pub fn encode(s: &str, errors: &E) -> Result, E::Error> { - let full_data = s; - let mut data = s; - let mut char_data_index = 0; - let mut out = Vec::::new(); - loop { - match data - .char_indices() - .enumerate() - .find(|(_, (_, c))| !c.is_ascii()) - { - None => { - out.extend_from_slice(data.as_bytes()); - break; - } - Some((char_i, (byte_i, _))) => { - out.extend_from_slice(&data.as_bytes()[..byte_i]); - let char_start = char_data_index + char_i; - // number of non-ascii chars between the first non-ascii char and the next ascii char - let non_ascii_run_length = - data[byte_i..].chars().take_while(|c| !c.is_ascii()).count(); - let char_range = char_start..char_start + non_ascii_run_length; - let (replace, char_restart) = - errors.handle_encode_error(full_data, char_range.clone(), ERR_REASON)?; - match replace { - EncodeReplace::Str(s) => { - if !s.is_ascii() { - return Err( - errors.error_encoding(full_data, char_range, ERR_REASON) - ); - } - out.extend_from_slice(s.as_ref().as_bytes()); - } - EncodeReplace::Bytes(b) => { - out.extend_from_slice(b.as_ref()); - } - } - data = crate::str::try_get_chars(full_data, char_restart..) - .ok_or_else(|| errors.error_oob_restart(char_restart))?; - char_data_index = char_restart; - continue; - } - } - } - Ok(out) + pub fn encode(s: &Wtf8, errors: &E) -> Result, E::Error> { + encode_utf8_compatible(s, errors, ERR_REASON, StrKind::Ascii) } pub fn decode(data: &[u8], errors: &E) -> Result<(Wtf8Buf, usize), E::Error> { diff --git a/common/src/str.rs b/common/src/str.rs index dbe4e22234..176b5d0f87 100644 --- a/common/src/str.rs +++ b/common/src/str.rs @@ -14,7 +14,7 @@ pub type wchar_t = libc::wchar_t; pub type wchar_t = u32; /// Utf8 + state.ascii (+ PyUnicode_Kind in future) -#[derive(Debug, Copy, Clone, PartialEq)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] pub enum StrKind { Ascii, Utf8, @@ -41,6 +41,15 @@ impl StrKind { pub fn is_utf8(&self) -> bool { matches!(self, Self::Ascii | Self::Utf8) } + + #[inline(always)] + pub fn can_encode(&self, code: CodePoint) -> bool { + match self { + StrKind::Ascii => code.is_ascii(), + StrKind::Utf8 => code.to_char().is_some(), + StrKind::Wtf8 => true, + } + } } pub trait DeduceStrKind { diff --git a/common/src/wtf8/mod.rs b/common/src/wtf8/mod.rs index 14ff8b5ef2..5b31fdc558 100644 --- a/common/src/wtf8/mod.rs +++ b/common/src/wtf8/mod.rs @@ -49,7 +49,7 @@ use std::collections::TryReserveError; use std::string::String; use std::vec::Vec; -use bstr::ByteSlice; +use bstr::{ByteSlice, ByteVec}; mod core_char; mod core_str; @@ -168,6 +168,10 @@ impl CodePoint { pub fn len_wtf8(&self) -> usize { len_utf8(self.value) } + + pub fn is_ascii(&self) -> bool { + self.is_char_and(|c| c.is_ascii()) + } } impl From for CodePoint { @@ -436,6 +440,13 @@ impl Wtf8Buf { self.push_wtf8(code_point.encode_wtf8(&mut [0; MAX_LEN_UTF8])) } + pub fn pop(&mut self) -> Option { + let ch = self.code_points().next_back()?; + let newlen = self.len() - ch.len_wtf8(); + self.bytes.truncate(newlen); + Some(ch) + } + /// Shortens a string to the specified length. /// /// # Panics @@ -448,6 +459,20 @@ impl Wtf8Buf { self.bytes.truncate(new_len) } + /// Inserts a codepoint into this `Wtf8Buf` at a byte position. + #[inline] + pub fn insert(&mut self, idx: usize, c: CodePoint) { + self.insert_wtf8(idx, c.encode_wtf8(&mut [0; MAX_LEN_UTF8])) + } + + /// Inserts a WTF-8 slice into this `Wtf8Buf` at a byte position. + #[inline] + pub fn insert_wtf8(&mut self, idx: usize, w: &Wtf8) { + assert!(is_code_point_boundary(self, idx)); + + self.bytes.insert_str(idx, w) + } + /// Consumes the WTF-8 string and tries to convert it to a vec of bytes. #[inline] pub fn into_bytes(self) -> Vec { @@ -914,6 +939,21 @@ impl Wtf8 { .map(|w| unsafe { Wtf8::from_bytes_unchecked(w) }) } + pub fn trim(&self) -> &Self { + let w = self.bytes.trim(); + unsafe { Wtf8::from_bytes_unchecked(w) } + } + + pub fn trim_start(&self) -> &Self { + let w = self.bytes.trim_start(); + unsafe { Wtf8::from_bytes_unchecked(w) } + } + + pub fn trim_end(&self) -> &Self { + let w = self.bytes.trim_end(); + unsafe { Wtf8::from_bytes_unchecked(w) } + } + pub fn trim_start_matches(&self, f: impl Fn(CodePoint) -> bool) -> &Self { let mut iter = self.code_points(); loop { @@ -958,6 +998,15 @@ impl Wtf8 { memchr::memmem::rfind(self.as_bytes(), pat.as_bytes()) } + pub fn contains(&self, pat: &Wtf8) -> bool { + self.bytes.contains_str(pat) + } + + pub fn contains_code_point(&self, pat: CodePoint) -> bool { + self.bytes + .contains_str(pat.encode_wtf8(&mut [0; MAX_LEN_UTF8])) + } + pub fn get(&self, range: impl ops::RangeBounds) -> Option<&Self> { let start = match range.start_bound() { ops::Bound::Included(&i) => i, @@ -977,6 +1026,26 @@ impl Wtf8 { None } } + + pub fn ends_with(&self, w: &Wtf8) -> bool { + self.bytes.ends_with_str(w) + } + + pub fn starts_with(&self, w: &Wtf8) -> bool { + self.bytes.starts_with_str(w) + } + + pub fn strip_prefix(&self, w: &Wtf8) -> Option<&Self> { + self.bytes + .strip_prefix(w.as_bytes()) + .map(|w| unsafe { Wtf8::from_bytes_unchecked(w) }) + } + + pub fn strip_suffix(&self, w: &Wtf8) -> Option<&Self> { + self.bytes + .strip_suffix(w.as_bytes()) + .map(|w| unsafe { Wtf8::from_bytes_unchecked(w) }) + } } impl AsRef for str { diff --git a/stdlib/src/csv.rs b/stdlib/src/csv.rs index f07b40b3a2..4b79130111 100644 --- a/stdlib/src/csv.rs +++ b/stdlib/src/csv.rs @@ -198,9 +198,9 @@ mod _csv { ) -> PyResult { match_class!(match obj.get_attr("lineterminator", vm)? { s @ PyStr => { - Ok(if s.as_str().as_bytes().eq(b"\r\n") { + Ok(if s.as_bytes().eq(b"\r\n") { csv_core::Terminator::CRLF - } else if let Some(t) = s.as_str().as_bytes().first() { + } else if let Some(t) = s.as_bytes().first() { // Due to limitations in the current implementation within csv_core // the support for multiple characters in lineterminator is not complete. // only capture the first character @@ -942,7 +942,7 @@ mod _csv { ), ) })?; - let input = string.as_str().as_bytes(); + let input = string.as_bytes(); if input.is_empty() || input.starts_with(b"\n") { return Ok(PyIterReturn::Return(vm.ctx.new_list(vec![]).into())); } @@ -1101,11 +1101,11 @@ mod _csv { let field: PyObjectRef = field?; let stringified; let data: &[u8] = match_class!(match field { - ref s @ PyStr => s.as_str().as_bytes(), + ref s @ PyStr => s.as_bytes(), crate::builtins::PyNone => b"", ref obj => { stringified = obj.str(vm)?; - stringified.as_str().as_bytes() + stringified.as_bytes() } }); let mut input_offset = 0; diff --git a/stdlib/src/pyexpat.rs b/stdlib/src/pyexpat.rs index 3cfe048f17..2363e6bed4 100644 --- a/stdlib/src/pyexpat.rs +++ b/stdlib/src/pyexpat.rs @@ -136,7 +136,7 @@ mod _pyexpat { #[pymethod(name = "Parse")] fn parse(&self, data: PyStrRef, _isfinal: OptionalArg, vm: &VirtualMachine) { - let reader = Cursor::>::new(data.as_str().as_bytes().to_vec()); + let reader = Cursor::>::new(data.as_bytes().to_vec()); let parser = self.create_config().create_reader(reader); self.do_parse(vm, parser); } diff --git a/stdlib/src/pystruct.rs b/stdlib/src/pystruct.rs index 702e5b6681..2c7aa1ebf7 100644 --- a/stdlib/src/pystruct.rs +++ b/stdlib/src/pystruct.rs @@ -47,7 +47,7 @@ pub(crate) mod _struct { impl IntoStructFormatBytes { fn format_spec(&self, vm: &VirtualMachine) -> PyResult { - FormatSpec::parse(self.0.as_str().as_bytes(), vm) + FormatSpec::parse(self.0.as_bytes(), vm) } } diff --git a/stdlib/src/re.rs b/stdlib/src/re.rs index 5417e03fc7..647f4c69ad 100644 --- a/stdlib/src/re.rs +++ b/stdlib/src/re.rs @@ -9,10 +9,11 @@ mod re { * system. */ use crate::vm::{ + PyObjectRef, PyPayload, PyResult, VirtualMachine, builtins::{PyInt, PyIntRef, PyStr, PyStrRef}, convert::{ToPyObject, TryFromObject}, function::{OptionalArg, PosArgs}, - match_class, PyObjectRef, PyResult, PyPayload, VirtualMachine, + match_class, }; use num_traits::Signed; use regex::bytes::{Captures, Regex, RegexBuilder}; @@ -158,11 +159,9 @@ mod re { } fn do_sub(pattern: &PyPattern, repl: PyStrRef, search_text: PyStrRef, limit: usize) -> String { - let out = pattern.regex.replacen( - search_text.as_str().as_bytes(), - limit, - repl.as_str().as_bytes(), - ); + let out = pattern + .regex + .replacen(search_text.as_bytes(), limit, repl.as_bytes()); String::from_utf8_lossy(&out).into_owned() } @@ -172,21 +171,21 @@ mod re { regex_text.push_str(pattern.regex.as_str()); let regex = Regex::new(®ex_text).unwrap(); regex - .captures(search_text.as_str().as_bytes()) + .captures(search_text.as_bytes()) .map(|captures| create_match(search_text.clone(), captures)) } fn do_search(regex: &PyPattern, search_text: PyStrRef) -> Option { regex .regex - .captures(search_text.as_str().as_bytes()) + .captures(search_text.as_bytes()) .map(|captures| create_match(search_text.clone(), captures)) } fn do_findall(vm: &VirtualMachine, pattern: &PyPattern, search_text: PyStrRef) -> PyResult { let out = pattern .regex - .captures_iter(search_text.as_str().as_bytes()) + .captures_iter(search_text.as_bytes()) .map(|captures| match captures.len() { 1 => { let full = captures.get(0).unwrap().as_bytes(); @@ -232,7 +231,7 @@ mod re { .map(|i| i.try_to_primitive::(vm)) .transpose()? .unwrap_or(0); - let text = search_text.as_str().as_bytes(); + let text = search_text.as_bytes(); // essentially Regex::split, but it outputs captures as well let mut output = Vec::new(); let mut last = 0; @@ -332,9 +331,7 @@ mod re { #[pymethod] fn sub(&self, repl: PyStrRef, text: PyStrRef, vm: &VirtualMachine) -> PyResult { - let replaced_text = self - .regex - .replace_all(text.as_str().as_bytes(), repl.as_str().as_bytes()); + let replaced_text = self.regex.replace_all(text.as_bytes(), repl.as_bytes()); let replaced_text = String::from_utf8_lossy(&replaced_text).into_owned(); Ok(vm.ctx.new_str(replaced_text)) } diff --git a/stdlib/src/sqlite.rs b/stdlib/src/sqlite.rs index 85ce8d80fd..d20324b5ef 100644 --- a/stdlib/src/sqlite.rs +++ b/stdlib/src/sqlite.rs @@ -3000,7 +3000,7 @@ mod _sqlite { ) -> PyResult<*const libc::c_char> { BEGIN_STATEMENTS .iter() - .find(|&&x| x[6..].eq_ignore_ascii_case(s.as_str().as_bytes())) + .find(|&&x| x[6..].eq_ignore_ascii_case(s.as_bytes())) .map(|&x| x.as_ptr().cast()) .ok_or_else(|| { vm.new_value_error( diff --git a/stdlib/src/ssl.rs b/stdlib/src/ssl.rs index 2b8ffc7d8c..9bfdf883ca 100644 --- a/stdlib/src/ssl.rs +++ b/stdlib/src/ssl.rs @@ -708,7 +708,7 @@ mod _ssl { if !s.is_ascii() { return Err(invalid_cadata(vm)); } - X509::stack_from_pem(s.as_str().as_bytes()) + X509::stack_from_pem(s.as_bytes()) } Either::B(b) => b.with_ref(x509_stack_from_der), }; diff --git a/vm/src/builtins/str.rs b/vm/src/builtins/str.rs index 75921bac71..e8c19c39e0 100644 --- a/vm/src/builtins/str.rs +++ b/vm/src/builtins/str.rs @@ -429,6 +429,10 @@ impl PyStr { self.data.as_str_kind() } + pub fn is_utf8(&self) -> bool { + self.kind().is_utf8() + } + fn char_all(&self, test: F) -> bool where F: Fn(char) -> bool, @@ -725,7 +729,7 @@ impl PyStr { .py_strip( chars, |s, chars| s.trim_matches(|c| chars.code_points().contains(&c)), - |s| s.trim_matches(|c| c.is_char_and(char::is_whitespace)), + |s| s.trim(), ) .into(), } @@ -737,10 +741,10 @@ impl PyStr { chars: OptionalOption, vm: &VirtualMachine, ) -> PyRef { - let s = zelf.as_str(); + let s = zelf.as_wtf8(); let stripped = s.py_strip( chars, - |s, chars| s.trim_start_matches(|c| chars.contains(c)), + |s, chars| s.trim_start_matches(|c| chars.contains_code_point(c)), |s| s.trim_start(), ); if s == stripped { @@ -756,10 +760,10 @@ impl PyStr { chars: OptionalOption, vm: &VirtualMachine, ) -> PyRef { - let s = zelf.as_str(); + let s = zelf.as_wtf8(); let stripped = s.py_strip( chars, - |s, chars| s.trim_end_matches(|c| chars.contains(c)), + |s, chars| s.trim_end_matches(|c| chars.contains_code_point(c)), |s| s.trim_end(), ); if s == stripped { @@ -772,7 +776,7 @@ impl PyStr { #[pymethod] fn endswith(&self, options: anystr::StartsEndsWithArgs, vm: &VirtualMachine) -> PyResult { let (affix, substr) = - match options.prepare(self.as_str(), self.len(), |s, r| s.get_chars(r)) { + match options.prepare(self.as_wtf8(), self.len(), |s, r| s.get_chars(r)) { Some(x) => x, None => return Ok(false), }; @@ -780,7 +784,7 @@ impl PyStr { &affix, "endswith", "str", - |s, x: &Py| s.ends_with(x.as_str()), + |s, x: &Py| s.ends_with(x.as_wtf8()), vm, ) } @@ -792,7 +796,7 @@ impl PyStr { vm: &VirtualMachine, ) -> PyResult { let (affix, substr) = - match options.prepare(self.as_str(), self.len(), |s, r| s.get_chars(r)) { + match options.prepare(self.as_wtf8(), self.len(), |s, r| s.get_chars(r)) { Some(x) => x, None => return Ok(false), }; @@ -800,7 +804,7 @@ impl PyStr { &affix, "startswith", "str", - |s, x: &Py| s.starts_with(x.as_str()), + |s, x: &Py| s.starts_with(x.as_wtf8()), vm, ) } diff --git a/vm/src/builtins/type.rs b/vm/src/builtins/type.rs index e8ad67a666..969d6db937 100644 --- a/vm/src/builtins/type.rs +++ b/vm/src/builtins/type.rs @@ -768,7 +768,7 @@ impl PyType { value.class().slot_name(), )) })?; - if name.as_str().as_bytes().contains(&0) { + if name.as_bytes().contains(&0) { return Err(vm.new_value_error("type name must not contain null characters".to_owned())); } @@ -811,7 +811,7 @@ impl Constructor for PyType { let (name, bases, dict, kwargs): (PyStrRef, PyTupleRef, PyDictRef, KwArgs) = args.clone().bind(vm)?; - if name.as_str().as_bytes().contains(&0) { + if name.as_bytes().contains(&0) { return Err(vm.new_value_error("type name must not contain null characters".to_owned())); } diff --git a/vm/src/function/buffer.rs b/vm/src/function/buffer.rs index 91379e7a7f..80b36833e5 100644 --- a/vm/src/function/buffer.rs +++ b/vm/src/function/buffer.rs @@ -152,7 +152,7 @@ impl ArgStrOrBytesLike { pub fn borrow_bytes(&self) -> BorrowedValue<'_, [u8]> { match self { Self::Buf(b) => b.borrow_buf(), - Self::Str(s) => s.as_str().as_bytes().into(), + Self::Str(s) => s.as_bytes().into(), } } } @@ -195,7 +195,7 @@ impl ArgAsciiBuffer { #[inline] pub fn with_ref(&self, f: impl FnOnce(&[u8]) -> R) -> R { match self { - Self::String(s) => f(s.as_str().as_bytes()), + Self::String(s) => f(s.as_bytes()), Self::Buffer(buffer) => buffer.with_ref(f), } } diff --git a/vm/src/function/fspath.rs b/vm/src/function/fspath.rs index 69f11eb65d..ab5cd093b9 100644 --- a/vm/src/function/fspath.rs +++ b/vm/src/function/fspath.rs @@ -26,7 +26,7 @@ impl FsPath { let match1 = |obj: PyObjectRef| { let pathlike = match_class!(match obj { s @ PyStr => { - check_nul(s.as_str().as_bytes())?; + check_nul(s.as_bytes())?; FsPath::Str(s) } b @ PyBytes => { @@ -61,7 +61,11 @@ impl FsPath { pub fn as_os_str(&self, vm: &VirtualMachine) -> PyResult<&OsStr> { // TODO: FS encodings match self { - FsPath::Str(s) => Ok(s.as_str().as_ref()), + FsPath::Str(s) => { + // XXX RUSTPYTHON: this is sketchy on windows; it's not guaranteed that its + // OsStr encoding will always be compatible with WTF-8. + Ok(unsafe { OsStr::from_encoded_bytes_unchecked(s.as_wtf8().as_bytes()) }) + } FsPath::Bytes(b) => Self::bytes_as_osstr(b.as_bytes(), vm), } } @@ -69,7 +73,7 @@ impl FsPath { pub fn as_bytes(&self) -> &[u8] { // TODO: FS encodings match self { - FsPath::Str(s) => s.as_str().as_bytes(), + FsPath::Str(s) => s.as_bytes(), FsPath::Bytes(b) => b.as_bytes(), } } diff --git a/vm/src/stdlib/codecs.rs b/vm/src/stdlib/codecs.rs index 9f58c8693d..9018ad0e3f 100644 --- a/vm/src/stdlib/codecs.rs +++ b/vm/src/stdlib/codecs.rs @@ -2,9 +2,9 @@ pub(crate) use _codecs::make_module; #[pymodule] mod _codecs { - use rustpython_common::wtf8::Wtf8Buf; - use crate::common::encodings; + use crate::common::str::StrKind; + use crate::common::wtf8::{Wtf8, Wtf8Buf}; use crate::{ AsObject, PyObject, PyObjectRef, PyResult, TryFromBorrowedObject, VirtualMachine, builtins::{PyBaseExceptionRef, PyBytes, PyBytesRef, PyStr, PyStrRef, PyTuple}, @@ -105,8 +105,8 @@ mod _codecs { } } impl encodings::StrBuffer for PyStrRef { - fn is_ascii(&self) -> bool { - PyStr::is_ascii(self) + fn is_compatible_with(&self, kind: StrKind) -> bool { + self.kind() <= kind } } impl encodings::ErrorHandler for ErrorsHandler<'_> { @@ -116,7 +116,7 @@ mod _codecs { fn handle_encode_error( &self, - data: &str, + data: &Wtf8, char_range: Range, reason: &str, ) -> PyResult<(encodings::EncodeReplace, usize)> { @@ -219,7 +219,7 @@ mod _codecs { fn error_encoding( &self, - data: &str, + data: &Wtf8, char_range: Range, reason: &str, ) -> Self::Error { @@ -251,10 +251,10 @@ mod _codecs { #[inline] fn encode<'a, F>(self, name: &'a str, encode: F, vm: &'a VirtualMachine) -> EncodeResult where - F: FnOnce(&str, &ErrorsHandler<'a>) -> PyResult>, + F: FnOnce(&Wtf8, &ErrorsHandler<'a>) -> PyResult>, { let errors = ErrorsHandler::new(name, self.errors, vm); - let encoded = encode(self.s.as_str(), &errors)?; + let encoded = encode(self.s.as_wtf8(), &errors)?; Ok((encoded, self.s.char_len())) } } @@ -312,6 +312,9 @@ mod _codecs { #[pyfunction] fn utf_8_encode(args: EncodeArgs, vm: &VirtualMachine) -> EncodeResult { + if args.s.is_utf8() { + return Ok((args.s.as_bytes().to_vec(), args.s.byte_len())); + } do_codec!(utf8::encode, args, vm) } @@ -323,7 +326,7 @@ mod _codecs { #[pyfunction] fn latin_1_encode(args: EncodeArgs, vm: &VirtualMachine) -> EncodeResult { if args.s.is_ascii() { - return Ok((args.s.as_str().as_bytes().to_vec(), args.s.byte_len())); + return Ok((args.s.as_bytes().to_vec(), args.s.byte_len())); } do_codec!(latin_1::encode, args, vm) } @@ -336,7 +339,7 @@ mod _codecs { #[pyfunction] fn ascii_encode(args: EncodeArgs, vm: &VirtualMachine) -> EncodeResult { if args.s.is_ascii() { - return Ok((args.s.as_str().as_bytes().to_vec(), args.s.byte_len())); + return Ok((args.s.as_bytes().to_vec(), args.s.byte_len())); } do_codec!(ascii::encode, args, vm) } diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index 0f472fd940..84dddee872 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -147,6 +147,7 @@ mod _io { use crossbeam_utils::atomic::AtomicCell; use malachite_bigint::{BigInt, BigUint}; use num_traits::ToPrimitive; + use rustpython_common::wtf8::Wtf8Buf; use std::{ borrow::Cow, io::{self, Cursor, SeekFrom, prelude::*}, @@ -2082,7 +2083,7 @@ mod _io { impl PendingWrite { fn as_bytes(&self) -> &[u8] { match self { - Self::Utf8(s) => s.as_str().as_bytes(), + Self::Utf8(s) => s.as_bytes(), Self::Bytes(b) => b.as_bytes(), } } @@ -3305,14 +3306,14 @@ mod _io { }; let orig_output: PyStrRef = output.try_into_value(vm)?; // this being Cow::Owned means we need to allocate a new string - let mut output = Cow::Borrowed(orig_output.as_str()); + let mut output = Cow::Borrowed(orig_output.as_wtf8()); if self.pendingcr && (final_ || !output.is_empty()) { - output = ["\r", &*output].concat().into(); + output.to_mut().insert(0, '\r'.into()); self.pendingcr = false; } if !final_ { - if let Some(s) = output.strip_suffix('\r') { - output = s.to_owned().into(); + if let Some(s) = output.strip_suffix("\r".as_ref()) { + output = Cow::Owned(s.to_owned()); self.pendingcr = true; } } @@ -3321,19 +3322,21 @@ mod _io { return Ok(vm.ctx.empty_str.to_owned()); } - if (self.seennl == SeenNewline::LF || self.seennl.is_empty()) && !output.contains('\r') + if (self.seennl == SeenNewline::LF || self.seennl.is_empty()) + && !output.contains_code_point('\r'.into()) { - if self.seennl.is_empty() && output.contains('\n') { + if self.seennl.is_empty() && output.contains_code_point('\n'.into()) { self.seennl.insert(SeenNewline::LF); } } else if !self.translate { - let mut matches = output.match_indices(['\r', '\n']); + let output = output.as_bytes(); + let mut matches = memchr::memchr2_iter(b'\r', b'\n', output); while !self.seennl.is_all() { - let Some((i, c)) = matches.next() else { break }; - match c { - "\n" => self.seennl.insert(SeenNewline::LF), + let Some(i) = matches.next() else { break }; + match output[i] { + b'\n' => self.seennl.insert(SeenNewline::LF), // if c isn't \n, it can only be \r - _ if output[i + 1..].starts_with('\n') => { + _ if output.get(i + 1) == Some(&b'\n') => { matches.next(); self.seennl.insert(SeenNewline::CRLF); } @@ -3341,30 +3344,31 @@ mod _io { } } } else { - let mut chunks = output.match_indices(['\r', '\n']); - let mut new_string = String::with_capacity(output.len()); + let bytes = output.as_bytes(); + let mut matches = memchr::memchr2_iter(b'\r', b'\n', bytes); + let mut new_string = Wtf8Buf::with_capacity(output.len()); let mut last_modification_index = 0; - while let Some((cr_index, chunk)) = chunks.next() { - if chunk == "\r" { + while let Some(cr_index) = matches.next() { + if bytes[cr_index] == b'\r' { // skip copying the CR let mut next_chunk_index = cr_index + 1; - if output[cr_index + 1..].starts_with('\n') { - chunks.next(); + if bytes.get(cr_index + 1) == Some(&b'\n') { + matches.next(); self.seennl.insert(SeenNewline::CRLF); // skip the LF too next_chunk_index += 1; } else { self.seennl.insert(SeenNewline::CR); } - new_string.push_str(&output[last_modification_index..cr_index]); - new_string.push('\n'); + new_string.push_wtf8(&output[last_modification_index..cr_index]); + new_string.push_char('\n'); last_modification_index = next_chunk_index; } else { self.seennl.insert(SeenNewline::LF); } } - new_string.push_str(&output[last_modification_index..]); - output = new_string.into(); + new_string.push_wtf8(&output[last_modification_index..]); + output = Cow::Owned(new_string); } Ok(match output { @@ -3404,7 +3408,7 @@ mod _io { ) -> PyResult { let raw_bytes = object .flatten() - .map_or_else(Vec::new, |v| v.as_str().as_bytes().to_vec()); + .map_or_else(Vec::new, |v| v.as_bytes().to_vec()); StringIO { buffer: PyRwLock::new(BufferedIO::new(Cursor::new(raw_bytes))), @@ -3453,7 +3457,7 @@ mod _io { // write string to underlying vector #[pymethod] fn write(&self, data: PyStrRef, vm: &VirtualMachine) -> PyResult { - let bytes = data.as_str().as_bytes(); + let bytes = data.as_bytes(); self.buffer(vm)? .write(bytes) .ok_or_else(|| vm.new_type_error("Error Writing String".to_owned())) diff --git a/vm/src/stdlib/operator.rs b/vm/src/stdlib/operator.rs index d1a4b376e8..d8ff1715fa 100644 --- a/vm/src/stdlib/operator.rs +++ b/vm/src/stdlib/operator.rs @@ -328,7 +328,7 @@ mod _operator { "comparing strings with non-ASCII characters is not supported".to_owned(), )); } - cmp::timing_safe_cmp(a.as_str().as_bytes(), b.as_str().as_bytes()) + cmp::timing_safe_cmp(a.as_bytes(), b.as_bytes()) } (Either::B(a), Either::B(b)) => { a.with_ref(|a| b.with_ref(|b| cmp::timing_safe_cmp(a, b))) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 39701cb3a3..9b196b6d03 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -376,7 +376,7 @@ pub(super) mod _os { fn env_bytes_as_bytes(obj: &Either) -> &[u8] { match obj { - Either::A(s) => s.as_str().as_bytes(), + Either::A(s) => s.as_bytes(), Either::B(b) => b.as_bytes(), } } From 3945d3b2fe25496914f6a82233bbe7dfcbd36926 Mon Sep 17 00:00:00 2001 From: Noa Date: Fri, 21 Mar 2025 20:02:15 -0500 Subject: [PATCH 108/295] Make format wtf8-compatible --- Lib/test/test_difflib.py | 2 - common/src/format.rs | 318 ++++++++++++++++++++------------------- vm/src/builtins/str.rs | 21 ++- vm/src/format.rs | 21 +-- vm/src/stdlib/string.rs | 19 +-- 5 files changed, 201 insertions(+), 180 deletions(-) diff --git a/Lib/test/test_difflib.py b/Lib/test/test_difflib.py index 5592a2d5a3..0d669afe61 100644 --- a/Lib/test/test_difflib.py +++ b/Lib/test/test_difflib.py @@ -373,8 +373,6 @@ def test_byte_content(self): check(difflib.diff_bytes(context, a, a, b'a', b'a', b'2005', b'2013')) check(difflib.diff_bytes(context, a, b, b'a', b'b', b'2005', b'2013')) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_byte_filenames(self): # somebody renamed a file from ISO-8859-2 to UTF-8 fna = b'\xb3odz.txt' # "łodz.txt" diff --git a/common/src/format.rs b/common/src/format.rs index d9f821658b..75d0996796 100644 --- a/common/src/format.rs +++ b/common/src/format.rs @@ -7,8 +7,10 @@ use rustpython_literal::format::Case; use std::ops::Deref; use std::{cmp, str::FromStr}; +use crate::wtf8::{CodePoint, Wtf8, Wtf8Buf}; + trait FormatParse { - fn parse(text: &str) -> (Option, &str) + fn parse(text: &Wtf8) -> (Option, &Wtf8) where Self: Sized; } @@ -23,20 +25,20 @@ pub enum FormatConversion { } impl FormatParse for FormatConversion { - fn parse(text: &str) -> (Option, &str) { + fn parse(text: &Wtf8) -> (Option, &Wtf8) { let Some(conversion) = Self::from_string(text) else { return (None, text); }; - let mut chars = text.chars(); + let mut chars = text.code_points(); chars.next(); // Consume the bang chars.next(); // Consume one r,s,a char - (Some(conversion), chars.as_str()) + (Some(conversion), chars.as_wtf8()) } } impl FormatConversion { - pub fn from_char(c: char) -> Option { - match c { + pub fn from_char(c: CodePoint) -> Option { + match c.to_char_lossy() { 's' => Some(FormatConversion::Str), 'r' => Some(FormatConversion::Repr), 'a' => Some(FormatConversion::Ascii), @@ -45,9 +47,9 @@ impl FormatConversion { } } - fn from_string(text: &str) -> Option { - let mut chars = text.chars(); - if chars.next() != Some('!') { + fn from_string(text: &Wtf8) -> Option { + let mut chars = text.code_points(); + if chars.next()? != '!' { return None; } @@ -64,8 +66,8 @@ pub enum FormatAlign { } impl FormatAlign { - fn from_char(c: char) -> Option { - match c { + fn from_char(c: CodePoint) -> Option { + match c.to_char_lossy() { '<' => Some(FormatAlign::Left), '>' => Some(FormatAlign::Right), '=' => Some(FormatAlign::AfterSign), @@ -76,10 +78,10 @@ impl FormatAlign { } impl FormatParse for FormatAlign { - fn parse(text: &str) -> (Option, &str) { - let mut chars = text.chars(); + fn parse(text: &Wtf8) -> (Option, &Wtf8) { + let mut chars = text.code_points(); if let Some(maybe_align) = chars.next().and_then(Self::from_char) { - (Some(maybe_align), chars.as_str()) + (Some(maybe_align), chars.as_wtf8()) } else { (None, text) } @@ -94,12 +96,12 @@ pub enum FormatSign { } impl FormatParse for FormatSign { - fn parse(text: &str) -> (Option, &str) { - let mut chars = text.chars(); - match chars.next() { - Some('-') => (Some(Self::Minus), chars.as_str()), - Some('+') => (Some(Self::Plus), chars.as_str()), - Some(' ') => (Some(Self::MinusOrSpace), chars.as_str()), + fn parse(text: &Wtf8) -> (Option, &Wtf8) { + let mut chars = text.code_points(); + match chars.next().and_then(CodePoint::to_char) { + Some('-') => (Some(Self::Minus), chars.as_wtf8()), + Some('+') => (Some(Self::Plus), chars.as_wtf8()), + Some(' ') => (Some(Self::MinusOrSpace), chars.as_wtf8()), _ => (None, text), } } @@ -112,11 +114,11 @@ pub enum FormatGrouping { } impl FormatParse for FormatGrouping { - fn parse(text: &str) -> (Option, &str) { - let mut chars = text.chars(); - match chars.next() { - Some('_') => (Some(Self::Underscore), chars.as_str()), - Some(',') => (Some(Self::Comma), chars.as_str()), + fn parse(text: &Wtf8) -> (Option, &Wtf8) { + let mut chars = text.code_points(); + match chars.next().and_then(CodePoint::to_char) { + Some('_') => (Some(Self::Underscore), chars.as_wtf8()), + Some(',') => (Some(Self::Comma), chars.as_wtf8()), _ => (None, text), } } @@ -161,25 +163,25 @@ impl From<&FormatType> for char { } impl FormatParse for FormatType { - fn parse(text: &str) -> (Option, &str) { - let mut chars = text.chars(); - match chars.next() { - Some('s') => (Some(Self::String), chars.as_str()), - Some('b') => (Some(Self::Binary), chars.as_str()), - Some('c') => (Some(Self::Character), chars.as_str()), - Some('d') => (Some(Self::Decimal), chars.as_str()), - Some('o') => (Some(Self::Octal), chars.as_str()), - Some('n') => (Some(Self::Number(Case::Lower)), chars.as_str()), - Some('N') => (Some(Self::Number(Case::Upper)), chars.as_str()), - Some('x') => (Some(Self::Hex(Case::Lower)), chars.as_str()), - Some('X') => (Some(Self::Hex(Case::Upper)), chars.as_str()), - Some('e') => (Some(Self::Exponent(Case::Lower)), chars.as_str()), - Some('E') => (Some(Self::Exponent(Case::Upper)), chars.as_str()), - Some('f') => (Some(Self::FixedPoint(Case::Lower)), chars.as_str()), - Some('F') => (Some(Self::FixedPoint(Case::Upper)), chars.as_str()), - Some('g') => (Some(Self::GeneralFormat(Case::Lower)), chars.as_str()), - Some('G') => (Some(Self::GeneralFormat(Case::Upper)), chars.as_str()), - Some('%') => (Some(Self::Percentage), chars.as_str()), + fn parse(text: &Wtf8) -> (Option, &Wtf8) { + let mut chars = text.code_points(); + match chars.next().and_then(CodePoint::to_char) { + Some('s') => (Some(Self::String), chars.as_wtf8()), + Some('b') => (Some(Self::Binary), chars.as_wtf8()), + Some('c') => (Some(Self::Character), chars.as_wtf8()), + Some('d') => (Some(Self::Decimal), chars.as_wtf8()), + Some('o') => (Some(Self::Octal), chars.as_wtf8()), + Some('n') => (Some(Self::Number(Case::Lower)), chars.as_wtf8()), + Some('N') => (Some(Self::Number(Case::Upper)), chars.as_wtf8()), + Some('x') => (Some(Self::Hex(Case::Lower)), chars.as_wtf8()), + Some('X') => (Some(Self::Hex(Case::Upper)), chars.as_wtf8()), + Some('e') => (Some(Self::Exponent(Case::Lower)), chars.as_wtf8()), + Some('E') => (Some(Self::Exponent(Case::Upper)), chars.as_wtf8()), + Some('f') => (Some(Self::FixedPoint(Case::Lower)), chars.as_wtf8()), + Some('F') => (Some(Self::FixedPoint(Case::Upper)), chars.as_wtf8()), + Some('g') => (Some(Self::GeneralFormat(Case::Lower)), chars.as_wtf8()), + Some('G') => (Some(Self::GeneralFormat(Case::Upper)), chars.as_wtf8()), + Some('%') => (Some(Self::Percentage), chars.as_wtf8()), _ => (None, text), } } @@ -188,7 +190,7 @@ impl FormatParse for FormatType { #[derive(Debug, PartialEq)] pub struct FormatSpec { conversion: Option, - fill: Option, + fill: Option, align: Option, sign: Option, alternate_form: bool, @@ -198,17 +200,17 @@ pub struct FormatSpec { format_type: Option, } -fn get_num_digits(text: &str) -> usize { - for (index, character) in text.char_indices() { - if !character.is_ascii_digit() { +fn get_num_digits(text: &Wtf8) -> usize { + for (index, character) in text.code_point_indices() { + if !character.is_char_and(|c| c.is_ascii_digit()) { return index; } } text.len() } -fn parse_fill_and_align(text: &str) -> (Option, Option, &str) { - let char_indices: Vec<(usize, char)> = text.char_indices().take(3).collect(); +fn parse_fill_and_align(text: &Wtf8) -> (Option, Option, &Wtf8) { + let char_indices: Vec<(usize, CodePoint)> = text.code_point_indices().take(3).collect(); if char_indices.is_empty() { (None, None, text) } else if char_indices.len() == 1 { @@ -225,12 +227,12 @@ fn parse_fill_and_align(text: &str) -> (Option, Option, &str) } } -fn parse_number(text: &str) -> Result<(Option, &str), FormatSpecError> { +fn parse_number(text: &Wtf8) -> Result<(Option, &Wtf8), FormatSpecError> { let num_digits: usize = get_num_digits(text); if num_digits == 0 { return Ok((None, text)); } - if let Ok(num) = text[..num_digits].parse::() { + if let Some(num) = parse_usize(&text[..num_digits]) { Ok((Some(num), &text[num_digits..])) } else { // NOTE: this condition is different from CPython @@ -238,27 +240,27 @@ fn parse_number(text: &str) -> Result<(Option, &str), FormatSpecError> { } } -fn parse_alternate_form(text: &str) -> (bool, &str) { - let mut chars = text.chars(); - match chars.next() { - Some('#') => (true, chars.as_str()), +fn parse_alternate_form(text: &Wtf8) -> (bool, &Wtf8) { + let mut chars = text.code_points(); + match chars.next().and_then(CodePoint::to_char) { + Some('#') => (true, chars.as_wtf8()), _ => (false, text), } } -fn parse_zero(text: &str) -> (bool, &str) { - let mut chars = text.chars(); - match chars.next() { - Some('0') => (true, chars.as_str()), +fn parse_zero(text: &Wtf8) -> (bool, &Wtf8) { + let mut chars = text.code_points(); + match chars.next().and_then(CodePoint::to_char) { + Some('0') => (true, chars.as_wtf8()), _ => (false, text), } } -fn parse_precision(text: &str) -> Result<(Option, &str), FormatSpecError> { - let mut chars = text.chars(); - Ok(match chars.next() { +fn parse_precision(text: &Wtf8) -> Result<(Option, &Wtf8), FormatSpecError> { + let mut chars = text.code_points(); + Ok(match chars.next().and_then(CodePoint::to_char) { Some('.') => { - let (size, remaining) = parse_number(chars.as_str())?; + let (size, remaining) = parse_number(chars.as_wtf8())?; if let Some(size) = size { if size > i32::MAX as usize { return Err(FormatSpecError::PrecisionTooBig); @@ -273,7 +275,10 @@ fn parse_precision(text: &str) -> Result<(Option, &str), FormatSpecError> } impl FormatSpec { - pub fn parse(text: &str) -> Result { + pub fn parse(text: impl AsRef) -> Result { + Self::_parse(text.as_ref()) + } + fn _parse(text: &Wtf8) -> Result { // get_integer in CPython let (conversion, text) = FormatConversion::parse(text); let (mut fill, mut align, text) = parse_fill_and_align(text); @@ -289,7 +294,7 @@ impl FormatSpec { } if zero && fill.is_none() { - fill.replace('0'); + fill.replace('0'.into()); align = align.or(Some(FormatAlign::AfterSign)); } @@ -306,10 +311,8 @@ impl FormatSpec { }) } - fn compute_fill_string(fill_char: char, fill_chars_needed: i32) -> String { - (0..fill_chars_needed) - .map(|_| fill_char) - .collect::() + fn compute_fill_string(fill_char: CodePoint, fill_chars_needed: i32) -> Wtf8Buf { + (0..fill_chars_needed).map(|_| fill_char).collect() } fn add_magnitude_separators_for_char( @@ -625,7 +628,7 @@ impl FormatSpec { let align = self.align.unwrap_or(default_align); let num_chars = magnitude_str.char_len(); - let fill_char = self.fill.unwrap_or(' '); + let fill_char = self.fill.unwrap_or(' '.into()); let fill_chars_needed: i32 = self.width.map_or(0, |w| { cmp::max(0, (w as i32) - (num_chars as i32) - (sign_str.len() as i32)) }); @@ -726,20 +729,20 @@ impl FromStr for FormatSpec { #[derive(Debug, PartialEq)] pub enum FieldNamePart { - Attribute(String), + Attribute(Wtf8Buf), Index(usize), - StringIndex(String), + StringIndex(Wtf8Buf), } impl FieldNamePart { fn parse_part( - chars: &mut impl PeekingNext, + chars: &mut impl PeekingNext, ) -> Result, FormatParseError> { chars .next() - .map(|ch| match ch { + .map(|ch| match ch.to_char_lossy() { '.' => { - let mut attribute = String::new(); + let mut attribute = Wtf8Buf::new(); for ch in chars.peeking_take_while(|ch| *ch != '.' && *ch != '[') { attribute.push(ch); } @@ -750,12 +753,12 @@ impl FieldNamePart { } } '[' => { - let mut index = String::new(); + let mut index = Wtf8Buf::new(); for ch in chars { if ch == ']' { return if index.is_empty() { Err(FormatParseError::EmptyAttribute) - } else if let Ok(index) = index.parse::() { + } else if let Some(index) = parse_usize(&index) { Ok(FieldNamePart::Index(index)) } else { Ok(FieldNamePart::StringIndex(index)) @@ -775,7 +778,7 @@ impl FieldNamePart { pub enum FieldType { Auto, Index(usize), - Keyword(String), + Keyword(Wtf8Buf), } #[derive(Debug, PartialEq)] @@ -784,17 +787,20 @@ pub struct FieldName { pub parts: Vec, } +fn parse_usize(s: &Wtf8) -> Option { + s.as_str().ok().and_then(|s| s.parse().ok()) +} + impl FieldName { - pub fn parse(text: &str) -> Result { - let mut chars = text.chars().peekable(); - let mut first = String::new(); - for ch in chars.peeking_take_while(|ch| *ch != '.' && *ch != '[') { - first.push(ch); - } + pub fn parse(text: &Wtf8) -> Result { + let mut chars = text.code_points().peekable(); + let first: Wtf8Buf = chars + .peeking_take_while(|ch| *ch != '.' && *ch != '[') + .collect(); let field_type = if first.is_empty() { FieldType::Auto - } else if let Ok(index) = first.parse::() { + } else if let Some(index) = parse_usize(&first) { FieldType::Index(index) } else { FieldType::Keyword(first) @@ -812,11 +818,11 @@ impl FieldName { #[derive(Debug, PartialEq)] pub enum FormatPart { Field { - field_name: String, - conversion_spec: Option, - format_spec: String, + field_name: Wtf8Buf, + conversion_spec: Option, + format_spec: Wtf8Buf, }, - Literal(String), + Literal(Wtf8Buf), } #[derive(Debug, PartialEq)] @@ -825,8 +831,8 @@ pub struct FormatString { } impl FormatString { - fn parse_literal_single(text: &str) -> Result<(char, &str), FormatParseError> { - let mut chars = text.chars(); + fn parse_literal_single(text: &Wtf8) -> Result<(CodePoint, &Wtf8), FormatParseError> { + let mut chars = text.code_points(); // This should never be called with an empty str let first_char = chars.next().unwrap(); // isn't this detectable only with bytes operation? @@ -836,15 +842,15 @@ impl FormatString { return if maybe_next_char.is_none() || maybe_next_char.unwrap() != first_char { Err(FormatParseError::UnescapedStartBracketInLiteral) } else { - Ok((first_char, chars.as_str())) + Ok((first_char, chars.as_wtf8())) }; } - Ok((first_char, chars.as_str())) + Ok((first_char, chars.as_wtf8())) } - fn parse_literal(text: &str) -> Result<(FormatPart, &str), FormatParseError> { + fn parse_literal(text: &Wtf8) -> Result<(FormatPart, &Wtf8), FormatParseError> { let mut cur_text = text; - let mut result_string = String::new(); + let mut result_string = Wtf8Buf::new(); while !cur_text.is_empty() { match FormatString::parse_literal_single(cur_text) { Ok((next_char, remaining)) => { @@ -860,14 +866,14 @@ impl FormatString { } } } - Ok((FormatPart::Literal(result_string), "")) + Ok((FormatPart::Literal(result_string), "".as_ref())) } - fn parse_part_in_brackets(text: &str) -> Result { - let mut chars = text.chars().peekable(); + fn parse_part_in_brackets(text: &Wtf8) -> Result { + let mut chars = text.code_points().peekable(); - let mut left = String::new(); - let mut right = String::new(); + let mut left = Wtf8Buf::new(); + let mut right = Wtf8Buf::new(); let mut split = false; let mut selected = &mut left; @@ -899,12 +905,12 @@ impl FormatString { } // before the comma is a keyword or arg index, after the comma is maybe a spec. - let arg_part: &str = &left; + let arg_part: &Wtf8 = &left; - let format_spec = if split { right } else { String::new() }; + let format_spec = if split { right } else { Wtf8Buf::new() }; // left can still be the conversion (!r, !s, !a) - let parts: Vec<&str> = arg_part.splitn(2, '!').collect(); + let parts: Vec<&Wtf8> = arg_part.splitn(2, "!".as_ref()).collect(); // before the bang is a keyword or arg index, after the comma is maybe a conversion spec. let arg_part = parts[0]; @@ -913,7 +919,7 @@ impl FormatString { .map(|conversion| { // conversions are only every one character conversion - .chars() + .code_points() .exactly_one() .map_err(|_| FormatParseError::UnknownConversion) }) @@ -926,13 +932,13 @@ impl FormatString { }) } - fn parse_spec(text: &str) -> Result<(FormatPart, &str), FormatParseError> { + fn parse_spec(text: &Wtf8) -> Result<(FormatPart, &Wtf8), FormatParseError> { let mut nested = false; let mut end_bracket_pos = None; - let mut left = String::new(); + let mut left = Wtf8Buf::new(); // There may be one layer nesting brackets in spec - for (idx, c) in text.char_indices() { + for (idx, c) in text.code_point_indices() { if idx == 0 { if c != '{' { return Err(FormatParseError::MissingStartBracket); @@ -959,7 +965,7 @@ impl FormatString { } } if let Some(pos) = end_bracket_pos { - let (_, right) = text.split_at(pos); + let right = &text[pos..]; let format_part = FormatString::parse_part_in_brackets(&left)?; Ok((format_part, &right[1..])) } else { @@ -970,14 +976,14 @@ impl FormatString { pub trait FromTemplate<'a>: Sized { type Err; - fn from_str(s: &'a str) -> Result; + fn from_str(s: &'a Wtf8) -> Result; } impl<'a> FromTemplate<'a> for FormatString { type Err = FormatParseError; - fn from_str(text: &'a str) -> Result { - let mut cur_text: &str = text; + fn from_str(text: &'a Wtf8) -> Result { + let mut cur_text: &Wtf8 = text; let mut parts: Vec = Vec::new(); while !cur_text.is_empty() { // Try to parse both literals and bracketed format parts until we @@ -1001,6 +1007,14 @@ mod tests { #[test] fn test_fill_and_align() { + let parse_fill_and_align = |text| { + let (fill, align, rest) = parse_fill_and_align(str::as_ref(text)); + ( + fill.and_then(CodePoint::to_char), + align, + rest.as_str().unwrap(), + ) + }; assert_eq!( parse_fill_and_align(" <"), (Some(' '), Some(FormatAlign::Left), "") @@ -1043,7 +1057,7 @@ mod tests { fn test_fill_and_width() { let expected = Ok(FormatSpec { conversion: None, - fill: Some('<'), + fill: Some('<'.into()), align: Some(FormatAlign::Right), sign: None, alternate_form: false, @@ -1059,7 +1073,7 @@ mod tests { fn test_all() { let expected = Ok(FormatSpec { conversion: None, - fill: Some('<'), + fill: Some('<'.into()), align: Some(FormatAlign::Right), sign: Some(FormatSign::Minus), alternate_form: true, @@ -1167,33 +1181,33 @@ mod tests { fn test_format_parse() { let expected = Ok(FormatString { format_parts: vec![ - FormatPart::Literal("abcd".to_owned()), + FormatPart::Literal("abcd".into()), FormatPart::Field { - field_name: "1".to_owned(), + field_name: "1".into(), conversion_spec: None, - format_spec: String::new(), + format_spec: "".into(), }, - FormatPart::Literal(":".to_owned()), + FormatPart::Literal(":".into()), FormatPart::Field { - field_name: "key".to_owned(), + field_name: "key".into(), conversion_spec: None, - format_spec: String::new(), + format_spec: "".into(), }, ], }); - assert_eq!(FormatString::from_str("abcd{1}:{key}"), expected); + assert_eq!(FormatString::from_str("abcd{1}:{key}".as_ref()), expected); } #[test] fn test_format_parse_multi_byte_char() { - assert!(FormatString::from_str("{a:%ЫйЯЧ}").is_ok()); + assert!(FormatString::from_str("{a:%ЫйЯЧ}".as_ref()).is_ok()); } #[test] fn test_format_parse_fail() { assert_eq!( - FormatString::from_str("{s"), + FormatString::from_str("{s".as_ref()), Err(FormatParseError::UnmatchedBracket) ); } @@ -1201,27 +1215,27 @@ mod tests { #[test] fn test_square_brackets_inside_format() { assert_eq!( - FormatString::from_str("{[:123]}"), + FormatString::from_str("{[:123]}".as_ref()), Ok(FormatString { format_parts: vec![FormatPart::Field { - field_name: "[:123]".to_owned(), + field_name: "[:123]".into(), conversion_spec: None, - format_spec: "".to_owned(), + format_spec: "".into(), }], }), ); - assert_eq!(FormatString::from_str("{asdf[:123]asdf}"), { + assert_eq!(FormatString::from_str("{asdf[:123]asdf}".as_ref()), { Ok(FormatString { format_parts: vec![FormatPart::Field { - field_name: "asdf[:123]asdf".to_owned(), + field_name: "asdf[:123]asdf".into(), conversion_spec: None, - format_spec: "".to_owned(), + format_spec: "".into(), }], }) }); - assert_eq!(FormatString::from_str("{[1234}"), { + assert_eq!(FormatString::from_str("{[1234}".as_ref()), { Err(FormatParseError::MissingRightBracket) }); } @@ -1230,17 +1244,17 @@ mod tests { fn test_format_parse_escape() { let expected = Ok(FormatString { format_parts: vec![ - FormatPart::Literal("{".to_owned()), + FormatPart::Literal("{".into()), FormatPart::Field { - field_name: "key".to_owned(), + field_name: "key".into(), conversion_spec: None, - format_spec: String::new(), + format_spec: "".into(), }, - FormatPart::Literal("}ddfe".to_owned()), + FormatPart::Literal("}ddfe".into()), ], }); - assert_eq!(FormatString::from_str("{{{key}}}ddfe"), expected); + assert_eq!(FormatString::from_str("{{{key}}}ddfe".as_ref()), expected); } #[test] @@ -1277,52 +1291,44 @@ mod tests { #[test] fn test_parse_field_name() { + let parse = |s: &str| FieldName::parse(s.as_ref()); assert_eq!( - FieldName::parse(""), + parse(""), Ok(FieldName { field_type: FieldType::Auto, parts: Vec::new(), }) ); assert_eq!( - FieldName::parse("0"), + parse("0"), Ok(FieldName { field_type: FieldType::Index(0), parts: Vec::new(), }) ); assert_eq!( - FieldName::parse("key"), + parse("key"), Ok(FieldName { - field_type: FieldType::Keyword("key".to_owned()), + field_type: FieldType::Keyword("key".into()), parts: Vec::new(), }) ); assert_eq!( - FieldName::parse("key.attr[0][string]"), + parse("key.attr[0][string]"), Ok(FieldName { - field_type: FieldType::Keyword("key".to_owned()), + field_type: FieldType::Keyword("key".into()), parts: vec![ - FieldNamePart::Attribute("attr".to_owned()), + FieldNamePart::Attribute("attr".into()), FieldNamePart::Index(0), - FieldNamePart::StringIndex("string".to_owned()) + FieldNamePart::StringIndex("string".into()) ], }) ); + assert_eq!(parse("key.."), Err(FormatParseError::EmptyAttribute)); + assert_eq!(parse("key[]"), Err(FormatParseError::EmptyAttribute)); + assert_eq!(parse("key["), Err(FormatParseError::MissingRightBracket)); assert_eq!( - FieldName::parse("key.."), - Err(FormatParseError::EmptyAttribute) - ); - assert_eq!( - FieldName::parse("key[]"), - Err(FormatParseError::EmptyAttribute) - ); - assert_eq!( - FieldName::parse("key["), - Err(FormatParseError::MissingRightBracket) - ); - assert_eq!( - FieldName::parse("key[0]after"), + parse("key[0]after"), Err(FormatParseError::InvalidCharacterAfterRightBracket) ); } diff --git a/vm/src/builtins/str.rs b/vm/src/builtins/str.rs index e8c19c39e0..4dc4e7a64e 100644 --- a/vm/src/builtins/str.rs +++ b/vm/src/builtins/str.rs @@ -871,8 +871,9 @@ impl PyStr { } #[pymethod] - fn format(&self, args: FuncArgs, vm: &VirtualMachine) -> PyResult { - let format_str = FormatString::from_str(self.as_str()).map_err(|e| e.to_pyexception(vm))?; + fn format(&self, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + let format_str = + FormatString::from_str(self.as_wtf8()).map_err(|e| e.to_pyexception(vm))?; format(&format_str, &args, vm) } @@ -881,9 +882,9 @@ impl PyStr { /// Return a formatted version of S, using substitutions from mapping. /// The substitutions are identified by braces ('{' and '}'). #[pymethod] - fn format_map(&self, mapping: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn format_map(&self, mapping: PyObjectRef, vm: &VirtualMachine) -> PyResult { let format_string = - FormatString::from_str(self.as_str()).map_err(|err| err.to_pyexception(vm))?; + FormatString::from_str(self.as_wtf8()).map_err(|err| err.to_pyexception(vm))?; format_map(&format_string, &mapping, vm) } @@ -1573,6 +1574,18 @@ impl ToPyObject for &String { } } +impl ToPyObject for &Wtf8 { + fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.new_str(self).into() + } +} + +impl ToPyObject for &Wtf8Buf { + fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.new_str(self.clone()).into() + } +} + impl ToPyObject for &AsciiStr { fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { vm.ctx.new_str(self).into() diff --git a/vm/src/format.rs b/vm/src/format.rs index 1dbfd46779..3349ee854e 100644 --- a/vm/src/format.rs +++ b/vm/src/format.rs @@ -7,6 +7,7 @@ use crate::{ }; use crate::common::format::*; +use crate::common::wtf8::{Wtf8, Wtf8Buf}; impl IntoPyException for FormatSpecError { fn into_pyexception(self, vm: &VirtualMachine) -> PyBaseExceptionRef { @@ -62,18 +63,18 @@ fn format_internal( vm: &VirtualMachine, format: &FormatString, field_func: &mut impl FnMut(FieldType) -> PyResult, -) -> PyResult { - let mut final_string = String::new(); +) -> PyResult { + let mut final_string = Wtf8Buf::new(); for part in &format.format_parts { let pystr; - let result_string: &str = match part { + let result_string: &Wtf8 = match part { FormatPart::Field { field_name, conversion_spec, format_spec, } => { let FieldName { field_type, parts } = - FieldName::parse(field_name.as_str()).map_err(|e| e.to_pyexception(vm))?; + FieldName::parse(field_name).map_err(|e| e.to_pyexception(vm))?; let mut argument = field_func(field_type)?; @@ -113,7 +114,7 @@ fn format_internal( } FormatPart::Literal(literal) => literal, }; - final_string.push_str(result_string); + final_string.push_wtf8(result_string); } Ok(final_string) } @@ -122,7 +123,7 @@ pub(crate) fn format( format: &FormatString, arguments: &FuncArgs, vm: &VirtualMachine, -) -> PyResult { +) -> PyResult { let mut auto_argument_index: usize = 0; let mut seen_index = false; format_internal(vm, format, &mut |field_type| match field_type { @@ -154,8 +155,10 @@ pub(crate) fn format( .cloned() .ok_or_else(|| vm.new_index_error("tuple index out of range".to_owned())) } - FieldType::Keyword(keyword) => arguments - .get_optional_kwarg(&keyword) + FieldType::Keyword(keyword) => keyword + .as_str() + .ok() + .and_then(|keyword| arguments.get_optional_kwarg(keyword)) .ok_or_else(|| vm.new_key_error(vm.ctx.new_str(keyword).into())), }) } @@ -164,7 +167,7 @@ pub(crate) fn format_map( format: &FormatString, dict: &PyObject, vm: &VirtualMachine, -) -> PyResult { +) -> PyResult { format_internal(vm, format, &mut |field_type| match field_type { FieldType::Auto | FieldType::Index(_) => { Err(vm.new_value_error("Format string contains positional fields".to_owned())) diff --git a/vm/src/stdlib/string.rs b/vm/src/stdlib/string.rs index 3c399e1c37..576cae6277 100644 --- a/vm/src/stdlib/string.rs +++ b/vm/src/stdlib/string.rs @@ -9,6 +9,7 @@ mod _string { use crate::common::format::{ FieldName, FieldNamePart, FieldType, FormatPart, FormatString, FromTemplate, }; + use crate::common::wtf8::{CodePoint, Wtf8Buf}; use crate::{ PyObjectRef, PyResult, VirtualMachine, builtins::{PyList, PyStrRef}, @@ -18,10 +19,10 @@ mod _string { use std::mem; fn create_format_part( - literal: String, - field_name: Option, - format_spec: Option, - conversion_spec: Option, + literal: Wtf8Buf, + field_name: Option, + format_spec: Option, + conversion_spec: Option, vm: &VirtualMachine, ) -> PyObjectRef { let tuple = ( @@ -36,10 +37,10 @@ mod _string { #[pyfunction] fn formatter_parser(text: PyStrRef, vm: &VirtualMachine) -> PyResult { let format_string = - FormatString::from_str(text.as_str()).map_err(|e| e.to_pyexception(vm))?; + FormatString::from_str(text.as_wtf8()).map_err(|e| e.to_pyexception(vm))?; - let mut result = Vec::new(); - let mut literal = String::new(); + let mut result: Vec = Vec::new(); + let mut literal = Wtf8Buf::new(); for part in format_string.format_parts { match part { FormatPart::Field { @@ -55,7 +56,7 @@ mod _string { vm, )); } - FormatPart::Literal(text) => literal.push_str(&text), + FormatPart::Literal(text) => literal.push_wtf8(&text), } } if !literal.is_empty() { @@ -75,7 +76,7 @@ mod _string { text: PyStrRef, vm: &VirtualMachine, ) -> PyResult<(PyObjectRef, PyList)> { - let field_name = FieldName::parse(text.as_str()).map_err(|e| e.to_pyexception(vm))?; + let field_name = FieldName::parse(text.as_wtf8()).map_err(|e| e.to_pyexception(vm))?; let first = match field_name.field_type { FieldType::Auto => vm.ctx.new_str(ascii!("")).into(), From b36b32bfe810ab96cf945ff6ec04f85fcf46dd03 Mon Sep 17 00:00:00 2001 From: Noa Date: Fri, 21 Mar 2025 20:13:15 -0500 Subject: [PATCH 109/295] Make re wtf8-compatible --- Cargo.lock | 1 + Lib/test/test_re.py | 2 -- Lib/test/test_smtplib.py | 2 -- vm/src/stdlib/sre.rs | 16 ++++++--- vm/sre_engine/Cargo.toml | 5 +++ vm/sre_engine/src/string.rs | 70 +++++++++++++++++++++++++++++++++++++ 6 files changed, 87 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 94b0732b8a..1fa60eac29 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2474,6 +2474,7 @@ dependencies = [ "criterion", "num_enum", "optional", + "rustpython-common", ] [[package]] diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index a060a3deef..f6af44df99 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -854,8 +854,6 @@ def test_string_boundaries(self): # Can match around the whitespace. self.assertEqual(len(re.findall(r"\B", " ")), 2) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_bigcharset(self): self.assertEqual(re.match("([\u2222\u2223])", "\u2222").group(1), "\u2222") diff --git a/Lib/test/test_smtplib.py b/Lib/test/test_smtplib.py index a36d7bbe2a..9b787950fc 100644 --- a/Lib/test/test_smtplib.py +++ b/Lib/test/test_smtplib.py @@ -1459,8 +1459,6 @@ def test_send_unicode_with_SMTPUTF8_via_low_level_API(self): self.assertIn('SMTPUTF8', self.serv.last_mail_options) self.assertEqual(self.serv.last_rcpt_options, []) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_send_message_uses_smtputf8_if_addrs_non_ascii(self): msg = EmailMessage() msg['From'] = "Páolo " diff --git a/vm/src/stdlib/sre.rs b/vm/src/stdlib/sre.rs index 193976a62d..038ac9934a 100644 --- a/vm/src/stdlib/sre.rs +++ b/vm/src/stdlib/sre.rs @@ -9,6 +9,7 @@ mod _sre { PyCallableIterator, PyDictRef, PyGenericAlias, PyInt, PyList, PyListRef, PyStr, PyStrRef, PyTuple, PyTupleRef, PyTypeRef, }, + common::wtf8::{Wtf8, Wtf8Buf}, common::{ascii, hash::PyHash}, convert::ToPyObject, function::{ArgCallable, OptionalArg, PosArgs, PyComparisonValue}, @@ -66,10 +67,15 @@ mod _sre { } } - impl SreStr for &str { + impl SreStr for &Wtf8 { fn slice(&self, start: usize, end: usize, vm: &VirtualMachine) -> PyObjectRef { vm.ctx - .new_str(self.chars().take(end).skip(start).collect::()) + .new_str( + self.code_points() + .take(end) + .skip(start) + .collect::(), + ) .into() } } @@ -206,12 +212,12 @@ mod _sre { impl Pattern { fn with_str(string: &PyObject, vm: &VirtualMachine, f: F) -> PyResult where - F: FnOnce(&str) -> PyResult, + F: FnOnce(&Wtf8) -> PyResult, { let string = string.payload::().ok_or_else(|| { vm.new_type_error(format!("expected string got '{}'", string.class())) })?; - f(string.as_str()) + f(string.as_wtf8()) } fn with_bytes(string: &PyObject, vm: &VirtualMachine, f: F) -> PyResult @@ -425,7 +431,7 @@ mod _sre { let is_template = if zelf.isbytes { Self::with_bytes(&repl, vm, |x| Ok(x.contains(&b'\\')))? } else { - Self::with_str(&repl, vm, |x| Ok(x.contains('\\')))? + Self::with_str(&repl, vm, |x| Ok(x.contains("\\".as_ref())))? }; if is_template { diff --git a/vm/sre_engine/Cargo.toml b/vm/sre_engine/Cargo.toml index 504652f3a7..55ce249903 100644 --- a/vm/sre_engine/Cargo.toml +++ b/vm/sre_engine/Cargo.toml @@ -14,7 +14,12 @@ license.workspace = true name = "benches" harness = false +[features] +default = ["wtf8"] +wtf8 = ["rustpython-common"] + [dependencies] +rustpython-common = { workspace = true, optional = true } num_enum = { workspace = true } bitflags = { workspace = true } optional = "0.5" diff --git a/vm/sre_engine/src/string.rs b/vm/sre_engine/src/string.rs index 77e0f3e772..551b1ca5e5 100644 --- a/vm/sre_engine/src/string.rs +++ b/vm/sre_engine/src/string.rs @@ -1,3 +1,6 @@ +#[cfg(feature = "wtf8")] +use rustpython_common::wtf8::Wtf8; + #[derive(Debug, Clone, Copy)] pub struct StringCursor { pub(crate) ptr: *const u8, @@ -148,6 +151,73 @@ impl StrDrive for &str { } } +#[cfg(feature = "wtf8")] +impl StrDrive for &Wtf8 { + #[inline] + fn count(&self) -> usize { + self.code_points().count() + } + + #[inline] + fn create_cursor(&self, n: usize) -> StringCursor { + let mut cursor = StringCursor { + ptr: self.as_bytes().as_ptr(), + position: 0, + }; + Self::skip(&mut cursor, n); + cursor + } + + #[inline] + fn adjust_cursor(&self, cursor: &mut StringCursor, n: usize) { + if cursor.ptr.is_null() || cursor.position > n { + *cursor = Self::create_cursor(self, n); + } else if cursor.position < n { + Self::skip(cursor, n - cursor.position); + } + } + + #[inline] + fn advance(cursor: &mut StringCursor) -> u32 { + cursor.position += 1; + unsafe { next_code_point(&mut cursor.ptr) } + } + + #[inline] + fn peek(cursor: &StringCursor) -> u32 { + let mut ptr = cursor.ptr; + unsafe { next_code_point(&mut ptr) } + } + + #[inline] + fn skip(cursor: &mut StringCursor, n: usize) { + cursor.position += n; + for _ in 0..n { + unsafe { next_code_point(&mut cursor.ptr) }; + } + } + + #[inline] + fn back_advance(cursor: &mut StringCursor) -> u32 { + cursor.position -= 1; + unsafe { next_code_point_reverse(&mut cursor.ptr) } + } + + #[inline] + fn back_peek(cursor: &StringCursor) -> u32 { + let mut ptr = cursor.ptr; + unsafe { next_code_point_reverse(&mut ptr) } + } + + #[inline] + fn back_skip(cursor: &mut StringCursor, n: usize) { + cursor.position -= n; + for _ in 0..n { + unsafe { next_code_point_reverse(&mut cursor.ptr) }; + } + } +} + /// Reads the next code point out of a byte iterator (assuming a /// UTF-8-like encoding). /// From cc6f3d30516708e595cbe642bec47101c6817b3c Mon Sep 17 00:00:00 2001 From: Noa Date: Fri, 21 Mar 2025 20:48:38 -0500 Subject: [PATCH 110/295] Make TextIOWrapper wtf8-compatible --- common/src/wtf8/mod.rs | 11 +++++++ vm/src/builtins/str.rs | 8 ++--- vm/src/dictdatatype.rs | 2 +- vm/src/stdlib/io.rs | 70 ++++++++++++++++++++++-------------------- 4 files changed, 53 insertions(+), 38 deletions(-) diff --git a/common/src/wtf8/mod.rs b/common/src/wtf8/mod.rs index 5b31fdc558..1ded88fc1f 100644 --- a/common/src/wtf8/mod.rs +++ b/common/src/wtf8/mod.rs @@ -613,6 +613,12 @@ impl ToOwned for Wtf8 { } } +impl PartialEq for Wtf8 { + fn eq(&self, other: &str) -> bool { + self.as_bytes().eq(other.as_bytes()) + } +} + /// Formats the string in double quotes, with characters escaped according to /// [`char::escape_debug`] and unpaired surrogates represented as `\u{xxxx}`, /// where each `x` is a hexadecimal digit. @@ -1046,6 +1052,11 @@ impl Wtf8 { .strip_suffix(w.as_bytes()) .map(|w| unsafe { Wtf8::from_bytes_unchecked(w) }) } + + pub fn replace(&self, from: &Wtf8, to: &Wtf8) -> Wtf8Buf { + let w = self.bytes.replace(from, to); + unsafe { Wtf8Buf::from_bytes_unchecked(w) } + } } impl AsRef for str { diff --git a/vm/src/builtins/str.rs b/vm/src/builtins/str.rs index 4dc4e7a64e..2f1cef075a 100644 --- a/vm/src/builtins/str.rs +++ b/vm/src/builtins/str.rs @@ -1408,14 +1408,14 @@ impl PyStrRef { (**self).is_empty() } - pub fn concat_in_place(&mut self, other: &str, vm: &VirtualMachine) { + pub fn concat_in_place(&mut self, other: &Wtf8, vm: &VirtualMachine) { // TODO: call [A]Rc::get_mut on the str to try to mutate the data in place if other.is_empty() { return; } - let mut s = String::with_capacity(self.byte_len() + other.len()); - s.push_str(self.as_ref()); - s.push_str(other); + let mut s = Wtf8Buf::with_capacity(self.byte_len() + other.len()); + s.push_wtf8(self.as_ref()); + s.push_wtf8(other); *self = PyStr::from(s).into_ref(&vm.ctx); } } diff --git a/vm/src/dictdatatype.rs b/vm/src/dictdatatype.rs index faf5152145..ab37b7dc85 100644 --- a/vm/src/dictdatatype.rs +++ b/vm/src/dictdatatype.rs @@ -835,7 +835,7 @@ impl DictKey for str { fn key_eq(&self, vm: &VirtualMachine, other_key: &PyObject) -> PyResult { if let Some(pystr) = other_key.payload_if_exact::(vm) { - Ok(pystr.as_wtf8() == self.as_ref()) + Ok(pystr.as_wtf8() == self) } else { // Fall back to PyObjectRef implementation. let s = vm.ctx.new_str(self); diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index 84dddee872..30d09b78ae 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -129,6 +129,7 @@ mod _io { PyMappedThreadMutexGuard, PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard, PyThreadMutex, PyThreadMutexGuard, }, + common::wtf8::{Wtf8, Wtf8Buf}, convert::ToPyObject, function::{ ArgBytesLike, ArgIterable, ArgMemoryBuffer, ArgSize, Either, FuncArgs, IntoFuncArgs, @@ -147,7 +148,6 @@ mod _io { use crossbeam_utils::atomic::AtomicCell; use malachite_bigint::{BigInt, BigUint}; use num_traits::ToPrimitive; - use rustpython_common::wtf8::Wtf8Buf; use std::{ borrow::Cow, io::{self, Cursor, SeekFrom, prelude::*}, @@ -1910,10 +1910,12 @@ mod _io { impl Newlines { /// returns position where the new line starts if found, otherwise position at which to /// continue the search after more is read into the buffer - fn find_newline(&self, s: &str) -> Result { + fn find_newline(&self, s: &Wtf8) -> Result { let len = s.len(); match self { - Newlines::Universal | Newlines::Lf => s.find('\n').map(|p| p + 1).ok_or(len), + Newlines::Universal | Newlines::Lf => { + s.find("\n".as_ref()).map(|p| p + 1).ok_or(len) + } Newlines::Passthrough => { let bytes = s.as_bytes(); memchr::memchr2(b'\n', b'\r', bytes) @@ -1928,7 +1930,7 @@ mod _io { }) .ok_or(len) } - Newlines::Cr => s.find('\n').map(|p| p + 1).ok_or(len), + Newlines::Cr => s.find("\n".as_ref()).map(|p| p + 1).ok_or(len), Newlines::Crlf => { // s[searched..] == remaining let mut searched = 0; @@ -1993,10 +1995,10 @@ mod _io { } } - fn len_str(s: &str) -> Self { + fn len_str(s: &Wtf8) -> Self { Utf8size { bytes: s.len(), - chars: s.chars().count(), + chars: s.code_points().count(), } } } @@ -2224,7 +2226,7 @@ mod _io { let encoding = match args.encoding { None if vm.state.settings.utf8_mode > 0 => PyStr::from("utf-8").into_ref(&vm.ctx), - Some(enc) if enc.as_str() != "locale" => enc, + Some(enc) if enc.as_wtf8() != "locale" => enc, _ => { // None without utf8_mode or "locale" encoding vm.import("locale", 0)? @@ -2534,9 +2536,10 @@ mod _io { *snapshot = Some((cookie.dec_flags, input_chunk.clone())); let decoded = vm.call_method(decoder, "decode", (input_chunk, cookie.need_eof))?; let decoded = check_decoded(decoded, vm)?; - let pos_is_valid = decoded - .as_str() - .is_char_boundary(cookie.bytes_to_skip as usize); + let pos_is_valid = crate::common::wtf8::is_code_point_boundary( + decoded.as_wtf8(), + cookie.bytes_to_skip as usize, + ); textio.set_decoded_chars(Some(decoded)); if !pos_is_valid { return Err(vm.new_os_error("can't restore logical file position".to_owned())); @@ -2715,9 +2718,9 @@ mod _io { } else if chunks.len() == 1 { chunks.pop().unwrap() } else { - let mut ret = String::with_capacity(chunks_bytes); + let mut ret = Wtf8Buf::with_capacity(chunks_bytes); for chunk in chunks { - ret.push_str(chunk.as_str()) + ret.push_wtf8(chunk.as_wtf8()) } PyStr::from(ret).into_ref(&vm.ctx) } @@ -2744,7 +2747,7 @@ mod _io { let char_len = obj.char_len(); - let data = obj.as_str(); + let data = obj.as_wtf8(); let replace_nl = match textio.newline { Newlines::Lf => Some("\n"), @@ -2753,11 +2756,12 @@ mod _io { Newlines::Universal if cfg!(windows) => Some("\r\n"), _ => None, }; - let has_lf = (replace_nl.is_some() || textio.line_buffering) && data.contains('\n'); - let flush = textio.line_buffering && (has_lf || data.contains('\r')); + let has_lf = (replace_nl.is_some() || textio.line_buffering) + && data.contains_code_point('\n'.into()); + let flush = textio.line_buffering && (has_lf || data.contains_code_point('\r'.into())); let chunk = if let Some(replace_nl) = replace_nl { if has_lf { - PyStr::from(data.replace('\n', replace_nl)).into_ref(&vm.ctx) + PyStr::from(data.replace("\n".as_ref(), replace_nl.as_ref())).into_ref(&vm.ctx) } else { obj } @@ -2834,7 +2838,7 @@ mod _io { if self.is_full_slice() { self.0.char_len() } else { - self.slice().chars().count() + self.slice().code_points().count() } } #[inline] @@ -2842,8 +2846,8 @@ mod _io { self.1.len() >= self.0.byte_len() } #[inline] - fn slice(&self) -> &str { - &self.0.as_str()[self.1.clone()] + fn slice(&self) -> &Wtf8 { + &self.0.as_wtf8()[self.1.clone()] } #[inline] fn slice_pystr(self, vm: &VirtualMachine) -> PyStrRef { @@ -2894,7 +2898,7 @@ mod _io { Some(remaining) => { assert_eq!(textio.decoded_chars_used.bytes, 0); offset_to_buffer = remaining.utf8_len(); - let decoded_chars = decoded_chars.as_str(); + let decoded_chars = decoded_chars.as_wtf8(); let line = if remaining.is_full_slice() { let mut line = remaining.0; line.concat_in_place(decoded_chars, vm); @@ -2902,16 +2906,16 @@ mod _io { } else { let remaining = remaining.slice(); let mut s = - String::with_capacity(remaining.len() + decoded_chars.len()); - s.push_str(remaining); - s.push_str(decoded_chars); + Wtf8Buf::with_capacity(remaining.len() + decoded_chars.len()); + s.push_wtf8(remaining); + s.push_wtf8(decoded_chars); PyStr::from(s).into_ref(&vm.ctx) }; start = Utf8size::default(); line } }; - let line_from_start = &line.as_str()[start.bytes..]; + let line_from_start = &line.as_wtf8()[start.bytes..]; let nl_res = textio.newline.find_newline(line_from_start); match nl_res { Ok(p) | Err(p) => { @@ -2922,7 +2926,7 @@ mod _io { endpos = start + Utf8size { chars: limit - chunked.chars, - bytes: crate::common::str::char_range_end( + bytes: crate::common::str::codepoint_range_end( line_from_start, limit - chunked.chars, ) @@ -2963,9 +2967,9 @@ mod _io { chunked += cur_line.byte_len(); chunks.push(cur_line); } - let mut s = String::with_capacity(chunked); + let mut s = Wtf8Buf::with_capacity(chunked); for chunk in chunks { - s.push_str(chunk.slice()) + s.push_wtf8(chunk.slice()) } PyStr::from(s).into_ref(&vm.ctx) } else if let Some(cur_line) = cur_line { @@ -3100,7 +3104,7 @@ mod _io { return None; } let decoded_chars = self.decoded_chars.as_ref()?; - let avail = &decoded_chars.as_str()[self.decoded_chars_used.bytes..]; + let avail = &decoded_chars.as_wtf8()[self.decoded_chars_used.bytes..]; if avail.is_empty() { return None; } @@ -3112,7 +3116,7 @@ mod _io { (PyStr::from(avail).into_ref(&vm.ctx), avail_chars) } } else { - let s = crate::common::str::get_chars(avail, 0..n); + let s = crate::common::str::get_codepoints(avail, 0..n); (PyStr::from(s).into_ref(&vm.ctx), n) }; self.decoded_chars_used += Utf8size { @@ -3142,11 +3146,11 @@ mod _io { return decoded_chars; } // TODO: in-place editing of `str` when refcount == 1 - let decoded_chars_unused = &decoded_chars.as_str()[chars_pos..]; - let mut s = String::with_capacity(decoded_chars_unused.len() + append_len); - s.push_str(decoded_chars_unused); + let decoded_chars_unused = &decoded_chars.as_wtf8()[chars_pos..]; + let mut s = Wtf8Buf::with_capacity(decoded_chars_unused.len() + append_len); + s.push_wtf8(decoded_chars_unused); if let Some(append) = append { - s.push_str(append.as_str()) + s.push_wtf8(append.as_wtf8()) } PyStr::from(s).into_ref(&vm.ctx) } From 5c2269734402cef5bbd89ebd6358a08faa64697c Mon Sep 17 00:00:00 2001 From: Noa Date: Fri, 21 Mar 2025 22:10:59 -0500 Subject: [PATCH 111/295] Implement fsencode/fsdecode for FsPath --- Lib/test/test_socket.py | 3 --- stdlib/src/socket.rs | 13 ++++++++---- vm/src/function/fspath.rs | 12 ++++------- vm/src/ospath.rs | 34 +++++++++---------------------- vm/src/stdlib/codecs.rs | 7 ++++++- vm/src/stdlib/io.rs | 4 ++-- vm/src/stdlib/nt.rs | 6 +++--- vm/src/stdlib/os.rs | 38 ++++++++++++++++------------------ vm/src/stdlib/sys.rs | 16 ++++----------- vm/src/vm/context.rs | 17 +++++++++++++--- vm/src/vm/mod.rs | 43 ++++++++++++++++++++++++++++++++++++++- 11 files changed, 111 insertions(+), 82 deletions(-) diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 5b82853102..1b8d968e4f 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -1578,7 +1578,6 @@ def test_getnameinfo(self): # only IP addresses are allowed self.assertRaises(OSError, socket.getnameinfo, ('mail.python.org',0), 0) - @unittest.expectedFailureIf(sys.platform != "darwin", "TODO: RUSTPYTHON; socket.gethostbyname_ex") @unittest.skipUnless(support.is_resource_enabled('network'), 'network is not enabled') def test_idna(self): @@ -5519,8 +5518,6 @@ def testBytesAddr(self): self.addCleanup(os_helper.unlink, path) self.assertEqual(self.sock.getsockname(), path) - # TODO: RUSTPYTHON, surrogateescape - @unittest.expectedFailure def testSurrogateescapeBind(self): # Test binding to a valid non-ASCII pathname, with the # non-ASCII bytes supplied using surrogateescape encoding. diff --git a/stdlib/src/socket.rs b/stdlib/src/socket.rs index 39bfde4bee..17daec7751 100644 --- a/stdlib/src/socket.rs +++ b/stdlib/src/socket.rs @@ -930,10 +930,15 @@ mod _socket { match family { #[cfg(unix)] c::AF_UNIX => { + use crate::vm::function::ArgStrOrBytesLike; use std::os::unix::ffi::OsStrExt; - let buf = crate::vm::function::ArgStrOrBytesLike::try_from_object(vm, addr)?; - let path = &*buf.borrow_bytes(); - socket2::SockAddr::unix(ffi::OsStr::from_bytes(path)) + let buf = ArgStrOrBytesLike::try_from_object(vm, addr)?; + let bytes = &*buf.borrow_bytes(); + let path = match &buf { + ArgStrOrBytesLike::Buf(_) => ffi::OsStr::from_bytes(bytes).into(), + ArgStrOrBytesLike::Str(s) => vm.fsencode(s)?, + }; + socket2::SockAddr::unix(path) .map_err(|_| vm.new_os_error("AF_UNIX path too long".to_owned()).into()) } c::AF_INET => { @@ -1704,7 +1709,7 @@ mod _socket { let path = ffi::OsStr::as_bytes(addr.as_pathname().unwrap_or("".as_ref()).as_ref()); let nul_pos = memchr::memchr(b'\0', path).unwrap_or(path.len()); let path = ffi::OsStr::from_bytes(&path[..nul_pos]); - return vm.ctx.new_str(path.to_string_lossy()).into(); + return vm.fsdecode(path).into(); } // TODO: support more address families (String::new(), 0).to_pyobject(vm) diff --git a/vm/src/function/fspath.rs b/vm/src/function/fspath.rs index ab5cd093b9..e034487e19 100644 --- a/vm/src/function/fspath.rs +++ b/vm/src/function/fspath.rs @@ -5,7 +5,7 @@ use crate::{ function::PyStr, protocol::PyBuffer, }; -use std::{ffi::OsStr, path::PathBuf}; +use std::{borrow::Cow, ffi::OsStr, path::PathBuf}; #[derive(Clone)] pub enum FsPath { @@ -58,15 +58,11 @@ impl FsPath { }) } - pub fn as_os_str(&self, vm: &VirtualMachine) -> PyResult<&OsStr> { + pub fn as_os_str(&self, vm: &VirtualMachine) -> PyResult> { // TODO: FS encodings match self { - FsPath::Str(s) => { - // XXX RUSTPYTHON: this is sketchy on windows; it's not guaranteed that its - // OsStr encoding will always be compatible with WTF-8. - Ok(unsafe { OsStr::from_encoded_bytes_unchecked(s.as_wtf8().as_bytes()) }) - } - FsPath::Bytes(b) => Self::bytes_as_osstr(b.as_bytes(), vm), + FsPath::Str(s) => vm.fsencode(s), + FsPath::Bytes(b) => Self::bytes_as_osstr(b.as_bytes(), vm).map(Cow::Borrowed), } } diff --git a/vm/src/ospath.rs b/vm/src/ospath.rs index 9dda60d621..c1b1859164 100644 --- a/vm/src/ospath.rs +++ b/vm/src/ospath.rs @@ -21,28 +21,14 @@ pub(super) enum OutputMode { } impl OutputMode { - pub(super) fn process_path(self, path: impl Into, vm: &VirtualMachine) -> PyResult { - fn inner(mode: OutputMode, path: PathBuf, vm: &VirtualMachine) -> PyResult { - let path_as_string = |p: PathBuf| { - p.into_os_string().into_string().map_err(|_| { - vm.new_unicode_decode_error( - "Can't convert OS path to valid UTF-8 string".into(), - ) - }) - }; + pub(super) fn process_path(self, path: impl Into, vm: &VirtualMachine) -> PyObjectRef { + fn inner(mode: OutputMode, path: PathBuf, vm: &VirtualMachine) -> PyObjectRef { match mode { - OutputMode::String => path_as_string(path).map(|s| vm.ctx.new_str(s).into()), - OutputMode::Bytes => { - #[cfg(any(unix, target_os = "wasi"))] - { - use rustpython_common::os::ffi::OsStringExt; - Ok(vm.ctx.new_bytes(path.into_os_string().into_vec()).into()) - } - #[cfg(windows)] - { - path_as_string(path).map(|s| vm.ctx.new_bytes(s.into_bytes()).into()) - } - } + OutputMode::String => vm.fsdecode(path).into(), + OutputMode::Bytes => vm + .ctx + .new_bytes(path.into_os_string().into_encoded_bytes()) + .into(), } } inner(self, path.into(), vm) @@ -59,7 +45,7 @@ impl OsPath { } pub(crate) fn from_fspath(fspath: FsPath, vm: &VirtualMachine) -> PyResult { - let path = fspath.as_os_str(vm)?.to_owned(); + let path = fspath.as_os_str(vm)?.into_owned(); let mode = match fspath { FsPath::Str(_) => OutputMode::String, FsPath::Bytes(_) => OutputMode::Bytes, @@ -88,7 +74,7 @@ impl OsPath { widestring::WideCString::from_os_str(&self.path).map_err(|err| err.to_pyexception(vm)) } - pub fn filename(&self, vm: &VirtualMachine) -> PyResult { + pub fn filename(&self, vm: &VirtualMachine) -> PyObjectRef { self.mode.process_path(self.path.clone(), vm) } } @@ -133,7 +119,7 @@ impl From for OsPathOrFd { impl OsPathOrFd { pub fn filename(&self, vm: &VirtualMachine) -> PyObjectRef { match self { - OsPathOrFd::Path(path) => path.filename(vm).unwrap_or_else(|_| vm.ctx.none()), + OsPathOrFd::Path(path) => path.filename(vm), OsPathOrFd::Fd(fd) => vm.ctx.new_int(*fd).into(), } } diff --git a/vm/src/stdlib/codecs.rs b/vm/src/stdlib/codecs.rs index 9018ad0e3f..664fe00616 100644 --- a/vm/src/stdlib/codecs.rs +++ b/vm/src/stdlib/codecs.rs @@ -312,7 +312,12 @@ mod _codecs { #[pyfunction] fn utf_8_encode(args: EncodeArgs, vm: &VirtualMachine) -> EncodeResult { - if args.s.is_utf8() { + if args.s.is_utf8() + || args + .errors + .as_ref() + .is_some_and(|s| s.is(identifier!(vm, surrogatepass))) + { return Ok((args.s.as_bytes().to_vec(), args.s.byte_len())); } do_codec!(utf8::encode, args, vm) diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index 30d09b78ae..0b680251fa 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -2225,7 +2225,7 @@ mod _io { *data = None; let encoding = match args.encoding { - None if vm.state.settings.utf8_mode > 0 => PyStr::from("utf-8").into_ref(&vm.ctx), + None if vm.state.settings.utf8_mode > 0 => identifier!(vm, utf_8).to_owned(), Some(enc) if enc.as_wtf8() != "locale" => enc, _ => { // None without utf8_mode or "locale" encoding @@ -2238,7 +2238,7 @@ mod _io { let errors = args .errors - .unwrap_or_else(|| PyStr::from("strict").into_ref(&vm.ctx)); + .unwrap_or_else(|| identifier!(vm, strict).to_owned()); let has_read1 = vm.get_attribute_opt(buffer.clone(), "read1")?.is_some(); let seekable = vm.call_method(&buffer, "seekable", ())?.try_to_bool(vm)?; diff --git a/vm/src/stdlib/nt.rs b/vm/src/stdlib/nt.rs index 48f1ab668b..624577b5ce 100644 --- a/vm/src/stdlib/nt.rs +++ b/vm/src/stdlib/nt.rs @@ -249,7 +249,7 @@ pub(crate) mod module { .as_ref() .canonicalize() .map_err(|e| e.to_pyexception(vm))?; - path.mode.process_path(real, vm) + Ok(path.mode.process_path(real, vm)) } #[pyfunction] @@ -282,7 +282,7 @@ pub(crate) mod module { } } let buffer = widestring::WideCString::from_vec_truncate(buffer); - path.mode.process_path(buffer.to_os_string(), vm) + Ok(path.mode.process_path(buffer.to_os_string(), vm)) } #[pyfunction] @@ -297,7 +297,7 @@ pub(crate) mod module { return Err(errno_err(vm)); } let buffer = widestring::WideCString::from_vec_truncate(buffer); - path.mode.process_path(buffer.to_os_string(), vm) + Ok(path.mode.process_path(buffer.to_os_string(), vm)) } #[pyfunction] diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 9b196b6d03..e1a5825b82 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -332,7 +332,7 @@ pub(super) mod _os { }; dir_iter .map(|entry| match entry { - Ok(entry_path) => path.mode.process_path(entry_path.file_name(), vm), + Ok(entry_path) => Ok(path.mode.process_path(entry_path.file_name(), vm)), Err(err) => Err(IOErrorBuilder::with_filename(&err, path.clone(), vm)), }) .collect::>()? @@ -352,22 +352,18 @@ pub(super) mod _os { let mut dir = nix::dir::Dir::from_fd(new_fd).map_err(|e| e.into_pyexception(vm))?; dir.iter() - .filter_map(|entry| { - entry - .map_err(|e| e.into_pyexception(vm)) - .and_then(|entry| { - let fname = entry.file_name().to_bytes(); - Ok(match fname { - b"." | b".." => None, - _ => Some( - OutputMode::String - .process_path(ffi::OsStr::from_bytes(fname), vm)?, - ), - }) - }) - .transpose() + .filter_map_ok(|entry| { + let fname = entry.file_name().to_bytes(); + match fname { + b"." | b".." => None, + _ => Some( + OutputMode::String + .process_path(ffi::OsStr::from_bytes(fname), vm), + ), + } }) - .collect::>()? + .collect::>() + .map_err(|e| e.into_pyexception(vm))? } } }; @@ -429,7 +425,7 @@ pub(super) mod _os { let [] = dir_fd.0; let path = fs::read_link(&path).map_err(|err| IOErrorBuilder::with_filename(&err, path, vm))?; - mode.process_path(path, vm) + Ok(mode.process_path(path, vm)) } #[pyattr] @@ -452,12 +448,12 @@ pub(super) mod _os { impl DirEntry { #[pygetset] fn name(&self, vm: &VirtualMachine) -> PyResult { - self.mode.process_path(&self.file_name, vm) + Ok(self.mode.process_path(&self.file_name, vm)) } #[pygetset] fn path(&self, vm: &VirtualMachine) -> PyResult { - self.mode.process_path(&self.pathval, vm) + Ok(self.mode.process_path(&self.pathval, vm)) } fn perform_on_metadata( @@ -908,12 +904,12 @@ pub(super) mod _os { #[pyfunction] fn getcwd(vm: &VirtualMachine) -> PyResult { - OutputMode::String.process_path(curdir_inner(vm)?, vm) + Ok(OutputMode::String.process_path(curdir_inner(vm)?, vm)) } #[pyfunction] fn getcwdb(vm: &VirtualMachine) -> PyResult { - OutputMode::Bytes.process_path(curdir_inner(vm)?, vm) + Ok(OutputMode::Bytes.process_path(curdir_inner(vm)?, vm)) } #[pyfunction] diff --git a/vm/src/stdlib/sys.rs b/vm/src/stdlib/sys.rs index 39c803a01b..fdfe2faf69 100644 --- a/vm/src/stdlib/sys.rs +++ b/vm/src/stdlib/sys.rs @@ -458,21 +458,13 @@ mod sys { } #[pyfunction] - fn getfilesystemencoding(_vm: &VirtualMachine) -> String { - // TODO: implement non-utf-8 mode. - "utf-8".to_owned() + fn getfilesystemencoding(vm: &VirtualMachine) -> PyStrRef { + vm.fs_encoding().to_owned() } - #[cfg(not(windows))] #[pyfunction] - fn getfilesystemencodeerrors(_vm: &VirtualMachine) -> String { - "surrogateescape".to_owned() - } - - #[cfg(windows)] - #[pyfunction] - fn getfilesystemencodeerrors(_vm: &VirtualMachine) -> String { - "surrogatepass".to_owned() + fn getfilesystemencodeerrors(vm: &VirtualMachine) -> PyStrRef { + vm.fs_encode_errors().to_owned() } #[pyfunction] diff --git a/vm/src/vm/context.rs b/vm/src/vm/context.rs index 54605704a5..a61484e6bc 100644 --- a/vm/src/vm/context.rs +++ b/vm/src/vm/context.rs @@ -51,7 +51,7 @@ pub struct Context { } macro_rules! declare_const_name { - ($($name:ident,)*) => { + ($($name:ident$(: $s:literal)?,)*) => { #[derive(Debug, Clone, Copy)] #[allow(non_snake_case)] pub struct ConstName { @@ -61,11 +61,13 @@ macro_rules! declare_const_name { impl ConstName { unsafe fn new(pool: &StringPool, typ: &PyTypeRef) -> Self { Self { - $($name: unsafe { pool.intern(stringify!($name), typ.clone()) },)* + $($name: unsafe { pool.intern(declare_const_name!(@string $name $($s)?), typ.clone()) },)* } } } - } + }; + (@string $name:ident) => { stringify!($name) }; + (@string $name:ident $string:literal) => { $string }; } declare_const_name! { @@ -236,6 +238,15 @@ declare_const_name! { flush, close, WarningMessage, + strict, + ignore, + replace, + xmlcharrefreplace, + backslashreplace, + namereplace, + surrogatepass, + surrogateescape, + utf_8: "utf-8", } // Basic objects: diff --git a/vm/src/vm/mod.rs b/vm/src/vm/mod.rs index eb4e846dea..9d7ecc2d54 100644 --- a/vm/src/vm/mod.rs +++ b/vm/src/vm/mod.rs @@ -41,11 +41,12 @@ use nix::{ sys::signal::{SaFlags, SigAction, SigSet, Signal::SIGINT, kill, sigaction}, unistd::getpid, }; -use std::sync::atomic::AtomicBool; use std::{ borrow::Cow, cell::{Cell, Ref, RefCell}, collections::{HashMap, HashSet}, + ffi::{OsStr, OsString}, + sync::atomic::AtomicBool, }; pub use context::Context; @@ -901,6 +902,46 @@ impl VirtualMachine { run_module_as_main.call((module,), self)?; Ok(()) } + + pub fn fs_encoding(&self) -> &'static PyStrInterned { + identifier!(self, utf_8) + } + + pub fn fs_encode_errors(&self) -> &'static PyStrInterned { + if cfg!(windows) { + identifier!(self, surrogatepass) + } else { + identifier!(self, surrogateescape) + } + } + + pub fn fsdecode(&self, s: impl Into) -> PyStrRef { + let bytes = self.ctx.new_bytes(s.into().into_encoded_bytes()); + let errors = self.fs_encode_errors().to_owned(); + self.state + .codec_registry + .decode_text(bytes.into(), "utf-8", Some(errors), self) + .unwrap() // this should never fail, since fsdecode should be lossless from the fs encoding + } + + pub fn fsencode<'a>(&self, s: &'a Py) -> PyResult> { + if cfg!(windows) || s.is_utf8() { + // XXX: this is sketchy on windows; it's not guaranteed that the + // OsStr encoding will always be compatible with WTF-8. + let s = unsafe { OsStr::from_encoded_bytes_unchecked(s.as_bytes()) }; + return Ok(Cow::Borrowed(s)); + } + let errors = self.fs_encode_errors().to_owned(); + let bytes = self + .state + .codec_registry + .encode_text(s.to_owned(), "utf-8", Some(errors), self)? + .to_vec(); + // XXX: this is sketchy on windows; it's not guaranteed that the + // OsStr encoding will always be compatible with WTF-8. + let s = unsafe { OsString::from_encoded_bytes_unchecked(bytes) }; + Ok(Cow::Owned(s)) + } } impl AsRef for VirtualMachine { From a86126419cd3eca03b535207cdbd5019bba85f5a Mon Sep 17 00:00:00 2001 From: Noa Date: Fri, 21 Mar 2025 23:10:51 -0500 Subject: [PATCH 112/295] Fix remaining tests --- Lib/test/string_tests.py | 2 - Lib/test/test_cmd_line_script.py | 3 +- Lib/test/test_import/__init__.py | 2 + Lib/test/test_json/test_scanstring.py | 6 -- Lib/test/test_ntpath.py | 6 -- Lib/test/test_re.py | 1 + Lib/test/test_socket.py | 1 + Lib/test/test_sqlite3/test_types.py | 2 - Lib/test/test_ucn.py | 2 - Lib/test/test_unicode.py | 2 - Lib/test/test_unicodedata.py | 6 -- common/src/wtf8/mod.rs | 19 ++++ stdlib/src/sqlite.rs | 7 +- stdlib/src/ssl.rs | 4 +- vm/src/builtins/str.rs | 141 ++++++++++++++++---------- vm/src/codecs.rs | 22 ++-- vm/src/function/fspath.rs | 6 +- vm/src/stdlib/builtins.rs | 4 +- vm/src/vm/mod.rs | 20 ++-- 19 files changed, 150 insertions(+), 106 deletions(-) diff --git a/Lib/test/string_tests.py b/Lib/test/string_tests.py index ea82b0166c..6f402513fd 100644 --- a/Lib/test/string_tests.py +++ b/Lib/test/string_tests.py @@ -1066,8 +1066,6 @@ def test_hash(self): hash(b) self.assertEqual(hash(a), hash(b)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_capitalize_nonascii(self): # check that titlecased chars are lowered correctly # \u1ffc is the titlecased char diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index d5fa1563ec..833dc6b15d 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -574,8 +574,7 @@ def test_pep_409_verbiage(self): self.assertTrue(text[1].startswith(' File ')) self.assertTrue(text[3].startswith('NameError')) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailureIf(sys.platform == "linux", "TODO: RUSTPYTHON") def test_non_ascii(self): # Mac OS X denies the creation of a file with an invalid UTF-8 name. # Windows allows creating a name with an arbitrary bytes name, but diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index c2f181cc86..89e5ec1534 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -1305,6 +1305,8 @@ def exec_module(*args): else: importlib.SourceLoader.exec_module = old_exec_module + # TODO: RUSTPYTHON + @unittest.expectedFailure @unittest.skipUnless(TESTFN_UNENCODABLE, 'need TESTFN_UNENCODABLE') def test_unencodable_filename(self): # Issue #11619: The Python parser and the import machinery must not diff --git a/Lib/test/test_json/test_scanstring.py b/Lib/test/test_json/test_scanstring.py index 140a7c12a2..682dc74999 100644 --- a/Lib/test/test_json/test_scanstring.py +++ b/Lib/test/test_json/test_scanstring.py @@ -143,10 +143,4 @@ def test_overflow(self): class TestPyScanstring(TestScanstring, PyTest): pass -# TODO: RUSTPYTHON -class TestPyScanstring(TestScanstring, PyTest): - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_bad_escapes(self): - super().test_bad_escapes() class TestCScanstring(TestScanstring, CTest): pass diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index 8e23b88676..7609ecea79 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -1032,12 +1032,6 @@ class NtCommonTest(test_genericpath.CommonTest, unittest.TestCase): pathmodule = ntpath attributes = ['relpath'] - # TODO: RUSTPYTHON - if sys.platform == "linux": - @unittest.expectedFailure - def test_nonascii_abspath(self): - super().test_nonascii_abspath() - # TODO: RUSTPYTHON if sys.platform == "win32": # TODO: RUSTPYTHON, ValueError: illegal environment variable name diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index f6af44df99..3c0b6af3c6 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -2231,6 +2231,7 @@ def test_bug_40736(self): with self.assertRaisesRegex(TypeError, "got 'type'"): re.search("x*", type) + @unittest.skip("TODO: RUSTPYTHON: flaky, improve perf") @requires_resource('cpu') def test_search_anchor_at_beginning(self): s = 'x'*10**7 diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 1b8d968e4f..0e3eb08b82 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -1578,6 +1578,7 @@ def test_getnameinfo(self): # only IP addresses are allowed self.assertRaises(OSError, socket.getnameinfo, ('mail.python.org',0), 0) + @unittest.skip("TODO: RUSTPYTHON: flaky on CI?") @unittest.skipUnless(support.is_resource_enabled('network'), 'network is not enabled') def test_idna(self): diff --git a/Lib/test/test_sqlite3/test_types.py b/Lib/test/test_sqlite3/test_types.py index d7631ec938..45f30824d0 100644 --- a/Lib/test/test_sqlite3/test_types.py +++ b/Lib/test/test_sqlite3/test_types.py @@ -95,8 +95,6 @@ def test_too_large_int(self): row = self.cur.fetchone() self.assertIsNone(row) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_string_with_surrogates(self): for value in 0xd8ff, 0xdcff: with self.assertRaises(UnicodeEncodeError): diff --git a/Lib/test/test_ucn.py b/Lib/test/test_ucn.py index 6d082a0942..f6d69540b9 100644 --- a/Lib/test/test_ucn.py +++ b/Lib/test/test_ucn.py @@ -102,8 +102,6 @@ def test_cjk_unified_ideographs(self): self.checkletter("CJK UNIFIED IDEOGRAPH-2B81D", "\U0002B81D") self.checkletter("CJK UNIFIED IDEOGRAPH-3134A", "\U0003134A") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_bmp_characters(self): for code in range(0x10000): char = chr(code) diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py index 17c9f01cd8..5c2c6c29b1 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -721,8 +721,6 @@ def test_isspace(self): '\U0001F40D', '\U0001F46F']: self.assertFalse(ch.isspace(), '{!a} is not space.'.format(ch)) - # TODO: RUSTPYTHON - @unittest.expectedFailure @support.requires_resource('cpu') def test_isspace_invariant(self): for codepoint in range(sys.maxunicode + 1): diff --git a/Lib/test/test_unicodedata.py b/Lib/test/test_unicodedata.py index f5b4b6218e..c9e0b234ef 100644 --- a/Lib/test/test_unicodedata.py +++ b/Lib/test/test_unicodedata.py @@ -99,8 +99,6 @@ def test_function_checksum(self): result = h.hexdigest() self.assertEqual(result, self.expectedchecksum) - # TODO: RUSTPYTHON - @unittest.expectedFailure @requires_resource('cpu') def test_name_inverse_lookup(self): for i in range(sys.maxunicode + 1): @@ -326,8 +324,6 @@ def test_ucd_510(self): self.assertTrue("\u1d79".upper()=='\ua77d') self.assertTrue(".".upper()=='.') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_bug_5828(self): self.assertEqual("\u1d79".lower(), "\u1d79") # Only U+0000 should have U+0000 as its upper/lower/titlecase variant @@ -347,8 +343,6 @@ def test_bug_4971(self): self.assertEqual("\u01c5".title(), "\u01c5") self.assertEqual("\u01c6".title(), "\u01c5") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_linebreak_7643(self): for i in range(0x10000): lines = (chr(i) + 'A').splitlines() diff --git a/common/src/wtf8/mod.rs b/common/src/wtf8/mod.rs index 1ded88fc1f..62fbc2ea39 100644 --- a/common/src/wtf8/mod.rs +++ b/common/src/wtf8/mod.rs @@ -552,6 +552,12 @@ impl Extend for Wtf8Buf { } } +impl Extend for Wtf8Buf { + fn extend>(&mut self, iter: T) { + self.extend(iter.into_iter().map(CodePoint::from)) + } +} + impl> Extend for Wtf8Buf { fn extend>(&mut self, iter: T) { iter.into_iter() @@ -1004,6 +1010,14 @@ impl Wtf8 { memchr::memmem::rfind(self.as_bytes(), pat.as_bytes()) } + pub fn find_iter(&self, pat: &Wtf8) -> impl Iterator { + memchr::memmem::find_iter(self.as_bytes(), pat.as_bytes()) + } + + pub fn rfind_iter(&self, pat: &Wtf8) -> impl Iterator { + memchr::memmem::rfind_iter(self.as_bytes(), pat.as_bytes()) + } + pub fn contains(&self, pat: &Wtf8) -> bool { self.bytes.contains_str(pat) } @@ -1057,6 +1071,11 @@ impl Wtf8 { let w = self.bytes.replace(from, to); unsafe { Wtf8Buf::from_bytes_unchecked(w) } } + + pub fn replacen(&self, from: &Wtf8, to: &Wtf8, n: usize) -> Wtf8Buf { + let w = self.bytes.replacen(from, to, n); + unsafe { Wtf8Buf::from_bytes_unchecked(w) } + } } impl AsRef for str { diff --git a/stdlib/src/sqlite.rs b/stdlib/src/sqlite.rs index d20324b5ef..67e94bd81b 100644 --- a/stdlib/src/sqlite.rs +++ b/stdlib/src/sqlite.rs @@ -2929,9 +2929,12 @@ mod _sqlite { } fn str_to_ptr_len(s: &PyStr, vm: &VirtualMachine) -> PyResult<(*const libc::c_char, i32)> { - let len = c_int::try_from(s.byte_len()) + let s = s + .to_str() + .ok_or_else(|| vm.new_unicode_encode_error("surrogates not allowed".to_owned()))?; + let len = c_int::try_from(s.len()) .map_err(|_| vm.new_overflow_error("TEXT longer than INT_MAX bytes".to_owned()))?; - let ptr = s.as_str().as_ptr().cast(); + let ptr = s.as_ptr().cast(); Ok((ptr, len)) } diff --git a/stdlib/src/ssl.rs b/stdlib/src/ssl.rs index 9bfdf883ca..10d1906448 100644 --- a/stdlib/src/ssl.rs +++ b/stdlib/src/ssl.rs @@ -403,8 +403,8 @@ mod _ssl { .to_str() .unwrap(); let (cert_file, cert_dir) = get_cert_file_dir(); - let cert_file = OsPath::new_str(cert_file).filename(vm)?; - let cert_dir = OsPath::new_str(cert_dir).filename(vm)?; + let cert_file = OsPath::new_str(cert_file).filename(vm); + let cert_dir = OsPath::new_str(cert_dir).filename(vm); Ok((cert_file_env, cert_file, cert_dir_env, cert_dir)) } diff --git a/vm/src/builtins/str.rs b/vm/src/builtins/str.rs index 2f1cef075a..55cefae4f7 100644 --- a/vm/src/builtins/str.rs +++ b/vm/src/builtins/str.rs @@ -38,7 +38,7 @@ use rustpython_common::{ str::DeduceStrKind, wtf8::{CodePoint, Wtf8, Wtf8Buf, Wtf8Chunk}, }; -use std::{char, fmt, ops::Range}; +use std::{borrow::Cow, char, fmt, ops::Range}; use unic_ucd_bidi::BidiClass; use unic_ucd_category::GeneralCategory; use unic_ucd_ident::{is_xid_continue, is_xid_start}; @@ -420,6 +420,16 @@ impl PyStr { self.data.as_str().expect("str has surrogates") } + pub fn to_str(&self) -> Option<&str> { + self.data.as_str() + } + + pub fn to_string_lossy(&self) -> Cow<'_, str> { + self.to_str() + .map(Cow::Borrowed) + .unwrap_or_else(|| self.as_wtf8().to_string_lossy()) + } + pub fn kind(&self) -> StrKind { self.data.kind() } @@ -629,16 +639,42 @@ impl PyStr { } #[pymethod] - fn capitalize(&self) -> String { - let mut chars = self.as_str().chars(); - if let Some(first_char) = chars.next() { - format!( - "{}{}", - first_char.to_uppercase(), - &chars.as_str().to_lowercase(), - ) - } else { - "".to_owned() + fn capitalize(&self) -> Wtf8Buf { + match self.as_str_kind() { + PyKindStr::Ascii(s) => { + let mut s = s.to_owned(); + if let [first, rest @ ..] = s.as_mut_slice() { + first.make_ascii_uppercase(); + ascii::AsciiStr::make_ascii_lowercase(rest.into()); + } + s.into() + } + PyKindStr::Utf8(s) => { + let mut chars = s.chars(); + let mut out = String::with_capacity(s.len()); + if let Some(c) = chars.next() { + out.extend(c.to_titlecase()); + out.push_str(&chars.as_str().to_lowercase()); + } + out.into() + } + PyKindStr::Wtf8(s) => { + let mut out = Wtf8Buf::with_capacity(s.len()); + let mut chars = s.code_points(); + if let Some(ch) = chars.next() { + match ch.to_char() { + Some(ch) => out.extend(ch.to_titlecase()), + None => out.push(ch), + } + for chunk in chars.as_wtf8().chunks() { + match chunk { + Wtf8Chunk::Utf8(s) => out.push_str(&s.to_lowercase()), + Wtf8Chunk::Surrogate(ch) => out.push(ch), + } + } + } + out + } } } @@ -844,14 +880,11 @@ impl PyStr { #[pymethod] fn isdigit(&self) -> bool { // python's isdigit also checks if exponents are digits, these are the unicode codepoints for exponents - let valid_codepoints: [u16; 10] = [ - 0x2070, 0x00B9, 0x00B2, 0x00B3, 0x2074, 0x2075, 0x2076, 0x2077, 0x2078, 0x2079, - ]; - let s = self.as_str(); - !s.is_empty() - && s.chars() - .filter(|c| !c.is_ascii_digit()) - .all(|c| valid_codepoints.contains(&(c as u16))) + !self.data.is_empty() + && self.char_all(|c| { + c.is_ascii_digit() + || matches!(c, '⁰' | '¹' | '²' | '³' | '⁴' | '⁵' | '⁶' | '⁷' | '⁸' | '⁹') + }) } #[pymethod] @@ -910,43 +943,45 @@ impl PyStr { /// Return a titlecased version of the string where words start with an /// uppercase character and the remaining characters are lowercase. #[pymethod] - fn title(&self) -> String { - let mut title = String::with_capacity(self.data.len()); + fn title(&self) -> Wtf8Buf { + let mut title = Wtf8Buf::with_capacity(self.data.len()); let mut previous_is_cased = false; - for c in self.as_str().chars() { + for c_orig in self.as_wtf8().code_points() { + let c = c_orig.to_char_lossy(); if c.is_lowercase() { if !previous_is_cased { title.extend(c.to_titlecase()); } else { - title.push(c); + title.push_char(c); } previous_is_cased = true; } else if c.is_uppercase() || c.is_titlecase() { if previous_is_cased { title.extend(c.to_lowercase()); } else { - title.push(c); + title.push_char(c); } previous_is_cased = true; } else { previous_is_cased = false; - title.push(c); + title.push(c_orig); } } title } #[pymethod] - fn swapcase(&self) -> String { - let mut swapped_str = String::with_capacity(self.data.len()); - for c in self.as_str().chars() { + fn swapcase(&self) -> Wtf8Buf { + let mut swapped_str = Wtf8Buf::with_capacity(self.data.len()); + for c_orig in self.as_wtf8().code_points() { + let c = c_orig.to_char_lossy(); // to_uppercase returns an iterator, to_ascii_uppercase returns the char if c.is_lowercase() { - swapped_str.push(c.to_ascii_uppercase()); + swapped_str.push_char(c.to_ascii_uppercase()); } else if c.is_uppercase() { - swapped_str.push(c.to_ascii_lowercase()); + swapped_str.push_char(c.to_ascii_lowercase()); } else { - swapped_str.push(c); + swapped_str.push(c_orig); } } swapped_str @@ -958,20 +993,20 @@ impl PyStr { } #[pymethod] - fn replace(&self, old: PyStrRef, new: PyStrRef, count: OptionalArg) -> String { - let s = self.as_str(); + fn replace(&self, old: PyStrRef, new: PyStrRef, count: OptionalArg) -> Wtf8Buf { + let s = self.as_wtf8(); match count { OptionalArg::Present(max_count) if max_count >= 0 => { if max_count == 0 || (s.is_empty() && !old.is_empty()) { // nothing to do; return the original bytes s.to_owned() } else if s.is_empty() && old.is_empty() { - new.as_str().to_owned() + new.as_wtf8().to_owned() } else { - s.replacen(old.as_str(), new.as_str(), max_count as usize) + s.replacen(old.as_wtf8(), new.as_wtf8(), max_count as usize) } } - _ => s.replace(old.as_str(), new.as_str()), + _ => s.replace(old.as_wtf8(), new.as_wtf8()), } } @@ -1148,7 +1183,7 @@ impl PyStr { if has_mid { sep } else { - vm.ctx.new_str(ascii!("")) + vm.ctx.empty_str.to_owned() }, self.new_substr(back), ) @@ -1165,7 +1200,7 @@ impl PyStr { let mut cased = false; let mut previous_is_cased = false; - for c in self.as_str().chars() { + for c in self.as_wtf8().code_points().map(CodePoint::to_char_lossy) { if c.is_uppercase() || c.is_titlecase() { if previous_is_cased { return false; @@ -1188,15 +1223,15 @@ impl PyStr { #[pymethod] fn count(&self, args: FindArgs) -> usize { let (needle, range) = args.get_value(self.len()); - self.as_str() - .py_count(needle.as_str(), range, |h, n| h.matches(n).count()) + self.as_wtf8() + .py_count(needle.as_wtf8(), range, |h, n| h.find_iter(n).count()) } #[pymethod] - fn zfill(&self, width: isize) -> String { + fn zfill(&self, width: isize) -> Wtf8Buf { unsafe { - // SAFETY: this is safe-guaranteed because the original self.as_str() is valid utf8 - String::from_utf8_unchecked(self.as_str().py_zfill(width)) + // SAFETY: this is safe-guaranteed because the original self.as_wtf8() is valid wtf8 + Wtf8Buf::from_bytes_unchecked(self.as_wtf8().py_zfill(width)) } } @@ -1205,20 +1240,20 @@ impl PyStr { &self, width: isize, fillchar: OptionalArg, - pad: fn(&str, usize, char, usize) -> String, + pad: fn(&Wtf8, usize, CodePoint, usize) -> Wtf8Buf, vm: &VirtualMachine, - ) -> PyResult { - let fillchar = fillchar.map_or(Ok(' '), |ref s| { - s.as_str().chars().exactly_one().map_err(|_| { + ) -> PyResult { + let fillchar = fillchar.map_or(Ok(' '.into()), |ref s| { + s.as_wtf8().code_points().exactly_one().map_err(|_| { vm.new_type_error( "The fill character must be exactly one character long".to_owned(), ) }) })?; Ok(if self.len() as isize >= width { - String::from(self.as_str()) + self.as_wtf8().to_owned() } else { - pad(self.as_str(), width as usize, fillchar, self.len()) + pad(self.as_wtf8(), width as usize, fillchar, self.len()) }) } @@ -1228,7 +1263,7 @@ impl PyStr { width: isize, fillchar: OptionalArg, vm: &VirtualMachine, - ) -> PyResult { + ) -> PyResult { self._pad(width, fillchar, AnyStr::py_center, vm) } @@ -1238,7 +1273,7 @@ impl PyStr { width: isize, fillchar: OptionalArg, vm: &VirtualMachine, - ) -> PyResult { + ) -> PyResult { self._pad(width, fillchar, AnyStr::py_ljust, vm) } @@ -1248,7 +1283,7 @@ impl PyStr { width: isize, fillchar: OptionalArg, vm: &VirtualMachine, - ) -> PyResult { + ) -> PyResult { self._pad(width, fillchar, AnyStr::py_rjust, vm) } @@ -2186,7 +2221,7 @@ mod tests { ("Greek ῼitlecases ...", "greek ῳitlecases ..."), ]; for (title, input) in tests { - assert_eq!(PyStr::from(input).title().as_str(), title); + assert_eq!(PyStr::from(input).title().as_str(), Ok(title)); } } diff --git a/vm/src/codecs.rs b/vm/src/codecs.rs index 8a690fcb58..bdb9b4b809 100644 --- a/vm/src/codecs.rs +++ b/vm/src/codecs.rs @@ -426,12 +426,13 @@ fn xmlcharrefreplace_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<( } let range = extract_unicode_error_range(&err, vm)?; let s = PyStrRef::try_from_object(vm, err.get_attr("object", vm)?)?; - let s_after_start = crate::common::str::try_get_chars(s.as_str(), range.start..).unwrap_or(""); + let s_after_start = + crate::common::str::try_get_codepoints(s.as_wtf8(), range.start..).unwrap_or_default(); let num_chars = range.len(); // capacity rough guess; assuming that the codepoints are 3 digits in decimal + the &#; let mut out = String::with_capacity(num_chars * 6); - for c in s_after_start.chars().take(num_chars) { - write!(out, "&#{};", c as u32).unwrap() + for c in s_after_start.code_points().take(num_chars) { + write!(out, "&#{};", c.to_u32()).unwrap() } Ok((out, range.end)) } @@ -450,12 +451,13 @@ fn backslashreplace_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<(S } let range = extract_unicode_error_range(&err, vm)?; let s = PyStrRef::try_from_object(vm, err.get_attr("object", vm)?)?; - let s_after_start = crate::common::str::try_get_chars(s.as_str(), range.start..).unwrap_or(""); + let s_after_start = + crate::common::str::try_get_codepoints(s.as_wtf8(), range.start..).unwrap_or_default(); let num_chars = range.len(); // minimum 4 output bytes per char: \xNN let mut out = String::with_capacity(num_chars * 4); - for c in s_after_start.chars().take(num_chars) { - let c = c as u32; + for c in s_after_start.code_points().take(num_chars) { + let c = c.to_u32(); if c >= 0x10000 { write!(out, "\\U{c:08x}").unwrap(); } else if c >= 0x100 { @@ -472,12 +474,12 @@ fn namereplace_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<(String let range = extract_unicode_error_range(&err, vm)?; let s = PyStrRef::try_from_object(vm, err.get_attr("object", vm)?)?; let s_after_start = - crate::common::str::try_get_chars(s.as_str(), range.start..).unwrap_or(""); + crate::common::str::try_get_codepoints(s.as_wtf8(), range.start..).unwrap_or_default(); let num_chars = range.len(); let mut out = String::with_capacity(num_chars * 4); - for c in s_after_start.chars().take(num_chars) { - let c_u32 = c as u32; - if let Some(c_name) = unicode_names2::name(c) { + for c in s_after_start.code_points().take(num_chars) { + let c_u32 = c.to_u32(); + if let Some(c_name) = unicode_names2::name(c.to_char_lossy()) { write!(out, "\\N{{{c_name}}}").unwrap(); } else if c_u32 >= 0x10000 { write!(out, "\\U{c_u32:08x}").unwrap(); diff --git a/vm/src/function/fspath.rs b/vm/src/function/fspath.rs index e034487e19..83bd452151 100644 --- a/vm/src/function/fspath.rs +++ b/vm/src/function/fspath.rs @@ -74,10 +74,10 @@ impl FsPath { } } - pub fn as_str(&self) -> &str { + pub fn to_string_lossy(&self) -> Cow<'_, str> { match self { - FsPath::Bytes(b) => std::str::from_utf8(b).unwrap(), - FsPath::Str(s) => s.as_str(), + FsPath::Str(s) => s.to_string_lossy(), + FsPath::Bytes(s) => String::from_utf8_lossy(s), } } diff --git a/vm/src/stdlib/builtins.rs b/vm/src/stdlib/builtins.rs index 6cae1bb7d5..f778d02ba3 100644 --- a/vm/src/stdlib/builtins.rs +++ b/vm/src/stdlib/builtins.rs @@ -155,7 +155,7 @@ mod builtins { return ast::compile( vm, args.source, - args.filename.as_str(), + &args.filename.to_string_lossy(), mode, Some(optimize), ); @@ -204,7 +204,7 @@ mod builtins { .compile_with_opts( source, mode, - args.filename.as_str().to_owned(), + args.filename.to_string_lossy().into_owned(), opts, ) .map_err(|err| (err, Some(source)).to_pyexception(vm))?; diff --git a/vm/src/vm/mod.rs b/vm/src/vm/mod.rs index 9d7ecc2d54..493789f510 100644 --- a/vm/src/vm/mod.rs +++ b/vm/src/vm/mod.rs @@ -916,12 +916,20 @@ impl VirtualMachine { } pub fn fsdecode(&self, s: impl Into) -> PyStrRef { - let bytes = self.ctx.new_bytes(s.into().into_encoded_bytes()); - let errors = self.fs_encode_errors().to_owned(); - self.state - .codec_registry - .decode_text(bytes.into(), "utf-8", Some(errors), self) - .unwrap() // this should never fail, since fsdecode should be lossless from the fs encoding + match s.into().into_string() { + Ok(s) => self.ctx.new_str(s), + Err(s) => { + let bytes = self.ctx.new_bytes(s.into_encoded_bytes()); + let errors = self.fs_encode_errors().to_owned(); + let res = self.state.codec_registry.decode_text( + bytes.into(), + "utf-8", + Some(errors), + self, + ); + self.expect_pyresult(res, "fsdecode should be lossless and never fail") + } + } } pub fn fsencode<'a>(&self, s: &'a Py) -> PyResult> { From bd55baefa6f39aff3c7e3a6e365c0ce1a187bccf Mon Sep 17 00:00:00 2001 From: Noa Date: Mon, 24 Mar 2025 22:52:25 -0500 Subject: [PATCH 113/295] Optimize Wtf8Codepoints::count --- common/src/wtf8/core_str_count.rs | 161 ++++++++++++++++++++++++++++++ common/src/wtf8/mod.rs | 14 ++- 2 files changed, 173 insertions(+), 2 deletions(-) create mode 100644 common/src/wtf8/core_str_count.rs diff --git a/common/src/wtf8/core_str_count.rs b/common/src/wtf8/core_str_count.rs new file mode 100644 index 0000000000..cff5a4b076 --- /dev/null +++ b/common/src/wtf8/core_str_count.rs @@ -0,0 +1,161 @@ +//! Modified from core::str::count + +use super::Wtf8; + +const USIZE_SIZE: usize = core::mem::size_of::(); +const UNROLL_INNER: usize = 4; + +#[inline] +pub(super) fn count_chars(s: &Wtf8) -> usize { + if s.len() < USIZE_SIZE * UNROLL_INNER { + // Avoid entering the optimized implementation for strings where the + // difference is not likely to matter, or where it might even be slower. + // That said, a ton of thought was not spent on the particular threshold + // here, beyond "this value seems to make sense". + char_count_general_case(s.as_bytes()) + } else { + do_count_chars(s) + } +} + +fn do_count_chars(s: &Wtf8) -> usize { + // For correctness, `CHUNK_SIZE` must be: + // + // - Less than or equal to 255, otherwise we'll overflow bytes in `counts`. + // - A multiple of `UNROLL_INNER`, otherwise our `break` inside the + // `body.chunks(CHUNK_SIZE)` loop is incorrect. + // + // For performance, `CHUNK_SIZE` should be: + // - Relatively cheap to `/` against (so some simple sum of powers of two). + // - Large enough to avoid paying for the cost of the `sum_bytes_in_usize` + // too often. + const CHUNK_SIZE: usize = 192; + + // Check the properties of `CHUNK_SIZE` and `UNROLL_INNER` that are required + // for correctness. + const _: () = assert!(CHUNK_SIZE < 256); + const _: () = assert!(CHUNK_SIZE % UNROLL_INNER == 0); + + // SAFETY: transmuting `[u8]` to `[usize]` is safe except for size + // differences which are handled by `align_to`. + let (head, body, tail) = unsafe { s.as_bytes().align_to::() }; + + // This should be quite rare, and basically exists to handle the degenerate + // cases where align_to fails (as well as miri under symbolic alignment + // mode). + // + // The `unlikely` helps discourage LLVM from inlining the body, which is + // nice, as we would rather not mark the `char_count_general_case` function + // as cold. + if unlikely(body.is_empty() || head.len() > USIZE_SIZE || tail.len() > USIZE_SIZE) { + return char_count_general_case(s.as_bytes()); + } + + let mut total = char_count_general_case(head) + char_count_general_case(tail); + // Split `body` into `CHUNK_SIZE` chunks to reduce the frequency with which + // we call `sum_bytes_in_usize`. + for chunk in body.chunks(CHUNK_SIZE) { + // We accumulate intermediate sums in `counts`, where each byte contains + // a subset of the sum of this chunk, like a `[u8; size_of::()]`. + let mut counts = 0; + + let (unrolled_chunks, remainder) = slice_as_chunks::<_, UNROLL_INNER>(chunk); + for unrolled in unrolled_chunks { + for &word in unrolled { + // Because `CHUNK_SIZE` is < 256, this addition can't cause the + // count in any of the bytes to overflow into a subsequent byte. + counts += contains_non_continuation_byte(word); + } + } + + // Sum the values in `counts` (which, again, is conceptually a `[u8; + // size_of::()]`), and accumulate the result into `total`. + total += sum_bytes_in_usize(counts); + + // If there's any data in `remainder`, then handle it. This will only + // happen for the last `chunk` in `body.chunks()` (because `CHUNK_SIZE` + // is divisible by `UNROLL_INNER`), so we explicitly break at the end + // (which seems to help LLVM out). + if !remainder.is_empty() { + // Accumulate all the data in the remainder. + let mut counts = 0; + for &word in remainder { + counts += contains_non_continuation_byte(word); + } + total += sum_bytes_in_usize(counts); + break; + } + } + total +} + +// Checks each byte of `w` to see if it contains the first byte in a UTF-8 +// sequence. Bytes in `w` which are continuation bytes are left as `0x00` (e.g. +// false), and bytes which are non-continuation bytes are left as `0x01` (e.g. +// true) +#[inline] +fn contains_non_continuation_byte(w: usize) -> usize { + const LSB: usize = usize_repeat_u8(0x01); + ((!w >> 7) | (w >> 6)) & LSB +} + +// Morally equivalent to `values.to_ne_bytes().into_iter().sum::()`, but +// more efficient. +#[inline] +fn sum_bytes_in_usize(values: usize) -> usize { + const LSB_SHORTS: usize = usize_repeat_u16(0x0001); + const SKIP_BYTES: usize = usize_repeat_u16(0x00ff); + + let pair_sum: usize = (values & SKIP_BYTES) + ((values >> 8) & SKIP_BYTES); + pair_sum.wrapping_mul(LSB_SHORTS) >> ((USIZE_SIZE - 2) * 8) +} + +// This is the most direct implementation of the concept of "count the number of +// bytes in the string which are not continuation bytes", and is used for the +// head and tail of the input string (the first and last item in the tuple +// returned by `slice::align_to`). +fn char_count_general_case(s: &[u8]) -> usize { + s.iter() + .filter(|&&byte| !super::core_str::utf8_is_cont_byte(byte)) + .count() +} + +// polyfills of unstable library features + +const fn usize_repeat_u8(x: u8) -> usize { + usize::from_ne_bytes([x; size_of::()]) +} + +const fn usize_repeat_u16(x: u16) -> usize { + let mut r = 0usize; + let mut i = 0; + while i < size_of::() { + // Use `wrapping_shl` to make it work on targets with 16-bit `usize` + r = r.wrapping_shl(16) | (x as usize); + i += 2; + } + r +} + +fn slice_as_chunks(slice: &[T]) -> (&[[T; N]], &[T]) { + assert!(N != 0, "chunk size must be non-zero"); + let len_rounded_down = slice.len() / N * N; + // SAFETY: The rounded-down value is always the same or smaller than the + // original length, and thus must be in-bounds of the slice. + let (multiple_of_n, remainder) = unsafe { slice.split_at_unchecked(len_rounded_down) }; + // SAFETY: We already panicked for zero, and ensured by construction + // that the length of the subslice is a multiple of N. + let array_slice = unsafe { slice_as_chunks_unchecked(multiple_of_n) }; + (array_slice, remainder) +} + +unsafe fn slice_as_chunks_unchecked(slice: &[T]) -> &[[T; N]] { + let new_len = slice.len() / N; + // SAFETY: We cast a slice of `new_len * N` elements into + // a slice of `new_len` many `N` elements chunks. + unsafe { std::slice::from_raw_parts(slice.as_ptr().cast(), new_len) } +} + +fn unlikely(x: bool) -> bool { + x +} diff --git a/common/src/wtf8/mod.rs b/common/src/wtf8/mod.rs index 62fbc2ea39..4125a9bf7c 100644 --- a/common/src/wtf8/mod.rs +++ b/common/src/wtf8/mod.rs @@ -53,6 +53,7 @@ use bstr::{ByteSlice, ByteVec}; mod core_char; mod core_str; +mod core_str_count; const UTF8_REPLACEMENT_CHARACTER: &str = "\u{FFFD}"; @@ -1256,6 +1257,10 @@ impl Iterator for Wtf8CodePoints<'_> { fn last(mut self) -> Option { self.next_back() } + + fn count(self) -> usize { + core_str_count::count_chars(self.as_wtf8()) + } } impl DoubleEndedIterator for Wtf8CodePoints<'_> { @@ -1277,8 +1282,8 @@ impl<'a> Wtf8CodePoints<'a> { #[derive(Clone)] pub struct Wtf8CodePointIndices<'a> { - pub(super) front_offset: usize, - pub(super) iter: Wtf8CodePoints<'a>, + front_offset: usize, + iter: Wtf8CodePoints<'a>, } impl Iterator for Wtf8CodePointIndices<'_> { @@ -1308,6 +1313,11 @@ impl Iterator for Wtf8CodePointIndices<'_> { // No need to go through the entire string. self.next_back() } + + #[inline] + fn count(self) -> usize { + self.iter.count() + } } impl DoubleEndedIterator for Wtf8CodePointIndices<'_> { From f3b8d5515a72884c4b01ec6adc3d6eb140fbabf8 Mon Sep 17 00:00:00 2001 From: Noa Date: Tue, 25 Mar 2025 21:06:56 -0500 Subject: [PATCH 114/295] Address comments --- common/src/encodings.rs | 1 - common/src/wtf8/mod.rs | 2 +- vm/src/anystr.rs | 5 +++-- vm/sre_engine/Cargo.toml | 6 +----- vm/sre_engine/src/string.rs | 2 -- 5 files changed, 5 insertions(+), 11 deletions(-) diff --git a/common/src/encodings.rs b/common/src/encodings.rs index 402e0db77f..7d99646c31 100644 --- a/common/src/encodings.rs +++ b/common/src/encodings.rs @@ -230,7 +230,6 @@ pub mod utf8 { } pub mod latin_1 { - use super::*; pub const ENCODING_NAME: &str = "latin-1"; diff --git a/common/src/wtf8/mod.rs b/common/src/wtf8/mod.rs index 4125a9bf7c..f209b98a3e 100644 --- a/common/src/wtf8/mod.rs +++ b/common/src/wtf8/mod.rs @@ -23,7 +23,7 @@ //! needing any copies or re-encoding. //! //! This implementation is mostly copied from the WTF-8 implentation in the -//! Rust standard library, which is used as the backing for [`OsStr`] on +//! Rust 1.85 standard library, which is used as the backing for [`OsStr`] on //! Windows targets. As previously mentioned, however, it is modified to not //! join two surrogates into one codepoint when concatenating strings, in order //! to match CPython's behavior. diff --git a/vm/src/anystr.rs b/vm/src/anystr.rs index 0a884fc732..89e7473441 100644 --- a/vm/src/anystr.rs +++ b/vm/src/anystr.rs @@ -403,9 +403,7 @@ pub trait AnyStr { // Unified form of CPython functions: // _Py_bytes_islower - // Py_bytes_isupper // unicode_islower_impl - // unicode_isupper_impl fn py_islower(&self) -> bool { let mut lower = false; for c in self.elements() { @@ -418,6 +416,9 @@ pub trait AnyStr { lower } + // Unified form of CPython functions: + // Py_bytes_isupper + // unicode_isupper_impl fn py_isupper(&self) -> bool { let mut upper = false; for c in self.elements() { diff --git a/vm/sre_engine/Cargo.toml b/vm/sre_engine/Cargo.toml index 55ce249903..b34b01a0e8 100644 --- a/vm/sre_engine/Cargo.toml +++ b/vm/sre_engine/Cargo.toml @@ -14,12 +14,8 @@ license.workspace = true name = "benches" harness = false -[features] -default = ["wtf8"] -wtf8 = ["rustpython-common"] - [dependencies] -rustpython-common = { workspace = true, optional = true } +rustpython-common = { workspace = true } num_enum = { workspace = true } bitflags = { workspace = true } optional = "0.5" diff --git a/vm/sre_engine/src/string.rs b/vm/sre_engine/src/string.rs index 551b1ca5e5..20cacbfbec 100644 --- a/vm/sre_engine/src/string.rs +++ b/vm/sre_engine/src/string.rs @@ -1,4 +1,3 @@ -#[cfg(feature = "wtf8")] use rustpython_common::wtf8::Wtf8; #[derive(Debug, Clone, Copy)] @@ -151,7 +150,6 @@ impl StrDrive for &str { } } -#[cfg(feature = "wtf8")] impl StrDrive for &Wtf8 { #[inline] fn count(&self) -> usize { From bc38e9dedd5d2a21ea588a72b904d70f31a22013 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Wed, 26 Mar 2025 14:39:14 +0900 Subject: [PATCH 115/295] Apply nightly clippy suggestions --- common/src/boxvec.rs | 2 +- common/src/str.rs | 2 +- compiler/codegen/src/compile.rs | 2 +- compiler/codegen/src/ir.rs | 3 +-- compiler/codegen/src/symboltable.rs | 2 +- src/settings.rs | 2 +- stdlib/src/binascii.rs | 2 +- stdlib/src/csv.rs | 4 ++-- vm/src/anystr.rs | 4 ++-- vm/src/builtins/range.rs | 6 +----- vm/src/builtins/staticmethod.rs | 3 +-- vm/src/builtins/tuple.rs | 2 +- vm/src/frame.rs | 2 +- vm/src/stdlib/ctypes/base.rs | 16 ++++++++-------- vm/src/stdlib/itertools.rs | 7 +++---- vm/src/stdlib/signal.rs | 5 +---- 16 files changed, 27 insertions(+), 37 deletions(-) diff --git a/common/src/boxvec.rs b/common/src/boxvec.rs index 25afbcebb3..1a1d57c169 100644 --- a/common/src/boxvec.rs +++ b/common/src/boxvec.rs @@ -550,7 +550,7 @@ impl Extend for BoxVec { }; let mut iter = iter.into_iter(); loop { - if ptr == end_ptr { + if std::ptr::eq(ptr, end_ptr) { break; } if let Some(elt) = iter.next() { diff --git a/common/src/str.rs b/common/src/str.rs index 176b5d0f87..fcc000fd2c 100644 --- a/common/src/str.rs +++ b/common/src/str.rs @@ -418,7 +418,7 @@ pub fn zfill(bytes: &[u8], width: usize) -> Vec { }; let mut filled = Vec::new(); filled.extend_from_slice(sign); - filled.extend(std::iter::repeat(b'0').take(width - bytes.len())); + filled.extend(std::iter::repeat_n(b'0', width - bytes.len())); filled.extend_from_slice(s); filled } diff --git a/compiler/codegen/src/compile.rs b/compiler/codegen/src/compile.rs index a6eb216e2a..c640b014a4 100644 --- a/compiler/codegen/src/compile.rs +++ b/compiler/codegen/src/compile.rs @@ -1519,7 +1519,7 @@ impl Compiler<'_> { loop_data: None, }; - let prev_class_name = std::mem::replace(&mut self.class_name, Some(name.to_owned())); + let prev_class_name = self.class_name.replace(name.to_owned()); // Check if the class is declared global let symbol_table = self.symbol_table_stack.last().unwrap(); diff --git a/compiler/codegen/src/ir.rs b/compiler/codegen/src/ir.rs index 2b3e49d036..39857e6fc5 100644 --- a/compiler/codegen/src/ir.rs +++ b/compiler/codegen/src/ir.rs @@ -136,8 +136,7 @@ impl CodeInfo { *arg = new_arg; } let (extras, lo_arg) = arg.split(); - locations - .extend(std::iter::repeat(info.location.clone()).take(arg.instr_size())); + locations.extend(std::iter::repeat_n(info.location.clone(), arg.instr_size())); instructions.extend( extras .map(|byte| CodeUnit::new(Instruction::ExtendedArg, byte)) diff --git a/compiler/codegen/src/symboltable.rs b/compiler/codegen/src/symboltable.rs index 4246700cec..4f42b3996f 100644 --- a/compiler/codegen/src/symboltable.rs +++ b/compiler/codegen/src/symboltable.rs @@ -730,7 +730,7 @@ impl SymbolTableBuilder<'_> { SymbolTableType::Class, self.line_index_start(*range), ); - let prev_class = std::mem::replace(&mut self.class_name, Some(name.to_string())); + let prev_class = self.class_name.replace(name.to_string()); self.register_name("__module__", SymbolUsage::Assigned, *range)?; self.register_name("__qualname__", SymbolUsage::Assigned, *range)?; self.register_name("__doc__", SymbolUsage::Assigned, *range)?; diff --git a/src/settings.rs b/src/settings.rs index bc279ba287..76c46ac43a 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -305,7 +305,7 @@ pub fn parse_opts() -> Result<(Settings, RunMode), lexopt::Error> { .then(|| get_env("PYTHONHASHSEED")) .flatten() { - Some(s) if s == "random" || s == "" => None, + Some(s) if s == "random" || s.is_empty() => None, Some(s) => { let seed = s.parse_with(|s| { s.parse::().map_err(|_| { diff --git a/stdlib/src/binascii.rs b/stdlib/src/binascii.rs index ce3f9febd5..65be2e0bdc 100644 --- a/stdlib/src/binascii.rs +++ b/stdlib/src/binascii.rs @@ -716,7 +716,7 @@ mod decl { vm, )); } - let mut res = Vec::::with_capacity(2 + ((length + 2) / 3) * 4); + let mut res = Vec::::with_capacity(2 + length.div_ceil(3) * 4); res.push(uu_b2a(length as u8, backtick)); for chunk in b.chunks(3) { diff --git a/stdlib/src/csv.rs b/stdlib/src/csv.rs index 4b79130111..e103e7697c 100644 --- a/stdlib/src/csv.rs +++ b/stdlib/src/csv.rs @@ -982,7 +982,7 @@ mod _csv { }; loop { let (res, nread, nwritten, nends) = reader.read_record( - input[input_offset..].as_bytes(), + &input.as_bytes()[input_offset..], &mut buffer[output_offset..], &mut output_ends[output_ends_offset..], ); @@ -999,7 +999,7 @@ mod _csv { } } } - let rest = input[input_offset..].as_bytes(); + let rest = &input.as_bytes()[input_offset..]; if !rest.iter().all(|&c| matches!(c, b'\r' | b'\n')) { return Err(new_csv_error( vm, diff --git a/vm/src/anystr.rs b/vm/src/anystr.rs index 89e7473441..efac49d917 100644 --- a/vm/src/anystr.rs +++ b/vm/src/anystr.rs @@ -283,9 +283,9 @@ pub trait AnyStr { let mut u = Self::Container::with_capacity( (left + right) * fillchar.bytes_len() + self.bytes_len(), ); - u.extend(std::iter::repeat(fillchar).take(left)); + u.extend(std::iter::repeat_n(fillchar, left)); u.push_str(self); - u.extend(std::iter::repeat(fillchar).take(right)); + u.extend(std::iter::repeat_n(fillchar, right)); u } diff --git a/vm/src/builtins/range.rs b/vm/src/builtins/range.rs index b542a5f191..36a826a1f8 100644 --- a/vm/src/builtins/range.rs +++ b/vm/src/builtins/range.rs @@ -608,11 +608,7 @@ impl PyRangeIterator { #[pymethod(magic)] fn length_hint(&self) -> usize { let index = self.index.load(); - if index < self.length { - self.length - index - } else { - 0 - } + self.length.saturating_sub(index) } #[pymethod(magic)] diff --git a/vm/src/builtins/staticmethod.rs b/vm/src/builtins/staticmethod.rs index 8e2333da7f..6c19a42a33 100644 --- a/vm/src/builtins/staticmethod.rs +++ b/vm/src/builtins/staticmethod.rs @@ -27,8 +27,7 @@ impl GetDescriptor for PyStaticMethod { vm: &VirtualMachine, ) -> PyResult { let (zelf, _obj) = Self::_unwrap(&zelf, obj, vm)?; - let x = Ok(zelf.callable.lock().clone()); - x + Ok(zelf.callable.lock().clone()) } } diff --git a/vm/src/builtins/tuple.rs b/vm/src/builtins/tuple.rs index c9af04d300..044550eb8b 100644 --- a/vm/src/builtins/tuple.rs +++ b/vm/src/builtins/tuple.rs @@ -430,7 +430,7 @@ impl Iterable for PyTuple { impl Representable for PyTuple { #[inline] fn repr(zelf: &Py, vm: &VirtualMachine) -> PyResult { - let s = if zelf.len() == 0 { + let s = if zelf.is_empty() { vm.ctx.intern_str("()").to_owned() } else if let Some(_guard) = ReprGuard::enter(vm, zelf.as_object()) { let s = if zelf.len() == 1 { diff --git a/vm/src/frame.rs b/vm/src/frame.rs index c4dc23c95f..b148934467 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -1147,7 +1147,7 @@ impl ExecutingFrame<'_> { bytecode::Instruction::FormatValue { conversion } => { self.format_value(conversion.get(arg), vm) } - bytecode::Instruction::PopException {} => { + bytecode::Instruction::PopException => { let block = self.pop_block(); if let BlockType::ExceptHandler { prev_exc } = block.typ { vm.set_exception(prev_exc); diff --git a/vm/src/stdlib/ctypes/base.rs b/vm/src/stdlib/ctypes/base.rs index d07ccba30b..e3e12bdf3b 100644 --- a/vm/src/stdlib/ctypes/base.rs +++ b/vm/src/stdlib/ctypes/base.rs @@ -278,21 +278,21 @@ impl PyCSimple { let value = unsafe { (*self.value.as_ptr()).clone() }; if let Ok(i) = value.try_int(vm) { let i = i.as_bigint(); - if ty.as_raw_ptr() == libffi::middle::Type::u8().as_raw_ptr() { + if std::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::u8().as_raw_ptr()) { return i.to_u8().map(|r: u8| libffi::middle::Arg::new(&r)); - } else if ty.as_raw_ptr() == libffi::middle::Type::i8().as_raw_ptr() { + } else if std::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::i8().as_raw_ptr()) { return i.to_i8().map(|r: i8| libffi::middle::Arg::new(&r)); - } else if ty.as_raw_ptr() == libffi::middle::Type::u16().as_raw_ptr() { + } else if std::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::u16().as_raw_ptr()) { return i.to_u16().map(|r: u16| libffi::middle::Arg::new(&r)); - } else if ty.as_raw_ptr() == libffi::middle::Type::i16().as_raw_ptr() { + } else if std::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::i16().as_raw_ptr()) { return i.to_i16().map(|r: i16| libffi::middle::Arg::new(&r)); - } else if ty.as_raw_ptr() == libffi::middle::Type::u32().as_raw_ptr() { + } else if std::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::u32().as_raw_ptr()) { return i.to_u32().map(|r: u32| libffi::middle::Arg::new(&r)); - } else if ty.as_raw_ptr() == libffi::middle::Type::i32().as_raw_ptr() { + } else if std::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::i32().as_raw_ptr()) { return i.to_i32().map(|r: i32| libffi::middle::Arg::new(&r)); - } else if ty.as_raw_ptr() == libffi::middle::Type::u64().as_raw_ptr() { + } else if std::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::u64().as_raw_ptr()) { return i.to_u64().map(|r: u64| libffi::middle::Arg::new(&r)); - } else if ty.as_raw_ptr() == libffi::middle::Type::i64().as_raw_ptr() { + } else if std::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::i64().as_raw_ptr()) { return i.to_i64().map(|r: i64| libffi::middle::Arg::new(&r)); } else { return None; diff --git a/vm/src/stdlib/itertools.rs b/vm/src/stdlib/itertools.rs index 65c1482057..dab62987d6 100644 --- a/vm/src/stdlib/itertools.rs +++ b/vm/src/stdlib/itertools.rs @@ -335,7 +335,7 @@ mod decl { item } else { let saved = zelf.saved.read(); - if saved.len() == 0 { + if saved.is_empty() { return Ok(PyIterReturn::StopIteration(None)); } @@ -1332,8 +1332,7 @@ mod decl { for arg in iterables.iter() { pools.push(arg.try_to_value(vm)?); } - let pools = std::iter::repeat(pools) - .take(repeat) + let pools = std::iter::repeat_n(pools, repeat) .flatten() .collect::>>(); @@ -1353,7 +1352,7 @@ mod decl { #[pyclass(with(IterNext, Iterable, Constructor))] impl PyItertoolsProduct { fn update_idxs(&self, mut idxs: PyRwLockWriteGuard<'_, Vec>) { - if idxs.len() == 0 { + if idxs.is_empty() { self.stop.store(true); return; } diff --git a/vm/src/stdlib/signal.rs b/vm/src/stdlib/signal.rs index 0c47ca082e..abec4535e5 100644 --- a/vm/src/stdlib/signal.rs +++ b/vm/src/stdlib/signal.rs @@ -182,10 +182,7 @@ pub(crate) mod _signal { siginterrupt(signalnum, 1); } - let old_handler = std::mem::replace( - &mut signal_handlers.borrow_mut()[signalnum as usize], - Some(handler), - ); + let old_handler = signal_handlers.borrow_mut()[signalnum as usize].replace(handler); Ok(old_handler) } From f323d14ed3d470ee81342f821906b963981537e9 Mon Sep 17 00:00:00 2001 From: Noa Date: Tue, 25 Mar 2025 19:04:57 -0500 Subject: [PATCH 116/295] Refactor codecs --- Cargo.lock | 1 + Lib/_pycodecs.py | 11 +- Lib/test/test_charmapcodec.py | 2 - Lib/test/test_codeccallbacks.py | 14 - Lib/test/test_logging.py | 2 - common/Cargo.toml | 1 + common/src/encodings.rs | 565 +++++++++++++---- common/src/str.rs | 15 + common/src/wtf8/mod.rs | 13 +- vm/src/codecs.rs | 1052 ++++++++++++++++++++++--------- vm/src/exceptions.rs | 164 ++++- vm/src/function/buffer.rs | 6 + vm/src/stdlib/codecs.rs | 196 +----- vm/src/stdlib/io.rs | 7 +- vm/src/vm/vm_new.rs | 65 +- 15 files changed, 1448 insertions(+), 666 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1fa60eac29..241d2a595d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2355,6 +2355,7 @@ dependencies = [ "rand 0.9.0", "rustpython-literal", "siphasher 0.3.11", + "unicode_names2", "volatile", "widestring", "windows-sys 0.59.0", diff --git a/Lib/_pycodecs.py b/Lib/_pycodecs.py index 0741504cc9..d0efa9ad6b 100644 --- a/Lib/_pycodecs.py +++ b/Lib/_pycodecs.py @@ -1086,11 +1086,13 @@ def charmapencode_output(c, mapping): rep = mapping[c] if isinstance(rep, int) or isinstance(rep, int): if rep < 256: - return rep + return [rep] else: raise TypeError("character mapping must be in range(256)") elif isinstance(rep, str): - return ord(rep) + return [ord(rep)] + elif isinstance(rep, bytes): + return rep elif rep == None: raise KeyError("character maps to ") else: @@ -1113,12 +1115,13 @@ def PyUnicode_EncodeCharmap(p, size, mapping='latin-1', errors='strict'): #/* try to encode it */ try: x = charmapencode_output(ord(p[inpos]), mapping) - res += [x] + res += x except KeyError: x = unicode_call_errorhandler(errors, "charmap", "character maps to ", p, inpos, inpos+1, False) try: - res += [charmapencode_output(ord(y), mapping) for y in x[0]] + for y in x[0]: + res += charmapencode_output(ord(y), mapping) except KeyError: raise UnicodeEncodeError("charmap", p, inpos, inpos+1, "character maps to ") diff --git a/Lib/test/test_charmapcodec.py b/Lib/test/test_charmapcodec.py index 8ea75d9129..0d4594d8c0 100644 --- a/Lib/test/test_charmapcodec.py +++ b/Lib/test/test_charmapcodec.py @@ -33,8 +33,6 @@ def test_constructorx(self): self.assertEqual(str(b'dxf', codecname), 'dabcf') self.assertEqual(str(b'dxfx', codecname), 'dabcfabc') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_encodex(self): self.assertEqual('abc'.encode(codecname), b'abc') self.assertEqual('xdef'.encode(codecname), b'abcdef') diff --git a/Lib/test/test_codeccallbacks.py b/Lib/test/test_codeccallbacks.py index 293b75a866..09a6d883f8 100644 --- a/Lib/test/test_codeccallbacks.py +++ b/Lib/test/test_codeccallbacks.py @@ -203,8 +203,6 @@ def relaxedutf8(exc): self.assertRaises(UnicodeDecodeError, sin.decode, "utf-8", "test.relaxedutf8") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_charmapencode(self): # For charmap encodings the replacement string will be # mapped through the encoding again. This means, that @@ -329,8 +327,6 @@ def check_exceptionobjectargs(self, exctype, args, msg): exc = exctype(*args) self.assertEqual(str(exc), msg) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_unicodeencodeerror(self): self.check_exceptionobjectargs( UnicodeEncodeError, @@ -363,8 +359,6 @@ def test_unicodeencodeerror(self): "'ascii' codec can't encode character '\\U00010000' in position 0: ouch" ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_unicodedecodeerror(self): self.check_exceptionobjectargs( UnicodeDecodeError, @@ -377,8 +371,6 @@ def test_unicodedecodeerror(self): "'ascii' codec can't decode bytes in position 1-2: ouch" ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_unicodetranslateerror(self): self.check_exceptionobjectargs( UnicodeTranslateError, @@ -467,8 +459,6 @@ def test_badandgoodignoreexceptions(self): ("", 2) ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_badandgoodreplaceexceptions(self): # "replace" complains about a non-exception passed in self.assertRaises( @@ -509,8 +499,6 @@ def test_badandgoodreplaceexceptions(self): ("\ufffd", 2) ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_badandgoodxmlcharrefreplaceexceptions(self): # "xmlcharrefreplace" complains about a non-exception passed in self.assertRaises( @@ -1017,8 +1005,6 @@ def __getitem__(self, key): self.assertRaises(ValueError, codecs.charmap_decode, b"\xff", "strict", D()) self.assertRaises(TypeError, codecs.charmap_decode, b"\xff", "strict", {0xff: sys.maxunicode+1}) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_encodehelper(self): # enhance coverage of: # Objects/unicodeobject.c::unicode_encode_call_errorhandler() diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index a570d65f6c..370685f1c6 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -5525,8 +5525,6 @@ def test_encoding_errors_default(self): self.assertEqual(data, r'\U0001f602: \u2603\ufe0f: The \xd8resund ' r'Bridge joins Copenhagen to Malm\xf6') - # TODO: RustPython - @unittest.expectedFailure def test_encoding_errors_none(self): # Specifying None should behave as 'strict' try: diff --git a/common/Cargo.toml b/common/Cargo.toml index e9aeba7459..3ebac23cbf 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -29,6 +29,7 @@ num-traits = { workspace = true } once_cell = { workspace = true } parking_lot = { workspace = true, optional = true } rand = { workspace = true } +unicode_names2 = { workspace = true } lock_api = "0.4" radium = "0.7" diff --git a/common/src/encodings.rs b/common/src/encodings.rs index 7d99646c31..9582d63b6c 100644 --- a/common/src/encodings.rs +++ b/common/src/encodings.rs @@ -1,13 +1,9 @@ -use std::ops::Range; +use std::ops::{self, Range}; use num_traits::ToPrimitive; use crate::str::StrKind; -use crate::wtf8::{Wtf8, Wtf8Buf}; - -pub type EncodeErrorResult = Result<(EncodeReplace, usize), E>; - -pub type DecodeErrorResult = Result<(S, Option, usize), E>; +use crate::wtf8::{CodePoint, Wtf8, Wtf8Buf}; pub trait StrBuffer: AsRef { fn is_compatible_with(&self, kind: StrKind) -> bool { @@ -20,28 +16,116 @@ pub trait StrBuffer: AsRef { } } -pub trait ErrorHandler { +pub trait CodecContext: Sized { type Error; type StrBuf: StrBuffer; type BytesBuf: AsRef<[u8]>; + + fn string(&self, s: Wtf8Buf) -> Self::StrBuf; + fn bytes(&self, b: Vec) -> Self::BytesBuf; +} + +pub trait EncodeContext: CodecContext { + fn full_data(&self) -> &Wtf8; + fn data_len(&self) -> StrSize; + + fn remaining_data(&self) -> &Wtf8; + fn position(&self) -> StrSize; + + fn restart_from(&mut self, pos: StrSize) -> Result<(), Self::Error>; + + fn error_encoding(&self, range: Range, reason: Option<&str>) -> Self::Error; + + fn handle_error( + &mut self, + errors: &E, + range: Range, + reason: Option<&str>, + ) -> Result, Self::Error> + where + E: EncodeErrorHandler, + { + let (replace, restart) = errors.handle_encode_error(self, range, reason)?; + self.restart_from(restart)?; + Ok(replace) + } +} + +pub trait DecodeContext: CodecContext { + fn full_data(&self) -> &[u8]; + + fn remaining_data(&self) -> &[u8]; + fn position(&self) -> usize; + + fn advance(&mut self, by: usize); + + fn restart_from(&mut self, pos: usize) -> Result<(), Self::Error>; + + fn error_decoding(&self, byte_range: Range, reason: Option<&str>) -> Self::Error; + + fn handle_error( + &mut self, + errors: &E, + byte_range: Range, + reason: Option<&str>, + ) -> Result + where + E: DecodeErrorHandler, + { + let (replace, restart) = errors.handle_decode_error(self, byte_range, reason)?; + self.restart_from(restart)?; + Ok(replace) + } +} + +pub trait EncodeErrorHandler { fn handle_encode_error( &self, - data: &Wtf8, - char_range: Range, - reason: &str, - ) -> EncodeErrorResult; + ctx: &mut Ctx, + range: Range, + reason: Option<&str>, + ) -> Result<(EncodeReplace, StrSize), Ctx::Error>; +} +pub trait DecodeErrorHandler { fn handle_decode_error( &self, - data: &[u8], + ctx: &mut Ctx, byte_range: Range, - reason: &str, - ) -> DecodeErrorResult; - fn error_oob_restart(&self, i: usize) -> Self::Error; - fn error_encoding(&self, data: &Wtf8, char_range: Range, reason: &str) -> Self::Error; + reason: Option<&str>, + ) -> Result<(Ctx::StrBuf, usize), Ctx::Error>; } -pub enum EncodeReplace { - Str(S), - Bytes(B), + +pub enum EncodeReplace { + Str(Ctx::StrBuf), + Bytes(Ctx::BytesBuf), +} + +#[derive(Copy, Clone, Default, Debug)] +pub struct StrSize { + pub bytes: usize, + pub chars: usize, +} + +fn iter_code_points(w: &Wtf8) -> impl Iterator { + w.code_point_indices() + .enumerate() + .map(|(chars, (bytes, c))| (StrSize { bytes, chars }, c)) +} + +impl ops::Add for StrSize { + type Output = Self; + fn add(self, rhs: Self) -> Self::Output { + Self { + bytes: self.bytes + rhs.bytes, + chars: self.chars + rhs.chars, + } + } +} +impl ops::AddAssign for StrSize { + fn add_assign(&mut self, rhs: Self) { + self.bytes += rhs.bytes; + self.chars += rhs.chars; + } } struct DecodeError<'a> { @@ -68,128 +152,344 @@ enum HandleResult<'a> { reason: &'a str, }, } -fn decode_utf8_compatible( - data: &[u8], +fn decode_utf8_compatible( + mut ctx: Ctx, errors: &E, decode: DecodeF, handle_error: ErrF, -) -> Result<(Wtf8Buf, usize), E::Error> +) -> Result<(Wtf8Buf, usize), Ctx::Error> where + Ctx: DecodeContext, + E: DecodeErrorHandler, DecodeF: Fn(&[u8]) -> Result<&str, DecodeError<'_>>, - ErrF: Fn(&[u8], Option) -> HandleResult<'_>, + ErrF: Fn(&[u8], Option) -> HandleResult<'static>, { - if data.is_empty() { + if ctx.remaining_data().is_empty() { return Ok((Wtf8Buf::new(), 0)); } - // we need to coerce the lifetime to that of the function body rather than the - // anonymous input lifetime, so that we can assign it data borrowed from data_from_err - let mut data = data; - let mut data_from_err: E::BytesBuf; - let mut out = Wtf8Buf::with_capacity(data.len()); - let mut remaining_index = 0; - let mut remaining_data = data; + let mut out = Wtf8Buf::with_capacity(ctx.remaining_data().len()); loop { - match decode(remaining_data) { + match decode(ctx.remaining_data()) { Ok(decoded) => { out.push_str(decoded); - remaining_index += decoded.len(); + ctx.advance(decoded.len()); break; } Err(e) => { out.push_str(e.valid_prefix); match handle_error(e.rest, e.err_len) { HandleResult::Done => { - remaining_index += e.valid_prefix.len(); + ctx.advance(e.valid_prefix.len()); break; } HandleResult::Error { err_len, reason } => { - let err_idx = remaining_index + e.valid_prefix.len(); - let err_range = - err_idx..err_len.map_or_else(|| data.len(), |len| err_idx + len); - let (replace, new_data, restart) = - errors.handle_decode_error(data, err_range, reason)?; + let err_start = ctx.position() + e.valid_prefix.len(); + let err_end = match err_len { + Some(len) => err_start + len, + None => ctx.full_data().len(), + }; + let err_range = err_start..err_end; + let replace = ctx.handle_error(errors, err_range, Some(reason))?; out.push_wtf8(replace.as_ref()); - if let Some(new_data) = new_data { - data_from_err = new_data; - data = data_from_err.as_ref(); - } - remaining_data = data - .get(restart..) - .ok_or_else(|| errors.error_oob_restart(restart))?; - remaining_index = restart; continue; } } } } } - Ok((out, remaining_index)) + Ok((out, ctx.position())) } #[inline] -fn encode_utf8_compatible( - s: &Wtf8, +fn encode_utf8_compatible( + mut ctx: Ctx, errors: &E, err_reason: &str, target_kind: StrKind, -) -> Result, E::Error> { - let full_data = s; - let mut data = s; - let mut char_data_index = 0; - let mut out = Vec::::new(); - while let Some((char_i, (byte_i, _))) = data - .code_point_indices() - .enumerate() - .find(|(_, (_, c))| !target_kind.can_encode(*c)) - { - out.extend_from_slice(&data.as_bytes()[..byte_i]); - let char_start = char_data_index + char_i; +) -> Result, Ctx::Error> +where + Ctx: EncodeContext, + E: EncodeErrorHandler, +{ + // let mut data = s.as_ref(); + // let mut char_data_index = 0; + let mut out = Vec::::with_capacity(ctx.remaining_data().len()); + loop { + let data = ctx.remaining_data(); + let mut iter = iter_code_points(data); + let Some((i, _)) = iter.find(|(_, c)| !target_kind.can_encode(*c)) else { + break; + }; + + out.extend_from_slice(&ctx.remaining_data().as_bytes()[..i.bytes]); + let err_start = ctx.position() + i; // number of non-compatible chars between the first non-compatible char and the next compatible char - let non_compat_run_length = data[byte_i..] - .code_points() - .take_while(|c| !target_kind.can_encode(*c)) - .count(); - let char_range = char_start..char_start + non_compat_run_length; - let (replace, char_restart) = - errors.handle_encode_error(full_data, char_range.clone(), err_reason)?; + let err_end = match { iter }.find(|(_, c)| target_kind.can_encode(*c)) { + Some((i, _)) => ctx.position() + i, + None => ctx.data_len(), + }; + + let range = err_start..err_end; + let replace = ctx.handle_error(errors, range.clone(), Some(err_reason))?; match replace { EncodeReplace::Str(s) => { if s.is_compatible_with(target_kind) { out.extend_from_slice(s.as_ref().as_bytes()); } else { - return Err(errors.error_encoding(full_data, char_range, err_reason)); + return Err(ctx.error_encoding(range, Some(err_reason))); } } EncodeReplace::Bytes(b) => { out.extend_from_slice(b.as_ref()); } } - data = crate::str::try_get_codepoints(full_data, char_restart..) - .ok_or_else(|| errors.error_oob_restart(char_restart))?; - char_data_index = char_restart; + // data = crate::str::try_get_codepoints(full_data.as_ref(), char_restart..) + // .ok_or_else(|| errors.error_oob_restart(char_restart))?; + // char_data_index = char_restart; } - out.extend_from_slice(data.as_bytes()); + out.extend_from_slice(ctx.remaining_data().as_bytes()); Ok(out) } +pub mod errors { + use crate::str::UnicodeEscapeCodepoint; + + use super::*; + use std::fmt::Write; + + pub struct Strict; + + impl EncodeErrorHandler for Strict { + fn handle_encode_error( + &self, + ctx: &mut Ctx, + range: Range, + reason: Option<&str>, + ) -> Result<(EncodeReplace, StrSize), Ctx::Error> { + Err(ctx.error_encoding(range, reason)) + } + } + + impl DecodeErrorHandler for Strict { + fn handle_decode_error( + &self, + ctx: &mut Ctx, + byte_range: Range, + reason: Option<&str>, + ) -> Result<(Ctx::StrBuf, usize), Ctx::Error> { + Err(ctx.error_decoding(byte_range, reason)) + } + } + + pub struct Ignore; + + impl EncodeErrorHandler for Ignore { + fn handle_encode_error( + &self, + ctx: &mut Ctx, + range: Range, + _reason: Option<&str>, + ) -> Result<(EncodeReplace, StrSize), Ctx::Error> { + Ok((EncodeReplace::Bytes(ctx.bytes(b"".into())), range.end)) + } + } + + impl DecodeErrorHandler for Ignore { + fn handle_decode_error( + &self, + ctx: &mut Ctx, + byte_range: Range, + _reason: Option<&str>, + ) -> Result<(Ctx::StrBuf, usize), Ctx::Error> { + Ok((ctx.string("".into()), byte_range.end)) + } + } + + pub struct Replace; + + impl EncodeErrorHandler for Replace { + fn handle_encode_error( + &self, + ctx: &mut Ctx, + range: Range, + _reason: Option<&str>, + ) -> Result<(EncodeReplace, StrSize), Ctx::Error> { + let replace = "?".repeat(range.end.chars - range.start.chars); + Ok((EncodeReplace::Str(ctx.string(replace.into())), range.end)) + } + } + + impl DecodeErrorHandler for Replace { + fn handle_decode_error( + &self, + ctx: &mut Ctx, + byte_range: Range, + _reason: Option<&str>, + ) -> Result<(Ctx::StrBuf, usize), Ctx::Error> { + Ok(( + ctx.string(char::REPLACEMENT_CHARACTER.to_string().into()), + byte_range.end, + )) + } + } + + pub struct XmlCharRefReplace; + + impl EncodeErrorHandler for XmlCharRefReplace { + fn handle_encode_error( + &self, + ctx: &mut Ctx, + range: Range, + _reason: Option<&str>, + ) -> Result<(EncodeReplace, StrSize), Ctx::Error> { + let err_str = &ctx.full_data()[range.start.bytes..range.end.bytes]; + let num_chars = range.end.chars - range.start.chars; + // capacity rough guess; assuming that the codepoints are 3 digits in decimal + the &#; + let mut out = String::with_capacity(num_chars * 6); + for c in err_str.code_points() { + write!(out, "&#{};", c.to_u32()).unwrap() + } + Ok((EncodeReplace::Str(ctx.string(out.into())), range.end)) + } + } + + pub struct BackslashReplace; + + impl EncodeErrorHandler for BackslashReplace { + fn handle_encode_error( + &self, + ctx: &mut Ctx, + range: Range, + _reason: Option<&str>, + ) -> Result<(EncodeReplace, StrSize), Ctx::Error> { + let err_str = &ctx.full_data()[range.start.bytes..range.end.bytes]; + let num_chars = range.end.chars - range.start.chars; + // minimum 4 output bytes per char: \xNN + let mut out = String::with_capacity(num_chars * 4); + for c in err_str.code_points() { + write!(out, "{}", UnicodeEscapeCodepoint(c)).unwrap(); + } + Ok((EncodeReplace::Str(ctx.string(out.into())), range.end)) + } + } + + impl DecodeErrorHandler for BackslashReplace { + fn handle_decode_error( + &self, + ctx: &mut Ctx, + byte_range: Range, + _reason: Option<&str>, + ) -> Result<(Ctx::StrBuf, usize), Ctx::Error> { + let err_bytes = &ctx.full_data()[byte_range.clone()]; + let mut replace = String::with_capacity(4 * err_bytes.len()); + for &c in err_bytes { + write!(replace, "\\x{c:02x}").unwrap(); + } + Ok((ctx.string(replace.into()), byte_range.end)) + } + } + + pub struct NameReplace; + + impl EncodeErrorHandler for NameReplace { + fn handle_encode_error( + &self, + ctx: &mut Ctx, + range: Range, + _reason: Option<&str>, + ) -> Result<(EncodeReplace, StrSize), Ctx::Error> { + let err_str = &ctx.full_data()[range.start.bytes..range.end.bytes]; + let num_chars = range.end.chars - range.start.chars; + let mut out = String::with_capacity(num_chars * 4); + for c in err_str.code_points() { + let c_u32 = c.to_u32(); + if let Some(c_name) = unicode_names2::name(c.to_char_lossy()) { + write!(out, "\\N{{{c_name}}}").unwrap(); + } else if c_u32 >= 0x10000 { + write!(out, "\\U{c_u32:08x}").unwrap(); + } else if c_u32 >= 0x100 { + write!(out, "\\u{c_u32:04x}").unwrap(); + } else { + write!(out, "\\x{c_u32:02x}").unwrap(); + } + } + Ok((EncodeReplace::Str(ctx.string(out.into())), range.end)) + } + } + + pub struct SurrogateEscape; + + impl EncodeErrorHandler for SurrogateEscape { + fn handle_encode_error( + &self, + ctx: &mut Ctx, + range: Range, + reason: Option<&str>, + ) -> Result<(EncodeReplace, StrSize), Ctx::Error> { + let err_str = &ctx.full_data()[range.start.bytes..range.end.bytes]; + let num_chars = range.end.chars - range.start.chars; + let mut out = Vec::with_capacity(num_chars); + for ch in err_str.code_points() { + let ch = ch.to_u32(); + if !(0xdc80..=0xdcff).contains(&ch) { + // Not a UTF-8b surrogate, fail with original exception + return Err(ctx.error_encoding(range, reason)); + } + out.push((ch - 0xdc00) as u8); + } + Ok((EncodeReplace::Bytes(ctx.bytes(out)), range.end)) + } + } + + impl DecodeErrorHandler for SurrogateEscape { + fn handle_decode_error( + &self, + ctx: &mut Ctx, + byte_range: Range, + reason: Option<&str>, + ) -> Result<(Ctx::StrBuf, usize), Ctx::Error> { + let err_bytes = &ctx.full_data()[byte_range.clone()]; + let mut consumed = 0; + let mut replace = Wtf8Buf::with_capacity(4 * byte_range.len()); + while consumed < 4 && consumed < byte_range.len() { + let c = err_bytes[consumed] as u16; + // Refuse to escape ASCII bytes + if c < 128 { + break; + } + replace.push(CodePoint::from(0xdc00 + c)); + consumed += 1; + } + if consumed == 0 { + return Err(ctx.error_decoding(byte_range, reason)); + } + Ok((ctx.string(replace), byte_range.start + consumed)) + } + } +} + pub mod utf8 { use super::*; pub const ENCODING_NAME: &str = "utf-8"; #[inline] - pub fn encode(s: &Wtf8, errors: &E) -> Result, E::Error> { - encode_utf8_compatible(s, errors, "surrogates not allowed", StrKind::Utf8) + pub fn encode(ctx: Ctx, errors: &E) -> Result, Ctx::Error> + where + Ctx: EncodeContext, + E: EncodeErrorHandler, + { + encode_utf8_compatible(ctx, errors, "surrogates not allowed", StrKind::Utf8) } - pub fn decode( - data: &[u8], + pub fn decode>( + ctx: Ctx, errors: &E, final_decode: bool, - ) -> Result<(Wtf8Buf, usize), E::Error> { + ) -> Result<(Wtf8Buf, usize), Ctx::Error> { decode_utf8_compatible( - data, + ctx, errors, |v| { core::str::from_utf8(v).map_err(|e| { @@ -237,67 +537,55 @@ pub mod latin_1 { const ERR_REASON: &str = "ordinal not in range(256)"; #[inline] - pub fn encode(s: &Wtf8, errors: &E) -> Result, E::Error> { - let full_data = s; - let mut data = s; - let mut char_data_index = 0; + pub fn encode(mut ctx: Ctx, errors: &E) -> Result, Ctx::Error> + where + Ctx: EncodeContext, + E: EncodeErrorHandler, + { let mut out = Vec::::new(); loop { - match data - .code_point_indices() - .enumerate() - .find(|(_, (_, c))| !c.is_ascii()) - { - None => { - out.extend_from_slice(data.as_bytes()); - break; - } - Some((char_i, (byte_i, ch))) => { - out.extend_from_slice(&data.as_bytes()[..byte_i]); - let char_start = char_data_index + char_i; - if let Some(byte) = ch.to_u32().to_u8() { - out.push(byte); - // if the codepoint is between 128..=255, it's utf8-length is 2 - data = &data[byte_i + 2..]; - char_data_index = char_start + 1; - } else { - // number of non-latin_1 chars between the first non-latin_1 char and the next latin_1 char - let non_latin_1_run_length = data[byte_i..] - .code_points() - .take_while(|c| c.to_u32() > 255) - .count(); - let char_range = char_start..char_start + non_latin_1_run_length; - let (replace, char_restart) = errors.handle_encode_error( - full_data, - char_range.clone(), - ERR_REASON, - )?; - match replace { - EncodeReplace::Str(s) => { - if s.as_ref().code_points().any(|c| c.to_u32() > 255) { - return Err( - errors.error_encoding(full_data, char_range, ERR_REASON) - ); - } - out.extend_from_slice(s.as_ref().as_bytes()); - } - EncodeReplace::Bytes(b) => { - out.extend_from_slice(b.as_ref()); - } + let data = ctx.remaining_data(); + let mut iter = iter_code_points(ctx.remaining_data()); + let Some((i, ch)) = iter.find(|(_, c)| !c.is_ascii()) else { + break; + }; + out.extend_from_slice(&data.as_bytes()[..i.bytes]); + let err_start = ctx.position() + i; + if let Some(byte) = ch.to_u32().to_u8() { + drop(iter); + out.push(byte); + // if the codepoint is between 128..=255, it's utf8-length is 2 + ctx.restart_from(err_start + StrSize { bytes: 2, chars: 1 })?; + } else { + // number of non-latin_1 chars between the first non-latin_1 char and the next latin_1 char + let err_end = match { iter }.find(|(_, c)| c.to_u32() <= 255) { + Some((i, _)) => ctx.position() + i, + None => ctx.data_len(), + }; + let err_range = err_start..err_end; + let replace = ctx.handle_error(errors, err_range.clone(), Some(ERR_REASON))?; + match replace { + EncodeReplace::Str(s) => { + if s.as_ref().code_points().any(|c| c.to_u32() > 255) { + return Err(ctx.error_encoding(err_range, Some(ERR_REASON))); } - data = crate::str::try_get_codepoints(full_data, char_restart..) - .ok_or_else(|| errors.error_oob_restart(char_restart))?; - char_data_index = char_restart; + out.extend(s.as_ref().code_points().map(|c| c.to_u32() as u8)); + } + EncodeReplace::Bytes(b) => { + out.extend_from_slice(b.as_ref()); } - continue; } } } + out.extend_from_slice(ctx.remaining_data().as_bytes()); Ok(out) } - pub fn decode(data: &[u8], _errors: &E) -> Result<(Wtf8Buf, usize), E::Error> { - let out: String = data.iter().map(|c| *c as char).collect(); + pub fn decode>( + ctx: Ctx, + _errors: &E, + ) -> Result<(Wtf8Buf, usize), Ctx::Error> { + let out: String = ctx.remaining_data().iter().map(|c| *c as char).collect(); let out_len = out.len(); Ok((out.into(), out_len)) } @@ -312,13 +600,20 @@ pub mod ascii { const ERR_REASON: &str = "ordinal not in range(128)"; #[inline] - pub fn encode(s: &Wtf8, errors: &E) -> Result, E::Error> { - encode_utf8_compatible(s, errors, ERR_REASON, StrKind::Ascii) + pub fn encode(ctx: Ctx, errors: &E) -> Result, Ctx::Error> + where + Ctx: EncodeContext, + E: EncodeErrorHandler, + { + encode_utf8_compatible(ctx, errors, ERR_REASON, StrKind::Ascii) } - pub fn decode(data: &[u8], errors: &E) -> Result<(Wtf8Buf, usize), E::Error> { + pub fn decode>( + ctx: Ctx, + errors: &E, + ) -> Result<(Wtf8Buf, usize), Ctx::Error> { decode_utf8_compatible( - data, + ctx, errors, |v| { AsciiStr::from_ascii(v).map(|s| s.as_str()).map_err(|e| { diff --git a/common/src/str.rs b/common/src/str.rs index 176b5d0f87..8a00dcf1d8 100644 --- a/common/src/str.rs +++ b/common/src/str.rs @@ -446,6 +446,21 @@ pub fn to_ascii(value: &str) -> AsciiString { unsafe { AsciiString::from_ascii_unchecked(ascii) } } +pub struct UnicodeEscapeCodepoint(pub CodePoint); + +impl fmt::Display for UnicodeEscapeCodepoint { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let c = self.0.to_u32(); + if c >= 0x10000 { + write!(f, "\\U{c:08x}") + } else if c >= 0x100 { + write!(f, "\\u{c:04x}") + } else { + write!(f, "\\x{c:02x}") + } + } +} + pub mod levenshtein { use std::{cell::RefCell, thread_local}; diff --git a/common/src/wtf8/mod.rs b/common/src/wtf8/mod.rs index f209b98a3e..f6ae628bad 100644 --- a/common/src/wtf8/mod.rs +++ b/common/src/wtf8/mod.rs @@ -618,6 +618,9 @@ impl ToOwned for Wtf8 { fn to_owned(&self) -> Self::Owned { self.to_wtf8_buf() } + fn clone_into(&self, buf: &mut Self::Owned) { + self.bytes.clone_into(&mut buf.bytes); + } } impl PartialEq for Wtf8 { @@ -872,8 +875,8 @@ impl Wtf8 { } } - pub fn clone_into(&self, buf: &mut Wtf8Buf) { - self.bytes.clone_into(&mut buf.bytes); + pub fn is_code_point_boundary(&self, index: usize) -> bool { + is_code_point_boundary(self, index) } /// Boxes this `Wtf8`. @@ -1101,6 +1104,7 @@ impl ops::Index> for Wtf8 { type Output = Wtf8; #[inline] + #[track_caller] fn index(&self, range: ops::Range) -> &Wtf8 { // is_code_point_boundary checks that the index is in [0, .len()] if range.start <= range.end @@ -1124,6 +1128,7 @@ impl ops::Index> for Wtf8 { type Output = Wtf8; #[inline] + #[track_caller] fn index(&self, range: ops::RangeFrom) -> &Wtf8 { // is_code_point_boundary checks that the index is in [0, .len()] if is_code_point_boundary(self, range.start) { @@ -1144,6 +1149,7 @@ impl ops::Index> for Wtf8 { type Output = Wtf8; #[inline] + #[track_caller] fn index(&self, range: ops::RangeTo) -> &Wtf8 { // is_code_point_boundary checks that the index is in [0, .len()] if is_code_point_boundary(self, range.end) { @@ -1171,7 +1177,7 @@ fn decode_surrogate(second_byte: u8, third_byte: u8) -> u16 { /// Copied from str::is_char_boundary #[inline] -pub fn is_code_point_boundary(slice: &Wtf8, index: usize) -> bool { +fn is_code_point_boundary(slice: &Wtf8, index: usize) -> bool { if index == 0 { return true; } @@ -1226,6 +1232,7 @@ pub unsafe fn slice_unchecked(s: &Wtf8, begin: usize, end: usize) -> &Wtf8 { /// Copied from core::str::raw::slice_error_fail #[inline(never)] +#[track_caller] pub fn slice_error_fail(s: &Wtf8, begin: usize, end: usize) -> ! { assert!(begin <= end); panic!("index {begin} and/or {end} in `{s:?}` do not lie on character boundary"); diff --git a/vm/src/codecs.rs b/vm/src/codecs.rs index bdb9b4b809..8d002916a6 100644 --- a/vm/src/codecs.rs +++ b/vm/src/codecs.rs @@ -1,13 +1,27 @@ -use rustpython_common::wtf8::{CodePoint, Wtf8Buf}; +use rustpython_common::{ + borrow::BorrowedValue, + encodings::{ + CodecContext, DecodeContext, DecodeErrorHandler, EncodeContext, EncodeErrorHandler, + EncodeReplace, StrBuffer, StrSize, errors, + }, + str::StrKind, + wtf8::{CodePoint, Wtf8, Wtf8Buf}, +}; use crate::{ - AsObject, Context, PyObject, PyObjectRef, PyPayload, PyResult, TryFromObject, VirtualMachine, - builtins::{PyBaseExceptionRef, PyBytesRef, PyStr, PyStrRef, PyTuple, PyTupleRef}, + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyResult, TryFromBorrowedObject, + TryFromObject, VirtualMachine, + builtins::{PyBaseExceptionRef, PyBytes, PyBytesRef, PyStr, PyStrRef, PyTuple, PyTupleRef}, common::{ascii, lock::PyRwLock}, convert::ToPyObject, - function::PyMethodDef, + function::{ArgBytesLike, PyMethodDef}, +}; +use once_cell::unsync::OnceCell; +use std::{ + borrow::Cow, + collections::HashMap, + ops::{self, Range}, }; -use std::{borrow::Cow, collections::HashMap, fmt::Write, ops::Range}; pub struct CodecsRegistry { inner: PyRwLock, @@ -360,6 +374,676 @@ fn normalize_encoding_name(encoding: &str) -> Cow<'_, str> { } } +#[derive(Eq, PartialEq)] +enum StandardEncoding { + Utf8, + Utf16Be, + Utf16Le, + Utf32Be, + Utf32Le, +} + +impl StandardEncoding { + #[cfg(target_endian = "little")] + const UTF_16_NE: Self = Self::Utf16Le; + #[cfg(target_endian = "big")] + const UTF_16_NE: Self = Self::Utf16Be; + + #[cfg(target_endian = "little")] + const UTF_32_NE: Self = Self::Utf32Le; + #[cfg(target_endian = "big")] + const UTF_32_NE: Self = Self::Utf32Be; + + fn parse(encoding: &str) -> Option { + if let Some(encoding) = encoding.to_lowercase().strip_prefix("utf") { + let encoding = encoding + .strip_prefix(|c| ['-', '_'].contains(&c)) + .unwrap_or(encoding); + if encoding == "8" { + Some(Self::Utf8) + } else if let Some(encoding) = encoding.strip_prefix("16") { + if encoding.is_empty() { + return Some(Self::UTF_16_NE); + } + let encoding = encoding.strip_prefix(['-', '_']).unwrap_or(encoding); + match encoding { + "be" => Some(Self::Utf16Be), + "le" => Some(Self::Utf16Le), + _ => None, + } + } else if let Some(encoding) = encoding.strip_prefix("32") { + if encoding.is_empty() { + return Some(Self::UTF_32_NE); + } + let encoding = encoding.strip_prefix(['-', '_']).unwrap_or(encoding); + match encoding { + "be" => Some(Self::Utf32Be), + "le" => Some(Self::Utf32Le), + _ => return None, + } + } else { + None + } + } else if encoding == "CP_UTF8" { + Some(Self::Utf8) + } else { + None + } + } +} + +struct SurrogatePass; + +impl<'a> EncodeErrorHandler> for SurrogatePass { + fn handle_encode_error( + &self, + ctx: &mut PyEncodeContext<'a>, + range: Range, + reason: Option<&str>, + ) -> PyResult<(EncodeReplace>, StrSize)> { + let standard_encoding = StandardEncoding::parse(ctx.encoding) + .ok_or_else(|| ctx.error_encoding(range.clone(), reason))?; + let err_str = &ctx.full_data()[range.start.bytes..range.end.bytes]; + let num_chars = range.end.chars - range.start.chars; + let mut out: Vec = Vec::with_capacity(num_chars * 4); + for ch in err_str.code_points() { + let c = ch.to_u32(); + let 0xd800..=0xdfff = c else { + // Not a surrogate, fail with original exception + return Err(ctx.error_encoding(range, reason)); + }; + match standard_encoding { + StandardEncoding::Utf8 => out.extend(ch.encode_wtf8(&mut [0; 4]).as_bytes()), + StandardEncoding::Utf16Le => out.extend((c as u16).to_le_bytes()), + StandardEncoding::Utf16Be => out.extend((c as u16).to_be_bytes()), + StandardEncoding::Utf32Le => out.extend(c.to_le_bytes()), + StandardEncoding::Utf32Be => out.extend(c.to_be_bytes()), + } + } + Ok((EncodeReplace::Bytes(ctx.bytes(out)), range.end)) + } +} + +impl<'a> DecodeErrorHandler> for SurrogatePass { + fn handle_decode_error( + &self, + ctx: &mut PyDecodeContext<'a>, + byte_range: Range, + reason: Option<&str>, + ) -> PyResult<(PyStrRef, usize)> { + let standard_encoding = StandardEncoding::parse(ctx.encoding) + .ok_or_else(|| ctx.error_decoding(byte_range.clone(), reason))?; + + let s = ctx.full_data(); + debug_assert!(byte_range.start <= 0.max(s.len() - 1)); + debug_assert!(byte_range.end >= 1.min(s.len())); + debug_assert!(byte_range.end <= s.len()); + + // Try decoding a single surrogate character. If there are more, + // let the codec call us again. + let p = &s[byte_range.start..]; + + fn slice(p: &[u8]) -> Option<[u8; N]> { + p.get(..N).map(|x| x.try_into().unwrap()) + } + + let c = match standard_encoding { + StandardEncoding::Utf8 => { + // it's a three-byte code + slice::<3>(p) + .filter(|&[a, b, c]| { + (u32::from(a) & 0xf0) == 0xe0 + && (u32::from(b) & 0xc0) == 0x80 + && (u32::from(c) & 0xc0) == 0x80 + }) + .map(|[a, b, c]| { + ((u32::from(a) & 0x0f) << 12) + + ((u32::from(b) & 0x3f) << 6) + + (u32::from(c) & 0x3f) + }) + } + StandardEncoding::Utf16Le => slice(p).map(u16::from_le_bytes).map(u32::from), + StandardEncoding::Utf16Be => slice(p).map(u16::from_be_bytes).map(u32::from), + StandardEncoding::Utf32Le => slice(p).map(u32::from_le_bytes), + StandardEncoding::Utf32Be => slice(p).map(u32::from_be_bytes), + }; + let byte_length = match standard_encoding { + StandardEncoding::Utf8 => 3, + StandardEncoding::Utf16Be | StandardEncoding::Utf16Le => 2, + StandardEncoding::Utf32Be | StandardEncoding::Utf32Le => 4, + }; + + // !Py_UNICODE_IS_SURROGATE + let c = c + .and_then(CodePoint::from_u32) + .filter(|c| matches!(c.to_u32(), 0xd800..=0xdfff)) + .ok_or_else(|| ctx.error_decoding(byte_range.clone(), reason))?; + + Ok((ctx.string(c.into()), byte_range.start + byte_length)) + } +} + +pub struct PyEncodeContext<'a> { + vm: &'a VirtualMachine, + encoding: &'a str, + data: &'a Py, + pos: StrSize, + exception: OnceCell, +} + +impl<'a> PyEncodeContext<'a> { + pub fn new(encoding: &'a str, data: &'a Py, vm: &'a VirtualMachine) -> Self { + Self { + vm, + encoding, + data, + pos: StrSize::default(), + exception: OnceCell::new(), + } + } +} + +impl CodecContext for PyEncodeContext<'_> { + type Error = PyBaseExceptionRef; + type StrBuf = PyStrRef; + type BytesBuf = PyBytesRef; + + fn string(&self, s: Wtf8Buf) -> Self::StrBuf { + self.vm.ctx.new_str(s) + } + + fn bytes(&self, b: Vec) -> Self::BytesBuf { + self.vm.ctx.new_bytes(b) + } +} +impl EncodeContext for PyEncodeContext<'_> { + fn full_data(&self) -> &Wtf8 { + self.data.as_wtf8() + } + + fn data_len(&self) -> StrSize { + StrSize { + bytes: self.data.byte_len(), + chars: self.data.char_len(), + } + } + + fn remaining_data(&self) -> &Wtf8 { + &self.full_data()[self.pos.bytes..] + } + + fn position(&self) -> StrSize { + self.pos + } + + fn restart_from(&mut self, pos: StrSize) -> Result<(), Self::Error> { + if pos.chars > self.data.char_len() { + return Err(self.vm.new_index_error(format!( + "position {} from error handler out of bounds", + pos.chars + ))); + } + assert!( + self.data.as_wtf8().is_code_point_boundary(pos.bytes), + "invalid pos {pos:?} for {:?}", + self.data.as_wtf8() + ); + self.pos = pos; + Ok(()) + } + + fn error_encoding(&self, range: Range, reason: Option<&str>) -> Self::Error { + let vm = self.vm; + match self.exception.get() { + Some(exc) => { + match update_unicode_error_attrs( + exc.as_object(), + range.start.chars, + range.end.chars, + reason, + vm, + ) { + Ok(()) => exc.clone(), + Err(e) => e, + } + } + None => self + .exception + .get_or_init(|| { + let reason = reason.expect( + "should only ever pass reason: None if an exception is already set", + ); + vm.new_unicode_encode_error_real( + vm.ctx.new_str(self.encoding), + self.data.to_owned(), + range.start.chars, + range.end.chars, + vm.ctx.new_str(reason), + ) + }) + .clone(), + } + } +} + +pub struct PyDecodeContext<'a> { + vm: &'a VirtualMachine, + encoding: &'a str, + data: PyDecodeData<'a>, + orig_bytes: Option<&'a Py>, + pos: usize, + exception: OnceCell, +} +enum PyDecodeData<'a> { + Original(BorrowedValue<'a, [u8]>), + Modified(PyBytesRef), +} +impl ops::Deref for PyDecodeData<'_> { + type Target = [u8]; + fn deref(&self) -> &Self::Target { + match self { + PyDecodeData::Original(data) => data, + PyDecodeData::Modified(data) => data, + } + } +} + +impl<'a> PyDecodeContext<'a> { + pub fn new(encoding: &'a str, data: &'a ArgBytesLike, vm: &'a VirtualMachine) -> Self { + Self { + vm, + encoding, + data: PyDecodeData::Original(data.borrow_buf()), + orig_bytes: data.as_object().downcast_ref(), + pos: 0, + exception: OnceCell::new(), + } + } +} + +impl CodecContext for PyDecodeContext<'_> { + type Error = PyBaseExceptionRef; + type StrBuf = PyStrRef; + type BytesBuf = PyBytesRef; + + fn string(&self, s: Wtf8Buf) -> Self::StrBuf { + self.vm.ctx.new_str(s) + } + + fn bytes(&self, b: Vec) -> Self::BytesBuf { + self.vm.ctx.new_bytes(b) + } +} +impl DecodeContext for PyDecodeContext<'_> { + fn full_data(&self) -> &[u8] { + &self.data + } + + fn remaining_data(&self) -> &[u8] { + &self.data[self.pos..] + } + + fn position(&self) -> usize { + self.pos + } + + fn advance(&mut self, by: usize) { + self.pos += by; + } + + fn restart_from(&mut self, pos: usize) -> Result<(), Self::Error> { + if pos > self.data.len() { + return Err(self + .vm + .new_index_error(format!("position {pos} from error handler out of bounds",))); + } + self.pos = pos; + Ok(()) + } + + fn error_decoding(&self, byte_range: Range, reason: Option<&str>) -> Self::Error { + let vm = self.vm; + + match self.exception.get() { + Some(exc) => { + match update_unicode_error_attrs( + exc.as_object(), + byte_range.start, + byte_range.end, + reason, + vm, + ) { + Ok(()) => exc.clone(), + Err(e) => e, + } + } + None => self + .exception + .get_or_init(|| { + let reason = reason.expect( + "should only ever pass reason: None if an exception is already set", + ); + let data = if let Some(bytes) = self.orig_bytes { + bytes.to_owned() + } else { + vm.ctx.new_bytes(self.data.to_vec()) + }; + vm.new_unicode_decode_error_real( + vm.ctx.new_str(self.encoding), + data, + byte_range.start, + byte_range.end, + vm.ctx.new_str(reason), + ) + }) + .clone(), + } + } +} + +#[derive(strum_macros::EnumString)] +#[strum(serialize_all = "lowercase")] +enum StandardError { + Strict, + Ignore, + Replace, + XmlCharRefReplace, + BackslashReplace, + SurrogatePass, + SurrogateEscape, +} + +impl<'a> EncodeErrorHandler> for StandardError { + fn handle_encode_error( + &self, + ctx: &mut PyEncodeContext<'a>, + range: Range, + reason: Option<&str>, + ) -> PyResult<(EncodeReplace>, StrSize)> { + use StandardError::*; + // use errors::*; + match self { + Strict => errors::Strict.handle_encode_error(ctx, range, reason), + Ignore => errors::Ignore.handle_encode_error(ctx, range, reason), + Replace => errors::Replace.handle_encode_error(ctx, range, reason), + XmlCharRefReplace => errors::XmlCharRefReplace.handle_encode_error(ctx, range, reason), + BackslashReplace => errors::BackslashReplace.handle_encode_error(ctx, range, reason), + SurrogatePass => SurrogatePass.handle_encode_error(ctx, range, reason), + SurrogateEscape => errors::SurrogateEscape.handle_encode_error(ctx, range, reason), + } + } +} + +impl<'a> DecodeErrorHandler> for StandardError { + fn handle_decode_error( + &self, + ctx: &mut PyDecodeContext<'a>, + byte_range: Range, + reason: Option<&str>, + ) -> PyResult<(PyStrRef, usize)> { + use StandardError::*; + match self { + Strict => errors::Strict.handle_decode_error(ctx, byte_range, reason), + Ignore => errors::Ignore.handle_decode_error(ctx, byte_range, reason), + Replace => errors::Replace.handle_decode_error(ctx, byte_range, reason), + XmlCharRefReplace => Err(ctx.vm.new_type_error( + "don't know how to handle UnicodeDecodeError in error callback".to_owned(), + )), + BackslashReplace => { + errors::BackslashReplace.handle_decode_error(ctx, byte_range, reason) + } + SurrogatePass => self::SurrogatePass.handle_decode_error(ctx, byte_range, reason), + SurrogateEscape => errors::SurrogateEscape.handle_decode_error(ctx, byte_range, reason), + } + } +} + +pub struct ErrorsHandler<'a> { + errors: &'a Py, + resolved: OnceCell, +} +enum ResolvedError { + Standard(StandardError), + Handler(PyObjectRef), +} + +impl<'a> ErrorsHandler<'a> { + #[inline] + pub fn new(errors: Option<&'a Py>, vm: &VirtualMachine) -> Self { + match errors { + Some(errors) => Self { + errors, + resolved: OnceCell::new(), + }, + None => Self { + errors: identifier!(vm, strict).as_ref(), + resolved: OnceCell::with_value(ResolvedError::Standard(StandardError::Strict)), + }, + } + } + #[inline] + fn resolve(&self, vm: &VirtualMachine) -> PyResult<&ResolvedError> { + self.resolved.get_or_try_init(|| { + if let Ok(standard) = self.errors.as_str().parse() { + Ok(ResolvedError::Standard(standard)) + } else { + vm.state + .codec_registry + .lookup_error(self.errors.as_str(), vm) + .map(ResolvedError::Handler) + } + }) + } +} +impl StrBuffer for PyStrRef { + fn is_compatible_with(&self, kind: StrKind) -> bool { + self.kind() <= kind + } +} +impl<'a> EncodeErrorHandler> for ErrorsHandler<'_> { + fn handle_encode_error( + &self, + ctx: &mut PyEncodeContext<'a>, + range: Range, + reason: Option<&str>, + ) -> PyResult<(EncodeReplace>, StrSize)> { + let vm = ctx.vm; + let handler = match self.resolve(vm)? { + ResolvedError::Standard(standard) => { + return standard.handle_encode_error(ctx, range, reason); + } + ResolvedError::Handler(handler) => handler, + }; + let encode_exc = ctx.error_encoding(range.clone(), reason); + let res = handler.call((encode_exc.clone(),), vm)?; + let tuple_err = || { + vm.new_type_error( + "encoding error handler must return (str/bytes, int) tuple".to_owned(), + ) + }; + let (replace, restart) = match res.payload::().map(|tup| tup.as_slice()) { + Some([replace, restart]) => (replace.clone(), restart), + _ => return Err(tuple_err()), + }; + let replace = match_class!(match replace { + s @ PyStr => EncodeReplace::Str(s), + b @ PyBytes => EncodeReplace::Bytes(b), + _ => return Err(tuple_err()), + }); + let restart = isize::try_from_borrowed_object(vm, restart).map_err(|_| tuple_err())?; + let restart = if restart < 0 { + // will still be out of bounds if it underflows ¯\_(ツ)_/¯ + ctx.data.char_len().wrapping_sub(restart.unsigned_abs()) + } else { + restart as usize + }; + let restart = if restart == range.end.chars { + range.end + } else { + StrSize { + chars: restart, + bytes: ctx + .data + .as_wtf8() + .code_point_indices() + .nth(restart) + .map_or(ctx.data.byte_len(), |(i, _)| i), + } + }; + Ok((replace, restart)) + } +} +impl<'a> DecodeErrorHandler> for ErrorsHandler<'_> { + fn handle_decode_error( + &self, + ctx: &mut PyDecodeContext<'a>, + byte_range: Range, + reason: Option<&str>, + ) -> PyResult<(PyStrRef, usize)> { + let vm = ctx.vm; + let handler = match self.resolve(vm)? { + ResolvedError::Standard(standard) => { + return standard.handle_decode_error(ctx, byte_range, reason); + } + ResolvedError::Handler(handler) => handler, + }; + let decode_exc = ctx.error_decoding(byte_range.clone(), reason); + let data_bytes: PyObjectRef = decode_exc.as_object().get_attr("object", vm)?; + let res = handler.call((decode_exc.clone(),), vm)?; + let new_data = decode_exc.as_object().get_attr("object", vm)?; + if !new_data.is(&data_bytes) { + let new_data: PyBytesRef = new_data + .downcast() + .map_err(|_| vm.new_type_error("object attribute must be bytes".to_owned()))?; + ctx.data = PyDecodeData::Modified(new_data); + } + let data = &*ctx.data; + let tuple_err = + || vm.new_type_error("decoding error handler must return (str, int) tuple".to_owned()); + match res.payload::().map(|tup| tup.as_slice()) { + Some([replace, restart]) => { + let replace = replace + .downcast_ref::() + .ok_or_else(tuple_err)? + .to_owned(); + let restart = + isize::try_from_borrowed_object(vm, restart).map_err(|_| tuple_err())?; + let restart = if restart < 0 { + // will still be out of bounds if it underflows ¯\_(ツ)_/¯ + data.len().wrapping_sub(restart.unsigned_abs()) + } else { + restart as usize + }; + Ok((replace, restart)) + } + _ => Err(tuple_err()), + } + } +} + +fn call_native_encode_error( + handler: E, + err: PyObjectRef, + vm: &VirtualMachine, +) -> PyResult<(PyObjectRef, usize)> +where + for<'a> E: EncodeErrorHandler>, +{ + // let err = err. + let range = extract_unicode_error_range(&err, vm)?; + let s = PyStrRef::try_from_object(vm, err.get_attr("object", vm)?)?; + let s_encoding = PyStrRef::try_from_object(vm, err.get_attr("encoding", vm)?)?; + let mut ctx = PyEncodeContext { + vm, + encoding: s_encoding.as_str(), + data: &s, + pos: StrSize::default(), + exception: OnceCell::with_value(err.downcast().unwrap()), + }; + let mut iter = s.as_wtf8().code_point_indices(); + let start = StrSize { + chars: range.start, + bytes: iter.nth(range.start).unwrap().0, + }; + let end = StrSize { + chars: range.end, + bytes: if let Some(n) = range.len().checked_sub(1) { + iter.nth(n).map_or(s.byte_len(), |(i, _)| i) + } else { + start.bytes + }, + }; + let (replace, restart) = handler.handle_encode_error(&mut ctx, start..end, None)?; + let replace = match replace { + EncodeReplace::Str(s) => s.into(), + EncodeReplace::Bytes(b) => b.into(), + }; + Ok((replace, restart.chars)) +} + +fn call_native_decode_error( + handler: E, + err: PyObjectRef, + vm: &VirtualMachine, +) -> PyResult<(PyObjectRef, usize)> +where + for<'a> E: DecodeErrorHandler>, +{ + let range = extract_unicode_error_range(&err, vm)?; + let s = ArgBytesLike::try_from_object(vm, err.get_attr("object", vm)?)?; + let s_encoding = PyStrRef::try_from_object(vm, err.get_attr("encoding", vm)?)?; + let mut ctx = PyDecodeContext { + vm, + encoding: s_encoding.as_str(), + data: PyDecodeData::Original(s.borrow_buf()), + orig_bytes: s.as_object().downcast_ref(), + pos: 0, + exception: OnceCell::with_value(err.downcast().unwrap()), + }; + let (replace, restart) = handler.handle_decode_error(&mut ctx, range, None)?; + Ok((replace.into(), restart)) +} + +// this is a hack, for now +fn call_native_translate_error( + handler: E, + err: PyObjectRef, + vm: &VirtualMachine, +) -> PyResult<(PyObjectRef, usize)> +where + for<'a> E: EncodeErrorHandler>, +{ + // let err = err. + let range = extract_unicode_error_range(&err, vm)?; + let s = PyStrRef::try_from_object(vm, err.get_attr("object", vm)?)?; + let mut ctx = PyEncodeContext { + vm, + encoding: "", + data: &s, + pos: StrSize::default(), + exception: OnceCell::with_value(err.downcast().unwrap()), + }; + let mut iter = s.as_wtf8().code_point_indices(); + let start = StrSize { + chars: range.start, + bytes: iter.nth(range.start).unwrap().0, + }; + let end = StrSize { + chars: range.end, + bytes: if let Some(n) = range.len().checked_sub(1) { + iter.nth(n).map_or(s.byte_len(), |(i, _)| i) + } else { + start.bytes + }, + }; + let (replace, restart) = handler.handle_encode_error(&mut ctx, start..end, None)?; + let replace = match replace { + EncodeReplace::Str(s) => s.into(), + EncodeReplace::Bytes(b) => b.into(), + }; + Ok((replace, restart.chars)) +} + // TODO: exceptions with custom payloads fn extract_unicode_error_range(err: &PyObject, vm: &VirtualMachine) -> PyResult> { let start = err.get_attr("start", vm)?; @@ -369,14 +1053,32 @@ fn extract_unicode_error_range(err: &PyObject, vm: &VirtualMachine) -> PyResult< Ok(Range { start, end }) } +fn update_unicode_error_attrs( + err: &PyObject, + start: usize, + end: usize, + reason: Option<&str>, + vm: &VirtualMachine, +) -> PyResult<()> { + err.set_attr("start", start.to_pyobject(vm), vm)?; + err.set_attr("end", end.to_pyobject(vm), vm)?; + if let Some(reason) = reason { + err.set_attr("reason", reason.to_pyobject(vm), vm)?; + } + Ok(()) +} + +#[inline] +fn is_encode_err(err: &PyObject, vm: &VirtualMachine) -> bool { + err.fast_isinstance(vm.ctx.exceptions.unicode_encode_error) +} #[inline] fn is_decode_err(err: &PyObject, vm: &VirtualMachine) -> bool { err.fast_isinstance(vm.ctx.exceptions.unicode_decode_error) } #[inline] -fn is_encode_ish_err(err: &PyObject, vm: &VirtualMachine) -> bool { - err.fast_isinstance(vm.ctx.exceptions.unicode_encode_error) - || err.fast_isinstance(vm.ctx.exceptions.unicode_translate_error) +fn is_translate_err(err: &PyObject, vm: &VirtualMachine) -> bool { + err.fast_isinstance(vm.ctx.exceptions.unicode_translate_error) } fn bad_err_type(err: PyObjectRef, vm: &VirtualMachine) -> PyBaseExceptionRef { @@ -394,7 +1096,7 @@ fn strict_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult { } fn ignore_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<(PyObjectRef, usize)> { - if is_encode_ish_err(&err, vm) || is_decode_err(&err, vm) { + if is_encode_err(&err, vm) || is_decode_err(&err, vm) || is_translate_err(&err, vm) { let range = extract_unicode_error_range(&err, vm)?; Ok((vm.ctx.new_str(ascii!("")).into(), range.end)) } else { @@ -402,325 +1104,71 @@ fn ignore_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<(PyObjectRef } } -fn replace_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<(String, usize)> { - // char::REPLACEMENT_CHARACTER as a str - let replacement_char = "\u{FFFD}"; - let replace = if err.fast_isinstance(vm.ctx.exceptions.unicode_encode_error) { - "?" - } else if err.fast_isinstance(vm.ctx.exceptions.unicode_decode_error) { +fn replace_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<(PyObjectRef, usize)> { + if is_encode_err(&err, vm) { + call_native_encode_error(errors::Replace, err, vm) + } else if is_decode_err(&err, vm) { + call_native_decode_error(errors::Replace, err, vm) + } else if is_translate_err(&err, vm) { + // char::REPLACEMENT_CHARACTER as a str + let replacement_char = "\u{FFFD}"; let range = extract_unicode_error_range(&err, vm)?; - return Ok((replacement_char.to_owned(), range.end)); - } else if err.fast_isinstance(vm.ctx.exceptions.unicode_translate_error) { - replacement_char + let replace = replacement_char.repeat(range.end - range.start); + Ok((replace.to_pyobject(vm), range.end)) } else { return Err(bad_err_type(err, vm)); - }; - let range = extract_unicode_error_range(&err, vm)?; - let replace = replace.repeat(range.end - range.start); - Ok((replace, range.end)) -} - -fn xmlcharrefreplace_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<(String, usize)> { - if !is_encode_ish_err(&err, vm) { - return Err(bad_err_type(err, vm)); - } - let range = extract_unicode_error_range(&err, vm)?; - let s = PyStrRef::try_from_object(vm, err.get_attr("object", vm)?)?; - let s_after_start = - crate::common::str::try_get_codepoints(s.as_wtf8(), range.start..).unwrap_or_default(); - let num_chars = range.len(); - // capacity rough guess; assuming that the codepoints are 3 digits in decimal + the &#; - let mut out = String::with_capacity(num_chars * 6); - for c in s_after_start.code_points().take(num_chars) { - write!(out, "&#{};", c.to_u32()).unwrap() } - Ok((out, range.end)) } -fn backslashreplace_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<(String, usize)> { - if is_decode_err(&err, vm) { - let range = extract_unicode_error_range(&err, vm)?; - let b = PyBytesRef::try_from_object(vm, err.get_attr("object", vm)?)?; - let mut replace = String::with_capacity(4 * range.len()); - for &c in &b[range.clone()] { - write!(replace, "\\x{c:02x}").unwrap(); - } - return Ok((replace, range.end)); - } else if !is_encode_ish_err(&err, vm) { - return Err(bad_err_type(err, vm)); - } - let range = extract_unicode_error_range(&err, vm)?; - let s = PyStrRef::try_from_object(vm, err.get_attr("object", vm)?)?; - let s_after_start = - crate::common::str::try_get_codepoints(s.as_wtf8(), range.start..).unwrap_or_default(); - let num_chars = range.len(); - // minimum 4 output bytes per char: \xNN - let mut out = String::with_capacity(num_chars * 4); - for c in s_after_start.code_points().take(num_chars) { - let c = c.to_u32(); - if c >= 0x10000 { - write!(out, "\\U{c:08x}").unwrap(); - } else if c >= 0x100 { - write!(out, "\\u{c:04x}").unwrap(); - } else { - write!(out, "\\x{c:02x}").unwrap(); - } +fn xmlcharrefreplace_errors( + err: PyObjectRef, + vm: &VirtualMachine, +) -> PyResult<(PyObjectRef, usize)> { + if is_encode_err(&err, vm) { + call_native_encode_error(errors::XmlCharRefReplace, err, vm) + } else { + Err(bad_err_type(err, vm)) } - Ok((out, range.end)) } -fn namereplace_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<(String, usize)> { - if err.fast_isinstance(vm.ctx.exceptions.unicode_encode_error) { - let range = extract_unicode_error_range(&err, vm)?; - let s = PyStrRef::try_from_object(vm, err.get_attr("object", vm)?)?; - let s_after_start = - crate::common::str::try_get_codepoints(s.as_wtf8(), range.start..).unwrap_or_default(); - let num_chars = range.len(); - let mut out = String::with_capacity(num_chars * 4); - for c in s_after_start.code_points().take(num_chars) { - let c_u32 = c.to_u32(); - if let Some(c_name) = unicode_names2::name(c.to_char_lossy()) { - write!(out, "\\N{{{c_name}}}").unwrap(); - } else if c_u32 >= 0x10000 { - write!(out, "\\U{c_u32:08x}").unwrap(); - } else if c_u32 >= 0x100 { - write!(out, "\\u{c_u32:04x}").unwrap(); - } else { - write!(out, "\\x{c_u32:02x}").unwrap(); - } - } - Ok((out, range.end)) +fn backslashreplace_errors( + err: PyObjectRef, + vm: &VirtualMachine, +) -> PyResult<(PyObjectRef, usize)> { + if is_decode_err(&err, vm) { + call_native_decode_error(errors::BackslashReplace, err, vm) + } else if is_encode_err(&err, vm) { + call_native_encode_error(errors::BackslashReplace, err, vm) + } else if is_translate_err(&err, vm) { + call_native_translate_error(errors::BackslashReplace, err, vm) } else { Err(bad_err_type(err, vm)) } } -#[derive(Eq, PartialEq)] -enum StandardEncoding { - Utf8, - Utf16Be, - Utf16Le, - Utf32Be, - Utf32Le, - Unknown, -} - -fn get_standard_encoding(encoding: &str) -> (usize, StandardEncoding) { - if let Some(encoding) = encoding.to_lowercase().strip_prefix("utf") { - let mut byte_length: usize = 0; - let mut standard_encoding = StandardEncoding::Unknown; - let encoding = encoding - .strip_prefix(|c| ['-', '_'].contains(&c)) - .unwrap_or(encoding); - if encoding == "8" { - byte_length = 3; - standard_encoding = StandardEncoding::Utf8; - } else if let Some(encoding) = encoding.strip_prefix("16") { - byte_length = 2; - if encoding.is_empty() { - if cfg!(target_endian = "little") { - standard_encoding = StandardEncoding::Utf16Le; - } else if cfg!(target_endian = "big") { - standard_encoding = StandardEncoding::Utf16Be; - } - if standard_encoding != StandardEncoding::Unknown { - return (byte_length, standard_encoding); - } - } - let encoding = encoding - .strip_prefix(|c| ['-', '_'].contains(&c)) - .unwrap_or(encoding); - standard_encoding = match encoding { - "be" => StandardEncoding::Utf16Be, - "le" => StandardEncoding::Utf16Le, - _ => StandardEncoding::Unknown, - } - } else if let Some(encoding) = encoding.strip_prefix("32") { - byte_length = 4; - if encoding.is_empty() { - if cfg!(target_endian = "little") { - standard_encoding = StandardEncoding::Utf32Le; - } else if cfg!(target_endian = "big") { - standard_encoding = StandardEncoding::Utf32Be; - } - if standard_encoding != StandardEncoding::Unknown { - return (byte_length, standard_encoding); - } - } - let encoding = encoding - .strip_prefix(|c| ['-', '_'].contains(&c)) - .unwrap_or(encoding); - standard_encoding = match encoding { - "be" => StandardEncoding::Utf32Be, - "le" => StandardEncoding::Utf32Le, - _ => StandardEncoding::Unknown, - } - } - return (byte_length, standard_encoding); - } else if encoding == "CP_UTF8" { - return (3, StandardEncoding::Utf8); +fn namereplace_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<(PyObjectRef, usize)> { + if is_encode_err(&err, vm) { + call_native_encode_error(errors::NameReplace, err, vm) + } else { + Err(bad_err_type(err, vm)) } - (0, StandardEncoding::Unknown) } fn surrogatepass_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<(PyObjectRef, usize)> { - if err.fast_isinstance(vm.ctx.exceptions.unicode_encode_error) { - let range = extract_unicode_error_range(&err, vm)?; - let s = PyStrRef::try_from_object(vm, err.get_attr("object", vm)?)?; - let s_encoding = PyStrRef::try_from_object(vm, err.get_attr("encoding", vm)?)?; - let (_, standard_encoding) = get_standard_encoding(s_encoding.as_str()); - if let StandardEncoding::Unknown = standard_encoding { - // Not supported, fail with original exception - return Err(err.downcast().unwrap()); - } - let s_after_start = - crate::common::str::try_get_codepoints(s.as_wtf8(), range.start..).unwrap_or_default(); - let num_chars = range.len(); - let mut out: Vec = Vec::with_capacity(num_chars * 4); - for c in s_after_start.code_points().take(num_chars) { - let c = c.to_u32(); - if !(0xd800..=0xdfff).contains(&c) { - // Not a surrogate, fail with original exception - return Err(err.downcast().unwrap()); - } - match standard_encoding { - StandardEncoding::Utf8 => { - out.push((0xe0 | (c >> 12)) as u8); - out.push((0x80 | ((c >> 6) & 0x3f)) as u8); - out.push((0x80 | (c & 0x3f)) as u8); - } - StandardEncoding::Utf16Le => { - out.push(c as u8); - out.push((c >> 8) as u8); - } - StandardEncoding::Utf16Be => { - out.push((c >> 8) as u8); - out.push(c as u8); - } - StandardEncoding::Utf32Le => { - out.push(c as u8); - out.push((c >> 8) as u8); - out.push((c >> 16) as u8); - out.push((c >> 24) as u8); - } - StandardEncoding::Utf32Be => { - out.push((c >> 24) as u8); - out.push((c >> 16) as u8); - out.push((c >> 8) as u8); - out.push(c as u8); - } - StandardEncoding::Unknown => { - unreachable!("NOTE: RUSTPYTHON, should've bailed out earlier") - } - } - } - Ok((vm.ctx.new_bytes(out).into(), range.end)) + if is_encode_err(&err, vm) { + call_native_encode_error(SurrogatePass, err, vm) } else if is_decode_err(&err, vm) { - let range = extract_unicode_error_range(&err, vm)?; - let s = PyBytesRef::try_from_object(vm, err.get_attr("object", vm)?)?; - let s_encoding = PyStrRef::try_from_object(vm, err.get_attr("encoding", vm)?)?; - let (byte_length, standard_encoding) = get_standard_encoding(s_encoding.as_str()); - if let StandardEncoding::Unknown = standard_encoding { - // Not supported, fail with original exception - return Err(err.downcast().unwrap()); - } - - debug_assert!(range.start <= 0.max(s.len() - 1)); - debug_assert!(range.end >= 1.min(s.len())); - debug_assert!(range.end <= s.len()); - - let mut c: u32 = 0; - // Try decoding a single surrogate character. If there are more, - // let the codec call us again. - let p = &s.as_bytes()[range.start..]; - if p.len().overflowing_sub(range.start).0 >= byte_length { - match standard_encoding { - StandardEncoding::Utf8 => { - if (p[0] as u32 & 0xf0) == 0xe0 - && (p[1] as u32 & 0xc0) == 0x80 - && (p[2] as u32 & 0xc0) == 0x80 - { - // it's a three-byte code - c = ((p[0] as u32 & 0x0f) << 12) - + ((p[1] as u32 & 0x3f) << 6) - + (p[2] as u32 & 0x3f); - } - } - StandardEncoding::Utf16Le => { - c = ((p[1] as u32) << 8) | p[0] as u32; - } - StandardEncoding::Utf16Be => { - c = ((p[0] as u32) << 8) | p[1] as u32; - } - StandardEncoding::Utf32Le => { - c = ((p[3] as u32) << 24) - | ((p[2] as u32) << 16) - | ((p[1] as u32) << 8) - | p[0] as u32; - } - StandardEncoding::Utf32Be => { - c = ((p[0] as u32) << 24) - | ((p[1] as u32) << 16) - | ((p[2] as u32) << 8) - | p[3] as u32; - } - StandardEncoding::Unknown => { - unreachable!("NOTE: RUSTPYTHON, should've bailed out earlier") - } - } - } - // !Py_UNICODE_IS_SURROGATE - if !(0xd800..=0xdfff).contains(&c) { - // Not a surrogate, fail with original exception - return Err(err.downcast().unwrap()); - } - - Ok(( - vm.new_pyobj(CodePoint::from_u32(c).unwrap()), - range.start + byte_length, - )) + call_native_decode_error(SurrogatePass, err, vm) } else { Err(bad_err_type(err, vm)) } } fn surrogateescape_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<(PyObjectRef, usize)> { - if err.fast_isinstance(vm.ctx.exceptions.unicode_encode_error) { - let range = extract_unicode_error_range(&err, vm)?; - let object = PyStrRef::try_from_object(vm, err.get_attr("object", vm)?)?; - let s_after_start = crate::common::str::try_get_codepoints(object.as_wtf8(), range.start..) - .unwrap_or_default(); - let mut out: Vec = Vec::with_capacity(range.len()); - for ch in s_after_start.code_points().take(range.len()) { - let ch = ch.to_u32(); - if !(0xdc80..=0xdcff).contains(&ch) { - // Not a UTF-8b surrogate, fail with original exception - return Err(err.downcast().unwrap()); - } - out.push((ch - 0xdc00) as u8); - } - let out = vm.ctx.new_bytes(out); - Ok((out.into(), range.end)) + if is_encode_err(&err, vm) { + call_native_encode_error(errors::SurrogateEscape, err, vm) } else if is_decode_err(&err, vm) { - let range = extract_unicode_error_range(&err, vm)?; - let object = err.get_attr("object", vm)?; - let object = PyBytesRef::try_from_object(vm, object)?; - let p = &object.as_bytes()[range.clone()]; - let mut consumed = 0; - let mut replace = Wtf8Buf::with_capacity(4 * range.len()); - while consumed < 4 && consumed < range.len() { - let c = p[consumed] as u16; - // Refuse to escape ASCII bytes - if c < 128 { - break; - } - replace.push(CodePoint::from(0xdc00 + c)); - consumed += 1; - } - if consumed == 0 { - return Err(err.downcast().unwrap()); - } - Ok((vm.new_pyobj(replace), range.start + consumed)) + call_native_decode_error(errors::SurrogateEscape, err, vm) } else { Err(bad_err_type(err, vm)) } diff --git a/vm/src/exceptions.rs b/vm/src/exceptions.rs index 9bf7372794..58f2a51b68 100644 --- a/vm/src/exceptions.rs +++ b/vm/src/exceptions.rs @@ -971,22 +971,10 @@ impl ExceptionZoo { extend_exception!(PySystemError, ctx, excs.system_error); extend_exception!(PyTypeError, ctx, excs.type_error); extend_exception!(PyValueError, ctx, excs.value_error); - extend_exception!(PyUnicodeError, ctx, excs.unicode_error, { - "encoding" => ctx.new_readonly_getset("encoding", excs.unicode_error, make_arg_getter(0)), - "object" => ctx.new_readonly_getset("object", excs.unicode_error, make_arg_getter(1)), - "start" => ctx.new_readonly_getset("start", excs.unicode_error, make_arg_getter(2)), - "end" => ctx.new_readonly_getset("end", excs.unicode_error, make_arg_getter(3)), - "reason" => ctx.new_readonly_getset("reason", excs.unicode_error, make_arg_getter(4)), - }); + extend_exception!(PyUnicodeError, ctx, excs.unicode_error); extend_exception!(PyUnicodeDecodeError, ctx, excs.unicode_decode_error); extend_exception!(PyUnicodeEncodeError, ctx, excs.unicode_encode_error); - extend_exception!(PyUnicodeTranslateError, ctx, excs.unicode_translate_error, { - "encoding" => ctx.new_readonly_getset("encoding", excs.unicode_translate_error, none_getter), - "object" => ctx.new_readonly_getset("object", excs.unicode_translate_error, make_arg_getter(0)), - "start" => ctx.new_readonly_getset("start", excs.unicode_translate_error, make_arg_getter(1)), - "end" => ctx.new_readonly_getset("end", excs.unicode_translate_error, make_arg_getter(2)), - "reason" => ctx.new_readonly_getset("reason", excs.unicode_translate_error, make_arg_getter(3)), - }); + extend_exception!(PyUnicodeTranslateError, ctx, excs.unicode_translate_error); #[cfg(feature = "jit")] extend_exception!(PyJitError, ctx, excs.jit_error); @@ -1010,10 +998,6 @@ impl ExceptionZoo { } } -fn none_getter(_obj: PyObjectRef, vm: &VirtualMachine) -> PyRef { - vm.ctx.none.clone() -} - fn make_arg_getter(idx: usize) -> impl Fn(PyBaseExceptionRef) -> Option { move |exc| exc.get_arg(idx) } @@ -1182,11 +1166,12 @@ pub(super) mod types { PyInt, PyStrRef, PyTupleRef, PyTypeRef, traceback::PyTracebackRef, tuple::IntoPyTuple, }, convert::ToPyResult, - function::FuncArgs, + function::{ArgBytesLike, FuncArgs}, types::{Constructor, Initializer}, }; use crossbeam_utils::atomic::AtomicCell; use itertools::Itertools; + use rustpython_common::str::UnicodeEscapeCodepoint; // This module is designed to be used as `use builtins::*;`. // Do not add any pub symbols not included in builtins module. @@ -1662,18 +1647,153 @@ pub(super) mod types { #[derive(Debug)] pub struct PyUnicodeError {} - #[pyexception(name, base = "PyUnicodeError", ctx = "unicode_decode_error", impl)] + #[pyexception(name, base = "PyUnicodeError", ctx = "unicode_decode_error")] #[derive(Debug)] pub struct PyUnicodeDecodeError {} - #[pyexception(name, base = "PyUnicodeError", ctx = "unicode_encode_error", impl)] + #[pyexception] + impl PyUnicodeDecodeError { + #[pyslot] + #[pymethod(name = "__init__")] + pub(crate) fn slot_init( + zelf: PyObjectRef, + args: FuncArgs, + vm: &VirtualMachine, + ) -> PyResult<()> { + type Args = (PyStrRef, ArgBytesLike, isize, isize, PyStrRef); + let (encoding, object, start, end, reason): Args = args.bind(vm)?; + zelf.set_attr("encoding", encoding, vm)?; + zelf.set_attr("object", object, vm)?; + zelf.set_attr("start", vm.ctx.new_int(start), vm)?; + zelf.set_attr("end", vm.ctx.new_int(end), vm)?; + zelf.set_attr("reason", reason, vm)?; + Ok(()) + } + + #[pymethod(magic)] + fn str(exc: PyBaseExceptionRef, vm: &VirtualMachine) -> PyResult { + let Ok(object) = exc.as_object().get_attr("object", vm) else { + return Ok("".to_owned()); + }; + let object: ArgBytesLike = object.try_into_value(vm)?; + let encoding: PyStrRef = exc + .as_object() + .get_attr("encoding", vm)? + .try_into_value(vm)?; + let start: usize = exc.as_object().get_attr("start", vm)?.try_into_value(vm)?; + let end: usize = exc.as_object().get_attr("end", vm)?.try_into_value(vm)?; + let reason: PyStrRef = exc.as_object().get_attr("reason", vm)?.try_into_value(vm)?; + if start < object.len() && end <= object.len() && end == start + 1 { + let b = object.borrow_buf()[start]; + Ok(format!( + "'{encoding}' codec can't decode byte {b:#02x} in position {start}: {reason}" + )) + } else { + Ok(format!( + "'{encoding}' codec can't decode bytes in position {start}-{}: {reason}", + end - 1, + )) + } + } + } + + #[pyexception(name, base = "PyUnicodeError", ctx = "unicode_encode_error")] #[derive(Debug)] pub struct PyUnicodeEncodeError {} - #[pyexception(name, base = "PyUnicodeError", ctx = "unicode_translate_error", impl)] + #[pyexception] + impl PyUnicodeEncodeError { + #[pyslot] + #[pymethod(name = "__init__")] + pub(crate) fn slot_init( + zelf: PyObjectRef, + args: FuncArgs, + vm: &VirtualMachine, + ) -> PyResult<()> { + type Args = (PyStrRef, PyStrRef, isize, isize, PyStrRef); + let (encoding, object, start, end, reason): Args = args.bind(vm)?; + zelf.set_attr("encoding", encoding, vm)?; + zelf.set_attr("object", object, vm)?; + zelf.set_attr("start", vm.ctx.new_int(start), vm)?; + zelf.set_attr("end", vm.ctx.new_int(end), vm)?; + zelf.set_attr("reason", reason, vm)?; + Ok(()) + } + + #[pymethod(magic)] + fn str(exc: PyBaseExceptionRef, vm: &VirtualMachine) -> PyResult { + let Ok(object) = exc.as_object().get_attr("object", vm) else { + return Ok("".to_owned()); + }; + let object: PyStrRef = object.try_into_value(vm)?; + let encoding: PyStrRef = exc + .as_object() + .get_attr("encoding", vm)? + .try_into_value(vm)?; + let start: usize = exc.as_object().get_attr("start", vm)?.try_into_value(vm)?; + let end: usize = exc.as_object().get_attr("end", vm)?.try_into_value(vm)?; + let reason: PyStrRef = exc.as_object().get_attr("reason", vm)?.try_into_value(vm)?; + if start < object.char_len() && end <= object.char_len() && end == start + 1 { + let ch = object.as_wtf8().code_points().nth(start).unwrap(); + Ok(format!( + "'{encoding}' codec can't encode character '{}' in position {start}: {reason}", + UnicodeEscapeCodepoint(ch) + )) + } else { + Ok(format!( + "'{encoding}' codec can't encode characters in position {start}-{}: {reason}", + end - 1, + )) + } + } + } + + #[pyexception(name, base = "PyUnicodeError", ctx = "unicode_translate_error")] #[derive(Debug)] pub struct PyUnicodeTranslateError {} + #[pyexception] + impl PyUnicodeTranslateError { + #[pyslot] + #[pymethod(name = "__init__")] + pub(crate) fn slot_init( + zelf: PyObjectRef, + args: FuncArgs, + vm: &VirtualMachine, + ) -> PyResult<()> { + type Args = (PyStrRef, isize, isize, PyStrRef); + let (object, start, end, reason): Args = args.bind(vm)?; + zelf.set_attr("object", object, vm)?; + zelf.set_attr("start", vm.ctx.new_int(start), vm)?; + zelf.set_attr("end", vm.ctx.new_int(end), vm)?; + zelf.set_attr("reason", reason, vm)?; + Ok(()) + } + + #[pymethod(magic)] + fn str(exc: PyBaseExceptionRef, vm: &VirtualMachine) -> PyResult { + let Ok(object) = exc.as_object().get_attr("object", vm) else { + return Ok("".to_owned()); + }; + let object: PyStrRef = object.try_into_value(vm)?; + let start: usize = exc.as_object().get_attr("start", vm)?.try_into_value(vm)?; + let end: usize = exc.as_object().get_attr("end", vm)?.try_into_value(vm)?; + let reason: PyStrRef = exc.as_object().get_attr("reason", vm)?.try_into_value(vm)?; + if start < object.char_len() && end <= object.char_len() && end == start + 1 { + let ch = object.as_wtf8().code_points().nth(start).unwrap(); + Ok(format!( + "can't translate character '{}' in position {start}: {reason}", + UnicodeEscapeCodepoint(ch) + )) + } else { + Ok(format!( + "can't translate characters in position {start}-{}: {reason}", + end - 1, + )) + } + } + } + /// JIT error. #[cfg(feature = "jit")] #[pyexception(name, base = "PyException", ctx = "jit_error", impl)] diff --git a/vm/src/function/buffer.rs b/vm/src/function/buffer.rs index 80b36833e5..40a0e04d7e 100644 --- a/vm/src/function/buffer.rs +++ b/vm/src/function/buffer.rs @@ -70,6 +70,12 @@ impl From for PyBuffer { } } +impl From for PyObjectRef { + fn from(buffer: ArgBytesLike) -> Self { + buffer.as_object().to_owned() + } +} + impl<'a> TryFromBorrowedObject<'a> for ArgBytesLike { fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult { let buffer = PyBuffer::try_from_borrowed_object(vm, obj)?; diff --git a/vm/src/stdlib/codecs.rs b/vm/src/stdlib/codecs.rs index 664fe00616..6ad2a74f4b 100644 --- a/vm/src/stdlib/codecs.rs +++ b/vm/src/stdlib/codecs.rs @@ -2,16 +2,15 @@ pub(crate) use _codecs::make_module; #[pymodule] mod _codecs { + use crate::codecs::{ErrorsHandler, PyDecodeContext, PyEncodeContext}; use crate::common::encodings; - use crate::common::str::StrKind; - use crate::common::wtf8::{Wtf8, Wtf8Buf}; + use crate::common::wtf8::Wtf8Buf; use crate::{ - AsObject, PyObject, PyObjectRef, PyResult, TryFromBorrowedObject, VirtualMachine, - builtins::{PyBaseExceptionRef, PyBytes, PyBytesRef, PyStr, PyStrRef, PyTuple}, + AsObject, PyObjectRef, PyResult, VirtualMachine, + builtins::PyStrRef, codecs, function::{ArgBytesLike, FuncArgs}, }; - use std::ops::Range; #[pyfunction] fn register(search_function: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { @@ -79,164 +78,6 @@ mod _codecs { vm.state.codec_registry.lookup_error(name.as_str(), vm) } - struct ErrorsHandler<'a> { - vm: &'a VirtualMachine, - encoding: &'a str, - errors: Option, - handler: once_cell::unsync::OnceCell, - } - impl<'a> ErrorsHandler<'a> { - #[inline] - fn new(encoding: &'a str, errors: Option, vm: &'a VirtualMachine) -> Self { - ErrorsHandler { - vm, - encoding, - errors, - handler: Default::default(), - } - } - #[inline] - fn handler_func(&self) -> PyResult<&PyObject> { - let vm = self.vm; - Ok(self.handler.get_or_try_init(|| { - let errors = self.errors.as_ref().map_or("strict", |s| s.as_str()); - vm.state.codec_registry.lookup_error(errors, vm) - })?) - } - } - impl encodings::StrBuffer for PyStrRef { - fn is_compatible_with(&self, kind: StrKind) -> bool { - self.kind() <= kind - } - } - impl encodings::ErrorHandler for ErrorsHandler<'_> { - type Error = PyBaseExceptionRef; - type StrBuf = PyStrRef; - type BytesBuf = PyBytesRef; - - fn handle_encode_error( - &self, - data: &Wtf8, - char_range: Range, - reason: &str, - ) -> PyResult<(encodings::EncodeReplace, usize)> { - let vm = self.vm; - let data_str = vm.ctx.new_str(data).into(); - let encode_exc = vm.new_exception( - vm.ctx.exceptions.unicode_encode_error.to_owned(), - vec![ - vm.ctx.new_str(self.encoding).into(), - data_str, - vm.ctx.new_int(char_range.start).into(), - vm.ctx.new_int(char_range.end).into(), - vm.ctx.new_str(reason).into(), - ], - ); - let res = self.handler_func()?.call((encode_exc,), vm)?; - let tuple_err = || { - vm.new_type_error( - "encoding error handler must return (str/bytes, int) tuple".to_owned(), - ) - }; - let (replace, restart) = match res.payload::().map(|tup| tup.as_slice()) { - Some([replace, restart]) => (replace.clone(), restart), - _ => return Err(tuple_err()), - }; - let replace = match_class!(match replace { - s @ PyStr => encodings::EncodeReplace::Str(s), - b @ PyBytes => encodings::EncodeReplace::Bytes(b), - _ => return Err(tuple_err()), - }); - let restart = isize::try_from_borrowed_object(vm, restart).map_err(|_| tuple_err())?; - let restart = if restart < 0 { - // will still be out of bounds if it underflows ¯\_(ツ)_/¯ - data.len().wrapping_sub(restart.unsigned_abs()) - } else { - restart as usize - }; - Ok((replace, restart)) - } - - fn handle_decode_error( - &self, - data: &[u8], - byte_range: Range, - reason: &str, - ) -> PyResult<(PyStrRef, Option, usize)> { - let vm = self.vm; - let data_bytes: PyObjectRef = vm.ctx.new_bytes(data.to_vec()).into(); - let decode_exc = vm.new_exception( - vm.ctx.exceptions.unicode_decode_error.to_owned(), - vec![ - vm.ctx.new_str(self.encoding).into(), - data_bytes.clone(), - vm.ctx.new_int(byte_range.start).into(), - vm.ctx.new_int(byte_range.end).into(), - vm.ctx.new_str(reason).into(), - ], - ); - let handler = self.handler_func()?; - let res = handler.call((decode_exc.clone(),), vm)?; - let new_data = decode_exc - .get_arg(1) - .ok_or_else(|| vm.new_type_error("object attribute not set".to_owned()))?; - let new_data = if new_data.is(&data_bytes) { - None - } else { - let new_data: PyBytesRef = new_data - .downcast() - .map_err(|_| vm.new_type_error("object attribute must be bytes".to_owned()))?; - Some(new_data) - }; - let data = new_data.as_ref().map_or(data, |s| s.as_ref()); - let tuple_err = || { - vm.new_type_error("decoding error handler must return (str, int) tuple".to_owned()) - }; - match res.payload::().map(|tup| tup.as_slice()) { - Some([replace, restart]) => { - let replace = replace - .downcast_ref::() - .ok_or_else(tuple_err)? - .to_owned(); - let restart = - isize::try_from_borrowed_object(vm, restart).map_err(|_| tuple_err())?; - let restart = if restart < 0 { - // will still be out of bounds if it underflows ¯\_(ツ)_/¯ - data.len().wrapping_sub(restart.unsigned_abs()) - } else { - restart as usize - }; - Ok((replace, new_data, restart)) - } - _ => Err(tuple_err()), - } - } - - fn error_oob_restart(&self, i: usize) -> PyBaseExceptionRef { - self.vm - .new_index_error(format!("position {i} from error handler out of bounds")) - } - - fn error_encoding( - &self, - data: &Wtf8, - char_range: Range, - reason: &str, - ) -> Self::Error { - let vm = self.vm; - vm.new_exception( - vm.ctx.exceptions.unicode_encode_error.to_owned(), - vec![ - vm.ctx.new_str(self.encoding).into(), - vm.ctx.new_str(data).into(), - vm.ctx.new_int(char_range.start).into(), - vm.ctx.new_int(char_range.end).into(), - vm.ctx.new_str(reason).into(), - ], - ) - } - } - type EncodeResult = PyResult<(Vec, usize)>; #[derive(FromArgs)] @@ -249,12 +90,13 @@ mod _codecs { impl EncodeArgs { #[inline] - fn encode<'a, F>(self, name: &'a str, encode: F, vm: &'a VirtualMachine) -> EncodeResult + fn encode<'a, F>(&'a self, name: &'a str, encode: F, vm: &'a VirtualMachine) -> EncodeResult where - F: FnOnce(&Wtf8, &ErrorsHandler<'a>) -> PyResult>, + F: FnOnce(PyEncodeContext<'a>, &ErrorsHandler<'a>) -> PyResult>, { - let errors = ErrorsHandler::new(name, self.errors, vm); - let encoded = encode(self.s.as_wtf8(), &errors)?; + let ctx = PyEncodeContext::new(name, &self.s, vm); + let errors = ErrorsHandler::new(self.errors.as_deref(), vm); + let encoded = encode(ctx, &errors)?; Ok((encoded, self.s.char_len())) } } @@ -273,13 +115,13 @@ mod _codecs { impl DecodeArgs { #[inline] - fn decode<'a, F>(self, name: &'a str, decode: F, vm: &'a VirtualMachine) -> DecodeResult + fn decode<'a, F>(&'a self, name: &'a str, decode: F, vm: &'a VirtualMachine) -> DecodeResult where - F: FnOnce(&[u8], &ErrorsHandler<'a>, bool) -> DecodeResult, + F: FnOnce(PyDecodeContext<'a>, &ErrorsHandler<'a>, bool) -> DecodeResult, { - let data = self.data.borrow_buf(); - let errors = ErrorsHandler::new(name, self.errors, vm); - decode(&data, &errors, self.final_decode) + let ctx = PyDecodeContext::new(name, &self.data, vm); + let errors = ErrorsHandler::new(self.errors.as_deref(), vm); + decode(ctx, &errors, self.final_decode) } } @@ -293,13 +135,13 @@ mod _codecs { impl DecodeArgsNoFinal { #[inline] - fn decode<'a, F>(self, name: &'a str, decode: F, vm: &'a VirtualMachine) -> DecodeResult + fn decode<'a, F>(&'a self, name: &'a str, decode: F, vm: &'a VirtualMachine) -> DecodeResult where - F: FnOnce(&[u8], &ErrorsHandler<'a>) -> DecodeResult, + F: FnOnce(PyDecodeContext<'a>, &ErrorsHandler<'a>) -> DecodeResult, { - let data = self.data.borrow_buf(); - let errors = ErrorsHandler::new(name, self.errors, vm); - decode(&data, &errors) + let ctx = PyDecodeContext::new(name, &self.data, vm); + let errors = ErrorsHandler::new(self.errors.as_deref(), vm); + decode(ctx, &errors) } } diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index 0b680251fa..77d9231724 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -2536,10 +2536,9 @@ mod _io { *snapshot = Some((cookie.dec_flags, input_chunk.clone())); let decoded = vm.call_method(decoder, "decode", (input_chunk, cookie.need_eof))?; let decoded = check_decoded(decoded, vm)?; - let pos_is_valid = crate::common::wtf8::is_code_point_boundary( - decoded.as_wtf8(), - cookie.bytes_to_skip as usize, - ); + let pos_is_valid = decoded + .as_wtf8() + .is_code_point_boundary(cookie.bytes_to_skip as usize); textio.set_decoded_chars(Some(decoded)); if !pos_is_valid { return Err(vm.new_os_error("can't restore logical file position".to_owned())); diff --git a/vm/src/vm/vm_new.rs b/vm/src/vm/vm_new.rs index 12241414a7..3ceb783a48 100644 --- a/vm/src/vm/vm_new.rs +++ b/vm/src/vm/vm_new.rs @@ -1,7 +1,8 @@ use crate::{ AsObject, Py, PyObject, PyObjectRef, PyRef, builtins::{ - PyBaseException, PyBaseExceptionRef, PyDictRef, PyModule, PyStrRef, PyType, PyTypeRef, + PyBaseException, PyBaseExceptionRef, PyBytesRef, PyDictRef, PyModule, PyStrRef, PyType, + PyTypeRef, builtin_func::PyNativeFunction, descriptor::PyMethodDescriptor, tuple::{IntoPyTuple, PyTupleRef}, @@ -203,16 +204,78 @@ impl VirtualMachine { self.new_exception_msg(sys_error, msg) } + // TODO: remove & replace with new_unicode_decode_error_real pub fn new_unicode_decode_error(&self, msg: String) -> PyBaseExceptionRef { let unicode_decode_error = self.ctx.exceptions.unicode_decode_error.to_owned(); self.new_exception_msg(unicode_decode_error, msg) } + pub fn new_unicode_decode_error_real( + &self, + encoding: PyStrRef, + object: PyBytesRef, + start: usize, + end: usize, + reason: PyStrRef, + ) -> PyBaseExceptionRef { + let start = self.ctx.new_int(start); + let end = self.ctx.new_int(end); + let exc = self.new_exception( + self.ctx.exceptions.unicode_decode_error.to_owned(), + vec![ + encoding.clone().into(), + object.clone().into(), + start.clone().into(), + end.clone().into(), + reason.clone().into(), + ], + ); + exc.as_object() + .set_attr("encoding", encoding, self) + .unwrap(); + exc.as_object().set_attr("object", object, self).unwrap(); + exc.as_object().set_attr("start", start, self).unwrap(); + exc.as_object().set_attr("end", end, self).unwrap(); + exc.as_object().set_attr("reason", reason, self).unwrap(); + exc + } + + // TODO: remove & replace with new_unicode_encode_error_real pub fn new_unicode_encode_error(&self, msg: String) -> PyBaseExceptionRef { let unicode_encode_error = self.ctx.exceptions.unicode_encode_error.to_owned(); self.new_exception_msg(unicode_encode_error, msg) } + pub fn new_unicode_encode_error_real( + &self, + encoding: PyStrRef, + object: PyStrRef, + start: usize, + end: usize, + reason: PyStrRef, + ) -> PyBaseExceptionRef { + let start = self.ctx.new_int(start); + let end = self.ctx.new_int(end); + let exc = self.new_exception( + self.ctx.exceptions.unicode_encode_error.to_owned(), + vec![ + encoding.clone().into(), + object.clone().into(), + start.clone().into(), + end.clone().into(), + reason.clone().into(), + ], + ); + exc.as_object() + .set_attr("encoding", encoding, self) + .unwrap(); + exc.as_object().set_attr("object", object, self).unwrap(); + exc.as_object().set_attr("start", start, self).unwrap(); + exc.as_object().set_attr("end", end, self).unwrap(); + exc.as_object().set_attr("reason", reason, self).unwrap(); + exc + } + /// Create a new python ValueError object. Useful for raising errors from /// python functions implemented in rust. pub fn new_value_error(&self, msg: String) -> PyBaseExceptionRef { From ec09599d84f23ab6e616a3986afcab4c8ff66b9e Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Wed, 26 Mar 2025 17:44:03 +0900 Subject: [PATCH 117/295] Remoce once_cell::Lazy usage --- Cargo.lock | 2 -- derive-impl/Cargo.toml | 1 - derive-impl/src/compile_bytecode.rs | 4 ++-- stdlib/Cargo.toml | 1 - stdlib/src/contextvars.rs | 4 ++-- stdlib/src/csv.rs | 6 +++--- stdlib/src/mmap.rs | 4 ++-- stdlib/src/sqlite.rs | 8 ++++---- vm/src/builtins/bytes.rs | 6 +++--- vm/src/builtins/dict.rs | 10 +++++----- vm/src/builtins/genericalias.rs | 4 ++-- vm/src/builtins/mappingproxy.rs | 6 +++--- vm/src/builtins/memory.rs | 4 ++-- vm/src/builtins/range.rs | 6 +++--- vm/src/builtins/set.rs | 6 +++--- vm/src/builtins/str.rs | 6 +++--- vm/src/builtins/tuple.rs | 7 +++---- vm/src/builtins/union.rs | 4 ++-- vm/src/builtins/weakproxy.rs | 4 ++-- vm/src/stdlib/sre.rs | 4 ++-- 20 files changed, 46 insertions(+), 51 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1fa60eac29..ab202d0b08 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2412,7 +2412,6 @@ version = "0.4.0" dependencies = [ "itertools 0.14.0", "maplit", - "once_cell", "proc-macro2", "quote", "rustpython-compiler-core", @@ -2515,7 +2514,6 @@ dependencies = [ "num-integer", "num-traits", "num_enum", - "once_cell", "openssl", "openssl-probe", "openssl-sys", diff --git a/derive-impl/Cargo.toml b/derive-impl/Cargo.toml index ca415dc543..eec7952df5 100644 --- a/derive-impl/Cargo.toml +++ b/derive-impl/Cargo.toml @@ -14,7 +14,6 @@ rustpython-compiler-core = { workspace = true } rustpython-doc = { workspace = true } itertools = { workspace = true } -once_cell = { workspace = true } syn = { workspace = true, features = ["full", "extra-traits"] } maplit = "1.0.2" diff --git a/derive-impl/src/compile_bytecode.rs b/derive-impl/src/compile_bytecode.rs index 2c702b35fe..fc349a7ed4 100644 --- a/derive-impl/src/compile_bytecode.rs +++ b/derive-impl/src/compile_bytecode.rs @@ -14,10 +14,10 @@ //! ``` use crate::Diagnostic; -use once_cell::sync::Lazy; use proc_macro2::{Span, TokenStream}; use quote::quote; use rustpython_compiler_core::{Mode, bytecode::CodeObject, frozen}; +use std::sync::LazyLock; use std::{ collections::HashMap, env, fs, @@ -29,7 +29,7 @@ use syn::{ spanned::Spanned, }; -static CARGO_MANIFEST_DIR: Lazy = Lazy::new(|| { +static CARGO_MANIFEST_DIR: LazyLock = LazyLock::new(|| { PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR is not present")) }); diff --git a/stdlib/Cargo.toml b/stdlib/Cargo.toml index b46fb16b8f..14fd0d39ec 100644 --- a/stdlib/Cargo.toml +++ b/stdlib/Cargo.toml @@ -40,7 +40,6 @@ malachite-bigint = { workspace = true } num-integer = { workspace = true } num-traits = { workspace = true } num_enum = { workspace = true } -once_cell = { workspace = true } parking_lot = { workspace = true } thread_local = { workspace = true } diff --git a/stdlib/src/contextvars.rs b/stdlib/src/contextvars.rs index bcc372ed55..4fd45842b9 100644 --- a/stdlib/src/contextvars.rs +++ b/stdlib/src/contextvars.rs @@ -33,7 +33,7 @@ mod _contextvars { }; use crossbeam_utils::atomic::AtomicCell; use indexmap::IndexMap; - use once_cell::sync::Lazy; + use std::sync::LazyLock; use std::{ cell::{Cell, RefCell, UnsafeCell}, sync::atomic::Ordering, @@ -274,7 +274,7 @@ mod _contextvars { impl AsSequence for PyContext { fn as_sequence() -> &'static PySequenceMethods { - static AS_SEQUENCE: Lazy = Lazy::new(|| PySequenceMethods { + static AS_SEQUENCE: LazyLock = LazyLock::new(|| PySequenceMethods { contains: atomic_func!(|seq, target, vm| { let target = target.try_to_value(vm)?; PyContext::sequence_downcast(seq).contains(target) diff --git a/stdlib/src/csv.rs b/stdlib/src/csv.rs index 4b79130111..04ebe49162 100644 --- a/stdlib/src/csv.rs +++ b/stdlib/src/csv.rs @@ -12,9 +12,9 @@ mod _csv { }; use csv_core::Terminator; use itertools::{self, Itertools}; - use once_cell::sync::Lazy; use parking_lot::Mutex; use rustpython_vm::match_class; + use std::sync::LazyLock; use std::{collections::HashMap, fmt}; #[pyattr] @@ -41,11 +41,11 @@ mod _csv { ) } - static GLOBAL_HASHMAP: Lazy>> = Lazy::new(|| { + static GLOBAL_HASHMAP: LazyLock>> = LazyLock::new(|| { let m = HashMap::new(); Mutex::new(m) }); - static GLOBAL_FIELD_LIMIT: Lazy> = Lazy::new(|| Mutex::new(131072)); + static GLOBAL_FIELD_LIMIT: LazyLock> = LazyLock::new(|| Mutex::new(131072)); fn new_csv_error(vm: &VirtualMachine, msg: String) -> PyBaseExceptionRef { vm.new_exception_msg(super::_csv::error(vm), msg) diff --git a/stdlib/src/mmap.rs b/stdlib/src/mmap.rs index cbc86bb2c9..f8600307b2 100644 --- a/stdlib/src/mmap.rs +++ b/stdlib/src/mmap.rs @@ -456,8 +456,8 @@ mod mmap { impl AsSequence for PyMmap { fn as_sequence() -> &'static PySequenceMethods { - use once_cell::sync::Lazy; - static AS_SEQUENCE: Lazy = Lazy::new(|| PySequenceMethods { + use std::sync::LazyLock; + static AS_SEQUENCE: LazyLock = LazyLock::new(|| PySequenceMethods { length: atomic_func!(|seq, _vm| Ok(PyMmap::sequence_downcast(seq).len())), item: atomic_func!(|seq, i, vm| { let zelf = PyMmap::sequence_downcast(seq); diff --git a/stdlib/src/sqlite.rs b/stdlib/src/sqlite.rs index 67e94bd81b..97ec193c46 100644 --- a/stdlib/src/sqlite.rs +++ b/stdlib/src/sqlite.rs @@ -1953,8 +1953,8 @@ mod _sqlite { impl AsMapping for Row { fn as_mapping() -> &'static PyMappingMethods { - static AS_MAPPING: once_cell::sync::Lazy = - once_cell::sync::Lazy::new(|| PyMappingMethods { + static AS_MAPPING: std::sync::LazyLock = + std::sync::LazyLock::new(|| PyMappingMethods { length: atomic_func!(|mapping, _vm| Ok(Row::mapping_downcast(mapping) .data .len())), @@ -1969,8 +1969,8 @@ mod _sqlite { impl AsSequence for Row { fn as_sequence() -> &'static PySequenceMethods { - static AS_SEQUENCE: once_cell::sync::Lazy = - once_cell::sync::Lazy::new(|| PySequenceMethods { + static AS_SEQUENCE: std::sync::LazyLock = + std::sync::LazyLock::new(|| PySequenceMethods { length: atomic_func!(|seq, _vm| Ok(Row::sequence_downcast(seq).data.len())), item: atomic_func!(|seq, i, vm| Row::sequence_downcast(seq) .data diff --git a/vm/src/builtins/bytes.rs b/vm/src/builtins/bytes.rs index e9f5adc8bb..eff4190eda 100644 --- a/vm/src/builtins/bytes.rs +++ b/vm/src/builtins/bytes.rs @@ -27,7 +27,7 @@ use crate::{ }, }; use bstr::ByteSlice; -use once_cell::sync::Lazy; +use std::sync::LazyLock; use std::{mem::size_of, ops::Deref}; #[pyclass(module = false, name = "bytes")] @@ -568,7 +568,7 @@ impl AsBuffer for PyBytes { impl AsMapping for PyBytes { fn as_mapping() -> &'static PyMappingMethods { - static AS_MAPPING: Lazy = Lazy::new(|| PyMappingMethods { + static AS_MAPPING: LazyLock = LazyLock::new(|| PyMappingMethods { length: atomic_func!(|mapping, _vm| Ok(PyBytes::mapping_downcast(mapping).len())), subscript: atomic_func!( |mapping, needle, vm| PyBytes::mapping_downcast(mapping)._getitem(needle, vm) @@ -581,7 +581,7 @@ impl AsMapping for PyBytes { impl AsSequence for PyBytes { fn as_sequence() -> &'static PySequenceMethods { - static AS_SEQUENCE: Lazy = Lazy::new(|| PySequenceMethods { + static AS_SEQUENCE: LazyLock = LazyLock::new(|| PySequenceMethods { length: atomic_func!(|seq, _vm| Ok(PyBytes::sequence_downcast(seq).len())), concat: atomic_func!(|seq, other, vm| { PyBytes::sequence_downcast(seq) diff --git a/vm/src/builtins/dict.rs b/vm/src/builtins/dict.rs index c8da40dc01..a19b11fcfb 100644 --- a/vm/src/builtins/dict.rs +++ b/vm/src/builtins/dict.rs @@ -23,9 +23,9 @@ use crate::{ }, vm::VirtualMachine, }; -use once_cell::sync::Lazy; use rustpython_common::lock::PyMutex; use std::fmt; +use std::sync::LazyLock; pub type DictContentType = dictdatatype::Dict; @@ -443,7 +443,7 @@ impl AsMapping for PyDict { impl AsSequence for PyDict { fn as_sequence() -> &'static PySequenceMethods { - static AS_SEQUENCE: Lazy = Lazy::new(|| PySequenceMethods { + static AS_SEQUENCE: LazyLock = LazyLock::new(|| PySequenceMethods { contains: atomic_func!(|seq, target, vm| PyDict::sequence_downcast(seq) .entries .contains(vm, target)), @@ -1133,7 +1133,7 @@ impl Comparable for PyDictKeys { impl AsSequence for PyDictKeys { fn as_sequence() -> &'static PySequenceMethods { - static AS_SEQUENCE: Lazy = Lazy::new(|| PySequenceMethods { + static AS_SEQUENCE: LazyLock = LazyLock::new(|| PySequenceMethods { length: atomic_func!(|seq, _vm| Ok(PyDictKeys::sequence_downcast(seq).len())), contains: atomic_func!(|seq, target, vm| { PyDictKeys::sequence_downcast(seq) @@ -1196,7 +1196,7 @@ impl Comparable for PyDictItems { impl AsSequence for PyDictItems { fn as_sequence() -> &'static PySequenceMethods { - static AS_SEQUENCE: Lazy = Lazy::new(|| PySequenceMethods { + static AS_SEQUENCE: LazyLock = LazyLock::new(|| PySequenceMethods { length: atomic_func!(|seq, _vm| Ok(PyDictItems::sequence_downcast(seq).len())), contains: atomic_func!(|seq, target, vm| { let needle: &Py = match target.downcast_ref() { @@ -1246,7 +1246,7 @@ impl Unconstructible for PyDictValues {} impl AsSequence for PyDictValues { fn as_sequence() -> &'static PySequenceMethods { - static AS_SEQUENCE: Lazy = Lazy::new(|| PySequenceMethods { + static AS_SEQUENCE: LazyLock = LazyLock::new(|| PySequenceMethods { length: atomic_func!(|seq, _vm| Ok(PyDictValues::sequence_downcast(seq).len())), ..PySequenceMethods::NOT_IMPLEMENTED }); diff --git a/vm/src/builtins/genericalias.rs b/vm/src/builtins/genericalias.rs index 0e0a34227b..549985bcfb 100644 --- a/vm/src/builtins/genericalias.rs +++ b/vm/src/builtins/genericalias.rs @@ -1,4 +1,4 @@ -use once_cell::sync::Lazy; +use std::sync::LazyLock; use super::type_; use crate::{ @@ -325,7 +325,7 @@ pub fn subs_parameters PyResult>( impl AsMapping for PyGenericAlias { fn as_mapping() -> &'static PyMappingMethods { - static AS_MAPPING: Lazy = Lazy::new(|| PyMappingMethods { + static AS_MAPPING: LazyLock = LazyLock::new(|| PyMappingMethods { subscript: atomic_func!(|mapping, needle, vm| { PyGenericAlias::mapping_downcast(mapping).getitem(needle.to_owned(), vm) }), diff --git a/vm/src/builtins/mappingproxy.rs b/vm/src/builtins/mappingproxy.rs index 385d18df0b..5dd31500fb 100644 --- a/vm/src/builtins/mappingproxy.rs +++ b/vm/src/builtins/mappingproxy.rs @@ -12,7 +12,7 @@ use crate::{ Representable, }, }; -use once_cell::sync::Lazy; +use std::sync::LazyLock; #[pyclass(module = false, name = "mappingproxy", traverse)] #[derive(Debug)] @@ -221,7 +221,7 @@ impl Comparable for PyMappingProxy { impl AsMapping for PyMappingProxy { fn as_mapping() -> &'static PyMappingMethods { - static AS_MAPPING: Lazy = Lazy::new(|| PyMappingMethods { + static AS_MAPPING: LazyLock = LazyLock::new(|| PyMappingMethods { length: atomic_func!(|mapping, vm| PyMappingProxy::mapping_downcast(mapping).len(vm)), subscript: atomic_func!(|mapping, needle, vm| { PyMappingProxy::mapping_downcast(mapping).getitem(needle.to_owned(), vm) @@ -234,7 +234,7 @@ impl AsMapping for PyMappingProxy { impl AsSequence for PyMappingProxy { fn as_sequence() -> &'static PySequenceMethods { - static AS_SEQUENCE: Lazy = Lazy::new(|| PySequenceMethods { + static AS_SEQUENCE: LazyLock = LazyLock::new(|| PySequenceMethods { contains: atomic_func!( |seq, target, vm| PyMappingProxy::sequence_downcast(seq)._contains(target, vm) ), diff --git a/vm/src/builtins/memory.rs b/vm/src/builtins/memory.rs index e411d312db..c5af12dc1f 100644 --- a/vm/src/builtins/memory.rs +++ b/vm/src/builtins/memory.rs @@ -28,8 +28,8 @@ use crate::{ }; use crossbeam_utils::atomic::AtomicCell; use itertools::Itertools; -use once_cell::sync::Lazy; use rustpython_common::lock::PyMutex; +use std::sync::LazyLock; use std::{cmp::Ordering, fmt::Debug, mem::ManuallyDrop, ops::Range}; #[derive(FromArgs)] @@ -993,7 +993,7 @@ impl AsMapping for PyMemoryView { impl AsSequence for PyMemoryView { fn as_sequence() -> &'static PySequenceMethods { - static AS_SEQUENCE: Lazy = Lazy::new(|| PySequenceMethods { + static AS_SEQUENCE: LazyLock = LazyLock::new(|| PySequenceMethods { length: atomic_func!(|seq, vm| { let zelf = PyMemoryView::sequence_downcast(seq); zelf.try_not_released(vm)?; diff --git a/vm/src/builtins/range.rs b/vm/src/builtins/range.rs index b542a5f191..2878e5f053 100644 --- a/vm/src/builtins/range.rs +++ b/vm/src/builtins/range.rs @@ -17,8 +17,8 @@ use crossbeam_utils::atomic::AtomicCell; use malachite_bigint::{BigInt, Sign}; use num_integer::Integer; use num_traits::{One, Signed, ToPrimitive, Zero}; -use once_cell::sync::Lazy; use std::cmp::max; +use std::sync::LazyLock; // Search flag passed to iter_search enum SearchType { @@ -385,7 +385,7 @@ impl PyRange { impl AsMapping for PyRange { fn as_mapping() -> &'static PyMappingMethods { - static AS_MAPPING: Lazy = Lazy::new(|| PyMappingMethods { + static AS_MAPPING: LazyLock = LazyLock::new(|| PyMappingMethods { length: atomic_func!( |mapping, vm| PyRange::mapping_downcast(mapping).protocol_length(vm) ), @@ -400,7 +400,7 @@ impl AsMapping for PyRange { impl AsSequence for PyRange { fn as_sequence() -> &'static PySequenceMethods { - static AS_SEQUENCE: Lazy = Lazy::new(|| PySequenceMethods { + static AS_SEQUENCE: LazyLock = LazyLock::new(|| PySequenceMethods { length: atomic_func!(|seq, vm| PyRange::sequence_downcast(seq).protocol_length(vm)), item: atomic_func!(|seq, i, vm| { PyRange::sequence_downcast(seq) diff --git a/vm/src/builtins/set.rs b/vm/src/builtins/set.rs index 40823aa37b..3e10e5c6b7 100644 --- a/vm/src/builtins/set.rs +++ b/vm/src/builtins/set.rs @@ -23,11 +23,11 @@ use crate::{ utils::collection_repr, vm::VirtualMachine, }; -use once_cell::sync::Lazy; use rustpython_common::{ atomic::{Ordering, PyAtomic, Radium}, hash, }; +use std::sync::LazyLock; use std::{fmt, ops::Deref}; pub type SetContentType = dictdatatype::Dict<()>; @@ -794,7 +794,7 @@ impl Initializer for PySet { impl AsSequence for PySet { fn as_sequence() -> &'static PySequenceMethods { - static AS_SEQUENCE: Lazy = Lazy::new(|| PySequenceMethods { + static AS_SEQUENCE: LazyLock = LazyLock::new(|| PySequenceMethods { length: atomic_func!(|seq, _vm| Ok(PySet::sequence_downcast(seq).len())), contains: atomic_func!(|seq, needle, vm| PySet::sequence_downcast(seq) .inner @@ -1112,7 +1112,7 @@ impl PyFrozenSet { impl AsSequence for PyFrozenSet { fn as_sequence() -> &'static PySequenceMethods { - static AS_SEQUENCE: Lazy = Lazy::new(|| PySequenceMethods { + static AS_SEQUENCE: LazyLock = LazyLock::new(|| PySequenceMethods { length: atomic_func!(|seq, _vm| Ok(PyFrozenSet::sequence_downcast(seq).len())), contains: atomic_func!(|seq, needle, vm| PyFrozenSet::sequence_downcast(seq) .inner diff --git a/vm/src/builtins/str.rs b/vm/src/builtins/str.rs index 55cefae4f7..c0bd556bf1 100644 --- a/vm/src/builtins/str.rs +++ b/vm/src/builtins/str.rs @@ -28,7 +28,6 @@ use ascii::{AsciiChar, AsciiStr, AsciiString}; use bstr::ByteSlice; use itertools::Itertools; use num_traits::ToPrimitive; -use once_cell::sync::Lazy; use rustpython_common::{ ascii, atomic::{self, PyAtomic, Radium}, @@ -38,6 +37,7 @@ use rustpython_common::{ str::DeduceStrKind, wtf8::{CodePoint, Wtf8, Wtf8Buf, Wtf8Chunk}, }; +use std::sync::LazyLock; use std::{borrow::Cow, char, fmt, ops::Range}; use unic_ucd_bidi::BidiClass; use unic_ucd_category::GeneralCategory; @@ -1495,7 +1495,7 @@ impl Iterable for PyStr { impl AsMapping for PyStr { fn as_mapping() -> &'static PyMappingMethods { - static AS_MAPPING: Lazy = Lazy::new(|| PyMappingMethods { + static AS_MAPPING: LazyLock = LazyLock::new(|| PyMappingMethods { length: atomic_func!(|mapping, _vm| Ok(PyStr::mapping_downcast(mapping).len())), subscript: atomic_func!( |mapping, needle, vm| PyStr::mapping_downcast(mapping)._getitem(needle, vm) @@ -1524,7 +1524,7 @@ impl AsNumber for PyStr { impl AsSequence for PyStr { fn as_sequence() -> &'static PySequenceMethods { - static AS_SEQUENCE: Lazy = Lazy::new(|| PySequenceMethods { + static AS_SEQUENCE: LazyLock = LazyLock::new(|| PySequenceMethods { length: atomic_func!(|seq, _vm| Ok(PyStr::sequence_downcast(seq).len())), concat: atomic_func!(|seq, other, vm| { let zelf = PyStr::sequence_downcast(seq); diff --git a/vm/src/builtins/tuple.rs b/vm/src/builtins/tuple.rs index c9af04d300..52577f0e2e 100644 --- a/vm/src/builtins/tuple.rs +++ b/vm/src/builtins/tuple.rs @@ -19,8 +19,7 @@ use crate::{ utils::collection_repr, vm::VirtualMachine, }; -use once_cell::sync::Lazy; -use std::{fmt, marker::PhantomData}; +use std::{fmt, marker::PhantomData, sync::LazyLock}; #[pyclass(module = false, name = "tuple", traverse)] pub struct PyTuple { @@ -351,7 +350,7 @@ impl PyTuple { impl AsMapping for PyTuple { fn as_mapping() -> &'static PyMappingMethods { - static AS_MAPPING: Lazy = Lazy::new(|| PyMappingMethods { + static AS_MAPPING: LazyLock = LazyLock::new(|| PyMappingMethods { length: atomic_func!(|mapping, _vm| Ok(PyTuple::mapping_downcast(mapping).len())), subscript: atomic_func!( |mapping, needle, vm| PyTuple::mapping_downcast(mapping)._getitem(needle, vm) @@ -364,7 +363,7 @@ impl AsMapping for PyTuple { impl AsSequence for PyTuple { fn as_sequence() -> &'static PySequenceMethods { - static AS_SEQUENCE: Lazy = Lazy::new(|| PySequenceMethods { + static AS_SEQUENCE: LazyLock = LazyLock::new(|| PySequenceMethods { length: atomic_func!(|seq, _vm| Ok(PyTuple::sequence_downcast(seq).len())), concat: atomic_func!(|seq, other, vm| { let zelf = PyTuple::sequence_downcast(seq); diff --git a/vm/src/builtins/union.rs b/vm/src/builtins/union.rs index 165113e216..83e2c86f08 100644 --- a/vm/src/builtins/union.rs +++ b/vm/src/builtins/union.rs @@ -10,8 +10,8 @@ use crate::{ protocol::{PyMappingMethods, PyNumberMethods}, types::{AsMapping, AsNumber, Comparable, GetAttr, Hashable, PyComparisonOp, Representable}, }; -use once_cell::sync::Lazy; use std::fmt; +use std::sync::LazyLock; const CLS_ATTRS: &[&str] = &["__module__"]; @@ -219,7 +219,7 @@ impl PyUnion { impl AsMapping for PyUnion { fn as_mapping() -> &'static PyMappingMethods { - static AS_MAPPING: Lazy = Lazy::new(|| PyMappingMethods { + static AS_MAPPING: LazyLock = LazyLock::new(|| PyMappingMethods { subscript: atomic_func!(|mapping, needle, vm| { PyUnion::mapping_downcast(mapping).getitem(needle.to_owned(), vm) }), diff --git a/vm/src/builtins/weakproxy.rs b/vm/src/builtins/weakproxy.rs index d17bc75118..49e38d2d66 100644 --- a/vm/src/builtins/weakproxy.rs +++ b/vm/src/builtins/weakproxy.rs @@ -11,7 +11,7 @@ use crate::{ PyComparisonOp, Representable, SetAttr, }, }; -use once_cell::sync::Lazy; +use std::sync::LazyLock; #[pyclass(module = false, name = "weakproxy", unhashable = true, traverse)] #[derive(Debug)] @@ -186,7 +186,7 @@ impl Comparable for PyWeakProxy { impl AsSequence for PyWeakProxy { fn as_sequence() -> &'static PySequenceMethods { - static AS_SEQUENCE: Lazy = Lazy::new(|| PySequenceMethods { + static AS_SEQUENCE: LazyLock = LazyLock::new(|| PySequenceMethods { length: atomic_func!(|seq, vm| PyWeakProxy::sequence_downcast(seq).len(vm)), contains: atomic_func!(|seq, needle, vm| { PyWeakProxy::sequence_downcast(seq).contains(needle.to_owned(), vm) diff --git a/vm/src/stdlib/sre.rs b/vm/src/stdlib/sre.rs index 038ac9934a..6a0a618e2f 100644 --- a/vm/src/stdlib/sre.rs +++ b/vm/src/stdlib/sre.rs @@ -814,8 +814,8 @@ mod _sre { impl AsMapping for Match { fn as_mapping() -> &'static PyMappingMethods { - static AS_MAPPING: once_cell::sync::Lazy = - once_cell::sync::Lazy::new(|| PyMappingMethods { + static AS_MAPPING: std::sync::LazyLock = + std::sync::LazyLock::new(|| PyMappingMethods { subscript: atomic_func!(|mapping, needle, vm| { Match::mapping_downcast(mapping) .getitem(needle.to_owned(), vm) From ad5788589bc5f6b058347ae0a7b4e76dfc9d476d Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Wed, 26 Mar 2025 18:20:43 +0900 Subject: [PATCH 118/295] Remove more direct use of OnceCell --- stdlib/src/uuid.rs | 4 ++-- vm/src/vm/mod.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/stdlib/src/uuid.rs b/stdlib/src/uuid.rs index 0bd7d0db8a..872f5bde0a 100644 --- a/stdlib/src/uuid.rs +++ b/stdlib/src/uuid.rs @@ -4,7 +4,7 @@ pub(crate) use _uuid::make_module; mod _uuid { use crate::{builtins::PyNone, vm::VirtualMachine}; use mac_address::get_mac_address; - use once_cell::sync::OnceCell; + use std::sync::OnceLock; use uuid::{Context, Uuid, timestamp::Timestamp}; fn get_node_id() -> [u8; 6] { @@ -19,7 +19,7 @@ mod _uuid { static CONTEXT: Context = Context::new(0); let ts = Timestamp::now(&CONTEXT); - static NODE_ID: OnceCell<[u8; 6]> = OnceCell::new(); + static NODE_ID: OnceLock<[u8; 6]> = OnceLock::new(); let unique_node_id = NODE_ID.get_or_init(get_node_id); (Uuid::new_v1(ts, unique_node_id).as_bytes().to_vec(), PyNone) diff --git a/vm/src/vm/mod.rs b/vm/src/vm/mod.rs index 493789f510..9aec244529 100644 --- a/vm/src/vm/mod.rs +++ b/vm/src/vm/mod.rs @@ -107,8 +107,8 @@ pub struct PyGlobalState { } pub fn process_hash_secret_seed() -> u32 { - use once_cell::sync::OnceCell; - static SEED: OnceCell = OnceCell::new(); + use std::sync::OnceLock; + static SEED: OnceLock = OnceLock::new(); *SEED.get_or_init(rand::random) } From 97fa11d5269d86f60a59aeb48634985c1a68b1d5 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Wed, 26 Mar 2025 15:10:23 +0900 Subject: [PATCH 119/295] Remove unused dependency --- Cargo.lock | 2 -- common/Cargo.toml | 1 - pylib/Cargo.toml | 6 +++--- stdlib/Cargo.toml | 1 - 4 files changed, 3 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1fa60eac29..bfb476fcb0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2347,7 +2347,6 @@ dependencies = [ "malachite-bigint", "malachite-q", "memchr", - "num-complex", "num-traits", "once_cell", "parking_lot", @@ -2536,7 +2535,6 @@ dependencies = [ "system-configuration", "tcl", "termios", - "thread_local", "tk", "ucd", "unic-char-property", diff --git a/common/Cargo.toml b/common/Cargo.toml index e9aeba7459..46cf7346be 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -24,7 +24,6 @@ malachite-bigint = { workspace = true } malachite-q = { workspace = true } malachite-base = { workspace = true } memchr = { workspace = true } -num-complex = { workspace = true } num-traits = { workspace = true } once_cell = { workspace = true } parking_lot = { workspace = true, optional = true } diff --git a/pylib/Cargo.toml b/pylib/Cargo.toml index 64a9da0219..dcbb592859 100644 --- a/pylib/Cargo.toml +++ b/pylib/Cargo.toml @@ -10,11 +10,11 @@ rust-version.workspace = true repository.workspace = true [features] -freeze-stdlib = [] +freeze-stdlib = ["dep:rustpython-compiler-core", "dep:rustpython-derive"] [dependencies] -rustpython-compiler-core = { workspace = true } -rustpython-derive = { version = "0.4.0", path = "../derive" } +rustpython-compiler-core = { workspace = true, optional = true } +rustpython-derive = { workspace = true, optional = true } [build-dependencies] glob = { workspace = true } diff --git a/stdlib/Cargo.toml b/stdlib/Cargo.toml index b46fb16b8f..7072c8753e 100644 --- a/stdlib/Cargo.toml +++ b/stdlib/Cargo.toml @@ -42,7 +42,6 @@ num-traits = { workspace = true } num_enum = { workspace = true } once_cell = { workspace = true } parking_lot = { workspace = true } -thread_local = { workspace = true } memchr = { workspace = true } base64 = "0.13.0" From 549cce24c8cf2a49bb7e0506dc44920825f6e90d Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 8 Sep 2024 23:20:34 +0900 Subject: [PATCH 120/295] Add #[pystruct(skip)] --- .cspell.json | 1 + derive-impl/src/pystructseq.rs | 103 ++++++++++++++++++++++++++------- derive/src/lib.rs | 4 +- 3 files changed, 84 insertions(+), 24 deletions(-) diff --git a/.cspell.json b/.cspell.json index e2723d6ce5..c1db1036a9 100644 --- a/.cspell.json +++ b/.cspell.json @@ -148,6 +148,7 @@ "origname", "posixsubprocess", "pyexpat", + "pytraverse", "PYTHONDEBUG", "PYTHONHOME", "PYTHONINSPECT", diff --git a/derive-impl/src/pystructseq.rs b/derive-impl/src/pystructseq.rs index dce8c4768a..4a4f9276f0 100644 --- a/derive-impl/src/pystructseq.rs +++ b/derive-impl/src/pystructseq.rs @@ -1,41 +1,93 @@ use proc_macro2::TokenStream; use quote::quote; use syn::{DeriveInput, Ident, Result}; +use syn_ext::ext::{AttributeExt, GetIdent}; +use syn_ext::types::Meta; -fn field_names(input: &DeriveInput) -> Result> { - let fields = if let syn::Data::Struct(ref struc) = input.data { - &struc.fields - } else { +// returning a pair of not-skipped and skipped field names +fn field_names(input: &mut DeriveInput) -> Result<(Vec, Vec)> { + let syn::Data::Struct(struc) = &mut input.data else { bail_span!( input, "#[pystruct_sequence] can only be on a struct declaration" ) }; - let field_names: Vec<_> = match fields { - syn::Fields::Named(fields) => fields - .named - .iter() - .map(|field| field.ident.as_ref().unwrap()) - .collect(), - _ => bail_span!( + let syn::Fields::Named(fields) = &mut struc.fields else { + bail_span!( input, "#[pystruct_sequence] can only be on a struct with named fields" - ), + ); }; - Ok(field_names) + let mut not_skipped = Vec::with_capacity(fields.named.len()); + let mut skipped = Vec::with_capacity(fields.named.len()); + for field in &mut fields.named { + let mut skip = false; + // Collect all attributes with pystruct and their indices + let mut attrs_to_remove = Vec::new(); + + for (i, attr) in field.attrs.iter().enumerate() { + if !attr.path().is_ident("pystruct") { + continue; + } + + let Ok(meta) = attr.parse_meta() else { + continue; + }; + + let Meta::List(l) = meta else { + bail_span!(input, "Only #[pystruct(...)] form is allowed"); + }; + + let idents: Vec<_> = l + .nested + .iter() + .filter_map(|n| n.get_ident()) + .cloned() + .collect(); + + // Follow #[serde(skip)] convention. + // Consider to add skip_serializing and skip_deserializing if required. + for ident in idents { + match ident.to_string().as_str() { + "skip" => { + skip = true; + } + _ => { + bail_span!(ident, "Unknown item for #[pystruct(...)]") + } + } + } + + attrs_to_remove.push(i); + } + + // Remove attributes in reverse order to maintain valid indices + attrs_to_remove.sort_unstable_by(|a, b| b.cmp(a)); // Sort in descending order + for index in attrs_to_remove { + field.attrs.remove(index); + } + let ident = field.ident.clone().unwrap(); + if skip { + skipped.push(ident.clone()); + } else { + not_skipped.push(ident.clone()); + } + } + + Ok((not_skipped, skipped)) } -pub(crate) fn impl_pystruct_sequence(input: DeriveInput) -> Result { - let field_names = field_names(&input)?; +pub(crate) fn impl_pystruct_sequence(mut input: DeriveInput) -> Result { + let (not_skipped_fields, _skipped_fields) = field_names(&mut input)?; let ty = &input.ident; let ret = quote! { impl ::rustpython_vm::types::PyStructSequence for #ty { - const FIELD_NAMES: &'static [&'static str] = &[#(stringify!(#field_names)),*]; + const FIELD_NAMES: &'static [&'static str] = &[#(stringify!(#not_skipped_fields)),*]; fn into_tuple(self, vm: &::rustpython_vm::VirtualMachine) -> ::rustpython_vm::builtins::PyTuple { let items = vec![#(::rustpython_vm::convert::ToPyObject::to_pyobject( - self.#field_names, + self.#not_skipped_fields, vm, )),*]; ::rustpython_vm::builtins::PyTuple::new_unchecked(items.into_boxed_slice()) @@ -50,8 +102,10 @@ pub(crate) fn impl_pystruct_sequence(input: DeriveInput) -> Result Ok(ret) } -pub(crate) fn impl_pystruct_sequence_try_from_object(input: DeriveInput) -> Result { - let field_names = field_names(&input)?; +pub(crate) fn impl_pystruct_sequence_try_from_object( + mut input: DeriveInput, +) -> Result { + let (not_skipped_fields, skipped_fields) = field_names(&mut input)?; let ty = &input.ident; let ret = quote! { impl ::rustpython_vm::TryFromObject for #ty { @@ -60,9 +114,14 @@ pub(crate) fn impl_pystruct_sequence_try_from_object(input: DeriveInput) -> Resu let seq = Self::try_elements_from::(seq, vm)?; // TODO: this is possible to be written without iterator let mut iter = seq.into_iter(); - Ok(Self {#( - #field_names: iter.next().unwrap().clone().try_into_value(vm)? - ),*}) + Ok(Self { + #( + #not_skipped_fields: iter.next().unwrap().clone().try_into_value(vm)?, + )* + #( + #skipped_fields: vm.ctx.none(), + )* + }) } } }; diff --git a/derive/src/lib.rs b/derive/src/lib.rs index a9a3123a5a..a96c2aef6e 100644 --- a/derive/src/lib.rs +++ b/derive/src/lib.rs @@ -225,13 +225,13 @@ pub fn pymodule(attr: TokenStream, item: TokenStream) -> TokenStream { derive_impl::pymodule(attr, item).into() } -#[proc_macro_derive(PyStructSequence)] +#[proc_macro_derive(PyStructSequence, attributes(pystruct))] pub fn pystruct_sequence(input: TokenStream) -> TokenStream { let input = parse_macro_input!(input); derive_impl::pystruct_sequence(input).into() } -#[proc_macro_derive(TryIntoPyStructSequence)] +#[proc_macro_derive(TryIntoPyStructSequence, attributes(pystruct))] pub fn pystruct_sequence_try_from_object(input: TokenStream) -> TokenStream { let input = parse_macro_input!(input); derive_impl::pystruct_sequence_try_from_object(input).into() From 7f94c10be78f4a3ddf5f0831ed5106a633b1d4c1 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 27 Mar 2025 01:13:36 +0900 Subject: [PATCH 121/295] patch typing not to reqiure contextlib This will be correctly fixed in future CPython --- Lib/typing.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Lib/typing.py b/Lib/typing.py index 086d0f3f95..75ec2a6a2e 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -22,7 +22,6 @@ from abc import abstractmethod, ABCMeta import collections import collections.abc -import contextlib import functools import operator import re as stdlib_re # Avoid confusion with the re we export. @@ -2138,8 +2137,13 @@ class Other(Leaf): # Error reported by type checker KeysView = _alias(collections.abc.KeysView, 1) ItemsView = _alias(collections.abc.ItemsView, 2) ValuesView = _alias(collections.abc.ValuesView, 1) -ContextManager = _alias(contextlib.AbstractContextManager, 1, name='ContextManager') -AsyncContextManager = _alias(contextlib.AbstractAsyncContextManager, 1, name='AsyncContextManager') +try: + # XXX: RUSTPYTHON; contextlib support for wasm + import contextlib + ContextManager = _alias(contextlib.AbstractContextManager, 1, name='ContextManager') + AsyncContextManager = _alias(contextlib.AbstractAsyncContextManager, 1, name='AsyncContextManager') +except ImportError: + pass Dict = _alias(dict, 2, inst=False, name='Dict') DefaultDict = _alias(collections.defaultdict, 2, name='DefaultDict') OrderedDict = _alias(collections.OrderedDict, 2) From 2d4eec88d3945d1fa310504cc21ec75a5c4de8a9 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 27 Mar 2025 01:14:30 +0900 Subject: [PATCH 122/295] better webdriver error printing --- wasm/demo/src/index.js | 9 ++++++++- wasm/tests/conftest.py | 6 ++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/wasm/demo/src/index.js b/wasm/demo/src/index.js index db3554c481..1af847d59d 100644 --- a/wasm/demo/src/index.js +++ b/wasm/demo/src/index.js @@ -21,7 +21,14 @@ import('rustpython') }) .catch((e) => { console.error('Error importing `rustpython`:', e); - document.getElementById('error').textContent = e; + let errorDetails = e.toString(); + if (window.__RUSTPYTHON_ERROR) { + errorDetails += '\nRustPython Error: ' + window.__RUSTPYTHON_ERROR; + } + if (window.__RUSTPYTHON_ERROR_STACK) { + errorDetails += '\nStack: ' + window.__RUSTPYTHON_ERROR_STACK; + } + document.getElementById('error').textContent = errorDetails; }); const fixedHeightEditor = EditorView.theme({ diff --git a/wasm/tests/conftest.py b/wasm/tests/conftest.py index ecb5b0007b..84a2530575 100644 --- a/wasm/tests/conftest.py +++ b/wasm/tests/conftest.py @@ -102,6 +102,12 @@ def wdriver(request): driver._print_panic() driver.quit() raise + except Exception as e: + print(f"Error waiting for page to load: {e}") + # Check the page source to see what's loaded + print("Page source:", driver.page_source[:500]) + driver.quit() + raise yield driver From e7fdfca5f5370f4056a102ca0bbdecd2821a44a0 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Mon, 22 Apr 2024 11:21:42 +0900 Subject: [PATCH 123/295] Add python-implemented ExceptionGroup --- Lib/test/test_baseexception.py | 2 - Lib/test/test_contextlib.py | 2 - src/interpreter.rs | 1 + vm/Lib/python_builtins/_py_exceptiongroup.py | 330 +++++++++++++++++++ vm/src/vm/mod.rs | 20 +- 5 files changed, 350 insertions(+), 5 deletions(-) create mode 100644 vm/Lib/python_builtins/_py_exceptiongroup.py diff --git a/Lib/test/test_baseexception.py b/Lib/test/test_baseexception.py index a7a20dc415..a73711c416 100644 --- a/Lib/test/test_baseexception.py +++ b/Lib/test/test_baseexception.py @@ -18,8 +18,6 @@ def verify_instance_interface(self, ins): "%s missing %s attribute" % (ins.__class__.__name__, attr)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_inheritance(self): # Make sure the inheritance hierarchy matches the documentation exc_set = set() diff --git a/Lib/test/test_contextlib.py b/Lib/test/test_contextlib.py index f6478b43e0..91764696ba 100644 --- a/Lib/test/test_contextlib.py +++ b/Lib/test/test_contextlib.py @@ -1271,8 +1271,6 @@ def test_cm_is_reentrant(self): 1/0 self.assertTrue(outer_continued) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_exception_groups(self): eg_ve = lambda: ExceptionGroup( "EG with ValueErrors only", diff --git a/src/interpreter.rs b/src/interpreter.rs index 35710ae829..b79a1a0ffb 100644 --- a/src/interpreter.rs +++ b/src/interpreter.rs @@ -18,6 +18,7 @@ pub type InitHook = Box; /// let mut settings = Settings::default(); /// settings.debug = 1; /// // You may want to add paths to `rustpython_vm::Settings::path_list` to allow import python libraries. +/// settings.path_list.push("Lib".to_owned()); // add standard library directory /// settings.path_list.push("".to_owned()); // add current working directory /// let interpreter = rustpython::InterpreterConfig::new() /// .settings(settings) diff --git a/vm/Lib/python_builtins/_py_exceptiongroup.py b/vm/Lib/python_builtins/_py_exceptiongroup.py new file mode 100644 index 0000000000..91e9354d8a --- /dev/null +++ b/vm/Lib/python_builtins/_py_exceptiongroup.py @@ -0,0 +1,330 @@ +# Copied from https://github.com/agronholm/ExceptionGroup/blob/1.2.1/src/exceptiongroup/_exceptions.py +# License: https://github.com/agronholm/exceptiongroup/blob/1.2.1/LICENSE +from __future__ import annotations + +from collections.abc import Callable, Sequence +from functools import partial +from typing import TYPE_CHECKING, Generic, Type, TypeVar, cast, overload + +_BaseExceptionT_co = TypeVar("_BaseExceptionT_co", bound=BaseException, covariant=True) +_BaseExceptionT = TypeVar("_BaseExceptionT", bound=BaseException) +_ExceptionT_co = TypeVar("_ExceptionT_co", bound=Exception, covariant=True) +_ExceptionT = TypeVar("_ExceptionT", bound=Exception) +# using typing.Self would require a typing_extensions dependency on py<3.11 +_ExceptionGroupSelf = TypeVar("_ExceptionGroupSelf", bound="ExceptionGroup") +_BaseExceptionGroupSelf = TypeVar("_BaseExceptionGroupSelf", bound="BaseExceptionGroup") + + +def check_direct_subclass( + exc: BaseException, parents: tuple[type[BaseException]] +) -> bool: + from inspect import getmro # requires rustpython-stdlib + + for cls in getmro(exc.__class__)[:-1]: + if cls in parents: + return True + + return False + + +def get_condition_filter( + condition: type[_BaseExceptionT] + | tuple[type[_BaseExceptionT], ...] + | Callable[[_BaseExceptionT_co], bool], +) -> Callable[[_BaseExceptionT_co], bool]: + from inspect import isclass # requires rustpython-stdlib + + if isclass(condition) and issubclass( + cast(Type[BaseException], condition), BaseException + ): + return partial(check_direct_subclass, parents=(condition,)) + elif isinstance(condition, tuple): + if all(isclass(x) and issubclass(x, BaseException) for x in condition): + return partial(check_direct_subclass, parents=condition) + elif callable(condition): + return cast("Callable[[BaseException], bool]", condition) + + raise TypeError("expected a function, exception type or tuple of exception types") + + +def _derive_and_copy_attributes(self, excs): + eg = self.derive(excs) + eg.__cause__ = self.__cause__ + eg.__context__ = self.__context__ + eg.__traceback__ = self.__traceback__ + if hasattr(self, "__notes__"): + # Create a new list so that add_note() only affects one exceptiongroup + eg.__notes__ = list(self.__notes__) + return eg + + +class BaseExceptionGroup(BaseException, Generic[_BaseExceptionT_co]): + """A combination of multiple unrelated exceptions.""" + + def __new__( + cls: type[_BaseExceptionGroupSelf], + __message: str, + __exceptions: Sequence[_BaseExceptionT_co], + ) -> _BaseExceptionGroupSelf: + if not isinstance(__message, str): + raise TypeError(f"argument 1 must be str, not {type(__message)}") + if not isinstance(__exceptions, Sequence): + raise TypeError("second argument (exceptions) must be a sequence") + if not __exceptions: + raise ValueError( + "second argument (exceptions) must be a non-empty sequence" + ) + + for i, exc in enumerate(__exceptions): + if not isinstance(exc, BaseException): + raise ValueError( + f"Item {i} of second argument (exceptions) is not an exception" + ) + + if cls is BaseExceptionGroup: + if all(isinstance(exc, Exception) for exc in __exceptions): + cls = ExceptionGroup + + if issubclass(cls, Exception): + for exc in __exceptions: + if not isinstance(exc, Exception): + if cls is ExceptionGroup: + raise TypeError( + "Cannot nest BaseExceptions in an ExceptionGroup" + ) + else: + raise TypeError( + f"Cannot nest BaseExceptions in {cls.__name__!r}" + ) + + instance = super().__new__(cls, __message, __exceptions) + instance._message = __message + instance._exceptions = __exceptions + return instance + + def add_note(self, note: str) -> None: + if not isinstance(note, str): + raise TypeError( + f"Expected a string, got note={note!r} (type {type(note).__name__})" + ) + + if not hasattr(self, "__notes__"): + self.__notes__: list[str] = [] + + self.__notes__.append(note) + + @property + def message(self) -> str: + return self._message + + @property + def exceptions( + self, + ) -> tuple[_BaseExceptionT_co | BaseExceptionGroup[_BaseExceptionT_co], ...]: + return tuple(self._exceptions) + + @overload + def subgroup( + self, __condition: type[_ExceptionT] | tuple[type[_ExceptionT], ...] + ) -> ExceptionGroup[_ExceptionT] | None: ... + + @overload + def subgroup( + self, __condition: type[_BaseExceptionT] | tuple[type[_BaseExceptionT], ...] + ) -> BaseExceptionGroup[_BaseExceptionT] | None: ... + + @overload + def subgroup( + self, + __condition: Callable[[_BaseExceptionT_co | _BaseExceptionGroupSelf], bool], + ) -> BaseExceptionGroup[_BaseExceptionT_co] | None: ... + + def subgroup( + self, + __condition: type[_BaseExceptionT] + | tuple[type[_BaseExceptionT], ...] + | Callable[[_BaseExceptionT_co | _BaseExceptionGroupSelf], bool], + ) -> BaseExceptionGroup[_BaseExceptionT] | None: + condition = get_condition_filter(__condition) + modified = False + if condition(self): + return self + + exceptions: list[BaseException] = [] + for exc in self.exceptions: + if isinstance(exc, BaseExceptionGroup): + subgroup = exc.subgroup(__condition) + if subgroup is not None: + exceptions.append(subgroup) + + if subgroup is not exc: + modified = True + elif condition(exc): + exceptions.append(exc) + else: + modified = True + + if not modified: + return self + elif exceptions: + group = _derive_and_copy_attributes(self, exceptions) + return group + else: + return None + + @overload + def split( + self, __condition: type[_ExceptionT] | tuple[type[_ExceptionT], ...] + ) -> tuple[ + ExceptionGroup[_ExceptionT] | None, + BaseExceptionGroup[_BaseExceptionT_co] | None, + ]: ... + + @overload + def split( + self, __condition: type[_BaseExceptionT] | tuple[type[_BaseExceptionT], ...] + ) -> tuple[ + BaseExceptionGroup[_BaseExceptionT] | None, + BaseExceptionGroup[_BaseExceptionT_co] | None, + ]: ... + + @overload + def split( + self, + __condition: Callable[[_BaseExceptionT_co | _BaseExceptionGroupSelf], bool], + ) -> tuple[ + BaseExceptionGroup[_BaseExceptionT_co] | None, + BaseExceptionGroup[_BaseExceptionT_co] | None, + ]: ... + + def split( + self, + __condition: type[_BaseExceptionT] + | tuple[type[_BaseExceptionT], ...] + | Callable[[_BaseExceptionT_co], bool], + ) -> ( + tuple[ + ExceptionGroup[_ExceptionT] | None, + BaseExceptionGroup[_BaseExceptionT_co] | None, + ] + | tuple[ + BaseExceptionGroup[_BaseExceptionT] | None, + BaseExceptionGroup[_BaseExceptionT_co] | None, + ] + | tuple[ + BaseExceptionGroup[_BaseExceptionT_co] | None, + BaseExceptionGroup[_BaseExceptionT_co] | None, + ] + ): + condition = get_condition_filter(__condition) + if condition(self): + return self, None + + matching_exceptions: list[BaseException] = [] + nonmatching_exceptions: list[BaseException] = [] + for exc in self.exceptions: + if isinstance(exc, BaseExceptionGroup): + matching, nonmatching = exc.split(condition) + if matching is not None: + matching_exceptions.append(matching) + + if nonmatching is not None: + nonmatching_exceptions.append(nonmatching) + elif condition(exc): + matching_exceptions.append(exc) + else: + nonmatching_exceptions.append(exc) + + matching_group: _BaseExceptionGroupSelf | None = None + if matching_exceptions: + matching_group = _derive_and_copy_attributes(self, matching_exceptions) + + nonmatching_group: _BaseExceptionGroupSelf | None = None + if nonmatching_exceptions: + nonmatching_group = _derive_and_copy_attributes( + self, nonmatching_exceptions + ) + + return matching_group, nonmatching_group + + @overload + def derive(self, __excs: Sequence[_ExceptionT]) -> ExceptionGroup[_ExceptionT]: ... + + @overload + def derive( + self, __excs: Sequence[_BaseExceptionT] + ) -> BaseExceptionGroup[_BaseExceptionT]: ... + + def derive( + self, __excs: Sequence[_BaseExceptionT] + ) -> BaseExceptionGroup[_BaseExceptionT]: + return BaseExceptionGroup(self.message, __excs) + + def __str__(self) -> str: + suffix = "" if len(self._exceptions) == 1 else "s" + return f"{self.message} ({len(self._exceptions)} sub-exception{suffix})" + + def __repr__(self) -> str: + return f"{self.__class__.__name__}({self.message!r}, {self._exceptions!r})" + + +class ExceptionGroup(BaseExceptionGroup[_ExceptionT_co], Exception): + def __new__( + cls: type[_ExceptionGroupSelf], + __message: str, + __exceptions: Sequence[_ExceptionT_co], + ) -> _ExceptionGroupSelf: + return super().__new__(cls, __message, __exceptions) + + if TYPE_CHECKING: + + @property + def exceptions( + self, + ) -> tuple[_ExceptionT_co | ExceptionGroup[_ExceptionT_co], ...]: ... + + @overload # type: ignore[override] + def subgroup( + self, __condition: type[_ExceptionT] | tuple[type[_ExceptionT], ...] + ) -> ExceptionGroup[_ExceptionT] | None: ... + + @overload + def subgroup( + self, __condition: Callable[[_ExceptionT_co | _ExceptionGroupSelf], bool] + ) -> ExceptionGroup[_ExceptionT_co] | None: ... + + def subgroup( + self, + __condition: type[_ExceptionT] + | tuple[type[_ExceptionT], ...] + | Callable[[_ExceptionT_co], bool], + ) -> ExceptionGroup[_ExceptionT] | None: + return super().subgroup(__condition) + + @overload + def split( + self, __condition: type[_ExceptionT] | tuple[type[_ExceptionT], ...] + ) -> tuple[ + ExceptionGroup[_ExceptionT] | None, ExceptionGroup[_ExceptionT_co] | None + ]: ... + + @overload + def split( + self, __condition: Callable[[_ExceptionT_co | _ExceptionGroupSelf], bool] + ) -> tuple[ + ExceptionGroup[_ExceptionT_co] | None, ExceptionGroup[_ExceptionT_co] | None + ]: ... + + def split( + self: _ExceptionGroupSelf, + __condition: type[_ExceptionT] + | tuple[type[_ExceptionT], ...] + | Callable[[_ExceptionT_co], bool], + ) -> tuple[ + ExceptionGroup[_ExceptionT_co] | None, ExceptionGroup[_ExceptionT_co] | None + ]: + return super().split(__condition) + + +BaseExceptionGroup.__module__ = 'builtins' +ExceptionGroup.__module__ = 'builtins' diff --git a/vm/src/vm/mod.rs b/vm/src/vm/mod.rs index 493789f510..b47c986cae 100644 --- a/vm/src/vm/mod.rs +++ b/vm/src/vm/mod.rs @@ -364,8 +364,11 @@ impl VirtualMachine { } } + let expect_stdlib = + cfg!(feature = "freeze-stdlib") || !self.state.settings.path_list.is_empty(); + #[cfg(feature = "encodings")] - if cfg!(feature = "freeze-stdlib") || !self.state.settings.path_list.is_empty() { + if expect_stdlib { if let Err(e) = self.import_encodings() { eprintln!( "encodings initialization failed. Only utf-8 encoding will be supported." @@ -382,6 +385,21 @@ impl VirtualMachine { ); } + if expect_stdlib { + // enable python-implemented ExceptionGroup when stdlib exists + let py_core_init = || -> PyResult<()> { + let exception_group = import::import_frozen(self, "_py_exceptiongroup")?; + let base_exception_group = exception_group.get_attr("BaseExceptionGroup", self)?; + self.builtins + .set_attr("BaseExceptionGroup", base_exception_group, self)?; + let exception_group = exception_group.get_attr("ExceptionGroup", self)?; + self.builtins + .set_attr("ExceptionGroup", exception_group, self)?; + Ok(()) + }; + self.expect_pyresult(py_core_init(), "exceptiongroup initialization failed"); + } + self.initialized = true; } From 5e0eace8d9a0c83a62324c6470065b0899fcb6df Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Mon, 22 Apr 2024 11:26:52 +0900 Subject: [PATCH 124/295] test_exception_group from CPython 3.12.2 --- Lib/test/test_exception_group.py | 92 +++++++++++--------------------- 1 file changed, 32 insertions(+), 60 deletions(-) diff --git a/Lib/test/test_exception_group.py b/Lib/test/test_exception_group.py index 03c721d56f..d0d81490df 100644 --- a/Lib/test/test_exception_group.py +++ b/Lib/test/test_exception_group.py @@ -1,12 +1,9 @@ import collections.abc -import traceback import types import unittest - +from test.support import C_RECURSION_LIMIT class TestExceptionGroupTypeHierarchy(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_exception_group_types(self): self.assertTrue(issubclass(ExceptionGroup, Exception)) self.assertTrue(issubclass(ExceptionGroup, BaseExceptionGroup)) @@ -38,8 +35,6 @@ def test_bad_EG_construction__too_many_args(self): with self.assertRaisesRegex(TypeError, MSG): ExceptionGroup('eg', [ValueError('too')], [TypeError('many')]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_bad_EG_construction__bad_message(self): MSG = 'argument 1 must be str, not ' with self.assertRaisesRegex(TypeError, MSG): @@ -47,8 +42,6 @@ def test_bad_EG_construction__bad_message(self): with self.assertRaisesRegex(TypeError, MSG): ExceptionGroup(None, [ValueError(12)]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_bad_EG_construction__bad_excs_sequence(self): MSG = r'second argument \(exceptions\) must be a sequence' with self.assertRaisesRegex(TypeError, MSG): @@ -60,8 +53,6 @@ def test_bad_EG_construction__bad_excs_sequence(self): with self.assertRaisesRegex(ValueError, MSG): ExceptionGroup("eg", []) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_bad_EG_construction__nested_non_exceptions(self): MSG = (r'Item [0-9]+ of second argument \(exceptions\)' ' is not an exception') @@ -78,16 +69,12 @@ def test_EG_wraps_Exceptions__creates_EG(self): type(ExceptionGroup("eg", excs)), ExceptionGroup) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_BEG_wraps_Exceptions__creates_EG(self): excs = [ValueError(1), TypeError(2)] self.assertIs( type(BaseExceptionGroup("beg", excs)), ExceptionGroup) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_EG_wraps_BaseException__raises_TypeError(self): MSG= "Cannot nest BaseExceptions in an ExceptionGroup" with self.assertRaisesRegex(TypeError, MSG): @@ -105,8 +92,6 @@ class MyEG(ExceptionGroup): type(MyEG("eg", [ValueError(12), TypeError(42)])), MyEG) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_EG_subclass_does_not_wrap_base_exceptions(self): class MyEG(ExceptionGroup): pass @@ -115,8 +100,6 @@ class MyEG(ExceptionGroup): with self.assertRaisesRegex(TypeError, msg): MyEG("eg", [ValueError(12), KeyboardInterrupt(42)]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_BEG_and_E_subclass_does_not_wrap_base_exceptions(self): class MyEG(BaseExceptionGroup, ValueError): pass @@ -125,6 +108,21 @@ class MyEG(BaseExceptionGroup, ValueError): with self.assertRaisesRegex(TypeError, msg): MyEG("eg", [ValueError(12), KeyboardInterrupt(42)]) + def test_EG_and_specific_subclass_can_wrap_any_nonbase_exception(self): + class MyEG(ExceptionGroup, ValueError): + pass + + # The restriction is specific to Exception, not "the other base class" + MyEG("eg", [ValueError(12), Exception()]) + + def test_BEG_and_specific_subclass_can_wrap_any_nonbase_exception(self): + class MyEG(BaseExceptionGroup, ValueError): + pass + + # The restriction is specific to Exception, not "the other base class" + MyEG("eg", [ValueError(12), Exception()]) + + def test_BEG_subclass_wraps_anything(self): class MyBEG(BaseExceptionGroup): pass @@ -138,8 +136,6 @@ class MyBEG(BaseExceptionGroup): class StrAndReprTests(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_ExceptionGroup(self): eg = BaseExceptionGroup( 'flat', [ValueError(1), TypeError(2)]) @@ -160,8 +156,6 @@ def test_ExceptionGroup(self): "ExceptionGroup('flat', " "[ValueError(1), TypeError(2)]), TypeError(2)])") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_BaseExceptionGroup(self): eg = BaseExceptionGroup( 'flat', [ValueError(1), KeyboardInterrupt(2)]) @@ -184,8 +178,6 @@ def test_BaseExceptionGroup(self): "BaseExceptionGroup('flat', " "[ValueError(1), KeyboardInterrupt(2)])])") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_custom_exception(self): class MyEG(ExceptionGroup): pass @@ -270,8 +262,6 @@ def test_basics_ExceptionGroup_fields(self): self.assertIsNone(tb.tb_next) self.assertEqual(tb.tb_lineno, tb_linenos[1][i]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_fields_are_readonly(self): eg = ExceptionGroup('eg', [TypeError(1), OSError(2)]) @@ -287,8 +277,6 @@ def test_fields_are_readonly(self): class ExceptionGroupTestBase(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def assertMatchesTemplate(self, exc, exc_type, template): """ Assert that the exception matches the template @@ -318,7 +306,6 @@ def setUp(self): self.eg = create_simple_eg() self.eg_template = [ValueError(1), TypeError(int), ValueError(2)] - @unittest.skip("TODO: RUSTPYTHON") def test_basics_subgroup_split__bad_arg_type(self): bad_args = ["bad arg", OSError('instance not type'), @@ -330,8 +317,6 @@ def test_basics_subgroup_split__bad_arg_type(self): with self.assertRaises(TypeError): self.eg.split(arg) - - @unittest.skip("TODO: RUSTPYTHON") def test_basics_subgroup_by_type__passthrough(self): eg = self.eg self.assertIs(eg, eg.subgroup(BaseException)) @@ -339,11 +324,9 @@ def test_basics_subgroup_by_type__passthrough(self): self.assertIs(eg, eg.subgroup(BaseExceptionGroup)) self.assertIs(eg, eg.subgroup(ExceptionGroup)) - @unittest.skip("TODO: RUSTPYTHON") def test_basics_subgroup_by_type__no_match(self): self.assertIsNone(self.eg.subgroup(OSError)) - @unittest.skip("TODO: RUSTPYTHON") def test_basics_subgroup_by_type__match(self): eg = self.eg testcases = [ @@ -358,15 +341,12 @@ def test_basics_subgroup_by_type__match(self): self.assertEqual(subeg.message, eg.message) self.assertMatchesTemplate(subeg, ExceptionGroup, template) - @unittest.skip("TODO: RUSTPYTHON") def test_basics_subgroup_by_predicate__passthrough(self): self.assertIs(self.eg, self.eg.subgroup(lambda e: True)) - @unittest.skip("TODO: RUSTPYTHON") def test_basics_subgroup_by_predicate__no_match(self): self.assertIsNone(self.eg.subgroup(lambda e: False)) - @unittest.skip("TODO: RUSTPYTHON") def test_basics_subgroup_by_predicate__match(self): eg = self.eg testcases = [ @@ -386,7 +366,6 @@ def setUp(self): self.eg = create_simple_eg() self.eg_template = [ValueError(1), TypeError(int), ValueError(2)] - @unittest.skip("TODO: RUSTPYTHON") def test_basics_split_by_type__passthrough(self): for E in [BaseException, Exception, BaseExceptionGroup, ExceptionGroup]: @@ -395,14 +374,12 @@ def test_basics_split_by_type__passthrough(self): match, ExceptionGroup, self.eg_template) self.assertIsNone(rest) - @unittest.skip("TODO: RUSTPYTHON") def test_basics_split_by_type__no_match(self): match, rest = self.eg.split(OSError) self.assertIsNone(match) self.assertMatchesTemplate( rest, ExceptionGroup, self.eg_template) - @unittest.skip("TODO: RUSTPYTHON") def test_basics_split_by_type__match(self): eg = self.eg VE = ValueError @@ -427,19 +404,16 @@ def test_basics_split_by_type__match(self): else: self.assertIsNone(rest) - @unittest.skip("TODO: RUSTPYTHON") def test_basics_split_by_predicate__passthrough(self): match, rest = self.eg.split(lambda e: True) self.assertMatchesTemplate(match, ExceptionGroup, self.eg_template) self.assertIsNone(rest) - @unittest.skip("TODO: RUSTPYTHON") def test_basics_split_by_predicate__no_match(self): match, rest = self.eg.split(lambda e: False) self.assertIsNone(match) self.assertMatchesTemplate(rest, ExceptionGroup, self.eg_template) - @unittest.skip("TODO: RUSTPYTHON") def test_basics_split_by_predicate__match(self): eg = self.eg VE = ValueError @@ -465,19 +439,15 @@ def test_basics_split_by_predicate__match(self): class DeepRecursionInSplitAndSubgroup(unittest.TestCase): def make_deep_eg(self): e = TypeError(1) - for i in range(2000): + for i in range(C_RECURSION_LIMIT + 1): e = ExceptionGroup('eg', [e]) return e - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_deep_split(self): e = self.make_deep_eg() with self.assertRaises(RecursionError): e.split(TypeError) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_deep_subgroup(self): e = self.make_deep_eg() with self.assertRaises(RecursionError): @@ -501,7 +471,7 @@ def leaf_generator(exc, tbs=None): class LeafGeneratorTest(unittest.TestCase): # The leaf_generator is mentioned in PEP 654 as a suggestion - # on how to iterate over leaf nodes of an EG. It is also + # on how to iterate over leaf nodes of an EG. Is is also # used below as a test utility. So we test it here. def test_leaf_generator(self): @@ -654,8 +624,6 @@ def tb_linenos(tbs): class NestedExceptionGroupSplitTest(ExceptionGroupSplitTestBase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_split_by_type(self): class MyExceptionGroup(ExceptionGroup): pass @@ -755,8 +723,6 @@ def level3(i): self.assertMatchesTemplate(match, ExceptionGroup, [eg_template[0]]) self.assertMatchesTemplate(rest, ExceptionGroup, [eg_template[1]]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_split_BaseExceptionGroup(self): def exc(ex): try: @@ -797,8 +763,6 @@ def exc(ex): self.assertMatchesTemplate( rest, ExceptionGroup, [ValueError(1)]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_split_copies_notes(self): # make sure each exception group after a split has its own __notes__ list eg = ExceptionGroup("eg", [ValueError(1), TypeError(2)]) @@ -830,11 +794,23 @@ def test_split_does_not_copy_non_sequence_notes(self): self.assertFalse(hasattr(match, '__notes__')) self.assertFalse(hasattr(rest, '__notes__')) + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_drive_invalid_return_value(self): + class MyEg(ExceptionGroup): + def derive(self, excs): + return 42 + + eg = MyEg('eg', [TypeError(1), ValueError(2)]) + msg = "derive must return an instance of BaseExceptionGroup" + with self.assertRaisesRegex(TypeError, msg): + eg.split(TypeError) + with self.assertRaisesRegex(TypeError, msg): + eg.subgroup(TypeError) + class NestedExceptionGroupSubclassSplitTest(ExceptionGroupSplitTestBase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_split_ExceptionGroup_subclass_no_derive_no_new_override(self): class EG(ExceptionGroup): pass @@ -877,8 +853,6 @@ class EG(ExceptionGroup): self.assertMatchesTemplate(match, ExceptionGroup, [[TypeError(2)]]) self.assertMatchesTemplate(rest, ExceptionGroup, [ValueError(1)]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_split_BaseExceptionGroup_subclass_no_derive_new_override(self): class EG(BaseExceptionGroup): def __new__(cls, message, excs, unused): @@ -921,8 +895,6 @@ def __new__(cls, message, excs, unused): match, BaseExceptionGroup, [KeyboardInterrupt(2)]) self.assertMatchesTemplate(rest, ExceptionGroup, [ValueError(1)]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_split_ExceptionGroup_subclass_derive_and_new_overrides(self): class EG(ExceptionGroup): def __new__(cls, message, excs, code): From 053583f5a0b9499c21f8bfa427d6066601bebd8e Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 25 Apr 2024 23:16:08 +0900 Subject: [PATCH 125/295] Add wasm/demo/.envrc --- wasm/demo/.envrc | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 wasm/demo/.envrc diff --git a/wasm/demo/.envrc b/wasm/demo/.envrc new file mode 100644 index 0000000000..928b937d32 --- /dev/null +++ b/wasm/demo/.envrc @@ -0,0 +1,2 @@ +export NODE_OPTIONS=--openssl-legacy-provider +export PATH=$PATH:`pwd`/../../geckodriver From 27bcba30275f2bb43f56160b76a56a3f671e6df0 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 25 Apr 2024 23:18:31 +0900 Subject: [PATCH 126/295] wasm test prints more info --- wasm/demo/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wasm/demo/package.json b/wasm/demo/package.json index 53ed264ea7..487f46fbc1 100644 --- a/wasm/demo/package.json +++ b/wasm/demo/package.json @@ -25,7 +25,7 @@ "dev": "webpack serve", "build": "webpack", "dist": "webpack --mode production", - "test": "webpack --mode production && cd ../tests && pytest" + "test": "webpack --mode production && cd ../tests && pytest -v" }, "repository": { "type": "git", From 5f6f6cce92aac1f9c142ced4108cc831fea38431 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 16 Mar 2023 21:31:58 +0900 Subject: [PATCH 127/295] add cspell to CI --- .github/workflows/ci.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1dac8d47c7..12399413ba 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -313,6 +313,11 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - name: spell checker + uses: streetsidesoftware/cspell-action@v2 + with: + files: '**/*.rs' + incremental_files_only: true - uses: dtolnay/rust-toolchain@stable with: components: rustfmt, clippy From 372e68306334adaeb0730b1b45a063714b1ec713 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 27 Mar 2025 01:43:45 +0900 Subject: [PATCH 128/295] disable cspell from files with a bunch of OS jargons --- .cspell.json | 1 + common/src/os.rs | 1 + stdlib/src/mmap.rs | 1 + stdlib/src/overlapped.rs | 2 ++ stdlib/src/posixsubprocess.rs | 2 ++ stdlib/src/resource.rs | 2 ++ stdlib/src/select.rs | 2 ++ stdlib/src/socket.rs | 2 ++ stdlib/src/ssl.rs | 2 ++ stdlib/src/termios.rs | 2 ++ vm/src/stdlib/ctypes.rs | 2 ++ vm/src/stdlib/ctypes/function.rs | 2 ++ vm/src/stdlib/errno.rs | 2 ++ vm/src/stdlib/msvcrt.rs | 2 ++ vm/src/stdlib/nt.rs | 2 ++ vm/src/stdlib/os.rs | 2 ++ vm/src/stdlib/posix.rs | 2 ++ vm/src/stdlib/posix_compat.rs | 2 ++ vm/src/stdlib/pwd.rs | 2 ++ vm/src/stdlib/signal.rs | 2 ++ vm/src/stdlib/winapi.rs | 2 ++ vm/src/stdlib/winreg.rs | 1 + 22 files changed, 40 insertions(+) diff --git a/.cspell.json b/.cspell.json index c1db1036a9..0d7df1a6b9 100644 --- a/.cspell.json +++ b/.cspell.json @@ -187,6 +187,7 @@ "warningregistry", "warnopts", "weakproxy", + "winver", "xopts", // RustPython "baseclass", diff --git a/common/src/os.rs b/common/src/os.rs index 8a832270bc..06ea1432e9 100644 --- a/common/src/os.rs +++ b/common/src/os.rs @@ -1,3 +1,4 @@ +// cspell:disable // TODO: we can move more os-specific bindings/interfaces from stdlib::{os, posix, nt} to here use std::{io, str::Utf8Error}; diff --git a/stdlib/src/mmap.rs b/stdlib/src/mmap.rs index cbc86bb2c9..c56369e979 100644 --- a/stdlib/src/mmap.rs +++ b/stdlib/src/mmap.rs @@ -1,3 +1,4 @@ +// cspell:disable //! mmap module pub(crate) use mmap::make_module; diff --git a/stdlib/src/overlapped.rs b/stdlib/src/overlapped.rs index 629461abe9..007fa67423 100644 --- a/stdlib/src/overlapped.rs +++ b/stdlib/src/overlapped.rs @@ -1,3 +1,5 @@ +// cspell:disable + pub(crate) use _overlapped::make_module; #[allow(non_snake_case)] diff --git a/stdlib/src/posixsubprocess.rs b/stdlib/src/posixsubprocess.rs index 595caf524b..346032fe79 100644 --- a/stdlib/src/posixsubprocess.rs +++ b/stdlib/src/posixsubprocess.rs @@ -1,3 +1,5 @@ +// cspell:disable + use crate::vm::{ builtins::PyListRef, function::ArgSequence, diff --git a/stdlib/src/resource.rs b/stdlib/src/resource.rs index e103cce779..3255bb3f61 100644 --- a/stdlib/src/resource.rs +++ b/stdlib/src/resource.rs @@ -1,3 +1,5 @@ +// cspell:disable + pub(crate) use resource::make_module; #[pymodule] diff --git a/stdlib/src/select.rs b/stdlib/src/select.rs index 4003856bb9..d4380b69fa 100644 --- a/stdlib/src/select.rs +++ b/stdlib/src/select.rs @@ -1,3 +1,5 @@ +// cspell:disable + use crate::vm::{ PyObject, PyObjectRef, PyRef, PyResult, TryFromObject, VirtualMachine, builtins::PyListRef, builtins::PyModule, diff --git a/stdlib/src/socket.rs b/stdlib/src/socket.rs index 17daec7751..988784856f 100644 --- a/stdlib/src/socket.rs +++ b/stdlib/src/socket.rs @@ -1,3 +1,5 @@ +// cspell:disable + use crate::vm::{PyRef, VirtualMachine, builtins::PyModule}; #[cfg(feature = "ssl")] pub(super) use _socket::{PySocket, SelectKind, sock_select, timeout_error_msg}; diff --git a/stdlib/src/ssl.rs b/stdlib/src/ssl.rs index 10d1906448..c1dcea354f 100644 --- a/stdlib/src/ssl.rs +++ b/stdlib/src/ssl.rs @@ -1,3 +1,5 @@ +// cspell:disable + use crate::vm::{PyRef, VirtualMachine, builtins::PyModule}; use openssl_probe::ProbeResult; diff --git a/stdlib/src/termios.rs b/stdlib/src/termios.rs index 5c49d62a3c..55cd45e651 100644 --- a/stdlib/src/termios.rs +++ b/stdlib/src/termios.rs @@ -1,3 +1,5 @@ +// cspell:disable + pub(crate) use self::termios::make_module; #[pymodule] diff --git a/vm/src/stdlib/ctypes.rs b/vm/src/stdlib/ctypes.rs index b6fd150889..38955fe391 100644 --- a/vm/src/stdlib/ctypes.rs +++ b/vm/src/stdlib/ctypes.rs @@ -1,3 +1,5 @@ +// cspell:disable + pub(crate) mod array; pub(crate) mod base; pub(crate) mod function; diff --git a/vm/src/stdlib/ctypes/function.rs b/vm/src/stdlib/ctypes/function.rs index c1c1230e03..21043da27d 100644 --- a/vm/src/stdlib/ctypes/function.rs +++ b/vm/src/stdlib/ctypes/function.rs @@ -1,3 +1,5 @@ +// cspell:disable + use crate::builtins::{PyStr, PyTupleRef, PyTypeRef}; use crate::convert::ToPyObject; use crate::function::FuncArgs; diff --git a/vm/src/stdlib/errno.rs b/vm/src/stdlib/errno.rs index 247e2a2340..c77fcbfefc 100644 --- a/vm/src/stdlib/errno.rs +++ b/vm/src/stdlib/errno.rs @@ -1,3 +1,5 @@ +// cspell:disable + use crate::{PyRef, VirtualMachine, builtins::PyModule}; #[pymodule] diff --git a/vm/src/stdlib/msvcrt.rs b/vm/src/stdlib/msvcrt.rs index 7b3620ad51..463f4566ae 100644 --- a/vm/src/stdlib/msvcrt.rs +++ b/vm/src/stdlib/msvcrt.rs @@ -1,3 +1,5 @@ +// cspell:disable + pub use msvcrt::*; #[pymodule] diff --git a/vm/src/stdlib/nt.rs b/vm/src/stdlib/nt.rs index 624577b5ce..b4899bb225 100644 --- a/vm/src/stdlib/nt.rs +++ b/vm/src/stdlib/nt.rs @@ -1,3 +1,5 @@ +// cspell:disable + use crate::{PyRef, VirtualMachine, builtins::PyModule}; pub use module::raw_set_handle_inheritable; diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index e1a5825b82..f53be8b01b 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -1,3 +1,5 @@ +// cspell:disable + use crate::{ AsObject, Py, PyPayload, PyResult, VirtualMachine, builtins::{PyBaseExceptionRef, PyModule, PySet}, diff --git a/vm/src/stdlib/posix.rs b/vm/src/stdlib/posix.rs index 05b1d8addd..dc5c74cef4 100644 --- a/vm/src/stdlib/posix.rs +++ b/vm/src/stdlib/posix.rs @@ -1,3 +1,5 @@ +// cspell:disable + use crate::{PyRef, VirtualMachine, builtins::PyModule}; use std::os::unix::io::RawFd; diff --git a/vm/src/stdlib/posix_compat.rs b/vm/src/stdlib/posix_compat.rs index 334aa597ce..696daf7f0f 100644 --- a/vm/src/stdlib/posix_compat.rs +++ b/vm/src/stdlib/posix_compat.rs @@ -1,3 +1,5 @@ +// cspell:disable + //! `posix` compatible module for `not(any(unix, windows))` use crate::{PyRef, VirtualMachine, builtins::PyModule}; diff --git a/vm/src/stdlib/pwd.rs b/vm/src/stdlib/pwd.rs index f6c277242c..b95910c73f 100644 --- a/vm/src/stdlib/pwd.rs +++ b/vm/src/stdlib/pwd.rs @@ -1,3 +1,5 @@ +// cspell:disable + pub(crate) use pwd::make_module; #[pymodule] diff --git a/vm/src/stdlib/signal.rs b/vm/src/stdlib/signal.rs index 0c47ca082e..cdd8d8c3b9 100644 --- a/vm/src/stdlib/signal.rs +++ b/vm/src/stdlib/signal.rs @@ -1,3 +1,5 @@ +// cspell:disable + use crate::{PyRef, VirtualMachine, builtins::PyModule}; pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { diff --git a/vm/src/stdlib/winapi.rs b/vm/src/stdlib/winapi.rs index 7ffc4227e6..2d853f4ed1 100644 --- a/vm/src/stdlib/winapi.rs +++ b/vm/src/stdlib/winapi.rs @@ -1,3 +1,5 @@ +// cspell:disable + #![allow(non_snake_case)] pub(crate) use _winapi::make_module; diff --git a/vm/src/stdlib/winreg.rs b/vm/src/stdlib/winreg.rs index b0dbbfceec..30fe016148 100644 --- a/vm/src/stdlib/winreg.rs +++ b/vm/src/stdlib/winreg.rs @@ -1,3 +1,4 @@ +// cspell:disable #![allow(non_snake_case)] use crate::{PyRef, VirtualMachine, builtins::PyModule}; From 10d2837041a721e97ad86a129195cc0d36335d2e Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 27 Mar 2025 02:15:03 +0900 Subject: [PATCH 129/295] rework dicts --- .cspell.dict/cpython.txt | 41 ++++++ .cspell.dict/python-more.txt | 116 +++++++++++++++++ .cspell.dict/rust-more.txt | 47 +++++++ .cspell.json | 239 ++++------------------------------- 4 files changed, 231 insertions(+), 212 deletions(-) create mode 100644 .cspell.dict/cpython.txt create mode 100644 .cspell.dict/python-more.txt create mode 100644 .cspell.dict/rust-more.txt diff --git a/.cspell.dict/cpython.txt b/.cspell.dict/cpython.txt new file mode 100644 index 0000000000..0ac387634d --- /dev/null +++ b/.cspell.dict/cpython.txt @@ -0,0 +1,41 @@ +argtypes +asdl +asname +augassign +badsyntax +basetype +boolop +bxor +cellarg +cellvar +cellvars +cmpop +dictoffset +elts +excepthandler +finalbody +freevar +freevars +fromlist +heaptype +IMMUTABLETYPE +kwonlyarg +kwonlyargs +linearise +maxdepth +mult +nkwargs +orelse +patma +posonlyarg +posonlyargs +prec +stackdepth +unaryop +unparse +unparser +VARKEYWORDS +varkwarg +wbits +withitem +withs \ No newline at end of file diff --git a/.cspell.dict/python-more.txt b/.cspell.dict/python-more.txt new file mode 100644 index 0000000000..a482c880cc --- /dev/null +++ b/.cspell.dict/python-more.txt @@ -0,0 +1,116 @@ +abstractmethods +aiter +anext +arrayiterator +arraytype +asend +athrow +basicsize +cformat +classcell +closesocket +codepoint +codepoints +cpython +decompressor +defaultaction +descr +dictcomp +dictitems +dictkeys +dictview +docstring +docstrings +dunder +eventmask +fdel +fget +fileencoding +fillchar +finallyhandler +frombytes +fromhex +fromunicode +fset +fspath +fstring +fstrings +genexpr +getattro +getformat +getnewargs +getweakrefcount +getweakrefs +hostnames +idiv +impls +infj +instancecheck +instanceof +isabstractmethod +itemiterator +itemsize +iternext +keyiterator +kwarg +kwargs +linearization +linearize +listcomp +mappingproxy +maxsplit +memoryview +memoryviewiterator +metaclass +metaclasses +metatype +mro +mros +nanj +ndigits +ndim +nonbytes +origname +posixsubprocess +pyexpat +pytraverse +PYTHONDEBUG +PYTHONHOME +PYTHONINSPECT +PYTHONOPTIMIZE +PYTHONPATH +PYTHONPATH +PYTHONVERBOSE +PYTHONWARNINGS +qualname +radd +rdiv +rdivmod +reconstructor +reversevalueiterator +rfloordiv +rlshift +rmod +rpow +rrshift +rsub +rtruediv +scproxy +setattro +setcomp +showwarnmsg +warnmsg +stacklevel +subclasscheck +subclasshook +unionable +unraisablehook +valueiterator +vararg +varargs +varnames +warningregistry +warnopts +weakproxy +winver +xopts \ No newline at end of file diff --git a/.cspell.dict/rust-more.txt b/.cspell.dict/rust-more.txt new file mode 100644 index 0000000000..f2177dd4c7 --- /dev/null +++ b/.cspell.dict/rust-more.txt @@ -0,0 +1,47 @@ +ahash +bidi +biguint +bindgen +bitflags +bstr +byteorder +chrono +consts +cstring +flate2 +fract +hasher +idents +indexmap +insta +keccak +lalrpop +libc +libz +longlong +Manually +maplit +memmap +metas +modpow +nanos +objclass +peekable +powc +powf +prepended +punct +replacen +rsplitn +rustc +rustfmt +seekfrom +splitn +subsec +timsort +trai +ulonglong +unic +unistd +winapi +winsock \ No newline at end of file diff --git a/.cspell.json b/.cspell.json index 0d7df1a6b9..562b300ffa 100644 --- a/.cspell.json +++ b/.cspell.json @@ -1,194 +1,51 @@ // See: https://github.com/streetsidesoftware/cspell/tree/master/packages/cspell { "version": "0.2", + "import": [ + "@cspell/dict-en_us/cspell-ext.json", + // "@cspell/dict-cpp/cspell-ext.json", + "@cspell/dict-python/cspell-ext.json", + "@cspell/dict-rust/cspell-ext.json", + "@cspell/dict-win32/cspell-ext.json", + "@cspell/dict-shell/cspell-ext.json", + ], // language - current active spelling language "language": "en", // dictionaries - list of the names of the dictionaries to use "dictionaries": [ + "cpython", // Sometimes keeping same terms with cpython is easy + "python-more", // Python API terms not listed in python + "rust-more", // Rust API terms not listed in rust "en_US", "softwareTerms", "c", "cpp", "python", - "python-custom", "rust", - "unix", - "posix", - "winapi" + "shell", + "win32" ], // dictionaryDefinitions - this list defines any custom dictionaries to use - "dictionaryDefinitions": [], + "dictionaryDefinitions": [ + { + "name": "cpython", + "path": "./.cspell.dict/cpython.txt" + }, + { + "name": "python-more", + "path": "./.cspell.dict/python-more.txt" + }, + { + "name": "rust-more", + "path": "./.cspell.dict/rust-more.txt" + } + ], "ignorePaths": [ "**/__pycache__/**", "Lib/**" ], // words - list of words to be always considered correct "words": [ - // Rust - "ahash", - "bidi", - "biguint", - "bindgen", - "bitflags", - "bstr", - "byteorder", - "chrono", - "consts", - "cstring", - "flate2", - "fract", - "hasher", - "idents", - "indexmap", - "insta", - "keccak", - "lalrpop", - "libc", - "libz", - "longlong", - "Manually", - "maplit", - "memmap", - "metas", - "modpow", - "nanos", - "objclass", - "peekable", - "powc", - "powf", - "prepended", - "punct", - "replacen", - "rsplitn", - "rustc", - "rustfmt", - "seekfrom", - "splitn", - "subsec", - "timsort", - "trai", - "ulonglong", - "unic", - "unistd", - "winapi", - "winsock", - // Python - "abstractmethods", - "aiter", - "anext", - "arrayiterator", - "arraytype", - "asend", - "athrow", - "basicsize", - "cformat", - "classcell", - "closesocket", - "codepoint", - "codepoints", - "cpython", - "decompressor", - "defaultaction", - "descr", - "dictcomp", - "dictitems", - "dictkeys", - "dictview", - "docstring", - "docstrings", - "dunder", - "eventmask", - "fdel", - "fget", - "fileencoding", - "fillchar", - "finallyhandler", - "frombytes", - "fromhex", - "fromunicode", - "fset", - "fspath", - "fstring", - "fstrings", - "genexpr", - "getattro", - "getformat", - "getnewargs", - "getweakrefcount", - "getweakrefs", - "hostnames", - "idiv", - "impls", - "infj", - "instancecheck", - "instanceof", - "isabstractmethod", - "itemiterator", - "itemsize", - "iternext", - "keyiterator", - "kwarg", - "kwargs", - "linearization", - "linearize", - "listcomp", - "mappingproxy", - "maxsplit", - "memoryview", - "memoryviewiterator", - "metaclass", - "metaclasses", - "metatype", - "mro", - "mros", - "nanj", - "ndigits", - "ndim", - "nonbytes", - "origname", - "posixsubprocess", - "pyexpat", - "pytraverse", - "PYTHONDEBUG", - "PYTHONHOME", - "PYTHONINSPECT", - "PYTHONOPTIMIZE", - "PYTHONPATH", - "PYTHONPATH", - "PYTHONVERBOSE", - "PYTHONWARNINGS", - "qualname", - "radd", - "rdiv", - "rdivmod", - "reconstructor", - "reversevalueiterator", - "rfloordiv", - "rlshift", - "rmod", - "rpow", - "rrshift", - "rsub", - "rtruediv", - "scproxy", - "setattro", - "setcomp", - "showwarnmsg", - "warnmsg", - "stacklevel", - "subclasscheck", - "subclasshook", - "unionable", - "unraisablehook", - "valueiterator", - "vararg", - "varargs", - "varnames", - "warningregistry", - "warnopts", - "weakproxy", - "winver", - "xopts", // RustPython "baseclass", "Bytecode", @@ -243,48 +100,6 @@ "unraisable", "wasi", "zelf", - // cpython - "argtypes", - "asdl", - "asname", - "augassign", - "badsyntax", - "basetype", - "boolop", - "bxor", - "cellarg", - "cellvar", - "cellvars", - "cmpop", - "dictoffset", - "elts", - "excepthandler", - "finalbody", - "freevar", - "freevars", - "fromlist", - "heaptype", - "IMMUTABLETYPE", - "kwonlyarg", - "kwonlyargs", - "linearise", - "maxdepth", - "mult", - "nkwargs", - "orelse", - "patma", - "posonlyarg", - "posonlyargs", - "prec", - "stackdepth", - "unaryop", - "unparse", - "unparser", - "VARKEYWORDS", - "varkwarg", - "wbits", - "withitem", - "withs" ], // flagWords - list of words to be always considered incorrect "flagWords": [ From e3d96aa3ca6d91293c87b022378b068b95da6c15 Mon Sep 17 00:00:00 2001 From: Noa Date: Wed, 26 Mar 2025 12:42:03 -0500 Subject: [PATCH 130/295] Remove commented-out code Co-authored-by: Jeong, YunWon <69878+youknowone@users.noreply.github.com> --- common/src/encodings.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/common/src/encodings.rs b/common/src/encodings.rs index 9582d63b6c..097dae17ba 100644 --- a/common/src/encodings.rs +++ b/common/src/encodings.rs @@ -244,9 +244,6 @@ where out.extend_from_slice(b.as_ref()); } } - // data = crate::str::try_get_codepoints(full_data.as_ref(), char_restart..) - // .ok_or_else(|| errors.error_oob_restart(char_restart))?; - // char_data_index = char_restart; } out.extend_from_slice(ctx.remaining_data().as_bytes()); Ok(out) From c6cab4c43af41368e03153afbd509bbd307de337 Mon Sep 17 00:00:00 2001 From: Noa Date: Wed, 26 Mar 2025 20:35:59 -0500 Subject: [PATCH 131/295] Parse surrogates in string literals properly --- Cargo.lock | 3 + Lib/test/test_codeccallbacks.py | 6 - common/src/encodings.rs | 2 +- common/src/wtf8/mod.rs | 53 +++++ compiler/codegen/Cargo.toml | 2 + compiler/codegen/src/compile.rs | 138 ++++++++----- compiler/codegen/src/lib.rs | 1 + compiler/codegen/src/string_parser.rs | 287 ++++++++++++++++++++++++++ compiler/core/Cargo.toml | 1 + compiler/core/src/bytecode.rs | 5 +- compiler/core/src/marshal.rs | 12 +- jit/tests/common.rs | 4 +- vm/src/builtins/code.rs | 2 +- vm/src/builtins/str.rs | 12 ++ vm/src/intern.rs | 53 +++-- vm/src/stdlib/marshal.rs | 5 +- 16 files changed, 506 insertions(+), 80 deletions(-) create mode 100644 compiler/codegen/src/string_parser.rs diff --git a/Cargo.lock b/Cargo.lock index 241d2a595d..70d58bc448 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2319,6 +2319,7 @@ dependencies = [ "itertools 0.14.0", "log", "malachite-bigint", + "memchr", "num-complex", "num-traits", "ruff_python_ast", @@ -2330,6 +2331,7 @@ dependencies = [ "rustpython-compiler-core", "rustpython-compiler-source", "thiserror 2.0.11", + "unicode_names2", ] [[package]] @@ -2387,6 +2389,7 @@ dependencies = [ "ruff_python_ast", "ruff_python_parser", "ruff_source_file", + "rustpython-common", "serde", ] diff --git a/Lib/test/test_codeccallbacks.py b/Lib/test/test_codeccallbacks.py index 09a6d883f8..bd1dbcd626 100644 --- a/Lib/test/test_codeccallbacks.py +++ b/Lib/test/test_codeccallbacks.py @@ -536,8 +536,6 @@ def test_badandgoodxmlcharrefreplaceexceptions(self): ("".join("&#%d;" % c for c in cs), 1 + len(s)) ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_badandgoodbackslashreplaceexceptions(self): # "backslashreplace" complains about a non-exception passed in self.assertRaises( @@ -596,8 +594,6 @@ def test_badandgoodbackslashreplaceexceptions(self): (r, 2) ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_badandgoodnamereplaceexceptions(self): # "namereplace" complains about a non-exception passed in self.assertRaises( @@ -644,8 +640,6 @@ def test_badandgoodnamereplaceexceptions(self): (r, 1 + len(s)) ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_badandgoodsurrogateescapeexceptions(self): surrogateescape_errors = codecs.lookup_error('surrogateescape') # "surrogateescape" complains about a non-exception passed in diff --git a/common/src/encodings.rs b/common/src/encodings.rs index 097dae17ba..c444e27a5a 100644 --- a/common/src/encodings.rs +++ b/common/src/encodings.rs @@ -401,7 +401,7 @@ pub mod errors { let mut out = String::with_capacity(num_chars * 4); for c in err_str.code_points() { let c_u32 = c.to_u32(); - if let Some(c_name) = unicode_names2::name(c.to_char_lossy()) { + if let Some(c_name) = c.to_char().and_then(unicode_names2::name) { write!(out, "\\N{{{c_name}}}").unwrap(); } else if c_u32 >= 0x10000 { write!(out, "\\U{c_u32:08x}").unwrap(); diff --git a/common/src/wtf8/mod.rs b/common/src/wtf8/mod.rs index f6ae628bad..21c5de28bd 100644 --- a/common/src/wtf8/mod.rs +++ b/common/src/wtf8/mod.rs @@ -574,6 +574,12 @@ impl> FromIterator for Wtf8Buf { } } +impl Hash for Wtf8Buf { + fn hash(&self, state: &mut H) { + Wtf8::hash(self, state) + } +} + impl AsRef for Wtf8Buf { fn as_ref(&self) -> &Wtf8 { self @@ -692,6 +698,13 @@ impl Default for &Wtf8 { } } +impl Hash for Wtf8 { + fn hash(&self, state: &mut H) { + state.write(self.as_bytes()); + state.write_u8(0xff); + } +} + impl Wtf8 { /// Creates a WTF-8 slice from a UTF-8 `&str` slice. /// @@ -722,6 +735,32 @@ impl Wtf8 { unsafe { &mut *(value as *mut [u8] as *mut Wtf8) } } + /// Create a WTF-8 slice from a WTF-8 byte slice. + // + // whooops! using WTF-8 for interchange! + #[inline] + pub fn from_bytes(b: &[u8]) -> Option<&Self> { + let mut rest = b; + while let Err(e) = std::str::from_utf8(rest) { + rest = &rest[e.valid_up_to()..]; + Self::decode_surrogate(rest)?; + rest = &rest[3..]; + } + Some(unsafe { Wtf8::from_bytes_unchecked(b) }) + } + + fn decode_surrogate(b: &[u8]) -> Option { + let [a, b, c, ..] = *b else { return None }; + if (a & 0xf0) == 0xe0 && (b & 0xc0) == 0x80 && (c & 0xc0) == 0x80 { + // it's a three-byte code + let c = ((a as u32 & 0x0f) << 12) + ((b as u32 & 0x3f) << 6) + (c as u32 & 0x3f); + let 0xD800..=0xDFFF = c else { return None }; + Some(CodePoint { value: c }) + } else { + None + } + } + /// Returns the length, in WTF-8 bytes. #[inline] pub fn len(&self) -> usize { @@ -875,6 +914,14 @@ impl Wtf8 { } } + #[inline] + fn final_lead_surrogate(&self) -> Option { + match self.bytes { + [.., 0xED, b2 @ 0xA0..=0xAF, b3] => Some(decode_surrogate(b2, b3)), + _ => None, + } + } + pub fn is_code_point_boundary(&self, index: usize) -> bool { is_code_point_boundary(self, index) } @@ -1481,6 +1528,12 @@ impl From for Box { } } +impl From> for Wtf8Buf { + fn from(w: Box) -> Self { + Wtf8Buf::from_box(w) + } +} + impl From for Box { fn from(s: String) -> Self { s.into_boxed_str().into() diff --git a/compiler/codegen/Cargo.toml b/compiler/codegen/Cargo.toml index 9cf93bc22b..c7ff439f78 100644 --- a/compiler/codegen/Cargo.toml +++ b/compiler/codegen/Cargo.toml @@ -30,6 +30,8 @@ num-complex = { workspace = true } num-traits = { workspace = true } thiserror = { workspace = true } malachite-bigint = { workspace = true } +memchr = { workspace = true } +unicode_names2 = { workspace = true } [dev-dependencies] # rustpython-parser = { workspace = true } diff --git a/compiler/codegen/src/compile.rs b/compiler/codegen/src/compile.rs index a6eb216e2a..a03a1fdb50 100644 --- a/compiler/codegen/src/compile.rs +++ b/compiler/codegen/src/compile.rs @@ -21,13 +21,14 @@ use ruff_python_ast::{ Alias, Arguments, BoolOp, CmpOp, Comprehension, ConversionFlag, DebugText, Decorator, DictItem, ExceptHandler, ExceptHandlerExceptHandler, Expr, ExprAttribute, ExprBoolOp, ExprFString, ExprList, ExprName, ExprStarred, ExprSubscript, ExprTuple, ExprUnaryOp, FString, - FStringElement, FStringElements, FStringPart, Int, Keyword, MatchCase, ModExpression, - ModModule, Operator, Parameters, Pattern, PatternMatchAs, PatternMatchValue, Stmt, StmtExpr, - TypeParam, TypeParamParamSpec, TypeParamTypeVar, TypeParamTypeVarTuple, TypeParams, UnaryOp, - WithItem, + FStringElement, FStringElements, FStringFlags, FStringPart, Int, Keyword, MatchCase, + ModExpression, ModModule, Operator, Parameters, Pattern, PatternMatchAs, PatternMatchValue, + Stmt, StmtExpr, TypeParam, TypeParamParamSpec, TypeParamTypeVar, TypeParamTypeVarTuple, + TypeParams, UnaryOp, WithItem, }; use ruff_source_file::OneIndexed; use ruff_text_size::{Ranged, TextRange}; +use rustpython_common::wtf8::Wtf8Buf; // use rustpython_ast::located::{self as located_ast, Located}; use rustpython_compiler_core::{ Mode, @@ -375,7 +376,9 @@ impl Compiler<'_> { let (doc, statements) = split_doc(&body.body, &self.opts); if let Some(value) = doc { - self.emit_load_const(ConstantData::Str { value }); + self.emit_load_const(ConstantData::Str { + value: value.into(), + }); let doc = self.name("__doc__"); emit!(self, Instruction::StoreGlobal(doc)) } @@ -636,14 +639,12 @@ impl Compiler<'_> { statement.range(), )); } - vec![ConstantData::Str { - value: "*".to_owned(), - }] + vec![ConstantData::Str { value: "*".into() }] } else { names .iter() .map(|n| ConstantData::Str { - value: n.name.to_string(), + value: n.name.as_str().into(), }) .collect() }; @@ -954,7 +955,7 @@ impl Compiler<'_> { self.pop_symbol_table(); } self.emit_load_const(ConstantData::Str { - value: name_string.clone(), + value: name_string.clone().into(), }); emit!(self, Instruction::TypeAlias); self.store_name(&name_string)?; @@ -1028,7 +1029,7 @@ impl Compiler<'_> { let default_kw_count = kw_with_defaults.len(); for (arg, default) in kw_with_defaults.iter() { self.emit_load_const(ConstantData::Str { - value: arg.name.to_string(), + value: arg.name.as_str().into(), }); self.compile_expression(default)?; } @@ -1101,7 +1102,7 @@ impl Compiler<'_> { if let Some(expr) = &bound { self.compile_expression(expr)?; self.emit_load_const(ConstantData::Str { - value: name.to_string(), + value: name.as_str().into(), }); emit!(self, Instruction::TypeVarWithBound); emit!(self, Instruction::Duplicate); @@ -1109,7 +1110,7 @@ impl Compiler<'_> { } else { // self.store_name(type_name.as_str())?; self.emit_load_const(ConstantData::Str { - value: name.to_string(), + value: name.as_str().into(), }); emit!(self, Instruction::TypeVar); emit!(self, Instruction::Duplicate); @@ -1118,7 +1119,7 @@ impl Compiler<'_> { } TypeParam::ParamSpec(TypeParamParamSpec { name, .. }) => { self.emit_load_const(ConstantData::Str { - value: name.to_string(), + value: name.as_str().into(), }); emit!(self, Instruction::ParamSpec); emit!(self, Instruction::Duplicate); @@ -1126,7 +1127,7 @@ impl Compiler<'_> { } TypeParam::TypeVarTuple(TypeParamTypeVarTuple { name, .. }) => { self.emit_load_const(ConstantData::Str { - value: name.to_string(), + value: name.as_str().into(), }); emit!(self, Instruction::TypeVarTuple); emit!(self, Instruction::Duplicate); @@ -1363,7 +1364,7 @@ impl Compiler<'_> { if let Some(annotation) = returns { // key: self.emit_load_const(ConstantData::Str { - value: "return".to_owned(), + value: "return".into(), }); // value: self.compile_annotation(annotation)?; @@ -1380,7 +1381,7 @@ impl Compiler<'_> { for param in parameters_iter { if let Some(annotation) = ¶m.annotation { self.emit_load_const(ConstantData::Str { - value: self.mangle(param.name.as_str()).into_owned(), + value: self.mangle(param.name.as_str()).into_owned().into(), }); self.compile_annotation(annotation)?; num_annotations += 1; @@ -1410,7 +1411,7 @@ impl Compiler<'_> { code: Box::new(code), }); self.emit_load_const(ConstantData::Str { - value: qualified_name, + value: qualified_name.into(), }); // Turn code object into function object: @@ -1418,7 +1419,9 @@ impl Compiler<'_> { if let Some(value) = doc_str { emit!(self, Instruction::Duplicate); - self.emit_load_const(ConstantData::Str { value }); + self.emit_load_const(ConstantData::Str { + value: value.into(), + }); emit!(self, Instruction::Rotate2); let doc = self.name("__doc__"); emit!(self, Instruction::StoreAttr { idx: doc }); @@ -1547,7 +1550,7 @@ impl Compiler<'_> { let dunder_module = self.name("__module__"); emit!(self, Instruction::StoreLocal(dunder_module)); self.emit_load_const(ConstantData::Str { - value: qualified_name, + value: qualified_name.into(), }); let qualname = self.name("__qualname__"); emit!(self, Instruction::StoreLocal(qualname)); @@ -1608,16 +1611,12 @@ impl Compiler<'_> { self.emit_load_const(ConstantData::Code { code: Box::new(code), }); - self.emit_load_const(ConstantData::Str { - value: name.to_owned(), - }); + self.emit_load_const(ConstantData::Str { value: name.into() }); // Turn code object into function object: emit!(self, Instruction::MakeFunction(func_flags)); - self.emit_load_const(ConstantData::Str { - value: name.to_owned(), - }); + self.emit_load_const(ConstantData::Str { value: name.into() }); // Call the __build_class__ builtin let call = if let Some(arguments) = arguments { @@ -1638,7 +1637,7 @@ impl Compiler<'_> { // Doc string value: self.emit_load_const(match doc_str { - Some(doc) => ConstantData::Str { value: doc }, + Some(doc) => ConstantData::Str { value: doc.into() }, None => ConstantData::None, // set docstring None if not declared }); } @@ -2031,7 +2030,7 @@ impl Compiler<'_> { let ident = Default::default(); let codegen = ruff_python_codegen::Generator::new(&ident, Default::default()); self.emit_load_const(ConstantData::Str { - value: codegen.expr(annotation), + value: codegen.expr(annotation).into(), }); } else { self.compile_expression(annotation)?; @@ -2063,7 +2062,7 @@ impl Compiler<'_> { let annotations = self.name("__annotations__"); emit!(self, Instruction::LoadNameAny(annotations)); self.emit_load_const(ConstantData::Str { - value: self.mangle(id.as_str()).into_owned(), + value: self.mangle(id.as_str()).into_owned().into(), }); emit!(self, Instruction::StoreSubscript); } else { @@ -2538,7 +2537,7 @@ impl Compiler<'_> { self.emit_load_const(ConstantData::Code { code: Box::new(code), }); - self.emit_load_const(ConstantData::Str { value: name }); + self.emit_load_const(ConstantData::Str { value: name.into() }); // Turn code object into function object: emit!(self, Instruction::MakeFunction(func_flags)); @@ -2679,9 +2678,23 @@ impl Compiler<'_> { self.compile_expr_fstring(fstring)?; } Expr::StringLiteral(string) => { - self.emit_load_const(ConstantData::Str { - value: string.value.to_str().to_owned(), - }); + let value = string.value.to_str(); + if value.contains(char::REPLACEMENT_CHARACTER) { + let value = string + .value + .iter() + .map(|lit| { + let source = self.source_code.get_range(lit.range); + crate::string_parser::parse_string_literal(source, lit.flags.into()) + }) + .collect(); + // might have a surrogate literal; should reparse to be sure + self.emit_load_const(ConstantData::Str { value }); + } else { + self.emit_load_const(ConstantData::Str { + value: value.into(), + }); + } } Expr::BytesLiteral(bytes) => { let iter = bytes.value.iter().flat_map(|x| x.iter().copied()); @@ -2732,7 +2745,7 @@ impl Compiler<'_> { for keyword in sub_keywords { if let Some(name) = &keyword.arg { self.emit_load_const(ConstantData::Str { - value: name.to_string(), + value: name.as_str().into(), }); self.compile_expression(&keyword.value)?; sub_size += 1; @@ -2822,7 +2835,7 @@ impl Compiler<'_> { for keyword in &arguments.keywords { if let Some(name) = &keyword.arg { kwarg_names.push(ConstantData::Str { - value: name.to_string(), + value: name.as_str().into(), }); } else { // This means **kwargs! @@ -3058,9 +3071,7 @@ impl Compiler<'_> { }); // List comprehension function name: - self.emit_load_const(ConstantData::Str { - value: name.to_owned(), - }); + self.emit_load_const(ConstantData::Str { value: name.into() }); // Turn code object into function object: emit!(self, Instruction::MakeFunction(func_flags)); @@ -3358,9 +3369,19 @@ impl Compiler<'_> { fn compile_fstring_part(&mut self, part: &FStringPart) -> CompileResult<()> { match part { FStringPart::Literal(string) => { - self.emit_load_const(ConstantData::Str { - value: string.value.to_string(), - }); + if string.value.contains(char::REPLACEMENT_CHARACTER) { + // might have a surrogate literal; should reparse to be sure + let source = self.source_code.get_range(string.range); + let value = + crate::string_parser::parse_string_literal(source, string.flags.into()); + self.emit_load_const(ConstantData::Str { + value: value.into(), + }); + } else { + self.emit_load_const(ConstantData::Str { + value: string.value.to_string().into(), + }); + } Ok(()) } FStringPart::FString(fstring) => self.compile_fstring(fstring), @@ -3368,19 +3389,32 @@ impl Compiler<'_> { } fn compile_fstring(&mut self, fstring: &FString) -> CompileResult<()> { - self.compile_fstring_elements(&fstring.elements) + self.compile_fstring_elements(fstring.flags, &fstring.elements) } fn compile_fstring_elements( &mut self, + flags: FStringFlags, fstring_elements: &FStringElements, ) -> CompileResult<()> { for element in fstring_elements { match element { FStringElement::Literal(string) => { - self.emit_load_const(ConstantData::Str { - value: string.value.to_string(), - }); + if string.value.contains(char::REPLACEMENT_CHARACTER) { + // might have a surrogate literal; should reparse to be sure + let source = self.source_code.get_range(string.range); + let value = crate::string_parser::parse_fstring_literal_element( + source.into(), + flags.into(), + ); + self.emit_load_const(ConstantData::Str { + value: value.into(), + }); + } else { + self.emit_load_const(ConstantData::Str { + value: string.value.to_string().into(), + }); + } } FStringElement::Expression(fstring_expr) => { let mut conversion = fstring_expr.conversion; @@ -3393,11 +3427,13 @@ impl Compiler<'_> { let source = source.to_string(); self.emit_load_const(ConstantData::Str { - value: leading.to_string(), + value: leading.to_string().into(), + }); + self.emit_load_const(ConstantData::Str { + value: source.into(), }); - self.emit_load_const(ConstantData::Str { value: source }); self.emit_load_const(ConstantData::Str { - value: trailing.to_string(), + value: trailing.to_string().into(), }); 3 @@ -3407,7 +3443,7 @@ impl Compiler<'_> { match &fstring_expr.format_spec { None => { self.emit_load_const(ConstantData::Str { - value: String::new(), + value: Wtf8Buf::new(), }); // Match CPython behavior: If debug text is present, apply repr conversion. // See: https://github.com/python/cpython/blob/f61afca262d3a0aa6a8a501db0b1936c60858e35/Parser/action_helpers.c#L1456 @@ -3416,7 +3452,7 @@ impl Compiler<'_> { } } Some(format_spec) => { - self.compile_fstring_elements(&format_spec.elements)?; + self.compile_fstring_elements(flags, &format_spec.elements)?; } } @@ -3449,7 +3485,7 @@ impl Compiler<'_> { if element_count == 0 { // ensure to put an empty string on the stack if there aren't any fstring elements self.emit_load_const(ConstantData::Str { - value: String::new(), + value: Wtf8Buf::new(), }); } else if element_count > 1 { emit!( diff --git a/compiler/codegen/src/lib.rs b/compiler/codegen/src/lib.rs index ceadb3c364..d44844543e 100644 --- a/compiler/codegen/src/lib.rs +++ b/compiler/codegen/src/lib.rs @@ -11,6 +11,7 @@ type IndexSet = indexmap::IndexSet; pub mod compile; pub mod error; pub mod ir; +mod string_parser; pub mod symboltable; pub use compile::CompileOpts; diff --git a/compiler/codegen/src/string_parser.rs b/compiler/codegen/src/string_parser.rs new file mode 100644 index 0000000000..7bdb86aa5e --- /dev/null +++ b/compiler/codegen/src/string_parser.rs @@ -0,0 +1,287 @@ +//! A stripped-down version of ruff's string literal parser, modified to +//! handle surrogates in string literals and output WTF-8. +//! +//! Any `unreachable!()` statements in this file are because we only get here +//! after ruff has already successfully parsed the string literal, meaning +//! we don't need to do any validation or error handling. + +use std::convert::Infallible; + +use ruff_python_ast::{AnyStringFlags, StringFlags}; +use rustpython_common::wtf8::{CodePoint, Wtf8, Wtf8Buf}; + +// use ruff_python_parser::{LexicalError, LexicalErrorType}; +type LexicalError = Infallible; + +enum EscapedChar { + Literal(CodePoint), + Escape(char), +} + +struct StringParser { + /// The raw content of the string e.g., the `foo` part in `"foo"`. + source: Box, + /// Current position of the parser in the source. + cursor: usize, + /// Flags that can be used to query information about the string. + flags: AnyStringFlags, +} + +impl StringParser { + fn new(source: Box, flags: AnyStringFlags) -> Self { + Self { + source, + cursor: 0, + flags, + } + } + + #[inline] + fn skip_bytes(&mut self, bytes: usize) -> &str { + let skipped_str = &self.source[self.cursor..self.cursor + bytes]; + self.cursor += bytes; + skipped_str + } + + /// Returns the next byte in the string, if there is one. + /// + /// # Panics + /// + /// When the next byte is a part of a multi-byte character. + #[inline] + fn next_byte(&mut self) -> Option { + self.source[self.cursor..].as_bytes().first().map(|&byte| { + self.cursor += 1; + byte + }) + } + + #[inline] + fn next_char(&mut self) -> Option { + self.source[self.cursor..].chars().next().inspect(|c| { + self.cursor += c.len_utf8(); + }) + } + + #[inline] + fn peek_byte(&self) -> Option { + self.source[self.cursor..].as_bytes().first().copied() + } + + fn parse_unicode_literal(&mut self, literal_number: usize) -> Result { + let mut p: u32 = 0u32; + for i in 1..=literal_number { + match self.next_char() { + Some(c) => match c.to_digit(16) { + Some(d) => p += d << ((literal_number - i) * 4), + None => unreachable!(), + }, + None => unreachable!(), + } + } + Ok(CodePoint::from_u32(p).unwrap()) + } + + fn parse_octet(&mut self, o: u8) -> char { + let mut radix_bytes = [o, 0, 0]; + let mut len = 1; + + while len < 3 { + let Some(b'0'..=b'7') = self.peek_byte() else { + break; + }; + + radix_bytes[len] = self.next_byte().unwrap(); + len += 1; + } + + // OK because radix_bytes is always going to be in the ASCII range. + let radix_str = std::str::from_utf8(&radix_bytes[..len]).expect("ASCII bytes"); + let value = u32::from_str_radix(radix_str, 8).unwrap(); + char::from_u32(value).unwrap() + } + + fn parse_unicode_name(&mut self) -> Result { + let Some('{') = self.next_char() else { + unreachable!() + }; + + let Some(close_idx) = self.source[self.cursor..].find('}') else { + unreachable!() + }; + + let name_and_ending = self.skip_bytes(close_idx + 1); + let name = &name_and_ending[..name_and_ending.len() - 1]; + + unicode_names2::character(name).ok_or_else(|| unreachable!()) + } + + /// Parse an escaped character, returning the new character. + fn parse_escaped_char(&mut self) -> Result, LexicalError> { + let Some(first_char) = self.next_char() else { + unreachable!() + }; + + let new_char = match first_char { + '\\' => '\\'.into(), + '\'' => '\''.into(), + '\"' => '"'.into(), + 'a' => '\x07'.into(), + 'b' => '\x08'.into(), + 'f' => '\x0c'.into(), + 'n' => '\n'.into(), + 'r' => '\r'.into(), + 't' => '\t'.into(), + 'v' => '\x0b'.into(), + o @ '0'..='7' => self.parse_octet(o as u8).into(), + 'x' => self.parse_unicode_literal(2)?, + 'u' if !self.flags.is_byte_string() => self.parse_unicode_literal(4)?, + 'U' if !self.flags.is_byte_string() => self.parse_unicode_literal(8)?, + 'N' if !self.flags.is_byte_string() => self.parse_unicode_name()?.into(), + // Special cases where the escape sequence is not a single character + '\n' => return Ok(None), + '\r' => { + if self.peek_byte() == Some(b'\n') { + self.next_byte(); + } + + return Ok(None); + } + _ => return Ok(Some(EscapedChar::Escape(first_char))), + }; + + Ok(Some(EscapedChar::Literal(new_char))) + } + + fn parse_fstring_middle(mut self) -> Result, LexicalError> { + // Fast-path: if the f-string doesn't contain any escape sequences, return the literal. + let Some(mut index) = memchr::memchr3(b'{', b'}', b'\\', self.source.as_bytes()) else { + return Ok(self.source.into()); + }; + + let mut value = Wtf8Buf::with_capacity(self.source.len()); + loop { + // Add the characters before the escape sequence (or curly brace) to the string. + let before_with_slash_or_brace = self.skip_bytes(index + 1); + let before = &before_with_slash_or_brace[..before_with_slash_or_brace.len() - 1]; + value.push_str(before); + + // Add the escaped character to the string. + match &self.source.as_bytes()[self.cursor - 1] { + // If there are any curly braces inside a `FStringMiddle` token, + // then they were escaped (i.e. `{{` or `}}`). This means that + // we need increase the location by 2 instead of 1. + b'{' => value.push_char('{'), + b'}' => value.push_char('}'), + // We can encounter a `\` as the last character in a `FStringMiddle` + // token which is valid in this context. For example, + // + // ```python + // f"\{foo} \{bar:\}" + // # ^ ^^ ^ + // ``` + // + // Here, the `FStringMiddle` token content will be "\" and " \" + // which is invalid if we look at the content in isolation: + // + // ```python + // "\" + // ``` + // + // However, the content is syntactically valid in the context of + // the f-string because it's a substring of the entire f-string. + // This is still an invalid escape sequence, but we don't want to + // raise a syntax error as is done by the CPython parser. It might + // be supported in the future, refer to point 3: https://peps.python.org/pep-0701/#rejected-ideas + b'\\' => { + if !self.flags.is_raw_string() && self.peek_byte().is_some() { + match self.parse_escaped_char()? { + None => {} + Some(EscapedChar::Literal(c)) => value.push(c), + Some(EscapedChar::Escape(c)) => { + value.push_char('\\'); + value.push_char(c); + } + } + } else { + value.push_char('\\'); + } + } + ch => { + unreachable!("Expected '{{', '}}', or '\\' but got {:?}", ch); + } + } + + let Some(next_index) = + memchr::memchr3(b'{', b'}', b'\\', self.source[self.cursor..].as_bytes()) + else { + // Add the rest of the string to the value. + let rest = &self.source[self.cursor..]; + value.push_str(rest); + break; + }; + + index = next_index; + } + + Ok(value.into()) + } + + fn parse_string(mut self) -> Result, LexicalError> { + if self.flags.is_raw_string() { + // For raw strings, no escaping is necessary. + return Ok(self.source.into()); + } + + let Some(mut escape) = memchr::memchr(b'\\', self.source.as_bytes()) else { + // If the string doesn't contain any escape sequences, return the owned string. + return Ok(self.source.into()); + }; + + // If the string contains escape sequences, we need to parse them. + let mut value = Wtf8Buf::with_capacity(self.source.len()); + + loop { + // Add the characters before the escape sequence to the string. + let before_with_slash = self.skip_bytes(escape + 1); + let before = &before_with_slash[..before_with_slash.len() - 1]; + value.push_str(before); + + // Add the escaped character to the string. + match self.parse_escaped_char()? { + None => {} + Some(EscapedChar::Literal(c)) => value.push(c), + Some(EscapedChar::Escape(c)) => { + value.push_char('\\'); + value.push_char(c); + } + } + + let Some(next_escape) = self.source[self.cursor..].find('\\') else { + // Add the rest of the string to the value. + let rest = &self.source[self.cursor..]; + value.push_str(rest); + break; + }; + + // Update the position of the next escape sequence. + escape = next_escape; + } + + Ok(value.into()) + } +} + +pub(crate) fn parse_string_literal(source: &str, flags: AnyStringFlags) -> Box { + let source = &source[flags.opener_len().to_usize()..]; + let source = &source[..source.len() - flags.quote_len().to_usize()]; + StringParser::new(source.into(), flags) + .parse_string() + .unwrap_or_else(|x| match x {}) +} + +pub(crate) fn parse_fstring_literal_element(source: Box, flags: AnyStringFlags) -> Box { + StringParser::new(source, flags) + .parse_fstring_middle() + .unwrap_or_else(|x| match x {}) +} diff --git a/compiler/core/Cargo.toml b/compiler/core/Cargo.toml index 7621c643d5..8ff0cd020d 100644 --- a/compiler/core/Cargo.toml +++ b/compiler/core/Cargo.toml @@ -13,6 +13,7 @@ license.workspace = true ruff_python_ast = { workspace = true } ruff_python_parser = { workspace = true } ruff_source_file = { workspace = true } +rustpython-common = { workspace = true } bitflags = { workspace = true } itertools = { workspace = true } diff --git a/compiler/core/src/bytecode.rs b/compiler/core/src/bytecode.rs index 4cb80020e7..7b018d1df1 100644 --- a/compiler/core/src/bytecode.rs +++ b/compiler/core/src/bytecode.rs @@ -8,6 +8,7 @@ use num_complex::Complex64; pub use ruff_python_ast::ConversionFlag; // use rustpython_parser_core::source_code::{OneIndexed, SourceLocation}; use ruff_source_file::{OneIndexed, SourceLocation}; +use rustpython_common::wtf8::{Wtf8, Wtf8Buf}; use std::marker::PhantomData; use std::{collections::BTreeSet, fmt, hash, mem}; @@ -678,7 +679,7 @@ pub enum ConstantData { Float { value: f64 }, Complex { value: Complex64 }, Boolean { value: bool }, - Str { value: String }, + Str { value: Wtf8Buf }, Bytes { value: Vec }, Code { code: Box }, None, @@ -738,7 +739,7 @@ pub enum BorrowedConstant<'a, C: Constant> { Float { value: f64 }, Complex { value: Complex64 }, Boolean { value: bool }, - Str { value: &'a str }, + Str { value: &'a Wtf8 }, Bytes { value: &'a [u8] }, Code { code: &'a CodeObject }, Tuple { elements: &'a [C] }, diff --git a/compiler/core/src/marshal.rs b/compiler/core/src/marshal.rs index 1e47a6cac5..0c8da17ff9 100644 --- a/compiler/core/src/marshal.rs +++ b/compiler/core/src/marshal.rs @@ -2,6 +2,7 @@ use crate::bytecode::*; use malachite_bigint::{BigInt, Sign}; use num_complex::Complex64; use ruff_source_file::{OneIndexed, SourceLocation}; +use rustpython_common::wtf8::Wtf8; use std::convert::Infallible; pub const FORMAT_VERSION: u32 = 4; @@ -117,6 +118,9 @@ pub trait Read { fn read_str(&mut self, len: u32) -> Result<&str> { Ok(std::str::from_utf8(self.read_slice(len)?)?) } + fn read_wtf8(&mut self, len: u32) -> Result<&Wtf8> { + Wtf8::from_bytes(self.read_slice(len)?).ok_or(MarshalError::InvalidUtf8) + } fn read_u8(&mut self) -> Result { Ok(u8::from_le_bytes(*self.read_array()?)) } @@ -262,7 +266,7 @@ pub trait MarshalBag: Copy { fn make_ellipsis(&self) -> Self::Value; fn make_float(&self, value: f64) -> Self::Value; fn make_complex(&self, value: Complex64) -> Self::Value; - fn make_str(&self, value: &str) -> Self::Value; + fn make_str(&self, value: &Wtf8) -> Self::Value; fn make_bytes(&self, value: &[u8]) -> Self::Value; fn make_int(&self, value: BigInt) -> Self::Value; fn make_tuple(&self, elements: impl Iterator) -> Self::Value; @@ -299,7 +303,7 @@ impl MarshalBag for Bag { fn make_complex(&self, value: Complex64) -> Self::Value { self.make_constant::(BorrowedConstant::Complex { value }) } - fn make_str(&self, value: &str) -> Self::Value { + fn make_str(&self, value: &Wtf8) -> Self::Value { self.make_constant::(BorrowedConstant::Str { value }) } fn make_bytes(&self, value: &[u8]) -> Self::Value { @@ -368,7 +372,7 @@ pub fn deserialize_value(rdr: &mut R, bag: Bag) -> Res } Type::Ascii | Type::Unicode => { let len = rdr.read_u32()?; - let value = rdr.read_str(len)?; + let value = rdr.read_wtf8(len)?; bag.make_str(value) } Type::Tuple => { @@ -422,7 +426,7 @@ pub enum DumpableValue<'a, D: Dumpable> { Float(f64), Complex(Complex64), Boolean(bool), - Str(&'a str), + Str(&'a Wtf8), Bytes(&'a [u8]), Code(&'a CodeObject), Tuple(&'a [D]), diff --git a/jit/tests/common.rs b/jit/tests/common.rs index a2d4fc3bc1..a4ac8a7967 100644 --- a/jit/tests/common.rs +++ b/jit/tests/common.rs @@ -53,7 +53,9 @@ enum StackValue { impl From for StackValue { fn from(value: ConstantData) -> Self { match value { - ConstantData::Str { value } => StackValue::String(value), + ConstantData::Str { value } => { + StackValue::String(value.into_string().expect("surrogate in test code")) + } ConstantData::None => StackValue::None, ConstantData::Code { code } => StackValue::Code(code), c => unimplemented!("constant {:?} isn't yet supported in py_function!", c), diff --git a/vm/src/builtins/code.rs b/vm/src/builtins/code.rs index ba2d2dd5c3..4bb209f6db 100644 --- a/vm/src/builtins/code.rs +++ b/vm/src/builtins/code.rs @@ -74,7 +74,7 @@ fn borrow_obj_constant(obj: &PyObject) -> BorrowedConstant<'_, Literal> { ref c @ super::complex::PyComplex => BorrowedConstant::Complex { value: c.to_complex() }, - ref s @ super::pystr::PyStr => BorrowedConstant::Str { value: s.as_str() }, + ref s @ super::pystr::PyStr => BorrowedConstant::Str { value: s.as_wtf8() }, ref b @ super::bytes::PyBytes => BorrowedConstant::Bytes { value: b.as_bytes() }, diff --git a/vm/src/builtins/str.rs b/vm/src/builtins/str.rs index 55cefae4f7..8fe3904945 100644 --- a/vm/src/builtins/str.rs +++ b/vm/src/builtins/str.rs @@ -1815,6 +1815,18 @@ impl AsRef for PyExact { } } +impl AsRef for PyRefExact { + fn as_ref(&self) -> &Wtf8 { + self.as_wtf8() + } +} + +impl AsRef for PyExact { + fn as_ref(&self) -> &Wtf8 { + self.as_wtf8() + } +} + impl AnyStrWrapper for PyStrRef { fn as_ref(&self) -> Option<&Wtf8> { Some(self.as_wtf8()) diff --git a/vm/src/intern.rs b/vm/src/intern.rs index 10aaa53454..bb9220d069 100644 --- a/vm/src/intern.rs +++ b/vm/src/intern.rs @@ -1,3 +1,5 @@ +use rustpython_common::wtf8::{Wtf8, Wtf8Buf}; + use crate::{ AsObject, Py, PyExact, PyObject, PyObjectRef, PyPayload, PyRef, PyRefExact, VirtualMachine, builtins::{PyStr, PyStrInterned, PyTypeRef}, @@ -86,29 +88,29 @@ pub struct CachedPyStrRef { impl std::hash::Hash for CachedPyStrRef { fn hash(&self, state: &mut H) { - self.inner.as_str().hash(state) + self.inner.as_wtf8().hash(state) } } impl PartialEq for CachedPyStrRef { fn eq(&self, other: &Self) -> bool { - self.inner.as_str() == other.inner.as_str() + self.inner.as_wtf8() == other.inner.as_wtf8() } } impl Eq for CachedPyStrRef {} -impl std::borrow::Borrow for CachedPyStrRef { +impl std::borrow::Borrow for CachedPyStrRef { #[inline] - fn borrow(&self) -> &str { - self.inner.as_str() + fn borrow(&self) -> &Wtf8 { + self.as_wtf8() } } -impl AsRef for CachedPyStrRef { +impl AsRef for CachedPyStrRef { #[inline] - fn as_ref(&self) -> &str { - self.as_str() + fn as_ref(&self) -> &Wtf8 { + self.as_wtf8() } } @@ -121,8 +123,8 @@ impl CachedPyStrRef { } #[inline] - fn as_str(&self) -> &str { - self.inner.as_str() + fn as_wtf8(&self) -> &Wtf8 { + self.inner.as_wtf8() } } @@ -209,6 +211,8 @@ impl ToPyObject for &'static PyInterned { } mod sealed { + use rustpython_common::wtf8::{Wtf8, Wtf8Buf}; + use crate::{ builtins::PyStr, object::{Py, PyExact, PyRefExact}, @@ -218,11 +222,14 @@ mod sealed { impl SealedInternable for String {} impl SealedInternable for &str {} + impl SealedInternable for Wtf8Buf {} + impl SealedInternable for &Wtf8 {} impl SealedInternable for PyRefExact {} pub trait SealedMaybeInterned {} impl SealedMaybeInterned for str {} + impl SealedMaybeInterned for Wtf8 {} impl SealedMaybeInterned for PyExact {} impl SealedMaybeInterned for Py {} } @@ -250,6 +257,21 @@ impl InternableString for &str { } } +impl InternableString for Wtf8Buf { + type Interned = Wtf8; + fn into_pyref_exact(self, str_type: PyTypeRef) -> PyRefExact { + let obj = PyRef::new_ref(PyStr::from(self), str_type, None); + unsafe { PyRefExact::new_unchecked(obj) } + } +} + +impl InternableString for &Wtf8 { + type Interned = Wtf8; + fn into_pyref_exact(self, str_type: PyTypeRef) -> PyRefExact { + self.to_owned().into_pyref_exact(str_type) + } +} + impl InternableString for PyRefExact { type Interned = Py; #[inline] @@ -259,7 +281,7 @@ impl InternableString for PyRefExact { } pub trait MaybeInternedString: - AsRef + crate::dictdatatype::DictKey + sealed::SealedMaybeInterned + AsRef + crate::dictdatatype::DictKey + sealed::SealedMaybeInterned { fn as_interned(&self) -> Option<&'static PyStrInterned>; } @@ -271,6 +293,13 @@ impl MaybeInternedString for str { } } +impl MaybeInternedString for Wtf8 { + #[inline(always)] + fn as_interned(&self) -> Option<&'static PyStrInterned> { + None + } +} + impl MaybeInternedString for PyExact { #[inline(always)] fn as_interned(&self) -> Option<&'static PyStrInterned> { @@ -296,7 +325,7 @@ impl PyObject { if self.is_interned() { s.unwrap().as_interned() } else if let Some(s) = s { - vm.ctx.interned_str(s.as_str()) + vm.ctx.interned_str(s.as_wtf8()) } else { None } diff --git a/vm/src/stdlib/marshal.rs b/vm/src/stdlib/marshal.rs index fd7332e7c2..564ee5bf6c 100644 --- a/vm/src/stdlib/marshal.rs +++ b/vm/src/stdlib/marshal.rs @@ -10,6 +10,7 @@ mod decl { PyBool, PyByteArray, PyBytes, PyCode, PyComplex, PyDict, PyEllipsis, PyFloat, PyFrozenSet, PyInt, PyList, PyNone, PySet, PyStopIteration, PyStr, PyTuple, }, + common::wtf8::Wtf8, convert::ToPyObject, function::{ArgBytesLike, OptionalArg}, object::AsObject, @@ -53,7 +54,7 @@ mod decl { f(Complex(pycomplex.to_complex64())) } ref pystr @ PyStr => { - f(Str(pystr.as_str())) + f(Str(pystr.as_wtf8())) } ref pylist @ PyList => { f(List(&pylist.borrow_vec())) @@ -139,7 +140,7 @@ mod decl { fn make_complex(&self, value: Complex64) -> Self::Value { self.0.ctx.new_complex(value).into() } - fn make_str(&self, value: &str) -> Self::Value { + fn make_str(&self, value: &Wtf8) -> Self::Value { self.0.ctx.new_str(value).into() } fn make_bytes(&self, value: &[u8]) -> Self::Value { From 0a07cd931f5437f3f24201ee2dec6e4e431e52f6 Mon Sep 17 00:00:00 2001 From: Noa Date: Wed, 26 Mar 2025 20:37:26 -0500 Subject: [PATCH 132/295] Fix more surrogate crashes --- Lib/test/test_codecs.py | 31 ++++++------ Lib/test/test_json/test_scanstring.py | 2 - Lib/test/test_regrtest.py | 2 - Lib/test/test_stringprep.py | 2 - Lib/test/test_subprocess.py | 2 - Lib/test/test_tarfile.py | 14 ----- Lib/test/test_unicode.py | 8 --- Lib/test/test_userstring.py | 4 -- Lib/test/test_zipimport.py | 1 + common/src/wtf8/mod.rs | 55 ++++++++++++-------- stdlib/src/json.rs | 5 +- stdlib/src/json/machinery.rs | 73 +++++++++++++-------------- vm/src/builtins/complex.rs | 9 ++-- vm/src/builtins/float.rs | 2 +- vm/src/builtins/int.rs | 2 +- vm/src/builtins/str.rs | 32 +++++++++--- vm/src/builtins/type.rs | 2 +- vm/src/protocol/number.rs | 2 +- vm/src/protocol/object.rs | 2 +- vm/src/stdlib/codecs.rs | 2 +- vm/src/stdlib/io.rs | 16 +++--- vm/src/stdlib/time.rs | 8 ++- vm/src/utils.rs | 6 ++- 23 files changed, 142 insertions(+), 140 deletions(-) diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index df04653c66..a12e5893dc 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -869,6 +869,11 @@ def test_bug691291(self): with reader: self.assertEqual(reader.read(), s1) + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_incremental_surrogatepass(self): + super().test_incremental_surrogatepass() + class UTF16LETest(ReadTest, unittest.TestCase): encoding = "utf-16-le" ill_formed_sequence = b"\x80\xdc" @@ -917,6 +922,11 @@ def test_nonbmp(self): self.assertEqual(b'\x00\xd8\x03\xde'.decode(self.encoding), "\U00010203") + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_incremental_surrogatepass(self): + super().test_incremental_surrogatepass() + class UTF16BETest(ReadTest, unittest.TestCase): encoding = "utf-16-be" ill_formed_sequence = b"\xdc\x80" @@ -965,6 +975,11 @@ def test_nonbmp(self): self.assertEqual(b'\xd8\x00\xde\x03'.decode(self.encoding), "\U00010203") + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_incremental_surrogatepass(self): + super().test_incremental_surrogatepass() + class UTF8Test(ReadTest, unittest.TestCase): encoding = "utf-8" ill_formed_sequence = b"\xed\xb2\x80" @@ -998,8 +1013,6 @@ def test_decoder_state(self): self.check_state_handling_decode(self.encoding, u, u.encode(self.encoding)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_decode_error(self): for data, error_handler, expected in ( (b'[\x80\xff]', 'ignore', '[]'), @@ -1026,8 +1039,6 @@ def test_lone_surrogates(self): exc = cm.exception self.assertEqual(exc.object[exc.start:exc.end], '\uD800\uDFFF') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_surrogatepass_handler(self): self.assertEqual("abc\ud800def".encode(self.encoding, "surrogatepass"), self.BOM + b"abc\xed\xa0\x80def") @@ -2884,8 +2895,6 @@ def test_escape_encode(self): class SurrogateEscapeTest(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_utf8(self): # Bad byte self.assertEqual(b"foo\x80bar".decode("utf-8", "surrogateescape"), @@ -2898,8 +2907,6 @@ def test_utf8(self): self.assertEqual("\udced\udcb0\udc80".encode("utf-8", "surrogateescape"), b"\xed\xb0\x80") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_ascii(self): # bad byte self.assertEqual(b"foo\x80bar".decode("ascii", "surrogateescape"), @@ -2916,8 +2923,6 @@ def test_charmap(self): self.assertEqual("foo\udca5bar".encode("iso-8859-3", "surrogateescape"), b"foo\xa5bar") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_latin1(self): # Issue6373 self.assertEqual("\udce4\udceb\udcef\udcf6\udcfc".encode("latin-1", "surrogateescape"), @@ -3561,8 +3566,6 @@ class ASCIITest(unittest.TestCase): def test_encode(self): self.assertEqual('abc123'.encode('ascii'), b'abc123') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_encode_error(self): for data, error_handler, expected in ( ('[\x80\xff\u20ac]', 'ignore', b'[]'), @@ -3585,8 +3588,6 @@ def test_encode_surrogateescape_error(self): def test_decode(self): self.assertEqual(b'abc'.decode('ascii'), 'abc') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_decode_error(self): for data, error_handler, expected in ( (b'[\x80\xff]', 'ignore', '[]'), @@ -3609,8 +3610,6 @@ def test_encode(self): with self.subTest(data=data, expected=expected): self.assertEqual(data.encode('latin1'), expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_encode_errors(self): for data, error_handler, expected in ( ('[\u20ac\udc80]', 'ignore', b'[]'), diff --git a/Lib/test/test_json/test_scanstring.py b/Lib/test/test_json/test_scanstring.py index 682dc74999..af4bb3a639 100644 --- a/Lib/test/test_json/test_scanstring.py +++ b/Lib/test/test_json/test_scanstring.py @@ -86,8 +86,6 @@ def test_scanstring(self): scanstring('["Bad value", truth]', 2, True), ('Bad value', 12)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_surrogates(self): scanstring = self.json.decoder.scanstring def assertScan(given, expect): diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index aab2d9f7ae..fc56ec4afc 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -945,7 +945,6 @@ def test_leak(self): """) self.check_leak(code, 'file descriptors') - @unittest.expectedFailureIfWindows('TODO: RUSTPYTHON Windows') def test_list_tests(self): # test --list-tests tests = [self.create_test() for i in range(5)] @@ -953,7 +952,6 @@ def test_list_tests(self): self.assertEqual(output.rstrip().splitlines(), tests) - @unittest.expectedFailureIfWindows('TODO: RUSTPYTHON Windows') def test_list_cases(self): # test --list-cases code = textwrap.dedent(""" diff --git a/Lib/test/test_stringprep.py b/Lib/test/test_stringprep.py index 118f3f0867..d4b4a13d0d 100644 --- a/Lib/test/test_stringprep.py +++ b/Lib/test/test_stringprep.py @@ -6,8 +6,6 @@ from stringprep import * class StringprepTests(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test(self): self.assertTrue(in_table_a1("\u0221")) self.assertFalse(in_table_a1("\u0222")) diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index d7507eb7f0..e5b18fe20f 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -1198,8 +1198,6 @@ def test_universal_newlines_communicate_encodings(self): stdout, stderr = popen.communicate(input='') self.assertEqual(stdout, '1\n2\n3\n4') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_communicate_errors(self): for errors, expected in [ ('ignore', ''), diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 4ae81cb99f..63f7b347ad 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -2086,11 +2086,6 @@ class UstarUnicodeTest(UnicodeTest, unittest.TestCase): format = tarfile.USTAR_FORMAT - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_uname_unicode(self): - super().test_uname_unicode() - # Test whether the utf-8 encoded version of a filename exceeds the 100 # bytes name field limit (every occurrence of '\xff' will be expanded to 2 # bytes). @@ -2170,13 +2165,6 @@ class GNUUnicodeTest(UnicodeTest, unittest.TestCase): format = tarfile.GNU_FORMAT - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_uname_unicode(self): - super().test_uname_unicode() - - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_bad_pax_header(self): # Test for issue #8633. GNU tar <= 1.23 creates raw binary fields # without a hdrcharset=BINARY header. @@ -2198,8 +2186,6 @@ class PAXUnicodeTest(UnicodeTest, unittest.TestCase): # PAX_FORMAT ignores encoding in write mode. test_unicode_filename_error = None - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_binary_header(self): # Test a POSIX.1-2008 compatible header with a hdrcharset=BINARY field. for encoding, name in ( diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py index 5c2c6c29b1..4da63c54d4 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -608,8 +608,6 @@ def test_bytes_comparison(self): self.assertEqual('abc' == bytearray(b'abc'), False) self.assertEqual('abc' != bytearray(b'abc'), True) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_comparison(self): # Comparisons: self.assertEqual('abc', 'abc') @@ -830,8 +828,6 @@ def test_isidentifier_legacy(self): warnings.simplefilter('ignore', DeprecationWarning) self.assertTrue(_testcapi.unicode_legacy_string(u).isidentifier()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_isprintable(self): self.assertTrue("".isprintable()) self.assertTrue(" ".isprintable()) @@ -847,8 +843,6 @@ def test_isprintable(self): self.assertTrue('\U0001F46F'.isprintable()) self.assertFalse('\U000E0020'.isprintable()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_surrogates(self): for s in ('a\uD800b\uDFFF', 'a\uDFFFb\uD800', 'a\uD800b\uDFFFa', 'a\uDFFFb\uD800a'): @@ -1827,8 +1821,6 @@ def test_codecs_utf7(self): 'ill-formed sequence'): b'+@'.decode('utf-7') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_codecs_utf8(self): self.assertEqual(''.encode('utf-8'), b'') self.assertEqual('\u20ac'.encode('utf-8'), b'\xe2\x82\xac') diff --git a/Lib/test/test_userstring.py b/Lib/test/test_userstring.py index c0017794e8..51b4f6041e 100644 --- a/Lib/test/test_userstring.py +++ b/Lib/test/test_userstring.py @@ -53,8 +53,6 @@ def __rmod__(self, other): str3 = ustr3('TEST') self.assertEqual(fmt2 % str3, 'value is TEST') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_encode_default_args(self): self.checkequal(b'hello', 'hello', 'encode') # Check that encoding defaults to utf-8 @@ -62,8 +60,6 @@ def test_encode_default_args(self): # Check that errors defaults to 'strict' self.checkraises(UnicodeError, '\ud800', 'encode') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_encode_explicit_none_args(self): self.checkequal(b'hello', 'hello', 'encode', None, None) # Check that encoding defaults to utf-8 diff --git a/Lib/test/test_zipimport.py b/Lib/test/test_zipimport.py index b291d53016..488a67e80f 100644 --- a/Lib/test/test_zipimport.py +++ b/Lib/test/test_zipimport.py @@ -730,6 +730,7 @@ def testTraceback(self): @unittest.skipIf(os_helper.TESTFN_UNENCODABLE is None, "need an unencodable filename") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def testUnencodable(self): filename = os_helper.TESTFN_UNENCODABLE + ".zip" self.addCleanup(os_helper.unlink, filename) diff --git a/common/src/wtf8/mod.rs b/common/src/wtf8/mod.rs index 21c5de28bd..57a1d6d7de 100644 --- a/common/src/wtf8/mod.rs +++ b/common/src/wtf8/mod.rs @@ -122,18 +122,18 @@ impl CodePoint { /// Returns the numeric value of the code point if it is a leading surrogate. #[inline] - pub fn to_lead_surrogate(self) -> Option { + pub fn to_lead_surrogate(self) -> Option { match self.value { - lead @ 0xD800..=0xDBFF => Some(lead as u16), + lead @ 0xD800..=0xDBFF => Some(LeadSurrogate(lead as u16)), _ => None, } } /// Returns the numeric value of the code point if it is a trailing surrogate. #[inline] - pub fn to_trail_surrogate(self) -> Option { + pub fn to_trail_surrogate(self) -> Option { match self.value { - trail @ 0xDC00..=0xDFFF => Some(trail as u16), + trail @ 0xDC00..=0xDFFF => Some(TrailSurrogate(trail as u16)), _ => None, } } @@ -216,6 +216,18 @@ impl PartialEq for char { } } +#[derive(Clone, Copy)] +pub struct LeadSurrogate(u16); + +#[derive(Clone, Copy)] +pub struct TrailSurrogate(u16); + +impl LeadSurrogate { + pub fn merge(self, trail: TrailSurrogate) -> char { + decode_surrogate_pair(self.0, trail.0) + } +} + /// An owned, growable string of well-formed WTF-8 data. /// /// Similar to `String`, but can additionally contain surrogate code points @@ -291,6 +303,14 @@ impl Wtf8Buf { Wtf8Buf { bytes: value } } + /// Create a WTF-8 string from a WTF-8 byte vec. + pub fn from_bytes(value: Vec) -> Result> { + match Wtf8::from_bytes(&value) { + Some(_) => Ok(unsafe { Self::from_bytes_unchecked(value) }), + None => Err(value), + } + } + /// Creates a WTF-8 string from a UTF-8 `String`. /// /// This takes ownership of the `String` and does not copy. @@ -750,15 +770,10 @@ impl Wtf8 { } fn decode_surrogate(b: &[u8]) -> Option { - let [a, b, c, ..] = *b else { return None }; - if (a & 0xf0) == 0xe0 && (b & 0xc0) == 0x80 && (c & 0xc0) == 0x80 { - // it's a three-byte code - let c = ((a as u32 & 0x0f) << 12) + ((b as u32 & 0x3f) << 6) + (c as u32 & 0x3f); - let 0xD800..=0xDFFF = c else { return None }; - Some(CodePoint { value: c }) - } else { - None - } + let [0xed, b2 @ (0xa0..), b3, ..] = *b else { + return None; + }; + Some(decode_surrogate(b2, b3).into()) } /// Returns the length, in WTF-8 bytes. @@ -914,14 +929,6 @@ impl Wtf8 { } } - #[inline] - fn final_lead_surrogate(&self) -> Option { - match self.bytes { - [.., 0xED, b2 @ 0xA0..=0xAF, b3] => Some(decode_surrogate(b2, b3)), - _ => None, - } - } - pub fn is_code_point_boundary(&self, index: usize) -> bool { is_code_point_boundary(self, index) } @@ -1222,6 +1229,12 @@ fn decode_surrogate(second_byte: u8, third_byte: u8) -> u16 { 0xD800 | (second_byte as u16 & 0x3F) << 6 | third_byte as u16 & 0x3F } +#[inline] +fn decode_surrogate_pair(lead: u16, trail: u16) -> char { + let code_point = 0x10000 + ((((lead - 0xD800) as u32) << 10) | (trail - 0xDC00) as u32); + unsafe { char::from_u32_unchecked(code_point) } +} + /// Copied from str::is_char_boundary #[inline] fn is_code_point_boundary(slice: &Wtf8, index: usize) -> bool { diff --git a/stdlib/src/json.rs b/stdlib/src/json.rs index aaac0b8bef..f970ef5dc2 100644 --- a/stdlib/src/json.rs +++ b/stdlib/src/json.rs @@ -13,6 +13,7 @@ mod _json { types::{Callable, Constructor}, }; use malachite_bigint::BigInt; + use rustpython_common::wtf8::Wtf8Buf; use std::str::FromStr; #[pyattr(name = "make_scanner")] @@ -253,8 +254,8 @@ mod _json { end: usize, strict: OptionalArg, vm: &VirtualMachine, - ) -> PyResult<(String, usize)> { - machinery::scanstring(s.as_str(), end, strict.unwrap_or(true)) + ) -> PyResult<(Wtf8Buf, usize)> { + machinery::scanstring(s.as_wtf8(), end, strict.unwrap_or(true)) .map_err(|e| py_decode_error(e, s, vm)) } } diff --git a/stdlib/src/json/machinery.rs b/stdlib/src/json/machinery.rs index 0614314f4f..4612b5263d 100644 --- a/stdlib/src/json/machinery.rs +++ b/stdlib/src/json/machinery.rs @@ -28,6 +28,9 @@ use std::io; +use itertools::Itertools; +use rustpython_common::wtf8::{CodePoint, Wtf8, Wtf8Buf}; + static ESCAPE_CHARS: [&str; 0x20] = [ "\\u0000", "\\u0001", "\\u0002", "\\u0003", "\\u0004", "\\u0005", "\\u0006", "\\u0007", "\\b", "\\t", "\\n", "\\u000", "\\f", "\\r", "\\u000e", "\\u000f", "\\u0010", "\\u0011", "\\u0012", @@ -111,22 +114,22 @@ impl DecodeError { } enum StrOrChar<'a> { - Str(&'a str), - Char(char), + Str(&'a Wtf8), + Char(CodePoint), } impl StrOrChar<'_> { fn len(&self) -> usize { match self { StrOrChar::Str(s) => s.len(), - StrOrChar::Char(c) => c.len_utf8(), + StrOrChar::Char(c) => c.len_wtf8(), } } } pub fn scanstring<'a>( - s: &'a str, + s: &'a Wtf8, end: usize, strict: bool, -) -> Result<(String, usize), DecodeError> { +) -> Result<(Wtf8Buf, usize), DecodeError> { let mut chunks: Vec> = Vec::new(); let mut output_len = 0usize; let mut push_chunk = |chunk: StrOrChar<'a>| { @@ -134,16 +137,16 @@ pub fn scanstring<'a>( chunks.push(chunk); }; let unterminated_err = || DecodeError::new("Unterminated string starting at", end - 1); - let mut chars = s.char_indices().enumerate().skip(end).peekable(); + let mut chars = s.code_point_indices().enumerate().skip(end).peekable(); let &(_, (mut chunk_start, _)) = chars.peek().ok_or_else(unterminated_err)?; while let Some((char_i, (byte_i, c))) = chars.next() { - match c { + match c.to_char_lossy() { '"' => { push_chunk(StrOrChar::Str(&s[chunk_start..byte_i])); - let mut out = String::with_capacity(output_len); + let mut out = Wtf8Buf::with_capacity(output_len); for x in chunks { match x { - StrOrChar::Str(s) => out.push_str(s), + StrOrChar::Str(s) => out.push_wtf8(s), StrOrChar::Char(c) => out.push(c), } } @@ -152,7 +155,7 @@ pub fn scanstring<'a>( '\\' => { push_chunk(StrOrChar::Str(&s[chunk_start..byte_i])); let (_, (_, c)) = chars.next().ok_or_else(unterminated_err)?; - let esc = match c { + let esc = match c.to_char_lossy() { '"' => "\"", '\\' => "\\", '/' => "/", @@ -162,41 +165,33 @@ pub fn scanstring<'a>( 'r' => "\r", 't' => "\t", 'u' => { - let surrogate_err = || DecodeError::new("unpaired surrogate", char_i); let mut uni = decode_unicode(&mut chars, char_i)?; chunk_start = byte_i + 6; - if (0xd800..=0xdbff).contains(&uni) { + if let Some(lead) = uni.to_lead_surrogate() { // uni is a surrogate -- try to find its pair - if let Some(&(pos2, (_, '\\'))) = chars.peek() { - // ok, the next char starts an escape - chars.next(); - if let Some((_, (_, 'u'))) = chars.peek() { - // ok, it's a unicode escape - chars.next(); - let uni2 = decode_unicode(&mut chars, pos2)?; + let mut chars2 = chars.clone(); + if let Some(((pos2, _), (_, _))) = chars2 + .next_tuple() + .filter(|((_, (_, c1)), (_, (_, c2)))| *c1 == '\\' && *c2 == 'u') + { + let uni2 = decode_unicode(&mut chars2, pos2)?; + if let Some(trail) = uni2.to_trail_surrogate() { + // ok, we found what we were looking for -- \uXXXX\uXXXX, both surrogates + uni = lead.merge(trail).into(); chunk_start = pos2 + 6; - if (0xdc00..=0xdfff).contains(&uni2) { - // ok, we found what we were looking for -- \uXXXX\uXXXX, both surrogates - uni = 0x10000 + (((uni - 0xd800) << 10) | (uni2 - 0xdc00)); - } else { - // if we don't find a matching surrogate, error -- until str - // isn't utf8 internally, we can't parse surrogates - return Err(surrogate_err()); - } - } else { - return Err(surrogate_err()); + chars = chars2; } } } - push_chunk(StrOrChar::Char( - std::char::from_u32(uni).ok_or_else(surrogate_err)?, - )); + push_chunk(StrOrChar::Char(uni)); continue; } - _ => return Err(DecodeError::new(format!("Invalid \\escape: {c:?}"), char_i)), + _ => { + return Err(DecodeError::new(format!("Invalid \\escape: {c:?}"), char_i)); + } }; chunk_start = byte_i + 2; - push_chunk(StrOrChar::Str(esc)); + push_chunk(StrOrChar::Str(esc.as_ref())); } '\x00'..='\x1f' if strict => { return Err(DecodeError::new( @@ -211,16 +206,16 @@ pub fn scanstring<'a>( } #[inline] -fn decode_unicode(it: &mut I, pos: usize) -> Result +fn decode_unicode(it: &mut I, pos: usize) -> Result where - I: Iterator, + I: Iterator, { let err = || DecodeError::new("Invalid \\uXXXX escape", pos); let mut uni = 0; for x in (0..4).rev() { let (_, (_, c)) = it.next().ok_or_else(err)?; - let d = c.to_digit(16).ok_or_else(err)?; - uni += d * 16u32.pow(x); + let d = c.to_char().and_then(|c| c.to_digit(16)).ok_or_else(err)? as u16; + uni += d * 16u16.pow(x); } - Ok(uni) + Ok(uni.into()) } diff --git a/vm/src/builtins/complex.rs b/vm/src/builtins/complex.rs index e665d1e27a..01dd65f519 100644 --- a/vm/src/builtins/complex.rs +++ b/vm/src/builtins/complex.rs @@ -179,9 +179,12 @@ impl Constructor for PyComplex { "complex() can't take second arg if first is a string".to_owned(), )); } - let value = parse_str(s.as_str().trim()).ok_or_else(|| { - vm.new_value_error("complex() arg is a malformed string".to_owned()) - })?; + let value = s + .to_str() + .and_then(|s| parse_str(s.trim())) + .ok_or_else(|| { + vm.new_value_error("complex() arg is a malformed string".to_owned()) + })?; return Self::from(value) .into_ref_with_type(vm, cls) .map(Into::into); diff --git a/vm/src/builtins/float.rs b/vm/src/builtins/float.rs index b4601fbb92..48ccd2c437 100644 --- a/vm/src/builtins/float.rs +++ b/vm/src/builtins/float.rs @@ -161,7 +161,7 @@ impl Constructor for PyFloat { fn float_from_string(val: PyObjectRef, vm: &VirtualMachine) -> PyResult { let (bytearray, buffer, buffer_lock); let b = if let Some(s) = val.payload_if_subclass::(vm) { - s.as_str().trim().as_bytes() + s.as_wtf8().trim().as_bytes() } else if let Some(bytes) = val.payload_if_subclass::(vm) { bytes.as_bytes() } else if let Some(buf) = val.payload_if_subclass::(vm) { diff --git a/vm/src/builtins/int.rs b/vm/src/builtins/int.rs index aa9613e9d7..f457bf5ed8 100644 --- a/vm/src/builtins/int.rs +++ b/vm/src/builtins/int.rs @@ -847,7 +847,7 @@ fn try_int_radix(obj: &PyObject, base: u32, vm: &VirtualMachine) -> PyResult { - let s = string.as_str().trim(); + let s = string.as_wtf8().trim(); bytes_to_int(s.as_bytes(), base) } bytes @ PyBytes => { diff --git a/vm/src/builtins/str.rs b/vm/src/builtins/str.rs index 8fe3904945..dfb9de9ba6 100644 --- a/vm/src/builtins/str.rs +++ b/vm/src/builtins/str.rs @@ -424,6 +424,23 @@ impl PyStr { self.data.as_str() } + pub fn try_to_str(&self, vm: &VirtualMachine) -> PyResult<&str> { + self.to_str().ok_or_else(|| { + let start = self + .as_wtf8() + .code_points() + .position(|c| c.to_char().is_none()) + .unwrap(); + vm.new_unicode_encode_error_real( + identifier!(vm, utf_8).to_owned(), + vm.ctx.new_str(self.data.clone()), + start, + start + 1, + vm.ctx.new_str("surrogates not allowed"), + ) + }) + } + pub fn to_string_lossy(&self) -> Cow<'_, str> { self.to_str() .map(Cow::Borrowed) @@ -850,9 +867,9 @@ impl PyStr { /// If the string starts with the prefix string, return string[len(prefix):] /// Otherwise, return a copy of the original string. #[pymethod] - fn removeprefix(&self, pref: PyStrRef) -> String { - self.as_str() - .py_removeprefix(pref.as_str(), pref.byte_len(), |s, p| s.starts_with(p)) + fn removeprefix(&self, pref: PyStrRef) -> Wtf8Buf { + self.as_wtf8() + .py_removeprefix(pref.as_wtf8(), pref.byte_len(), |s, p| s.starts_with(p)) .to_owned() } @@ -861,9 +878,9 @@ impl PyStr { /// If the string ends with the suffix string, return string[:len(suffix)] /// Otherwise, return a copy of the original string. #[pymethod] - fn removesuffix(&self, suffix: PyStrRef) -> String { - self.as_str() - .py_removesuffix(suffix.as_str(), suffix.byte_len(), |s, p| s.ends_with(p)) + fn removesuffix(&self, suffix: PyStrRef) -> Wtf8Buf { + self.as_wtf8() + .py_removesuffix(suffix.as_wtf8(), suffix.byte_len(), |s, p| s.ends_with(p)) .to_owned() } @@ -1294,7 +1311,8 @@ impl PyStr { #[pymethod] fn isidentifier(&self) -> bool { - let mut chars = self.as_str().chars(); + let Some(s) = self.to_str() else { return false }; + let mut chars = s.chars(); let is_identifier_start = chars.next().is_some_and(|c| c == '_' || is_xid_start(c)); // a string is not an identifier if it has whitespace or starts with a number is_identifier_start && chars.all(is_xid_continue) diff --git a/vm/src/builtins/type.rs b/vm/src/builtins/type.rs index 969d6db937..776c777cb3 100644 --- a/vm/src/builtins/type.rs +++ b/vm/src/builtins/type.rs @@ -884,7 +884,7 @@ impl Constructor for PyType { attributes .entry(identifier!(vm, __qualname__)) - .or_insert_with(|| vm.ctx.new_str(name.as_str()).into()); + .or_insert_with(|| name.clone().into()); if attributes.get(identifier!(vm, __eq__)).is_some() && attributes.get(identifier!(vm, __hash__)).is_none() diff --git a/vm/src/protocol/number.rs b/vm/src/protocol/number.rs index 2b6720e843..dd039c2733 100644 --- a/vm/src/protocol/number.rs +++ b/vm/src/protocol/number.rs @@ -77,7 +77,7 @@ impl PyObject { )) }) } else if let Some(s) = self.payload::() { - try_convert(self, s.as_str().trim().as_bytes(), vm) + try_convert(self, s.as_wtf8().trim().as_bytes(), vm) } else if let Some(bytes) = self.payload::() { try_convert(self, bytes, vm) } else if let Some(bytearray) = self.payload::() { diff --git a/vm/src/protocol/object.rs b/vm/src/protocol/object.rs index 4e69cf38a2..4cdcb68257 100644 --- a/vm/src/protocol/object.rs +++ b/vm/src/protocol/object.rs @@ -224,7 +224,7 @@ impl PyObject { dict: Option, vm: &VirtualMachine, ) -> PyResult> { - let name = name_str.as_str(); + let name = name_str.as_wtf8(); let obj_cls = self.class(); let cls_attr_name = vm.ctx.interned_str(name_str); let cls_attr = match cls_attr_name.and_then(|name| obj_cls.get_attr(name)) { diff --git a/vm/src/stdlib/codecs.rs b/vm/src/stdlib/codecs.rs index 6ad2a74f4b..320d839682 100644 --- a/vm/src/stdlib/codecs.rs +++ b/vm/src/stdlib/codecs.rs @@ -26,7 +26,7 @@ mod _codecs { fn lookup(encoding: PyStrRef, vm: &VirtualMachine) -> PyResult { vm.state .codec_registry - .lookup(encoding.as_str(), vm) + .lookup(encoding.try_to_str(vm)?, vm) .map(|codec| codec.into_tuple().into()) } diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index 77d9231724..4cf3c058df 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -2245,7 +2245,7 @@ mod _io { let newline = args.newline.unwrap_or_default(); let (encoder, decoder) = - Self::find_coder(&buffer, encoding.as_str(), &errors, newline, vm)?; + Self::find_coder(&buffer, encoding.try_to_str(vm)?, &errors, newline, vm)?; *data = Some(TextIOData { buffer, @@ -2345,7 +2345,7 @@ mod _io { if let Some(encoding) = args.encoding { let (encoder, decoder) = Self::find_coder( &data.buffer, - encoding.as_str(), + encoding.try_to_str(vm)?, &data.errors, data.newline, vm, @@ -3468,9 +3468,9 @@ mod _io { // return the entire contents of the underlying #[pymethod] - fn getvalue(&self, vm: &VirtualMachine) -> PyResult { + fn getvalue(&self, vm: &VirtualMachine) -> PyResult { let bytes = self.buffer(vm)?.getvalue(); - String::from_utf8(bytes) + Wtf8Buf::from_bytes(bytes) .map_err(|_| vm.new_value_error("Error Retrieving Value".to_owned())) } @@ -3491,10 +3491,10 @@ mod _io { // If k is undefined || k == -1, then we read all bytes until the end of the file. // This also increments the stream position by the value of k #[pymethod] - fn read(&self, size: OptionalSize, vm: &VirtualMachine) -> PyResult { + fn read(&self, size: OptionalSize, vm: &VirtualMachine) -> PyResult { let data = self.buffer(vm)?.read(size.to_usize()).unwrap_or_default(); - let value = String::from_utf8(data) + let value = Wtf8Buf::from_bytes(data) .map_err(|_| vm.new_value_error("Error Retrieving Value".to_owned()))?; Ok(value) } @@ -3505,11 +3505,11 @@ mod _io { } #[pymethod] - fn readline(&self, size: OptionalSize, vm: &VirtualMachine) -> PyResult { + fn readline(&self, size: OptionalSize, vm: &VirtualMachine) -> PyResult { // TODO size should correspond to the number of characters, at the moments its the number of // bytes. let input = self.buffer(vm)?.readline(size.to_usize(), vm)?; - String::from_utf8(input) + Wtf8Buf::from_bytes(input) .map_err(|_| vm.new_value_error("Error Retrieving Value".to_owned())) } diff --git a/vm/src/stdlib/time.rs b/vm/src/stdlib/time.rs index c464dc3abf..f98530e845 100644 --- a/vm/src/stdlib/time.rs +++ b/vm/src/stdlib/time.rs @@ -327,8 +327,12 @@ mod decl { * raises an error if unsupported format is supplied. * If error happens, we set result as input arg. */ - write!(&mut formatted_time, "{}", instant.format(format.as_str())) - .unwrap_or_else(|_| formatted_time = format.to_string()); + write!( + &mut formatted_time, + "{}", + instant.format(format.try_to_str(vm)?) + ) + .unwrap_or_else(|_| formatted_time = format.to_string()); Ok(vm.ctx.new_str(formatted_time).into()) } diff --git a/vm/src/utils.rs b/vm/src/utils.rs index e2bc993686..78edfb71cc 100644 --- a/vm/src/utils.rs +++ b/vm/src/utils.rs @@ -1,3 +1,5 @@ +use rustpython_common::wtf8::Wtf8; + use crate::{ PyObjectRef, PyResult, VirtualMachine, builtins::PyStr, @@ -18,9 +20,9 @@ impl ToPyObject for std::convert::Infallible { } } -pub trait ToCString: AsRef { +pub trait ToCString: AsRef { fn to_cstring(&self, vm: &VirtualMachine) -> PyResult { - std::ffi::CString::new(self.as_ref()).map_err(|err| err.to_pyexception(vm)) + std::ffi::CString::new(self.as_ref().as_bytes()).map_err(|err| err.to_pyexception(vm)) } fn ensure_no_nul(&self, vm: &VirtualMachine) -> PyResult<()> { if self.as_ref().as_bytes().contains(&b'\0') { From f27c1f7ea3300c223f39a1526d6b229eb4ea47e8 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" <69878+youknowone@users.noreply.github.com> Date: Thu, 27 Mar 2025 14:51:47 +0900 Subject: [PATCH 133/295] Merge pull request #5624 from youknowone/libffi-workspace common dependency in workspace --- Cargo.toml | 11 +++++++++++ compiler/core/Cargo.toml | 2 +- compiler/literal/Cargo.toml | 2 +- derive-impl/Cargo.toml | 4 ++-- jit/Cargo.toml | 5 +---- stdlib/Cargo.toml | 14 +++++++------- vm/Cargo.toml | 12 ++++++------ vm/sre_engine/Cargo.toml | 2 +- 8 files changed, 30 insertions(+), 22 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index b7e96c9b85..177e501040 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -167,6 +167,7 @@ itertools = "0.14.0" is-macro = "0.3.7" junction = "1.2.0" libc = "0.2.169" +libffi = "3.2" log = "0.4.25" nix = { version = "0.29", features = ["fs", "user", "process", "term", "time", "signal", "ioctl", "socket", "sched", "zerocopy", "dir", "hostname", "net", "poll"] } malachite-bigint = "0.5" @@ -177,9 +178,12 @@ num-complex = "0.4.6" num-integer = "0.1.46" num-traits = "0.2" num_enum = { version = "0.7", default-features = false } +optional = "0.5" once_cell = "1.20.3" parking_lot = "0.12.3" paste = "1.0.15" +proc-macro2 = "1.0.93" +quote = "1.0.38" rand = "0.9" rustix = { version = "0.38", features = ["event"] } rustyline = "15.0.0" @@ -191,6 +195,13 @@ strum_macros = "0.27" syn = "2" thiserror = "2.0" thread_local = "1.1.8" +unicode-casing = "0.1.0" +unic-char-property = "0.9.0" +unic-normal = "0.9.0" +unic-ucd-age = "0.9.0" +unic-ucd-bidi = "0.9.0" +unic-ucd-category = "0.9.0" +unic-ucd-ident = "0.9.0" unicode_names2 = "1.3.0" widestring = "1.1.0" windows-sys = "0.59.0" diff --git a/compiler/core/Cargo.toml b/compiler/core/Cargo.toml index 7621c643d5..d76052d93c 100644 --- a/compiler/core/Cargo.toml +++ b/compiler/core/Cargo.toml @@ -18,7 +18,7 @@ bitflags = { workspace = true } itertools = { workspace = true } malachite-bigint = { workspace = true } num-complex = { workspace = true } -serde = { version = "1.0.217", optional = true, default-features = false, features = ["derive"] } +serde = { workspace = true, optional = true, default-features = false, features = ["derive"] } lz4_flex = "0.11" diff --git a/compiler/literal/Cargo.toml b/compiler/literal/Cargo.toml index de7e89c0b4..5837f7a1ad 100644 --- a/compiler/literal/Cargo.toml +++ b/compiler/literal/Cargo.toml @@ -13,7 +13,7 @@ hexf-parse = "0.2.1" is-macro.workspace = true lexical-parse-float = { version = "0.8.0", features = ["format"] } num-traits = { workspace = true } -unic-ucd-category = "0.9" +unic-ucd-category = { workspace = true } [dev-dependencies] rand = { workspace = true } diff --git a/derive-impl/Cargo.toml b/derive-impl/Cargo.toml index eec7952df5..66ed67cef8 100644 --- a/derive-impl/Cargo.toml +++ b/derive-impl/Cargo.toml @@ -17,8 +17,8 @@ itertools = { workspace = true } syn = { workspace = true, features = ["full", "extra-traits"] } maplit = "1.0.2" -proc-macro2 = "1.0.93" -quote = "1.0.38" +proc-macro2 = { workspace = true } +quote = { workspace = true } syn-ext = { version = "0.5.0", features = ["full"] } textwrap = { version = "0.16.1", default-features = false } diff --git a/jit/Cargo.toml b/jit/Cargo.toml index f59293a7ea..e153b0eca8 100644 --- a/jit/Cargo.toml +++ b/jit/Cargo.toml @@ -15,15 +15,12 @@ rustpython-compiler-core = { workspace = true } num-traits = { workspace = true } thiserror = { workspace = true } +libffi = { workspace = true, features = ["system"] } cranelift = "0.116.1" cranelift-jit = "0.116.1" cranelift-module = "0.116.1" -[dependencies.libffi] -version = "3.1.0" -features = ["system"] - [dev-dependencies] rustpython-derive = { path = "../derive", version = "0.4.0" } diff --git a/stdlib/Cargo.toml b/stdlib/Cargo.toml index 01ad37165a..d29dc1474e 100644 --- a/stdlib/Cargo.toml +++ b/stdlib/Cargo.toml @@ -65,14 +65,14 @@ blake2 = "0.10.4" unicode_names2 = { workspace = true } # TODO: use unic for this; needed for title case: # https://github.com/RustPython/RustPython/pull/832#discussion_r275428939 -unicode-casing = "0.1.0" +unicode-casing = { workspace = true } # update version all at the same time -unic-char-property = "0.9.0" -unic-normal = "0.9.0" -unic-ucd-bidi = "0.9.0" -unic-ucd-category = "0.9.0" -unic-ucd-age = "0.9.0" -unic-ucd-ident = "0.9.0" +unic-char-property = { workspace = true } +unic-normal = { workspace = true } +unic-ucd-bidi = { workspace = true } +unic-ucd-category = { workspace = true } +unic-ucd-age = { workspace = true } +unic-ucd-ident = { workspace = true } ucd = "0.1.1" # compression diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 559390d278..a59327420f 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -81,7 +81,7 @@ caseless = "0.2.2" flamer = { version = "0.4", optional = true } half = "2" memoffset = "0.9.1" -optional = "0.5.0" +optional = { workspace = true } result-like = "0.5.0" timsort = "0.1.2" @@ -89,11 +89,11 @@ timsort = "0.1.2" unicode_names2 = { workspace = true } # TODO: use unic for this; needed for title case: # https://github.com/RustPython/RustPython/pull/832#discussion_r275428939 -unicode-casing = "0.1.0" +unicode-casing = { workspace = true } # update version all at the same time -unic-ucd-bidi = "0.9.0" -unic-ucd-category = "0.9.0" -unic-ucd-ident = "0.9.0" +unic-ucd-bidi = { workspace = true } +unic-ucd-category = { workspace = true } +unic-ucd-ident = { workspace = true } [target.'cfg(unix)'.dependencies] rustix = { workspace = true } @@ -107,7 +107,7 @@ errno = "0.3" widestring = { workspace = true } [target.'cfg(all(any(target_os = "linux", target_os = "macos", target_os = "windows"), not(any(target_env = "musl", target_env = "sgx"))))'.dependencies] -libffi = "3.2" +libffi = { workspace = true, features = ["system"] } libloading = "0.8" [target.'cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))'.dependencies] diff --git a/vm/sre_engine/Cargo.toml b/vm/sre_engine/Cargo.toml index b34b01a0e8..70c448d07f 100644 --- a/vm/sre_engine/Cargo.toml +++ b/vm/sre_engine/Cargo.toml @@ -18,7 +18,7 @@ harness = false rustpython-common = { workspace = true } num_enum = { workspace = true } bitflags = { workspace = true } -optional = "0.5" +optional = { workspace = true } [dev-dependencies] criterion = { workspace = true } From cd89aa51f0cdc1482dc12f5560b0560a49f9ee01 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" <69878+youknowone@users.noreply.github.com> Date: Thu, 27 Mar 2025 19:45:04 +0900 Subject: [PATCH 134/295] Fix _ctypes.Array base and metaclass (#5620) --- .../{builtins_ctypes.py => stdlib_ctypes.py} | 5 ++++- vm/src/stdlib/ctypes.rs | 6 +++--- vm/src/stdlib/ctypes/array.rs | 18 +++++++++++++----- vm/src/stdlib/ctypes/base.rs | 8 ++++---- vm/src/stdlib/ctypes/structure.rs | 3 ++- vm/src/stdlib/ctypes/union.rs | 5 ++++- 6 files changed, 30 insertions(+), 15 deletions(-) rename extra_tests/snippets/{builtins_ctypes.py => stdlib_ctypes.py} (98%) diff --git a/extra_tests/snippets/builtins_ctypes.py b/extra_tests/snippets/stdlib_ctypes.py similarity index 98% rename from extra_tests/snippets/builtins_ctypes.py rename to extra_tests/snippets/stdlib_ctypes.py index 2108ce41a4..95ee9900fb 100644 --- a/extra_tests/snippets/builtins_ctypes.py +++ b/extra_tests/snippets/stdlib_ctypes.py @@ -3,12 +3,15 @@ from _ctypes import RTLD_LOCAL, RTLD_GLOBAL from _ctypes import sizeof -from _ctypes import _SimpleCData +from _ctypes import _SimpleCData, Array from _ctypes import CFuncPtr as _CFuncPtr from struct import calcsize as _calcsize +assert Array.__class__.__name__ == 'PyCArrayType' +assert Array.__base__.__name__ == '_CData' + DEFAULT_MODE = RTLD_LOCAL if _os.name == "posix" and _sys.platform == "darwin": # On OS X 10.3, we use RTLD_GLOBAL as default mode diff --git a/vm/src/stdlib/ctypes.rs b/vm/src/stdlib/ctypes.rs index 38955fe391..235e089e3a 100644 --- a/vm/src/stdlib/ctypes.rs +++ b/vm/src/stdlib/ctypes.rs @@ -10,16 +10,16 @@ pub(crate) mod union; use crate::builtins::PyModule; use crate::class::PyClassImpl; -use crate::stdlib::ctypes::base::{PyCData, PyCSimple, PySimpleMeta}; +use crate::stdlib::ctypes::base::{PyCData, PyCSimple, PyCSimpleType}; use crate::{Py, PyRef, VirtualMachine}; pub fn extend_module_nodes(vm: &VirtualMachine, module: &Py) { let ctx = &vm.ctx; - PySimpleMeta::make_class(ctx); + PyCSimpleType::make_class(ctx); + array::PyCArrayType::make_class(ctx); extend_module!(vm, module, { "_CData" => PyCData::make_class(ctx), "_SimpleCData" => PyCSimple::make_class(ctx), - "ArrayType" => array::PyCArrayType::make_class(ctx), "Array" => array::PyCArray::make_class(ctx), "CFuncPtr" => function::PyCFuncPtr::make_class(ctx), "_Pointer" => pointer::PyCPointer::make_class(ctx), diff --git a/vm/src/stdlib/ctypes/array.rs b/vm/src/stdlib/ctypes/array.rs index 44d237cd7e..0880c6b63b 100644 --- a/vm/src/stdlib/ctypes/array.rs +++ b/vm/src/stdlib/ctypes/array.rs @@ -1,13 +1,16 @@ use crate::builtins::PyBytes; use crate::types::Callable; use crate::{Py, PyObjectRef, PyPayload}; -use crate::{PyResult, VirtualMachine, builtins::PyTypeRef, types::Constructor}; +use crate::{ + PyResult, VirtualMachine, + builtins::{PyType, PyTypeRef}, + types::Constructor, +}; use crossbeam_utils::atomic::AtomicCell; use rustpython_common::lock::PyRwLock; -use rustpython_vm::stdlib::ctypes::base::PyCSimple; +use rustpython_vm::stdlib::ctypes::base::PyCData; -// TODO: make it metaclass -#[pyclass(name = "ArrayType", module = "_ctypes")] +#[pyclass(name = "PyCArrayType", base = "PyType", module = "_ctypes")] #[derive(PyPayload)] pub struct PyCArrayType { pub(super) inner: PyCArray, @@ -44,7 +47,12 @@ impl Constructor for PyCArrayType { #[pyclass(flags(IMMUTABLETYPE), with(Callable, Constructor))] impl PyCArrayType {} -#[pyclass(name = "Array", base = "PyCSimple", module = "_ctypes")] +#[pyclass( + name = "Array", + base = "PyCData", + metaclass = "PyCArrayType", + module = "_ctypes" +)] #[derive(PyPayload)] pub struct PyCArray { pub(super) typ: PyRwLock, diff --git a/vm/src/stdlib/ctypes/base.rs b/vm/src/stdlib/ctypes/base.rs index e3e12bdf3b..6cc19be3df 100644 --- a/vm/src/stdlib/ctypes/base.rs +++ b/vm/src/stdlib/ctypes/base.rs @@ -160,10 +160,10 @@ pub struct PyCData { impl PyCData {} #[pyclass(module = "_ctypes", name = "PyCSimpleType", base = "PyType")] -pub struct PySimpleMeta {} +pub struct PyCSimpleType {} #[pyclass(flags(BASETYPE))] -impl PySimpleMeta { +impl PyCSimpleType { #[allow(clippy::new_ret_no_self)] #[pymethod] fn new(cls: PyTypeRef, _: OptionalArg, vm: &VirtualMachine) -> PyResult { @@ -176,10 +176,10 @@ impl PySimpleMeta { } #[pyclass( + module = "_ctypes", name = "_SimpleCData", base = "PyCData", - module = "_ctypes", - metaclass = "PySimpleMeta" + metaclass = "PyCSimpleType" )] #[derive(PyPayload)] pub struct PyCSimple { diff --git a/vm/src/stdlib/ctypes/structure.rs b/vm/src/stdlib/ctypes/structure.rs index c6cf158953..10c1fa3df8 100644 --- a/vm/src/stdlib/ctypes/structure.rs +++ b/vm/src/stdlib/ctypes/structure.rs @@ -1,3 +1,4 @@ +use super::base::PyCData; use crate::builtins::{PyList, PyStr, PyTuple, PyTypeRef}; use crate::function::FuncArgs; use crate::types::GetAttr; @@ -7,7 +8,7 @@ use rustpython_vm::types::Constructor; use std::collections::HashMap; use std::fmt::Debug; -#[pyclass(name = "Structure", module = "_ctypes")] +#[pyclass(module = "_ctypes", name = "Structure", base = "PyCData")] #[derive(PyPayload, Debug)] pub struct PyCStructure { #[allow(dead_code)] diff --git a/vm/src/stdlib/ctypes/union.rs b/vm/src/stdlib/ctypes/union.rs index 5a39d9062e..2d76dbc9ca 100644 --- a/vm/src/stdlib/ctypes/union.rs +++ b/vm/src/stdlib/ctypes/union.rs @@ -1,4 +1,7 @@ -#[pyclass(name = "Union", module = "_ctypes")] +use super::base::PyCData; + +// TODO: metaclass = "UnionType" +#[pyclass(module = "_ctypes", name = "Union", base = "PyCData")] pub struct PyCUnion {} #[pyclass(flags(BASETYPE, IMMUTABLETYPE))] From dd467f6c73bc31e04c11f990b919014e298ee631 Mon Sep 17 00:00:00 2001 From: Noa Date: Thu, 27 Mar 2025 10:15:18 -0500 Subject: [PATCH 135/295] Update common/src/wtf8/mod.rs Co-authored-by: Jeong, YunWon <69878+youknowone@users.noreply.github.com> --- common/src/wtf8/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/src/wtf8/mod.rs b/common/src/wtf8/mod.rs index 57a1d6d7de..ff4dcf8900 100644 --- a/common/src/wtf8/mod.rs +++ b/common/src/wtf8/mod.rs @@ -763,7 +763,7 @@ impl Wtf8 { let mut rest = b; while let Err(e) = std::str::from_utf8(rest) { rest = &rest[e.valid_up_to()..]; - Self::decode_surrogate(rest)?; + let _ = Self::decode_surrogate(rest)?; rest = &rest[3..]; } Some(unsafe { Wtf8::from_bytes_unchecked(b) }) From 6b72d2ef5d49bb25ba8d17acf85a785366e5e05b Mon Sep 17 00:00:00 2001 From: Noa Date: Thu, 27 Mar 2025 20:44:18 -0500 Subject: [PATCH 136/295] Check+lint examples, tests, and benches in CI --- .github/workflows/ci.yaml | 2 +- benches/execution.rs | 4 +--- compiler/codegen/src/compile.rs | 6 +++--- compiler/src/lib.rs | 30 +++++++++++++++--------------- examples/dis.rs | 19 ++++++++----------- examples/mini_repl.rs | 11 ++++++----- examples/parse_folder.rs | 11 +---------- vm/sre_engine/tests/tests.rs | 2 ++ 8 files changed, 37 insertions(+), 48 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 12399413ba..869ccb530a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -141,7 +141,7 @@ jobs: if: runner.os == 'macOS' - name: run clippy - run: cargo clippy ${{ env.CARGO_ARGS }} --workspace --exclude rustpython_wasm -- -Dwarnings + run: cargo clippy ${{ env.CARGO_ARGS }} --workspace --all-targets --exclude rustpython_wasm -- -Dwarnings - name: run rust tests run: cargo test --workspace --exclude rustpython_wasm --verbose --features threading ${{ env.CARGO_ARGS }} diff --git a/benches/execution.rs b/benches/execution.rs index 57529bbb3a..d38dab0890 100644 --- a/benches/execution.rs +++ b/benches/execution.rs @@ -4,8 +4,6 @@ use criterion::{ criterion_main, }; use rustpython_compiler::Mode; -use rustpython_parser::Parse; -use rustpython_parser::ast; use rustpython_vm::{Interpreter, PyResult, Settings}; use std::collections::HashMap; use std::path::Path; @@ -50,7 +48,7 @@ pub fn benchmark_file_execution(group: &mut BenchmarkGroup, name: &str pub fn benchmark_file_parsing(group: &mut BenchmarkGroup, name: &str, contents: &str) { group.throughput(Throughput::Bytes(contents.len() as u64)); group.bench_function(BenchmarkId::new("rustpython", name), |b| { - b.iter(|| ast::Suite::parse(contents, name).unwrap()) + b.iter(|| ruff_python_parser::parse_module(contents).unwrap()) }); group.bench_function(BenchmarkId::new("cpython", name), |b| { use pyo3::types::PyAnyMethods; diff --git a/compiler/codegen/src/compile.rs b/compiler/codegen/src/compile.rs index ccce28953d..95d304a0ea 100644 --- a/compiler/codegen/src/compile.rs +++ b/compiler/codegen/src/compile.rs @@ -3660,7 +3660,7 @@ mod tests { flags, }), }); - assert_eq!(Compiler::contains_await(not_present), false); + assert!(!Compiler::contains_await(not_present)); // f'{await x}' let expr_await_x = Expr::Await(ExprAwait { @@ -3686,7 +3686,7 @@ mod tests { flags, }), }); - assert_eq!(Compiler::contains_await(present), true); + assert!(Compiler::contains_await(present)); // f'{x:{await y}}' let expr_x = Expr::Name(ExprName { @@ -3727,7 +3727,7 @@ mod tests { flags, }), }); - assert_eq!(Compiler::contains_await(present), true); + assert!(Compiler::contains_await(present)); } } diff --git a/compiler/src/lib.rs b/compiler/src/lib.rs index 9df7a2c5a1..0ebfbbf416 100644 --- a/compiler/src/lib.rs +++ b/compiler/src/lib.rs @@ -159,7 +159,7 @@ pub fn _compile_symtable( #[test] fn test_compile() { let code = "x = 'abc'"; - let compiled = compile(&code, Mode::Single, "<>", CompileOpts::default()); + let compiled = compile(code, Mode::Single, "<>", CompileOpts::default()); dbg!(compiled.expect("compile error")); } @@ -172,7 +172,7 @@ def main(): if __name__ == '__main__': main() "#; - let compiled = compile(&code, Mode::Exec, "<>", CompileOpts::default()); + let compiled = compile(code, Mode::Exec, "<>", CompileOpts::default()); dbg!(compiled.expect("compile error")); } @@ -188,7 +188,7 @@ elif False: else: pass "#; - let compiled = compile(&code, Mode::Exec, "<>", CompileOpts::default()); + let compiled = compile(code, Mode::Exec, "<>", CompileOpts::default()); dbg!(compiled.expect("compile error")); } @@ -197,7 +197,7 @@ fn test_compile_lambda() { let code = r#" lambda: 'a' "#; - let compiled = compile(&code, Mode::Exec, "<>", CompileOpts::default()); + let compiled = compile(code, Mode::Exec, "<>", CompileOpts::default()); dbg!(compiled.expect("compile error")); } @@ -206,7 +206,7 @@ fn test_compile_lambda2() { let code = r#" (lambda x: f'hello, {x}')('world}') "#; - let compiled = compile(&code, Mode::Exec, "<>", CompileOpts::default()); + let compiled = compile(code, Mode::Exec, "<>", CompileOpts::default()); dbg!(compiled.expect("compile error")); } @@ -223,7 +223,7 @@ def f(): else: return g "#; - let compiled = compile(&code, Mode::Exec, "<>", CompileOpts::default()); + let compiled = compile(code, Mode::Exec, "<>", CompileOpts::default()); dbg!(compiled.expect("compile error")); } @@ -232,7 +232,7 @@ fn test_compile_int() { let code = r#" a = 0xFF "#; - let compiled = compile(&code, Mode::Exec, "<>", CompileOpts::default()); + let compiled = compile(code, Mode::Exec, "<>", CompileOpts::default()); dbg!(compiled.expect("compile error")); } @@ -241,7 +241,7 @@ fn test_compile_bigint() { let code = r#" a = 0xFFFFFFFFFFFFFFFFFFFFFFFF "#; - let compiled = compile(&code, Mode::Exec, "<>", CompileOpts::default()); + let compiled = compile(code, Mode::Exec, "<>", CompileOpts::default()); dbg!(compiled.expect("compile error")); } @@ -250,36 +250,36 @@ fn test_compile_fstring() { let code1 = r#" assert f"1" == '1' "#; - let compiled = compile(&code1, Mode::Exec, "<>", CompileOpts::default()); + let compiled = compile(code1, Mode::Exec, "<>", CompileOpts::default()); dbg!(compiled.expect("compile error")); let code2 = r#" assert f"{1}" == '1' "#; - let compiled = compile(&code2, Mode::Exec, "<>", CompileOpts::default()); + let compiled = compile(code2, Mode::Exec, "<>", CompileOpts::default()); dbg!(compiled.expect("compile error")); let code3 = r#" assert f"{1+1}" == '2' "#; - let compiled = compile(&code3, Mode::Exec, "<>", CompileOpts::default()); + let compiled = compile(code3, Mode::Exec, "<>", CompileOpts::default()); dbg!(compiled.expect("compile error")); let code4 = r#" assert f"{{{(lambda: f'{1}')}" == '{1' "#; - let compiled = compile(&code4, Mode::Exec, "<>", CompileOpts::default()); + let compiled = compile(code4, Mode::Exec, "<>", CompileOpts::default()); dbg!(compiled.expect("compile error")); let code5 = r#" assert f"a{1}" == 'a1' "#; - let compiled = compile(&code5, Mode::Exec, "<>", CompileOpts::default()); + let compiled = compile(code5, Mode::Exec, "<>", CompileOpts::default()); dbg!(compiled.expect("compile error")); let code6 = r#" assert f"{{{(lambda x: f'hello, {x}')('world}')}" == '{hello, world}' "#; - let compiled = compile(&code6, Mode::Exec, "<>", CompileOpts::default()); + let compiled = compile(code6, Mode::Exec, "<>", CompileOpts::default()); dbg!(compiled.expect("compile error")); } @@ -293,6 +293,6 @@ class RegexFlag: DEBUG = 1 print(RegexFlag.NOFLAG & RegexFlag.DEBUG) "#; - let compiled = compile(&code, Mode::Exec, "", CompileOpts::default()); + let compiled = compile(code, Mode::Exec, "", CompileOpts::default()); dbg!(compiled.expect("compile error")); } diff --git a/examples/dis.rs b/examples/dis.rs index 5b8826fdf1..c79b5e258c 100644 --- a/examples/dis.rs +++ b/examples/dis.rs @@ -1,10 +1,10 @@ -/// This an example usage of the rustpython_compiler crate. -/// This program reads, parses, and compiles a file you provide -/// to RustPython bytecode, and then displays the output in the -/// `dis.dis` format. -/// -/// example usage: -/// $ cargo run --release --example dis demo*.py +//! This an example usage of the rustpython_compiler crate. +//! This program reads, parses, and compiles a file you provide +//! to RustPython bytecode, and then displays the output in the +//! `dis.dis` format. +//! +//! example usage: +//! $ cargo run --release --example dis demo*.py #[macro_use] extern crate log; @@ -53,10 +53,7 @@ fn main() -> Result<(), lexopt::Error> { return Err("expected at least one argument".into()); } - let opts = compiler::CompileOpts { - optimize, - ..Default::default() - }; + let opts = compiler::CompileOpts { optimize }; for script in &scripts { if script.exists() && script.is_file() { diff --git a/examples/mini_repl.rs b/examples/mini_repl.rs index fe3b13dcfd..d7baa4692c 100644 --- a/examples/mini_repl.rs +++ b/examples/mini_repl.rs @@ -1,8 +1,9 @@ -///! This example show cases a very simple REPL. -///! While a much better REPL can be found in ../src/shell, -///! This much smaller REPL is still a useful example because it showcases inserting -///! values and functions into the Python runtime's scope, and showcases use -///! of the compilation mode "Single". +//! This example show cases a very simple REPL. +//! While a much better REPL can be found in ../src/shell, +//! This much smaller REPL is still a useful example because it showcases inserting +//! values and functions into the Python runtime's scope, and showcases use +//! of the compilation mode "Single". + use rustpython_vm as vm; // these are needed for special memory shenanigans to let us share a variable with Python and Rust use std::sync::atomic::{AtomicBool, Ordering}; diff --git a/examples/parse_folder.rs b/examples/parse_folder.rs index acad76ad76..e05b34c265 100644 --- a/examples/parse_folder.rs +++ b/examples/parse_folder.rs @@ -69,8 +69,6 @@ fn parse_python_file(filename: &Path) -> ParsedFile { info!("Parsing file {:?}", filename); match std::fs::read_to_string(filename) { Err(e) => ParsedFile { - filename: Box::new(filename.to_path_buf()), - code: "".to_owned(), num_lines: 0, result: Err(e.to_string()), }, @@ -79,12 +77,7 @@ fn parse_python_file(filename: &Path) -> ParsedFile { let result = parse_module(&source) .map(|x| x.into_suite()) .map_err(|e| e.to_string()); - ParsedFile { - filename: Box::new(filename.to_path_buf()), - code: source.to_string(), - num_lines, - result, - } + ParsedFile { num_lines, result } } } } @@ -134,8 +127,6 @@ struct ScanResult { } struct ParsedFile { - filename: Box, - code: String, num_lines: usize, result: ParseResult, } diff --git a/vm/sre_engine/tests/tests.rs b/vm/sre_engine/tests/tests.rs index 0946fd64ca..5499afa281 100644 --- a/vm/sre_engine/tests/tests.rs +++ b/vm/sre_engine/tests/tests.rs @@ -1,6 +1,7 @@ use rustpython_sre_engine::{Request, State, StrDrive}; struct Pattern { + #[allow(unused)] pattern: &'static str, code: &'static [u32], } @@ -178,6 +179,7 @@ fn test_bigcharset() { #[test] fn test_search_nonascii() { + #[allow(unused)] // pattern p = re.compile('\xe0+') // START GENERATED by generate_tests.py #[rustfmt::skip] let p = Pattern { pattern: "\u{e0}+", code: &[14, 4, 0, 1, 4294967295, 24, 6, 1, 4294967295, 16, 224, 1, 1] }; From 030243a6f9120010f53a9c4371ae981e691dd8d0 Mon Sep 17 00:00:00 2001 From: Noa Date: Thu, 27 Mar 2025 00:08:29 -0500 Subject: [PATCH 137/295] Split out wtf8 into its own crate --- Cargo.lock | 17 +++++++-- Cargo.toml | 3 +- common/Cargo.toml | 1 + common/src/lib.rs | 3 +- compiler/codegen/Cargo.toml | 2 +- compiler/codegen/src/compile.rs | 35 +++++++++++++++++-- compiler/codegen/src/string_parser.rs | 2 +- compiler/core/Cargo.toml | 2 +- compiler/core/src/bytecode.rs | 2 +- compiler/core/src/marshal.rs | 2 +- vm/sre_engine/Cargo.toml | 2 +- vm/sre_engine/src/string.rs | 2 +- wtf8/Cargo.toml | 15 ++++++++ {common/src/wtf8 => wtf8/src}/core_char.rs | 0 {common/src/wtf8 => wtf8/src}/core_str.rs | 0 .../src/wtf8 => wtf8/src}/core_str_count.rs | 0 common/src/wtf8/mod.rs => wtf8/src/lib.rs | 0 17 files changed, 74 insertions(+), 14 deletions(-) create mode 100644 wtf8/Cargo.toml rename {common/src/wtf8 => wtf8/src}/core_char.rs (100%) rename {common/src/wtf8 => wtf8/src}/core_str.rs (100%) rename {common/src/wtf8 => wtf8/src}/core_str_count.rs (100%) rename common/src/wtf8/mod.rs => wtf8/src/lib.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index 2003707be0..0bed6154ff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2327,9 +2327,9 @@ dependencies = [ "ruff_python_parser", "ruff_source_file", "ruff_text_size", - "rustpython-common", "rustpython-compiler-core", "rustpython-compiler-source", + "rustpython-wtf8", "thiserror 2.0.11", "unicode_names2", ] @@ -2355,6 +2355,7 @@ dependencies = [ "radium", "rand 0.9.0", "rustpython-literal", + "rustpython-wtf8", "siphasher 0.3.11", "unicode_names2", "volatile", @@ -2388,7 +2389,7 @@ dependencies = [ "ruff_python_ast", "ruff_python_parser", "ruff_source_file", - "rustpython-common", + "rustpython-wtf8", "serde", ] @@ -2476,7 +2477,7 @@ dependencies = [ "criterion", "num_enum", "optional", - "rustpython-common", + "rustpython-wtf8", ] [[package]] @@ -2634,6 +2635,16 @@ dependencies = [ "winreg", ] +[[package]] +name = "rustpython-wtf8" +version = "0.4.0" +dependencies = [ + "ascii", + "bstr", + "itertools 0.14.0", + "memchr", +] + [[package]] name = "rustpython_wasm" version = "0.4.0" diff --git a/Cargo.toml b/Cargo.toml index 177e501040..c9f9c96525 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -115,7 +115,7 @@ template = "installer-config/installer.wxs" resolver = "2" members = [ "compiler", "compiler/core", "compiler/codegen", "compiler/literal", "compiler/source", - ".", "common", "derive", "jit", "vm", "vm/sre_engine", "pylib", "stdlib", "derive-impl", + ".", "common", "derive", "jit", "vm", "vm/sre_engine", "pylib", "stdlib", "derive-impl", "wtf8", "wasm/lib", ] @@ -141,6 +141,7 @@ rustpython-vm = { path = "vm", default-features = false, version = "0.4.0" } rustpython-pylib = { path = "pylib", version = "0.4.0" } rustpython-stdlib = { path = "stdlib", default-features = false, version = "0.4.0" } rustpython-sre_engine = { path = "vm/sre_engine", version = "0.4.0" } +rustpython-wtf8 = { path = "wtf8", version = "0.4.0" } rustpython-doc = { git = "https://github.com/RustPython/__doc__", tag = "0.3.0", version = "0.3.0" } ruff_python_parser = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" } diff --git a/common/Cargo.toml b/common/Cargo.toml index 4163a9c284..1cf637faa4 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -13,6 +13,7 @@ threading = ["parking_lot"] [dependencies] rustpython-literal = { workspace = true } +rustpython-wtf8 = { workspace = true } ascii = { workspace = true } bitflags = { workspace = true } diff --git a/common/src/lib.rs b/common/src/lib.rs index e83a9af43a..4b50b31c2b 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -29,7 +29,8 @@ pub mod static_cell; pub mod str; #[cfg(windows)] pub mod windows; -pub mod wtf8; + +pub use rustpython_wtf8 as wtf8; pub mod vendored { pub use ascii; diff --git a/compiler/codegen/Cargo.toml b/compiler/codegen/Cargo.toml index c7ff439f78..2ee9277098 100644 --- a/compiler/codegen/Cargo.toml +++ b/compiler/codegen/Cargo.toml @@ -11,10 +11,10 @@ license.workspace = true [dependencies] # rustpython-ast = { workspace = true, features=["unparse", "constant-optimization"] } -rustpython-common = { workspace = true } # rustpython-parser-core = { workspace = true } rustpython-compiler-core = { workspace = true } rustpython-compiler-source = {workspace = true } +rustpython-wtf8 = { workspace = true } ruff_python_parser = { workspace = true } ruff_python_ast = { workspace = true } ruff_text_size = { workspace = true } diff --git a/compiler/codegen/src/compile.rs b/compiler/codegen/src/compile.rs index 95d304a0ea..6140d354ab 100644 --- a/compiler/codegen/src/compile.rs +++ b/compiler/codegen/src/compile.rs @@ -28,7 +28,7 @@ use ruff_python_ast::{ }; use ruff_source_file::OneIndexed; use ruff_text_size::{Ranged, TextRange}; -use rustpython_common::wtf8::Wtf8Buf; +use rustpython_wtf8::Wtf8Buf; // use rustpython_ast::located::{self as located_ast, Located}; use rustpython_compiler_core::{ Mode, @@ -3529,7 +3529,7 @@ impl EmitArg for ir::BlockIdx { /// The code has been ported from `_PyCompile_CleanDoc` in cpython. /// `inspect.cleandoc` is also a good reference, but has a few incompatibilities. fn clean_doc(doc: &str) -> String { - let doc = rustpython_common::str::expandtabs(doc, 8); + let doc = expandtabs(doc, 8); // First pass: find minimum indentation of any non-blank lines // after first line. let margin = doc @@ -3558,6 +3558,37 @@ fn clean_doc(doc: &str) -> String { } } +// copied from rustpython_common::str, so we don't have to depend on it just for this function +fn expandtabs(input: &str, tab_size: usize) -> String { + let tab_stop = tab_size; + let mut expanded_str = String::with_capacity(input.len()); + let mut tab_size = tab_stop; + let mut col_count = 0usize; + for ch in input.chars() { + match ch { + '\t' => { + let num_spaces = tab_size - col_count; + col_count += num_spaces; + let expand = " ".repeat(num_spaces); + expanded_str.push_str(&expand); + } + '\r' | '\n' => { + expanded_str.push(ch); + col_count = 0; + tab_size = 0; + } + _ => { + expanded_str.push(ch); + col_count += 1; + } + } + if col_count >= tab_size { + tab_size += tab_stop; + } + } + expanded_str +} + fn split_doc<'a>(body: &'a [Stmt], opts: &CompileOpts) -> (Option, &'a [Stmt]) { if let Some((Stmt::Expr(expr), body_rest)) = body.split_first() { let doc_comment = match &*expr.value { diff --git a/compiler/codegen/src/string_parser.rs b/compiler/codegen/src/string_parser.rs index 7bdb86aa5e..74f8e30012 100644 --- a/compiler/codegen/src/string_parser.rs +++ b/compiler/codegen/src/string_parser.rs @@ -8,7 +8,7 @@ use std::convert::Infallible; use ruff_python_ast::{AnyStringFlags, StringFlags}; -use rustpython_common::wtf8::{CodePoint, Wtf8, Wtf8Buf}; +use rustpython_wtf8::{CodePoint, Wtf8, Wtf8Buf}; // use ruff_python_parser::{LexicalError, LexicalErrorType}; type LexicalError = Infallible; diff --git a/compiler/core/Cargo.toml b/compiler/core/Cargo.toml index f38487e68f..e79d937683 100644 --- a/compiler/core/Cargo.toml +++ b/compiler/core/Cargo.toml @@ -13,7 +13,7 @@ license.workspace = true ruff_python_ast = { workspace = true } ruff_python_parser = { workspace = true } ruff_source_file = { workspace = true } -rustpython-common = { workspace = true } +rustpython-wtf8 = { workspace = true } bitflags = { workspace = true } itertools = { workspace = true } diff --git a/compiler/core/src/bytecode.rs b/compiler/core/src/bytecode.rs index 7b018d1df1..2745a5e9f7 100644 --- a/compiler/core/src/bytecode.rs +++ b/compiler/core/src/bytecode.rs @@ -8,7 +8,7 @@ use num_complex::Complex64; pub use ruff_python_ast::ConversionFlag; // use rustpython_parser_core::source_code::{OneIndexed, SourceLocation}; use ruff_source_file::{OneIndexed, SourceLocation}; -use rustpython_common::wtf8::{Wtf8, Wtf8Buf}; +use rustpython_wtf8::{Wtf8, Wtf8Buf}; use std::marker::PhantomData; use std::{collections::BTreeSet, fmt, hash, mem}; diff --git a/compiler/core/src/marshal.rs b/compiler/core/src/marshal.rs index 0c8da17ff9..700bb48230 100644 --- a/compiler/core/src/marshal.rs +++ b/compiler/core/src/marshal.rs @@ -2,7 +2,7 @@ use crate::bytecode::*; use malachite_bigint::{BigInt, Sign}; use num_complex::Complex64; use ruff_source_file::{OneIndexed, SourceLocation}; -use rustpython_common::wtf8::Wtf8; +use rustpython_wtf8::Wtf8; use std::convert::Infallible; pub const FORMAT_VERSION: u32 = 4; diff --git a/vm/sre_engine/Cargo.toml b/vm/sre_engine/Cargo.toml index 70c448d07f..4f899e6b3e 100644 --- a/vm/sre_engine/Cargo.toml +++ b/vm/sre_engine/Cargo.toml @@ -15,7 +15,7 @@ name = "benches" harness = false [dependencies] -rustpython-common = { workspace = true } +rustpython-wtf8 = { workspace = true } num_enum = { workspace = true } bitflags = { workspace = true } optional = { workspace = true } diff --git a/vm/sre_engine/src/string.rs b/vm/sre_engine/src/string.rs index 20cacbfbec..9ca97069f8 100644 --- a/vm/sre_engine/src/string.rs +++ b/vm/sre_engine/src/string.rs @@ -1,4 +1,4 @@ -use rustpython_common::wtf8::Wtf8; +use rustpython_wtf8::Wtf8; #[derive(Debug, Clone, Copy)] pub struct StringCursor { diff --git a/wtf8/Cargo.toml b/wtf8/Cargo.toml new file mode 100644 index 0000000000..110b54ad0c --- /dev/null +++ b/wtf8/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "rustpython-wtf8" +description = "An implementation of WTF-8 for use in RustPython" +version.workspace = true +authors.workspace = true +edition.workspace = true +rust-version.workspace = true +repository.workspace = true +license.workspace = true + +[dependencies] +ascii = { workspace = true } +bstr = { workspace = true } +itertools = { workspace = true } +memchr = { workspace = true } diff --git a/common/src/wtf8/core_char.rs b/wtf8/src/core_char.rs similarity index 100% rename from common/src/wtf8/core_char.rs rename to wtf8/src/core_char.rs diff --git a/common/src/wtf8/core_str.rs b/wtf8/src/core_str.rs similarity index 100% rename from common/src/wtf8/core_str.rs rename to wtf8/src/core_str.rs diff --git a/common/src/wtf8/core_str_count.rs b/wtf8/src/core_str_count.rs similarity index 100% rename from common/src/wtf8/core_str_count.rs rename to wtf8/src/core_str_count.rs diff --git a/common/src/wtf8/mod.rs b/wtf8/src/lib.rs similarity index 100% rename from common/src/wtf8/mod.rs rename to wtf8/src/lib.rs From 7d05f881c4b523c4371a5bbe3c92f0155ee4b8d6 Mon Sep 17 00:00:00 2001 From: Noa Date: Thu, 27 Mar 2025 00:55:03 -0500 Subject: [PATCH 138/295] Have rustpython_literal::escape support wtf8 --- Cargo.lock | 1 + Lib/test/test_bigmem.py | 2 - Lib/test/test_builtin.py | 2 - Lib/test/test_subprocess.py | 2 - compiler/literal/Cargo.toml | 2 + compiler/literal/src/escape.rs | 68 ++++++++++++++++++++++------------ stdlib/src/array.rs | 55 +++++++++++---------------- vm/src/builtins/str.rs | 13 ++++--- 8 files changed, 78 insertions(+), 67 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0bed6154ff..04729d30ab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2457,6 +2457,7 @@ dependencies = [ "lexical-parse-float", "num-traits", "rand 0.9.0", + "rustpython-wtf8", "unic-ucd-category", ] diff --git a/Lib/test/test_bigmem.py b/Lib/test/test_bigmem.py index e360ec15a8..aaa9972bc4 100644 --- a/Lib/test/test_bigmem.py +++ b/Lib/test/test_bigmem.py @@ -710,8 +710,6 @@ def test_repr_large(self, size): # original (Py_UCS2) one # There's also some overallocation when resizing the ascii() result # that isn't taken into account here. - # TODO: RUSTPYTHON - @unittest.expectedFailure @bigmemtest(size=_2G // 5 + 1, memuse=ucs2_char_size + ucs4_char_size + ascii_char_size * 6) def test_unicode_repr(self, size): diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 13d67e1855..b4119305f9 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -226,8 +226,6 @@ def test_any(self): S = [10, 20, 30] self.assertEqual(any(x > 42 for x in S), False) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_ascii(self): self.assertEqual(ascii(''), '\'\'') self.assertEqual(ascii(0), '0') diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index e5b18fe20f..5a75971be6 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -2784,8 +2784,6 @@ def prepare(): else: self.fail("Expected ValueError or subprocess.SubprocessError") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_undecodable_env(self): for key, value in (('test', 'abc\uDCFF'), ('test\uDCFF', '42')): encoded_value = value.encode("ascii", "surrogateescape") diff --git a/compiler/literal/Cargo.toml b/compiler/literal/Cargo.toml index 5837f7a1ad..b4fa6229f1 100644 --- a/compiler/literal/Cargo.toml +++ b/compiler/literal/Cargo.toml @@ -9,6 +9,8 @@ license = { workspace = true } rust-version = { workspace = true } [dependencies] +rustpython-wtf8 = { workspace = true } + hexf-parse = "0.2.1" is-macro.workspace = true lexical-parse-float = { version = "0.8.0", features = ["format"] } diff --git a/compiler/literal/src/escape.rs b/compiler/literal/src/escape.rs index ba8e3ecf17..920e0a8cb1 100644 --- a/compiler/literal/src/escape.rs +++ b/compiler/literal/src/escape.rs @@ -1,3 +1,5 @@ +use rustpython_wtf8::{CodePoint, Wtf8}; + #[derive(Debug, PartialEq, Eq, Copy, Clone, Hash, is_macro::Is)] pub enum Quote { Single, @@ -35,20 +37,32 @@ pub struct EscapeLayout { pub len: Option, } -pub trait Escape { +/// Represents string types that can be escape-printed. +/// +/// # Safety +/// +/// `source_len` and `layout` must be accurate, and `layout.len` must not be equal +/// to `Some(source_len)` if the string contains non-printable characters. +pub unsafe trait Escape { fn source_len(&self) -> usize; fn layout(&self) -> &EscapeLayout; fn changed(&self) -> bool { self.layout().len != Some(self.source_len()) } - fn write_source(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result; + /// Write the body of the string directly to the formatter. + /// + /// # Safety + /// + /// This string must only contain printable characters. + unsafe fn write_source(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result; fn write_body_slow(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result; fn write_body(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result { if self.changed() { self.write_body_slow(formatter) } else { - self.write_source(formatter) + // SAFETY: verified the string contains only printable characters. + unsafe { self.write_source(formatter) } } } } @@ -75,23 +89,23 @@ pub(crate) const fn choose_quote( } pub struct UnicodeEscape<'a> { - source: &'a str, + source: &'a Wtf8, layout: EscapeLayout, } impl<'a> UnicodeEscape<'a> { #[inline] - pub fn with_forced_quote(source: &'a str, quote: Quote) -> Self { + pub fn with_forced_quote(source: &'a Wtf8, quote: Quote) -> Self { let layout = EscapeLayout { quote, len: None }; Self { source, layout } } #[inline] - pub fn with_preferred_quote(source: &'a str, quote: Quote) -> Self { + pub fn with_preferred_quote(source: &'a Wtf8, quote: Quote) -> Self { let layout = Self::repr_layout(source, quote); Self { source, layout } } #[inline] - pub fn new_repr(source: &'a str) -> Self { + pub fn new_repr(source: &'a Wtf8) -> Self { Self::with_preferred_quote(source, Quote::Single) } #[inline] @@ -126,14 +140,14 @@ impl std::fmt::Display for StrRepr<'_, '_> { impl UnicodeEscape<'_> { const REPR_RESERVED_LEN: usize = 2; // for quotes - pub fn repr_layout(source: &str, preferred_quote: Quote) -> EscapeLayout { + pub fn repr_layout(source: &Wtf8, preferred_quote: Quote) -> EscapeLayout { Self::output_layout_with_checker(source, preferred_quote, |a, b| { Some((a as isize).checked_add(b as isize)? as usize) }) } fn output_layout_with_checker( - source: &str, + source: &Wtf8, preferred_quote: Quote, length_add: impl Fn(usize, usize) -> Option, ) -> EscapeLayout { @@ -141,17 +155,17 @@ impl UnicodeEscape<'_> { let mut single_count = 0; let mut double_count = 0; - for ch in source.chars() { - let incr = match ch { - '\'' => { + for ch in source.code_points() { + let incr = match ch.to_char() { + Some('\'') => { single_count += 1; 1 } - '"' => { + Some('"') => { double_count += 1; 1 } - c => Self::escaped_char_len(c), + _ => Self::escaped_char_len(ch), }; let Some(new_len) = length_add(out_len, incr) else { #[cold] @@ -182,7 +196,9 @@ impl UnicodeEscape<'_> { } } - fn escaped_char_len(ch: char) -> usize { + fn escaped_char_len(ch: CodePoint) -> usize { + // surrogates are \uHHHH + let Some(ch) = ch.to_char() else { return 6 }; match ch { '\\' | '\t' | '\r' | '\n' => 2, ch if ch < ' ' || ch as u32 == 0x7f => 4, // \xHH @@ -198,10 +214,13 @@ impl UnicodeEscape<'_> { } fn write_char( - ch: char, + ch: CodePoint, quote: Quote, formatter: &mut impl std::fmt::Write, ) -> std::fmt::Result { + let Some(ch) = ch.to_char() else { + return write!(formatter, "\\u{:04x}", ch.to_u32()); + }; match ch { '\n' => formatter.write_str("\\n"), '\t' => formatter.write_str("\\t"), @@ -232,7 +251,7 @@ impl UnicodeEscape<'_> { } } -impl Escape for UnicodeEscape<'_> { +unsafe impl Escape for UnicodeEscape<'_> { fn source_len(&self) -> usize { self.source.len() } @@ -241,13 +260,16 @@ impl Escape for UnicodeEscape<'_> { &self.layout } - fn write_source(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result { - formatter.write_str(self.source) + unsafe fn write_source(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result { + formatter.write_str(unsafe { + // SAFETY: this function must be called only when source is printable characters (i.e. no surrogates) + std::str::from_utf8_unchecked(self.source.as_bytes()) + }) } #[cold] fn write_body_slow(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result { - for ch in self.source.chars() { + for ch in self.source.code_points() { Self::write_char(ch, self.layout().quote, formatter)?; } Ok(()) @@ -373,7 +395,7 @@ impl AsciiEscape<'_> { } } -impl Escape for AsciiEscape<'_> { +unsafe impl Escape for AsciiEscape<'_> { fn source_len(&self) -> usize { self.source.len() } @@ -382,7 +404,7 @@ impl Escape for AsciiEscape<'_> { &self.layout } - fn write_source(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result { + unsafe fn write_source(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result { formatter.write_str(unsafe { // SAFETY: this function must be called only when source is printable ascii characters std::str::from_utf8_unchecked(self.source) @@ -429,7 +451,7 @@ mod unicode_escape_tests { #[test] fn changed() { fn test(s: &str) -> bool { - UnicodeEscape::new_repr(s).changed() + UnicodeEscape::new_repr(s.as_ref()).changed() } assert!(!test("hello")); assert!(!test("'hello'")); diff --git a/stdlib/src/array.rs b/stdlib/src/array.rs index 494e98dc1b..fd83f0a5ad 100644 --- a/stdlib/src/array.rs +++ b/stdlib/src/array.rs @@ -69,6 +69,7 @@ mod array { }; use itertools::Itertools; use num_traits::ToPrimitive; + use rustpython_common::wtf8::{CodePoint, Wtf8, Wtf8Buf}; use std::{cmp::Ordering, fmt, os::raw}; macro_rules! def_array_enum { @@ -590,21 +591,12 @@ mod array { } } - fn u32_to_char(ch: u32) -> Result { - if ch > 0x10ffff { - return Err(format!( - "character U+{ch:4x} is not in range [U+0000; U+10ffff]" - )); - }; - char::from_u32(ch).ok_or_else(|| { - format!( - "'utf-8' codec can't encode character '\\u{ch:x}' \ - in position 0: surrogates not allowed" - ) - }) + fn u32_to_char(ch: u32) -> Result { + CodePoint::from_u32(ch) + .ok_or_else(|| format!("character U+{ch:4x} is not in range [U+0000; U+10ffff]")) } - impl TryFrom for char { + impl TryFrom for CodePoint { type Error = String; fn try_from(ch: WideChar) -> Result { @@ -615,10 +607,9 @@ mod array { impl ToPyResult for WideChar { fn to_pyresult(self, vm: &VirtualMachine) -> PyResult { - Ok( - String::from(char::try_from(self).map_err(|e| vm.new_unicode_encode_error(e))?) - .to_pyobject(vm), - ) + Ok(CodePoint::try_from(self) + .map_err(|e| vm.new_unicode_encode_error(e))? + .to_pyobject(vm)) } } @@ -694,9 +685,9 @@ mod array { } } } - } else if let Some(utf8) = init.payload::() { + } else if let Some(wtf8) = init.payload::() { if spec == 'u' { - let bytes = Self::_unicode_to_wchar_bytes(utf8.as_str(), array.itemsize()); + let bytes = Self::_unicode_to_wchar_bytes(wtf8.as_wtf8(), array.itemsize()); array.frombytes_move(bytes); } else { return Err(vm.new_type_error(format!( @@ -794,7 +785,7 @@ mod array { bytes: &[u8], item_size: usize, vm: &VirtualMachine, - ) -> PyResult { + ) -> PyResult { if item_size == 2 { // safe because every configuration of bytes for the types we support are valid let utf16 = unsafe { @@ -803,7 +794,7 @@ mod array { bytes.len() / std::mem::size_of::(), ) }; - Ok(String::from_utf16_lossy(utf16)) + Ok(Wtf8Buf::from_wide(utf16)) } else { // safe because every configuration of bytes for the types we support are valid let chars = unsafe { @@ -822,21 +813,19 @@ mod array { } } - fn _unicode_to_wchar_bytes(utf8: &str, item_size: usize) -> Vec { + fn _unicode_to_wchar_bytes(wtf8: &Wtf8, item_size: usize) -> Vec { if item_size == 2 { - utf8.encode_utf16() - .flat_map(|ch| ch.to_ne_bytes()) - .collect() + wtf8.encode_wide().flat_map(|ch| ch.to_ne_bytes()).collect() } else { - utf8.chars() - .flat_map(|ch| (ch as u32).to_ne_bytes()) + wtf8.code_points() + .flat_map(|ch| ch.to_u32().to_ne_bytes()) .collect() } } #[pymethod] fn fromunicode(zelf: &Py, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - let utf8: &str = obj.try_to_value(vm).map_err(|_| { + let wtf8: &Wtf8 = obj.try_to_value(vm).map_err(|_| { vm.new_type_error(format!( "fromunicode() argument must be str, not {}", obj.class().name() @@ -848,13 +837,13 @@ mod array { )); } let mut w = zelf.try_resizable(vm)?; - let bytes = Self::_unicode_to_wchar_bytes(utf8, w.itemsize()); + let bytes = Self::_unicode_to_wchar_bytes(wtf8, w.itemsize()); w.frombytes_move(bytes); Ok(()) } #[pymethod] - fn tounicode(&self, vm: &VirtualMachine) -> PyResult { + fn tounicode(&self, vm: &VirtualMachine) -> PyResult { let array = self.array.read(); if array.typecode() != 'u' { return Err(vm.new_value_error( @@ -1184,7 +1173,7 @@ mod array { let typecode = vm.ctx.new_str(array.typecode_str()); let values = if array.typecode() == 'u' { let s = Self::_wchar_bytes_to_string(array.get_bytes(), array.itemsize(), vm)?; - s.chars().map(|x| x.to_pyobject(vm)).collect() + s.code_points().map(|x| x.to_pyobject(vm)).collect() } else { array.get_objects(vm) }; @@ -1656,11 +1645,11 @@ mod array { let s = String::from_utf16(&utf16).map_err(|_| { vm.new_unicode_encode_error("items cannot decode as utf16".into()) })?; - let bytes = PyArray::_unicode_to_wchar_bytes(&s, array.itemsize()); + let bytes = PyArray::_unicode_to_wchar_bytes((*s).as_ref(), array.itemsize()); array.frombytes_move(bytes); } MachineFormatCode::Utf32 { big_endian } => { - let s: String = chunks + let s: Wtf8Buf = chunks .map(|b| chunk_to_obj!(b, u32, big_endian)) .map(|ch| u32_to_char(ch).map_err(|msg| vm.new_value_error(msg))) .try_collect()?; diff --git a/vm/src/builtins/str.rs b/vm/src/builtins/str.rs index fe12b35f74..8aafc63c3b 100644 --- a/vm/src/builtins/str.rs +++ b/vm/src/builtins/str.rs @@ -57,6 +57,13 @@ impl<'a> TryFromBorrowedObject<'a> for &'a str { } } +impl<'a> TryFromBorrowedObject<'a> for &'a Wtf8 { + fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult { + let pystr: &Py = TryFromBorrowedObject::try_from_borrowed_object(vm, obj)?; + Ok(pystr.as_wtf8()) + } +} + #[pyclass(module = false, name = "str")] pub struct PyStr { data: StrData, @@ -607,11 +614,7 @@ impl PyStr { #[inline] pub(crate) fn repr(&self, vm: &VirtualMachine) -> PyResult { use crate::literal::escape::UnicodeEscape; - if !self.kind().is_utf8() { - return Ok(format!("{:?}", self.as_wtf8())); - } - let escape = UnicodeEscape::new_repr(self.as_str()); - escape + UnicodeEscape::new_repr(self.as_wtf8()) .str_repr() .to_string() .ok_or_else(|| vm.new_overflow_error("string is too long to generate repr".to_owned())) From f398321b1f2f0f850738c08b8fb83257c1e442be Mon Sep 17 00:00:00 2001 From: Noa Date: Thu, 27 Mar 2025 15:50:45 -0500 Subject: [PATCH 139/295] Remove parser dependency from codegen --- Cargo.lock | 3 --- compiler/codegen/Cargo.toml | 1 - compiler/codegen/src/compile.rs | 13 +++++++------ compiler/core/Cargo.toml | 2 -- compiler/core/src/bytecode.rs | 16 +++++++++++++--- compiler/core/src/mode.rs | 19 ++++++++----------- compiler/src/lib.rs | 9 ++++++++- vm/src/stdlib/ast/other.rs | 9 ++++++++- 8 files changed, 44 insertions(+), 28 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 04729d30ab..55872f1d68 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2324,7 +2324,6 @@ dependencies = [ "num-traits", "ruff_python_ast", "ruff_python_codegen", - "ruff_python_parser", "ruff_source_file", "ruff_text_size", "rustpython-compiler-core", @@ -2386,8 +2385,6 @@ dependencies = [ "lz4_flex", "malachite-bigint", "num-complex", - "ruff_python_ast", - "ruff_python_parser", "ruff_source_file", "rustpython-wtf8", "serde", diff --git a/compiler/codegen/Cargo.toml b/compiler/codegen/Cargo.toml index 2ee9277098..f54d6476af 100644 --- a/compiler/codegen/Cargo.toml +++ b/compiler/codegen/Cargo.toml @@ -15,7 +15,6 @@ license.workspace = true rustpython-compiler-core = { workspace = true } rustpython-compiler-source = {workspace = true } rustpython-wtf8 = { workspace = true } -ruff_python_parser = { workspace = true } ruff_python_ast = { workspace = true } ruff_text_size = { workspace = true } ruff_source_file = { workspace = true } diff --git a/compiler/codegen/src/compile.rs b/compiler/codegen/src/compile.rs index 6140d354ab..03eff74ba9 100644 --- a/compiler/codegen/src/compile.rs +++ b/compiler/codegen/src/compile.rs @@ -3458,12 +3458,13 @@ impl Compiler<'_> { self.compile_expression(&fstring_expr.expression)?; - emit!( - self, - Instruction::FormatValue { - conversion: conversion - } - ); + let conversion = match conversion { + ConversionFlag::None => bytecode::ConversionFlag::None, + ConversionFlag::Str => bytecode::ConversionFlag::Str, + ConversionFlag::Ascii => bytecode::ConversionFlag::Ascii, + ConversionFlag::Repr => bytecode::ConversionFlag::Repr, + }; + emit!(self, Instruction::FormatValue { conversion }); // concatenate formatted string and debug text (if present) if debug_text_count > 0 { diff --git a/compiler/core/Cargo.toml b/compiler/core/Cargo.toml index e79d937683..837f9e4866 100644 --- a/compiler/core/Cargo.toml +++ b/compiler/core/Cargo.toml @@ -10,8 +10,6 @@ license.workspace = true [dependencies] # rustpython-parser-core = { workspace = true, features=["location"] } -ruff_python_ast = { workspace = true } -ruff_python_parser = { workspace = true } ruff_source_file = { workspace = true } rustpython-wtf8 = { workspace = true } diff --git a/compiler/core/src/bytecode.rs b/compiler/core/src/bytecode.rs index 2745a5e9f7..2e8ff29014 100644 --- a/compiler/core/src/bytecode.rs +++ b/compiler/core/src/bytecode.rs @@ -5,14 +5,24 @@ use bitflags::bitflags; use itertools::Itertools; use malachite_bigint::BigInt; use num_complex::Complex64; -pub use ruff_python_ast::ConversionFlag; -// use rustpython_parser_core::source_code::{OneIndexed, SourceLocation}; use ruff_source_file::{OneIndexed, SourceLocation}; use rustpython_wtf8::{Wtf8, Wtf8Buf}; use std::marker::PhantomData; use std::{collections::BTreeSet, fmt, hash, mem}; -// pub use rustpython_parser_core::ConversionFlag; +#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq)] +#[repr(i8)] +#[allow(clippy::cast_possible_wrap)] +pub enum ConversionFlag { + /// No conversion + None = -1, // CPython uses -1 + /// Converts by calling `str()`. + Str = b's' as i8, + /// Converts by calling `ascii()`. + Ascii = b'a' as i8, + /// Converts by calling `repr()`. + Repr = b'r' as i8, +} pub trait Constant: Sized { type Name: AsRef; diff --git a/compiler/core/src/mode.rs b/compiler/core/src/mode.rs index 13cea42b13..0c14c3c495 100644 --- a/compiler/core/src/mode.rs +++ b/compiler/core/src/mode.rs @@ -1,10 +1,9 @@ -pub use ruff_python_parser::ModeParseError; - #[derive(Clone, Copy)] pub enum Mode { Exec, Eval, Single, + /// Returns the value of the last statement in the statement list. BlockExpr, } @@ -22,14 +21,12 @@ impl std::str::FromStr for Mode { } } -impl From for ruff_python_parser::Mode { - fn from(mode: Mode) -> Self { - match mode { - Mode::Exec => Self::Module, - Mode::Eval => Self::Expression, - // TODO: Improve ruff API - // ruff does not have an interactive mode - Mode::Single | Mode::BlockExpr => Self::Ipython, - } +/// Returned when a given mode is not valid. +#[derive(Debug)] +pub struct ModeParseError; + +impl std::fmt::Display for ModeParseError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, r#"mode must be "exec", "eval", or "single""#) } } diff --git a/compiler/src/lib.rs b/compiler/src/lib.rs index 0ebfbbf416..390a2d5669 100644 --- a/compiler/src/lib.rs +++ b/compiler/src/lib.rs @@ -119,7 +119,14 @@ fn _compile( mode: Mode, opts: CompileOpts, ) -> Result { - let parsed = parser::parse(source_code.text, parser::Mode::from(mode).into()) + let parser_mode = match mode { + Mode::Exec => parser::Mode::Module, + Mode::Eval => parser::Mode::Expression, + // ruff does not have an interactive mode, which is fine, + // since these are only different in terms of compilation + Mode::Single | Mode::BlockExpr => parser::Mode::Module, + }; + let parsed = parser::parse(source_code.text, parser_mode.into()) .map_err(|err| CompileError::from_ruff_parse_error(err, &source_code))?; let ast = parsed.into_syntax(); compile::compile_top(ast, source_code, mode, opts).map_err(|e| e.into()) diff --git a/vm/src/stdlib/ast/other.rs b/vm/src/stdlib/ast/other.rs index 2b9d292c1c..f7d6981332 100644 --- a/vm/src/stdlib/ast/other.rs +++ b/vm/src/stdlib/ast/other.rs @@ -1,5 +1,6 @@ use super::*; use num_traits::ToPrimitive; +use rustpython_compiler_core::bytecode; impl Node for ruff::ConversionFlag { fn ast_to_object(self, vm: &VirtualMachine, _source_code: &SourceCodeOwned) -> PyObjectRef { @@ -13,7 +14,13 @@ impl Node for ruff::ConversionFlag { ) -> PyResult { i32::try_from_object(vm, object)? .to_u32() - .and_then(ruff::ConversionFlag::from_op_arg) + .and_then(bytecode::ConversionFlag::from_op_arg) + .map(|flag| match flag { + bytecode::ConversionFlag::None => Self::None, + bytecode::ConversionFlag::Str => Self::Str, + bytecode::ConversionFlag::Ascii => Self::Ascii, + bytecode::ConversionFlag::Repr => Self::Repr, + }) .ok_or_else(|| vm.new_value_error("invalid conversion flag".to_owned())) } } From 8081e0d281429846799bc42ace8de285202baf51 Mon Sep 17 00:00:00 2001 From: Noa Date: Thu, 27 Mar 2025 17:34:09 -0500 Subject: [PATCH 140/295] Copy unparse.rs from rustpython-parser --- compiler/codegen/src/unparse.rs | 633 ++++++++++++++++++++++++++++++++ 1 file changed, 633 insertions(+) create mode 100644 compiler/codegen/src/unparse.rs diff --git a/compiler/codegen/src/unparse.rs b/compiler/codegen/src/unparse.rs new file mode 100644 index 0000000000..ae4a2d1616 --- /dev/null +++ b/compiler/codegen/src/unparse.rs @@ -0,0 +1,633 @@ +use crate::{ + Arg, ArgWithDefault, Arguments, BoolOp, Comprehension, Constant, ConversionFlag, Expr, + Identifier, Operator, PythonArguments, +}; +use std::fmt; + +mod precedence { + macro_rules! precedence { + ($($op:ident,)*) => { + precedence!(@0, $($op,)*); + }; + (@$i:expr, $op1:ident, $($op:ident,)*) => { + pub const $op1: u8 = $i; + precedence!(@$i + 1, $($op,)*); + }; + (@$i:expr,) => {}; + } + precedence!( + TUPLE, TEST, OR, AND, NOT, CMP, // "EXPR" = + BOR, BXOR, BAND, SHIFT, ARITH, TERM, FACTOR, POWER, AWAIT, ATOM, + ); + pub const EXPR: u8 = BOR; +} + +#[repr(transparent)] +struct Unparser<'a> { + f: fmt::Formatter<'a>, +} +impl<'a> Unparser<'a> { + fn new<'b>(f: &'b mut fmt::Formatter<'a>) -> &'b mut Unparser<'a> { + unsafe { &mut *(f as *mut fmt::Formatter<'a> as *mut Unparser<'a>) } + } + + fn p(&mut self, s: &str) -> fmt::Result { + self.f.write_str(s) + } + fn p_id(&mut self, s: &Identifier) -> fmt::Result { + self.f.write_str(s.as_str()) + } + fn p_if(&mut self, cond: bool, s: &str) -> fmt::Result { + if cond { + self.f.write_str(s)?; + } + Ok(()) + } + fn p_delim(&mut self, first: &mut bool, s: &str) -> fmt::Result { + self.p_if(!std::mem::take(first), s) + } + fn write_fmt(&mut self, f: fmt::Arguments<'_>) -> fmt::Result { + self.f.write_fmt(f) + } + + fn unparse_expr(&mut self, ast: &Expr, level: u8) -> fmt::Result { + macro_rules! op_prec { + ($op_ty:ident, $x:expr, $enu:path, $($var:ident($op:literal, $prec:ident)),*$(,)?) => { + match $x { + $(<$enu>::$var => (op_prec!(@space $op_ty, $op), precedence::$prec),)* + } + }; + (@space bin, $op:literal) => { + concat!(" ", $op, " ") + }; + (@space un, $op:literal) => { + $op + }; + } + macro_rules! group_if { + ($lvl:expr, $body:block) => {{ + let group = level > $lvl; + self.p_if(group, "(")?; + let ret = $body; + self.p_if(group, ")")?; + ret + }}; + } + match &ast { + Expr::BoolOp(crate::ExprBoolOp { + op, + values, + range: _range, + }) => { + let (op, prec) = op_prec!(bin, op, BoolOp, And("and", AND), Or("or", OR)); + group_if!(prec, { + let mut first = true; + for val in values { + self.p_delim(&mut first, op)?; + self.unparse_expr(val, prec + 1)?; + } + }) + } + Expr::NamedExpr(crate::ExprNamedExpr { + target, + value, + range: _range, + }) => { + group_if!(precedence::TUPLE, { + self.unparse_expr(target, precedence::ATOM)?; + self.p(" := ")?; + self.unparse_expr(value, precedence::ATOM)?; + }) + } + Expr::BinOp(crate::ExprBinOp { + left, + op, + right, + range: _range, + }) => { + let right_associative = matches!(op, Operator::Pow); + let (op, prec) = op_prec!( + bin, + op, + Operator, + Add("+", ARITH), + Sub("-", ARITH), + Mult("*", TERM), + MatMult("@", TERM), + Div("/", TERM), + Mod("%", TERM), + Pow("**", POWER), + LShift("<<", SHIFT), + RShift(">>", SHIFT), + BitOr("|", BOR), + BitXor("^", BXOR), + BitAnd("&", BAND), + FloorDiv("//", TERM), + ); + group_if!(prec, { + self.unparse_expr(left, prec + right_associative as u8)?; + self.p(op)?; + self.unparse_expr(right, prec + !right_associative as u8)?; + }) + } + Expr::UnaryOp(crate::ExprUnaryOp { + op, + operand, + range: _range, + }) => { + let (op, prec) = op_prec!( + un, + op, + crate::UnaryOp, + Invert("~", FACTOR), + Not("not ", NOT), + UAdd("+", FACTOR), + USub("-", FACTOR) + ); + group_if!(prec, { + self.p(op)?; + self.unparse_expr(operand, prec)?; + }) + } + Expr::Lambda(crate::ExprLambda { + args, + body, + range: _range, + }) => { + group_if!(precedence::TEST, { + let pos = args.args.len() + args.posonlyargs.len(); + self.p(if pos > 0 { "lambda " } else { "lambda" })?; + self.unparse_arguments(args)?; + write!(self, ": {}", **body)?; + }) + } + Expr::IfExp(crate::ExprIfExp { + test, + body, + orelse, + range: _range, + }) => { + group_if!(precedence::TEST, { + self.unparse_expr(body, precedence::TEST + 1)?; + self.p(" if ")?; + self.unparse_expr(test, precedence::TEST + 1)?; + self.p(" else ")?; + self.unparse_expr(orelse, precedence::TEST)?; + }) + } + Expr::Dict(crate::ExprDict { + keys, + values, + range: _range, + }) => { + self.p("{")?; + let mut first = true; + let (packed, unpacked) = values.split_at(keys.len()); + for (k, v) in keys.iter().zip(packed) { + self.p_delim(&mut first, ", ")?; + if let Some(k) = k { + write!(self, "{}: {}", *k, *v)?; + } else { + write!(self, "**{}", *v)?; + } + } + for d in unpacked { + self.p_delim(&mut first, ", ")?; + write!(self, "**{}", *d)?; + } + self.p("}")?; + } + Expr::Set(crate::ExprSet { + elts, + range: _range, + }) => { + self.p("{")?; + let mut first = true; + for v in elts { + self.p_delim(&mut first, ", ")?; + self.unparse_expr(v, precedence::TEST)?; + } + self.p("}")?; + } + Expr::ListComp(crate::ExprListComp { + elt, + generators, + range: _range, + }) => { + self.p("[")?; + self.unparse_expr(elt, precedence::TEST)?; + self.unparse_comp(generators)?; + self.p("]")?; + } + Expr::SetComp(crate::ExprSetComp { + elt, + generators, + range: _range, + }) => { + self.p("{")?; + self.unparse_expr(elt, precedence::TEST)?; + self.unparse_comp(generators)?; + self.p("}")?; + } + Expr::DictComp(crate::ExprDictComp { + key, + value, + generators, + range: _range, + }) => { + self.p("{")?; + self.unparse_expr(key, precedence::TEST)?; + self.p(": ")?; + self.unparse_expr(value, precedence::TEST)?; + self.unparse_comp(generators)?; + self.p("}")?; + } + Expr::GeneratorExp(crate::ExprGeneratorExp { + elt, + generators, + range: _range, + }) => { + self.p("(")?; + self.unparse_expr(elt, precedence::TEST)?; + self.unparse_comp(generators)?; + self.p(")")?; + } + Expr::Await(crate::ExprAwait { + value, + range: _range, + }) => { + group_if!(precedence::AWAIT, { + self.p("await ")?; + self.unparse_expr(value, precedence::ATOM)?; + }) + } + Expr::Yield(crate::ExprYield { + value, + range: _range, + }) => { + if let Some(value) = value { + write!(self, "(yield {})", **value)?; + } else { + self.p("(yield)")?; + } + } + Expr::YieldFrom(crate::ExprYieldFrom { + value, + range: _range, + }) => { + write!(self, "(yield from {})", **value)?; + } + Expr::Compare(crate::ExprCompare { + left, + ops, + comparators, + range: _range, + }) => { + group_if!(precedence::CMP, { + let new_lvl = precedence::CMP + 1; + self.unparse_expr(left, new_lvl)?; + for (op, cmp) in ops.iter().zip(comparators) { + self.p(" ")?; + self.p(op.as_str())?; + self.p(" ")?; + self.unparse_expr(cmp, new_lvl)?; + } + }) + } + Expr::Call(crate::ExprCall { + func, + args, + keywords, + range: _range, + }) => { + self.unparse_expr(func, precedence::ATOM)?; + self.p("(")?; + if let ( + [Expr::GeneratorExp(crate::ExprGeneratorExp { + elt, + generators, + range: _range, + })], + [], + ) = (&**args, &**keywords) + { + // make sure a single genexpr doesn't get double parens + self.unparse_expr(elt, precedence::TEST)?; + self.unparse_comp(generators)?; + } else { + let mut first = true; + for arg in args { + self.p_delim(&mut first, ", ")?; + self.unparse_expr(arg, precedence::TEST)?; + } + for kw in keywords { + self.p_delim(&mut first, ", ")?; + if let Some(arg) = &kw.arg { + self.p_id(arg)?; + self.p("=")?; + } else { + self.p("**")?; + } + self.unparse_expr(&kw.value, precedence::TEST)?; + } + } + self.p(")")?; + } + Expr::FormattedValue(crate::ExprFormattedValue { + value, + conversion, + format_spec, + range: _range, + }) => self.unparse_formatted(value, *conversion, format_spec.as_deref())?, + Expr::JoinedStr(crate::ExprJoinedStr { + values, + range: _range, + }) => self.unparse_joined_str(values, false)?, + Expr::Constant(crate::ExprConstant { + value, + kind, + range: _range, + }) => { + if let Some(kind) = kind { + self.p(kind)?; + } + assert_eq!(f64::MAX_10_EXP, 308); + let inf_str = "1e309"; + match value { + Constant::Float(f) if f.is_infinite() => self.p(inf_str)?, + Constant::Complex { real, imag } + if real.is_infinite() || imag.is_infinite() => + { + self.p(&value.to_string().replace("inf", inf_str))? + } + _ => fmt::Display::fmt(value, &mut self.f)?, + } + } + Expr::Attribute(crate::ExprAttribute { value, attr, .. }) => { + self.unparse_expr(value, precedence::ATOM)?; + let period = if let Expr::Constant(crate::ExprConstant { + value: Constant::Int(_), + .. + }) = value.as_ref() + { + " ." + } else { + "." + }; + self.p(period)?; + self.p_id(attr)?; + } + Expr::Subscript(crate::ExprSubscript { value, slice, .. }) => { + self.unparse_expr(value, precedence::ATOM)?; + let lvl = precedence::TUPLE; + self.p("[")?; + self.unparse_expr(slice, lvl)?; + self.p("]")?; + } + Expr::Starred(crate::ExprStarred { value, .. }) => { + self.p("*")?; + self.unparse_expr(value, precedence::EXPR)?; + } + Expr::Name(crate::ExprName { id, .. }) => self.p_id(id)?, + Expr::List(crate::ExprList { elts, .. }) => { + self.p("[")?; + let mut first = true; + for elt in elts { + self.p_delim(&mut first, ", ")?; + self.unparse_expr(elt, precedence::TEST)?; + } + self.p("]")?; + } + Expr::Tuple(crate::ExprTuple { elts, .. }) => { + if elts.is_empty() { + self.p("()")?; + } else { + group_if!(precedence::TUPLE, { + let mut first = true; + for elt in elts { + self.p_delim(&mut first, ", ")?; + self.unparse_expr(elt, precedence::TEST)?; + } + self.p_if(elts.len() == 1, ",")?; + }) + } + } + Expr::Slice(crate::ExprSlice { + lower, + upper, + step, + range: _range, + }) => { + if let Some(lower) = lower { + self.unparse_expr(lower, precedence::TEST)?; + } + self.p(":")?; + if let Some(upper) = upper { + self.unparse_expr(upper, precedence::TEST)?; + } + if let Some(step) = step { + self.p(":")?; + self.unparse_expr(step, precedence::TEST)?; + } + } + } + Ok(()) + } + + fn unparse_arguments(&mut self, args: &Arguments) -> fmt::Result { + let mut first = true; + for (i, arg) in args.posonlyargs.iter().chain(&args.args).enumerate() { + self.p_delim(&mut first, ", ")?; + self.unparse_function_arg(arg)?; + self.p_if(i + 1 == args.posonlyargs.len(), ", /")?; + } + if args.vararg.is_some() || !args.kwonlyargs.is_empty() { + self.p_delim(&mut first, ", ")?; + self.p("*")?; + } + if let Some(vararg) = &args.vararg { + self.unparse_arg(vararg)?; + } + for kwarg in args.kwonlyargs.iter() { + self.p_delim(&mut first, ", ")?; + self.unparse_function_arg(kwarg)?; + } + if let Some(kwarg) = &args.kwarg { + self.p_delim(&mut first, ", ")?; + self.p("**")?; + self.unparse_arg(kwarg)?; + } + Ok(()) + } + fn unparse_function_arg(&mut self, arg: &ArgWithDefault) -> fmt::Result { + self.p_id(&arg.def.arg)?; + if let Some(ann) = &arg.def.annotation { + write!(self, ": {}", **ann)?; + } + if let Some(default) = &arg.default { + write!(self, "={}", default)?; + } + Ok(()) + } + + #[allow(dead_code)] + fn unparse_python_arguments(&mut self, args: &PythonArguments) -> fmt::Result { + let mut first = true; + let defaults_start = args.posonlyargs.len() + args.args.len() - args.defaults.len(); + for (i, arg) in args.posonlyargs.iter().chain(&args.args).enumerate() { + self.p_delim(&mut first, ", ")?; + self.unparse_arg(arg)?; + if let Some(i) = i.checked_sub(defaults_start) { + write!(self, "={}", &args.defaults[i])?; + } + self.p_if(i + 1 == args.posonlyargs.len(), ", /")?; + } + if args.vararg.is_some() || !args.kwonlyargs.is_empty() { + self.p_delim(&mut first, ", ")?; + self.p("*")?; + } + if let Some(vararg) = &args.vararg { + self.unparse_arg(vararg)?; + } + let defaults_start = args.kwonlyargs.len() - args.kw_defaults.len(); + for (i, kwarg) in args.kwonlyargs.iter().enumerate() { + self.p_delim(&mut first, ", ")?; + self.unparse_arg(kwarg)?; + if let Some(default) = i + .checked_sub(defaults_start) + .and_then(|i| args.kw_defaults.get(i)) + { + write!(self, "={default}")?; + } + } + if let Some(kwarg) = &args.kwarg { + self.p_delim(&mut first, ", ")?; + self.p("**")?; + self.unparse_arg(kwarg)?; + } + Ok(()) + } + fn unparse_arg(&mut self, arg: &Arg) -> fmt::Result { + self.p_id(&arg.arg)?; + if let Some(ann) = &arg.annotation { + write!(self, ": {}", **ann)?; + } + Ok(()) + } + + fn unparse_comp(&mut self, generators: &[Comprehension]) -> fmt::Result { + for comp in generators { + self.p(if comp.is_async { + " async for " + } else { + " for " + })?; + self.unparse_expr(&comp.target, precedence::TUPLE)?; + self.p(" in ")?; + self.unparse_expr(&comp.iter, precedence::TEST + 1)?; + for cond in &comp.ifs { + self.p(" if ")?; + self.unparse_expr(cond, precedence::TEST + 1)?; + } + } + Ok(()) + } + + fn unparse_fstring_body(&mut self, values: &[Expr], is_spec: bool) -> fmt::Result { + for value in values { + self.unparse_fstring_elem(value, is_spec)?; + } + Ok(()) + } + + fn unparse_formatted( + &mut self, + val: &Expr, + conversion: ConversionFlag, + spec: Option<&Expr>, + ) -> fmt::Result { + let buffered = to_string_fmt(|f| Unparser::new(f).unparse_expr(val, precedence::TEST + 1)); + let brace = if buffered.starts_with('{') { + // put a space to avoid escaping the bracket + "{ " + } else { + "{" + }; + self.p(brace)?; + self.p(&buffered)?; + drop(buffered); + + if conversion != ConversionFlag::None { + self.p("!")?; + let buf = &[conversion as u8]; + let c = std::str::from_utf8(buf).unwrap(); + self.p(c)?; + } + + if let Some(spec) = spec { + self.p(":")?; + self.unparse_fstring_elem(spec, true)?; + } + + self.p("}")?; + + Ok(()) + } + + fn unparse_fstring_elem(&mut self, expr: &Expr, is_spec: bool) -> fmt::Result { + match &expr { + Expr::Constant(crate::ExprConstant { value, .. }) => { + if let Constant::Str(s) = value { + self.unparse_fstring_str(s) + } else { + unreachable!() + } + } + Expr::JoinedStr(crate::ExprJoinedStr { + values, + range: _range, + }) => self.unparse_joined_str(values, is_spec), + Expr::FormattedValue(crate::ExprFormattedValue { + value, + conversion, + format_spec, + range: _range, + }) => self.unparse_formatted(value, *conversion, format_spec.as_deref()), + _ => unreachable!(), + } + } + + fn unparse_fstring_str(&mut self, s: &str) -> fmt::Result { + let s = s.replace('{', "{{").replace('}', "}}"); + self.p(&s) + } + + fn unparse_joined_str(&mut self, values: &[Expr], is_spec: bool) -> fmt::Result { + if is_spec { + self.unparse_fstring_body(values, is_spec) + } else { + self.p("f")?; + let body = to_string_fmt(|f| Unparser::new(f).unparse_fstring_body(values, is_spec)); + rustpython_literal::escape::UnicodeEscape::new_repr(&body) + .str_repr() + .write(&mut self.f) + } + } +} + +impl fmt::Display for Expr { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + Unparser::new(f).unparse_expr(self, precedence::TEST) + } +} + +fn to_string_fmt(f: impl FnOnce(&mut fmt::Formatter) -> fmt::Result) -> String { + use std::cell::Cell; + struct Fmt(Cell>); + impl fmt::Result> fmt::Display for Fmt { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.take().unwrap()(f) + } + } + Fmt(Cell::new(Some(f))).to_string() +} From d3d92bbb6f7b16930e233e7c216d259585060f03 Mon Sep 17 00:00:00 2001 From: Noa Date: Thu, 27 Mar 2025 17:33:21 -0500 Subject: [PATCH 141/295] Update unparse to work with ruff & remove ruff_python_codegen --- Cargo.lock | 25 +- Cargo.toml | 1 - Lib/test/test_future_stmt/test_future.py | 6 - compiler/codegen/Cargo.toml | 2 +- compiler/codegen/src/compile.rs | 54 ++-- compiler/codegen/src/lib.rs | 1 + compiler/codegen/src/unparse.rs | 341 +++++++++++------------ compiler/literal/src/float.rs | 33 +++ vm/src/builtins/complex.rs | 32 +-- 9 files changed, 219 insertions(+), 276 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 55872f1d68..5de6ad43ae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2185,29 +2185,6 @@ dependencies = [ "rustc-hash 2.1.1", ] -[[package]] -name = "ruff_python_codegen" -version = "0.0.0" -source = "git+https://github.com/astral-sh/ruff.git?tag=0.11.0#2cd25ef6410fb5fca96af1578728a3d828d2d53a" -dependencies = [ - "ruff_python_ast", - "ruff_python_literal", - "ruff_python_parser", - "ruff_source_file", - "ruff_text_size", -] - -[[package]] -name = "ruff_python_literal" -version = "0.0.0" -source = "git+https://github.com/astral-sh/ruff.git?tag=0.11.0#2cd25ef6410fb5fca96af1578728a3d828d2d53a" -dependencies = [ - "bitflags 2.8.0", - "itertools 0.14.0", - "ruff_python_ast", - "unic-ucd-category", -] - [[package]] name = "ruff_python_parser" version = "0.0.0" @@ -2323,11 +2300,11 @@ dependencies = [ "num-complex", "num-traits", "ruff_python_ast", - "ruff_python_codegen", "ruff_source_file", "ruff_text_size", "rustpython-compiler-core", "rustpython-compiler-source", + "rustpython-literal", "rustpython-wtf8", "thiserror 2.0.11", "unicode_names2", diff --git a/Cargo.toml b/Cargo.toml index c9f9c96525..ccfec120f2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -148,7 +148,6 @@ ruff_python_parser = { git = "https://github.com/astral-sh/ruff.git", tag = "0.1 ruff_python_ast = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" } ruff_text_size = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" } ruff_source_file = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" } -ruff_python_codegen = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" } ahash = "0.8.11" ascii = "1.1" diff --git a/Lib/test/test_future_stmt/test_future.py b/Lib/test/test_future_stmt/test_future.py index ad9b5d5a87..0e08051038 100644 --- a/Lib/test/test_future_stmt/test_future.py +++ b/Lib/test/test_future_stmt/test_future.py @@ -198,8 +198,6 @@ def _exec_future(self, code): ) return scope - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_annotations(self): eq = self.assertAnnotationEqual eq('...') @@ -364,8 +362,6 @@ def test_annotations(self): eq('(((a, b)))', '(a, b)') eq("1 + 2 + 3") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_fstring_debug_annotations(self): # f-strings with '=' don't round trip very well, so set the expected # result explicitly. @@ -376,8 +372,6 @@ def test_fstring_debug_annotations(self): self.assertAnnotationEqual("f'{x=!a}'", expected="f'x={x!a}'") self.assertAnnotationEqual("f'{x=!s:*^20}'", expected="f'x={x!s:*^20}'") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_infinity_numbers(self): inf = "1e" + repr(sys.float_info.max_10_exp + 1) infj = f"{inf}j" diff --git a/compiler/codegen/Cargo.toml b/compiler/codegen/Cargo.toml index f54d6476af..53469b9f6e 100644 --- a/compiler/codegen/Cargo.toml +++ b/compiler/codegen/Cargo.toml @@ -14,11 +14,11 @@ license.workspace = true # rustpython-parser-core = { workspace = true } rustpython-compiler-core = { workspace = true } rustpython-compiler-source = {workspace = true } +rustpython-literal = {workspace = true } rustpython-wtf8 = { workspace = true } ruff_python_ast = { workspace = true } ruff_text_size = { workspace = true } ruff_source_file = { workspace = true } -ruff_python_codegen = { workspace = true } ahash = { workspace = true } bitflags = { workspace = true } diff --git a/compiler/codegen/src/compile.rs b/compiler/codegen/src/compile.rs index 03eff74ba9..38d5b8fb12 100644 --- a/compiler/codegen/src/compile.rs +++ b/compiler/codegen/src/compile.rs @@ -12,6 +12,7 @@ use crate::{ error::{CodegenError, CodegenErrorType}, ir, symboltable::{self, SymbolFlags, SymbolScope, SymbolTable}, + unparse::unparse_expr, }; use itertools::Itertools; use malachite_bigint::BigInt; @@ -2026,11 +2027,10 @@ impl Compiler<'_> { fn compile_annotation(&mut self, annotation: &Expr) -> CompileResult<()> { if self.future_annotations { - // FIXME: codegen? - let ident = Default::default(); - let codegen = ruff_python_codegen::Generator::new(&ident, Default::default()); self.emit_load_const(ConstantData::Str { - value: codegen.expr(annotation).into(), + value: unparse_expr(annotation, &self.source_code) + .to_string() + .into(), }); } else { self.compile_expression(annotation)?; @@ -3397,7 +3397,9 @@ impl Compiler<'_> { flags: FStringFlags, fstring_elements: &FStringElements, ) -> CompileResult<()> { + let mut element_count = 0; for element in fstring_elements { + element_count += 1; match element { FStringElement::Literal(string) => { if string.value.contains(char::REPLACEMENT_CHARACTER) { @@ -3419,26 +3421,14 @@ impl Compiler<'_> { FStringElement::Expression(fstring_expr) => { let mut conversion = fstring_expr.conversion; - let debug_text_count = match &fstring_expr.debug_text { - None => 0, - Some(DebugText { leading, trailing }) => { - let range = fstring_expr.expression.range(); - let source = self.source_code.get_range(range); - let source = source.to_string(); + if let Some(DebugText { leading, trailing }) = &fstring_expr.debug_text { + let range = fstring_expr.expression.range(); + let source = self.source_code.get_range(range); + let text = [leading, source, trailing].concat(); - self.emit_load_const(ConstantData::Str { - value: leading.to_string().into(), - }); - self.emit_load_const(ConstantData::Str { - value: source.into(), - }); - self.emit_load_const(ConstantData::Str { - value: trailing.to_string().into(), - }); - - 3 - } - }; + self.emit_load_const(ConstantData::Str { value: text.into() }); + element_count += 1; + } match &fstring_expr.format_spec { None => { @@ -3447,7 +3437,9 @@ impl Compiler<'_> { }); // Match CPython behavior: If debug text is present, apply repr conversion. // See: https://github.com/python/cpython/blob/f61afca262d3a0aa6a8a501db0b1936c60858e35/Parser/action_helpers.c#L1456 - if conversion == ConversionFlag::None && debug_text_count > 0 { + if conversion == ConversionFlag::None + && fstring_expr.debug_text.is_some() + { conversion = ConversionFlag::Repr; } } @@ -3465,24 +3457,10 @@ impl Compiler<'_> { ConversionFlag::Repr => bytecode::ConversionFlag::Repr, }; emit!(self, Instruction::FormatValue { conversion }); - - // concatenate formatted string and debug text (if present) - if debug_text_count > 0 { - emit!( - self, - Instruction::BuildString { - size: debug_text_count + 1 - } - ); - } } } } - let element_count: u32 = fstring_elements - .len() - .try_into() - .expect("BuildString size overflowed"); if element_count == 0 { // ensure to put an empty string on the stack if there aren't any fstring elements self.emit_load_const(ConstantData::Str { diff --git a/compiler/codegen/src/lib.rs b/compiler/codegen/src/lib.rs index d44844543e..90e11c5b84 100644 --- a/compiler/codegen/src/lib.rs +++ b/compiler/codegen/src/lib.rs @@ -13,6 +13,7 @@ pub mod error; pub mod ir; mod string_parser; pub mod symboltable; +mod unparse; pub use compile::CompileOpts; use ruff_python_ast::Expr; diff --git a/compiler/codegen/src/unparse.rs b/compiler/codegen/src/unparse.rs index ae4a2d1616..458ff76fc7 100644 --- a/compiler/codegen/src/unparse.rs +++ b/compiler/codegen/src/unparse.rs @@ -1,8 +1,13 @@ -use crate::{ - Arg, ArgWithDefault, Arguments, BoolOp, Comprehension, Constant, ConversionFlag, Expr, - Identifier, Operator, PythonArguments, +use ruff_python_ast as ruff; +use ruff_text_size::Ranged; +use rustpython_compiler_source::SourceCode; +use rustpython_literal::escape::{AsciiEscape, UnicodeEscape}; +use std::fmt::{self, Display as _}; + +use ruff::{ + Arguments, BoolOp, Comprehension, ConversionFlag, Expr, Identifier, Operator, Parameter, + ParameterWithDefault, Parameters, }; -use std::fmt; mod precedence { macro_rules! precedence { @@ -22,13 +27,13 @@ mod precedence { pub const EXPR: u8 = BOR; } -#[repr(transparent)] -struct Unparser<'a> { - f: fmt::Formatter<'a>, +struct Unparser<'a, 'b, 'c> { + f: &'b mut fmt::Formatter<'a>, + source: &'c SourceCode<'c>, } -impl<'a> Unparser<'a> { - fn new<'b>(f: &'b mut fmt::Formatter<'a>) -> &'b mut Unparser<'a> { - unsafe { &mut *(f as *mut fmt::Formatter<'a> as *mut Unparser<'a>) } +impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { + fn new(f: &'b mut fmt::Formatter<'a>, source: &'c SourceCode<'c>) -> Self { + Unparser { f, source } } fn p(&mut self, s: &str) -> fmt::Result { @@ -50,7 +55,7 @@ impl<'a> Unparser<'a> { self.f.write_fmt(f) } - fn unparse_expr(&mut self, ast: &Expr, level: u8) -> fmt::Result { + fn unparse_expr(&mut self, ast: &Expr, level: u8) -> fmt::Result { macro_rules! op_prec { ($op_ty:ident, $x:expr, $enu:path, $($var:ident($op:literal, $prec:ident)),*$(,)?) => { match $x { @@ -74,7 +79,7 @@ impl<'a> Unparser<'a> { }}; } match &ast { - Expr::BoolOp(crate::ExprBoolOp { + Expr::BoolOp(ruff::ExprBoolOp { op, values, range: _range, @@ -88,7 +93,7 @@ impl<'a> Unparser<'a> { } }) } - Expr::NamedExpr(crate::ExprNamedExpr { + Expr::Named(ruff::ExprNamed { target, value, range: _range, @@ -99,7 +104,7 @@ impl<'a> Unparser<'a> { self.unparse_expr(value, precedence::ATOM)?; }) } - Expr::BinOp(crate::ExprBinOp { + Expr::BinOp(ruff::ExprBinOp { left, op, right, @@ -130,7 +135,7 @@ impl<'a> Unparser<'a> { self.unparse_expr(right, prec + !right_associative as u8)?; }) } - Expr::UnaryOp(crate::ExprUnaryOp { + Expr::UnaryOp(ruff::ExprUnaryOp { op, operand, range: _range, @@ -138,7 +143,7 @@ impl<'a> Unparser<'a> { let (op, prec) = op_prec!( un, op, - crate::UnaryOp, + ruff::UnaryOp, Invert("~", FACTOR), Not("not ", NOT), UAdd("+", FACTOR), @@ -149,19 +154,22 @@ impl<'a> Unparser<'a> { self.unparse_expr(operand, prec)?; }) } - Expr::Lambda(crate::ExprLambda { - args, + Expr::Lambda(ruff::ExprLambda { + parameters, body, range: _range, }) => { group_if!(precedence::TEST, { - let pos = args.args.len() + args.posonlyargs.len(); - self.p(if pos > 0 { "lambda " } else { "lambda" })?; - self.unparse_arguments(args)?; - write!(self, ": {}", **body)?; + if let Some(parameters) = parameters { + self.p("lambda ")?; + self.unparse_arguments(parameters)?; + } else { + self.p("lambda")?; + } + write!(self, ": {}", unparse_expr(body, self.source))?; }) } - Expr::IfExp(crate::ExprIfExp { + Expr::If(ruff::ExprIf { test, body, orelse, @@ -175,29 +183,24 @@ impl<'a> Unparser<'a> { self.unparse_expr(orelse, precedence::TEST)?; }) } - Expr::Dict(crate::ExprDict { - keys, - values, + Expr::Dict(ruff::ExprDict { + items, range: _range, }) => { self.p("{")?; let mut first = true; - let (packed, unpacked) = values.split_at(keys.len()); - for (k, v) in keys.iter().zip(packed) { + for item in items { self.p_delim(&mut first, ", ")?; - if let Some(k) = k { - write!(self, "{}: {}", *k, *v)?; + if let Some(k) = &item.key { + write!(self, "{}: ", unparse_expr(k, self.source))?; } else { - write!(self, "**{}", *v)?; + self.p("**")?; } - } - for d in unpacked { - self.p_delim(&mut first, ", ")?; - write!(self, "**{}", *d)?; + self.unparse_expr(&item.value, level)?; } self.p("}")?; } - Expr::Set(crate::ExprSet { + Expr::Set(ruff::ExprSet { elts, range: _range, }) => { @@ -209,7 +212,7 @@ impl<'a> Unparser<'a> { } self.p("}")?; } - Expr::ListComp(crate::ExprListComp { + Expr::ListComp(ruff::ExprListComp { elt, generators, range: _range, @@ -219,7 +222,7 @@ impl<'a> Unparser<'a> { self.unparse_comp(generators)?; self.p("]")?; } - Expr::SetComp(crate::ExprSetComp { + Expr::SetComp(ruff::ExprSetComp { elt, generators, range: _range, @@ -229,7 +232,7 @@ impl<'a> Unparser<'a> { self.unparse_comp(generators)?; self.p("}")?; } - Expr::DictComp(crate::ExprDictComp { + Expr::DictComp(ruff::ExprDictComp { key, value, generators, @@ -242,7 +245,8 @@ impl<'a> Unparser<'a> { self.unparse_comp(generators)?; self.p("}")?; } - Expr::GeneratorExp(crate::ExprGeneratorExp { + Expr::Generator(ruff::ExprGenerator { + parenthesized: _, elt, generators, range: _range, @@ -252,7 +256,7 @@ impl<'a> Unparser<'a> { self.unparse_comp(generators)?; self.p(")")?; } - Expr::Await(crate::ExprAwait { + Expr::Await(ruff::ExprAwait { value, range: _range, }) => { @@ -261,23 +265,23 @@ impl<'a> Unparser<'a> { self.unparse_expr(value, precedence::ATOM)?; }) } - Expr::Yield(crate::ExprYield { + Expr::Yield(ruff::ExprYield { value, range: _range, }) => { if let Some(value) = value { - write!(self, "(yield {})", **value)?; + write!(self, "(yield {})", unparse_expr(value, self.source))?; } else { self.p("(yield)")?; } } - Expr::YieldFrom(crate::ExprYieldFrom { + Expr::YieldFrom(ruff::ExprYieldFrom { value, range: _range, }) => { - write!(self, "(yield from {})", **value)?; + write!(self, "(yield from {})", unparse_expr(value, self.source))?; } - Expr::Compare(crate::ExprCompare { + Expr::Compare(ruff::ExprCompare { left, ops, comparators, @@ -294,20 +298,22 @@ impl<'a> Unparser<'a> { } }) } - Expr::Call(crate::ExprCall { + Expr::Call(ruff::ExprCall { func, - args, - keywords, + arguments: Arguments { args, keywords, .. }, range: _range, }) => { self.unparse_expr(func, precedence::ATOM)?; self.p("(")?; if let ( - [Expr::GeneratorExp(crate::ExprGeneratorExp { - elt, - generators, - range: _range, - })], + [ + Expr::Generator(ruff::ExprGenerator { + elt, + generators, + range: _range, + .. + }), + ], [], ) = (&**args, &**keywords) { @@ -333,40 +339,46 @@ impl<'a> Unparser<'a> { } self.p(")")?; } - Expr::FormattedValue(crate::ExprFormattedValue { - value, - conversion, - format_spec, - range: _range, - }) => self.unparse_formatted(value, *conversion, format_spec.as_deref())?, - Expr::JoinedStr(crate::ExprJoinedStr { - values, - range: _range, - }) => self.unparse_joined_str(values, false)?, - Expr::Constant(crate::ExprConstant { - value, - kind, - range: _range, - }) => { - if let Some(kind) = kind { - self.p(kind)?; + Expr::FString(ruff::ExprFString { value, .. }) => self.unparse_fstring(value)?, + Expr::StringLiteral(ruff::ExprStringLiteral { value, .. }) => { + if value.is_unicode() { + self.p("u")? } - assert_eq!(f64::MAX_10_EXP, 308); + UnicodeEscape::new_repr(value.to_str().as_ref()) + .str_repr() + .fmt(self.f)? + } + Expr::BytesLiteral(ruff::ExprBytesLiteral { value, .. }) => { + AsciiEscape::new_repr(&value.bytes().collect::>()) + .bytes_repr() + .fmt(self.f)? + } + Expr::NumberLiteral(ruff::ExprNumberLiteral { value, .. }) => { + const { assert!(f64::MAX_10_EXP == 308) }; let inf_str = "1e309"; match value { - Constant::Float(f) if f.is_infinite() => self.p(inf_str)?, - Constant::Complex { real, imag } - if real.is_infinite() || imag.is_infinite() => - { - self.p(&value.to_string().replace("inf", inf_str))? + ruff::Number::Int(int) => int.fmt(self.f)?, + &ruff::Number::Float(fp) => { + if fp.is_infinite() { + self.p(inf_str)? + } else { + self.p(&rustpython_literal::float::to_string(fp))? + } } - _ => fmt::Display::fmt(value, &mut self.f)?, + &ruff::Number::Complex { real, imag } => self + .p(&rustpython_literal::float::complex_to_string(real, imag) + .replace("inf", inf_str))?, } } - Expr::Attribute(crate::ExprAttribute { value, attr, .. }) => { + Expr::BooleanLiteral(ruff::ExprBooleanLiteral { value, .. }) => { + self.p(if *value { "True" } else { "False" })? + } + Expr::NoneLiteral(ruff::ExprNoneLiteral { .. }) => self.p("None")?, + Expr::EllipsisLiteral(ruff::ExprEllipsisLiteral { .. }) => self.p("...")?, + Expr::Attribute(ruff::ExprAttribute { value, attr, .. }) => { self.unparse_expr(value, precedence::ATOM)?; - let period = if let Expr::Constant(crate::ExprConstant { - value: Constant::Int(_), + let period = if let Expr::NumberLiteral(ruff::ExprNumberLiteral { + value: ruff::Number::Int(_), .. }) = value.as_ref() { @@ -377,19 +389,19 @@ impl<'a> Unparser<'a> { self.p(period)?; self.p_id(attr)?; } - Expr::Subscript(crate::ExprSubscript { value, slice, .. }) => { + Expr::Subscript(ruff::ExprSubscript { value, slice, .. }) => { self.unparse_expr(value, precedence::ATOM)?; let lvl = precedence::TUPLE; self.p("[")?; self.unparse_expr(slice, lvl)?; self.p("]")?; } - Expr::Starred(crate::ExprStarred { value, .. }) => { + Expr::Starred(ruff::ExprStarred { value, .. }) => { self.p("*")?; self.unparse_expr(value, precedence::EXPR)?; } - Expr::Name(crate::ExprName { id, .. }) => self.p_id(id)?, - Expr::List(crate::ExprList { elts, .. }) => { + Expr::Name(ruff::ExprName { id, .. }) => self.p(id.as_str())?, + Expr::List(ruff::ExprList { elts, .. }) => { self.p("[")?; let mut first = true; for elt in elts { @@ -398,7 +410,7 @@ impl<'a> Unparser<'a> { } self.p("]")?; } - Expr::Tuple(crate::ExprTuple { elts, .. }) => { + Expr::Tuple(ruff::ExprTuple { elts, .. }) => { if elts.is_empty() { self.p("()")?; } else { @@ -412,7 +424,7 @@ impl<'a> Unparser<'a> { }) } } - Expr::Slice(crate::ExprSlice { + Expr::Slice(ruff::ExprSlice { lower, upper, step, @@ -430,11 +442,12 @@ impl<'a> Unparser<'a> { self.unparse_expr(step, precedence::TEST)?; } } + Expr::IpyEscapeCommand(_) => {} } Ok(()) } - fn unparse_arguments(&mut self, args: &Arguments) -> fmt::Result { + fn unparse_arguments(&mut self, args: &Parameters) -> fmt::Result { let mut first = true; for (i, arg) in args.posonlyargs.iter().chain(&args.args).enumerate() { self.p_delim(&mut first, ", ")?; @@ -459,63 +472,23 @@ impl<'a> Unparser<'a> { } Ok(()) } - fn unparse_function_arg(&mut self, arg: &ArgWithDefault) -> fmt::Result { - self.p_id(&arg.def.arg)?; - if let Some(ann) = &arg.def.annotation { - write!(self, ": {}", **ann)?; - } + fn unparse_function_arg(&mut self, arg: &ParameterWithDefault) -> fmt::Result { + self.unparse_arg(&arg.parameter)?; if let Some(default) = &arg.default { - write!(self, "={}", default)?; + write!(self, "={}", unparse_expr(default, self.source))?; } Ok(()) } - #[allow(dead_code)] - fn unparse_python_arguments(&mut self, args: &PythonArguments) -> fmt::Result { - let mut first = true; - let defaults_start = args.posonlyargs.len() + args.args.len() - args.defaults.len(); - for (i, arg) in args.posonlyargs.iter().chain(&args.args).enumerate() { - self.p_delim(&mut first, ", ")?; - self.unparse_arg(arg)?; - if let Some(i) = i.checked_sub(defaults_start) { - write!(self, "={}", &args.defaults[i])?; - } - self.p_if(i + 1 == args.posonlyargs.len(), ", /")?; - } - if args.vararg.is_some() || !args.kwonlyargs.is_empty() { - self.p_delim(&mut first, ", ")?; - self.p("*")?; - } - if let Some(vararg) = &args.vararg { - self.unparse_arg(vararg)?; - } - let defaults_start = args.kwonlyargs.len() - args.kw_defaults.len(); - for (i, kwarg) in args.kwonlyargs.iter().enumerate() { - self.p_delim(&mut first, ", ")?; - self.unparse_arg(kwarg)?; - if let Some(default) = i - .checked_sub(defaults_start) - .and_then(|i| args.kw_defaults.get(i)) - { - write!(self, "={default}")?; - } - } - if let Some(kwarg) = &args.kwarg { - self.p_delim(&mut first, ", ")?; - self.p("**")?; - self.unparse_arg(kwarg)?; - } - Ok(()) - } - fn unparse_arg(&mut self, arg: &Arg) -> fmt::Result { - self.p_id(&arg.arg)?; + fn unparse_arg(&mut self, arg: &Parameter) -> fmt::Result { + self.p_id(&arg.name)?; if let Some(ann) = &arg.annotation { - write!(self, ": {}", **ann)?; + write!(self, ": {}", unparse_expr(ann, self.source))?; } Ok(()) } - fn unparse_comp(&mut self, generators: &[Comprehension]) -> fmt::Result { + fn unparse_comp(&mut self, generators: &[Comprehension]) -> fmt::Result { for comp in generators { self.p(if comp.is_async { " async for " @@ -533,20 +506,28 @@ impl<'a> Unparser<'a> { Ok(()) } - fn unparse_fstring_body(&mut self, values: &[Expr], is_spec: bool) -> fmt::Result { - for value in values { - self.unparse_fstring_elem(value, is_spec)?; + fn unparse_fstring_body(&mut self, elements: &[ruff::FStringElement]) -> fmt::Result { + for elem in elements { + self.unparse_fstring_elem(elem)?; } Ok(()) } - fn unparse_formatted( + fn unparse_formatted( &mut self, - val: &Expr, + val: &Expr, + debug_text: Option<&ruff::DebugText>, conversion: ConversionFlag, - spec: Option<&Expr>, + spec: Option<&ruff::FStringFormatSpec>, ) -> fmt::Result { - let buffered = to_string_fmt(|f| Unparser::new(f).unparse_expr(val, precedence::TEST + 1)); + let buffered = to_string_fmt(|f| { + Unparser::new(f, self.source).unparse_expr(val, precedence::TEST + 1) + }); + if let Some(ruff::DebugText { leading, trailing }) = debug_text { + self.p(leading)?; + self.p(self.source.get_range(val.range()))?; + self.p(trailing)?; + } let brace = if buffered.starts_with('{') { // put a space to avoid escaping the bracket "{ " @@ -566,7 +547,7 @@ impl<'a> Unparser<'a> { if let Some(spec) = spec { self.p(":")?; - self.unparse_fstring_elem(spec, true)?; + self.unparse_fstring_body(&spec.elements)?; } self.p("}")?; @@ -574,26 +555,23 @@ impl<'a> Unparser<'a> { Ok(()) } - fn unparse_fstring_elem(&mut self, expr: &Expr, is_spec: bool) -> fmt::Result { - match &expr { - Expr::Constant(crate::ExprConstant { value, .. }) => { - if let Constant::Str(s) = value { - self.unparse_fstring_str(s) - } else { - unreachable!() - } - } - Expr::JoinedStr(crate::ExprJoinedStr { - values, - range: _range, - }) => self.unparse_joined_str(values, is_spec), - Expr::FormattedValue(crate::ExprFormattedValue { - value, + fn unparse_fstring_elem(&mut self, elem: &ruff::FStringElement) -> fmt::Result { + match elem { + ruff::FStringElement::Expression(ruff::FStringExpressionElement { + expression, + debug_text, conversion, format_spec, - range: _range, - }) => self.unparse_formatted(value, *conversion, format_spec.as_deref()), - _ => unreachable!(), + .. + }) => self.unparse_formatted( + expression, + debug_text.as_ref(), + *conversion, + format_spec.as_deref(), + ), + ruff::FStringElement::Literal(ruff::FStringLiteralElement { value, .. }) => { + self.unparse_fstring_str(value) + } } } @@ -602,29 +580,42 @@ impl<'a> Unparser<'a> { self.p(&s) } - fn unparse_joined_str(&mut self, values: &[Expr], is_spec: bool) -> fmt::Result { - if is_spec { - self.unparse_fstring_body(values, is_spec) - } else { - self.p("f")?; - let body = to_string_fmt(|f| Unparser::new(f).unparse_fstring_body(values, is_spec)); - rustpython_literal::escape::UnicodeEscape::new_repr(&body) - .str_repr() - .write(&mut self.f) - } + fn unparse_fstring(&mut self, value: &ruff::FStringValue) -> fmt::Result { + self.p("f")?; + let body = to_string_fmt(|f| { + value.iter().try_for_each(|part| match part { + ruff::FStringPart::Literal(lit) => f.write_str(lit), + ruff::FStringPart::FString(ruff::FString { elements, .. }) => { + Unparser::new(f, self.source).unparse_fstring_body(elements) + } + }) + }); + // .unparse_fstring_body(elements)); + UnicodeEscape::new_repr(body.as_str().as_ref()) + .str_repr() + .write(self.f) } } -impl fmt::Display for Expr { +pub struct UnparseExpr<'a> { + expr: &'a Expr, + source: &'a SourceCode<'a>, +} + +pub fn unparse_expr<'a>(expr: &'a Expr, source: &'a SourceCode<'a>) -> UnparseExpr<'a> { + UnparseExpr { expr, source } +} + +impl fmt::Display for UnparseExpr<'_> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - Unparser::new(f).unparse_expr(self, precedence::TEST) + Unparser::new(f, self.source).unparse_expr(self.expr, precedence::TEST) } } -fn to_string_fmt(f: impl FnOnce(&mut fmt::Formatter) -> fmt::Result) -> String { +fn to_string_fmt(f: impl FnOnce(&mut fmt::Formatter<'_>) -> fmt::Result) -> String { use std::cell::Cell; struct Fmt(Cell>); - impl fmt::Result> fmt::Display for Fmt { + impl) -> fmt::Result> fmt::Display for Fmt { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { self.0.take().unwrap()(f) } diff --git a/compiler/literal/src/float.rs b/compiler/literal/src/float.rs index a58a3f48a7..e05a105fd4 100644 --- a/compiler/literal/src/float.rs +++ b/compiler/literal/src/float.rs @@ -223,6 +223,39 @@ pub fn to_string(value: f64) -> String { } } +pub fn complex_to_string(re: f64, im: f64) -> String { + // integer => drop ., fractional => float_ops + let mut im_part = if im.fract() == 0.0 { + im.to_string() + } else { + to_string(im) + }; + im_part.push('j'); + + // positive empty => return im_part, integer => drop ., fractional => float_ops + let re_part = if re == 0.0 { + if re.is_sign_positive() { + return im_part; + } else { + re.to_string() + } + } else if re.fract() == 0.0 { + re.to_string() + } else { + to_string(re) + }; + let mut result = + String::with_capacity(re_part.len() + im_part.len() + 2 + im.is_sign_positive() as usize); + result.push('('); + result.push_str(&re_part); + if im.is_sign_positive() || im.is_nan() { + result.push('+'); + } + result.push_str(&im_part); + result.push(')'); + result +} + pub fn from_hex(s: &str) -> Option { if let Ok(f) = hexf_parse::parse_hexf64(s, false) { return Some(f); diff --git a/vm/src/builtins/complex.rs b/vm/src/builtins/complex.rs index 01dd65f519..a3a6d4d681 100644 --- a/vm/src/builtins/complex.rs +++ b/vm/src/builtins/complex.rs @@ -494,37 +494,7 @@ impl Representable for PyComplex { // TODO: when you fix this, move it to rustpython_common::complex::repr and update // ast/src/unparse.rs + impl Display for Constant in ast/src/constant.rs let Complex64 { re, im } = zelf.value; - // integer => drop ., fractional => float_ops - let mut im_part = if im.fract() == 0.0 { - im.to_string() - } else { - crate::literal::float::to_string(im) - }; - im_part.push('j'); - - // positive empty => return im_part, integer => drop ., fractional => float_ops - let re_part = if re == 0.0 { - if re.is_sign_positive() { - return Ok(im_part); - } else { - re.to_string() - } - } else if re.fract() == 0.0 { - re.to_string() - } else { - crate::literal::float::to_string(re) - }; - let mut result = String::with_capacity( - re_part.len() + im_part.len() + 2 + im.is_sign_positive() as usize, - ); - result.push('('); - result.push_str(&re_part); - if im.is_sign_positive() || im.is_nan() { - result.push('+'); - } - result.push_str(&im_part); - result.push(')'); - Ok(result) + Ok(rustpython_literal::float::complex_to_string(re, im)) } } From 0abd8b1d8706e2e3a496ff9cfd0a7c1446f547a3 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 27 Mar 2025 20:35:08 +0900 Subject: [PATCH 142/295] Fix pystructseq --- derive-impl/src/pystructseq.rs | 30 ++++++++++++------------ vm/src/types/structseq.rs | 43 ++++++++++++++++++++-------------- 2 files changed, 40 insertions(+), 33 deletions(-) diff --git a/derive-impl/src/pystructseq.rs b/derive-impl/src/pystructseq.rs index 4a4f9276f0..89dee8da10 100644 --- a/derive-impl/src/pystructseq.rs +++ b/derive-impl/src/pystructseq.rs @@ -80,16 +80,19 @@ fn field_names(input: &mut DeriveInput) -> Result<(Vec, Vec)> { } pub(crate) fn impl_pystruct_sequence(mut input: DeriveInput) -> Result { - let (not_skipped_fields, _skipped_fields) = field_names(&mut input)?; + let (not_skipped_fields, skipped_fields) = field_names(&mut input)?; let ty = &input.ident; let ret = quote! { impl ::rustpython_vm::types::PyStructSequence for #ty { - const FIELD_NAMES: &'static [&'static str] = &[#(stringify!(#not_skipped_fields)),*]; + const REQUIRED_FIELD_NAMES: &'static [&'static str] = &[#(stringify!(#not_skipped_fields),)*]; + const OPTIONAL_FIELD_NAMES: &'static [&'static str] = &[#(stringify!(#skipped_fields),)*]; fn into_tuple(self, vm: &::rustpython_vm::VirtualMachine) -> ::rustpython_vm::builtins::PyTuple { - let items = vec![#(::rustpython_vm::convert::ToPyObject::to_pyobject( - self.#not_skipped_fields, - vm, - )),*]; + let items = vec![ + #(::rustpython_vm::convert::ToPyObject::to_pyobject( + self.#not_skipped_fields, + vm, + ),)* + ]; ::rustpython_vm::builtins::PyTuple::new_unchecked(items.into_boxed_slice()) } } @@ -110,17 +113,14 @@ pub(crate) fn impl_pystruct_sequence_try_from_object( let ret = quote! { impl ::rustpython_vm::TryFromObject for #ty { fn try_from_object(vm: &::rustpython_vm::VirtualMachine, seq: ::rustpython_vm::PyObjectRef) -> ::rustpython_vm::PyResult { - const LEN: usize = #ty::FIELD_NAMES.len(); - let seq = Self::try_elements_from::(seq, vm)?; - // TODO: this is possible to be written without iterator + let seq = Self::try_elements_from(seq, vm)?; let mut iter = seq.into_iter(); Ok(Self { - #( - #not_skipped_fields: iter.next().unwrap().clone().try_into_value(vm)?, - )* - #( - #skipped_fields: vm.ctx.none(), - )* + #(#not_skipped_fields: iter.next().unwrap().clone().try_into_value(vm)?,)* + #(#skipped_fields: match iter.next() { + Some(v) => v.clone().try_into_value(vm)?, + None => vm.ctx.none(), + },)* }) } } diff --git a/vm/src/types/structseq.rs b/vm/src/types/structseq.rs index a0b445ce7d..7b7f5fb9b5 100644 --- a/vm/src/types/structseq.rs +++ b/vm/src/types/structseq.rs @@ -1,13 +1,14 @@ use crate::{ AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, - builtins::{PyTuple, PyTupleRef, PyType}, + builtins::{PyBaseExceptionRef, PyTuple, PyTupleRef, PyType}, class::{PyClassImpl, StaticType}, vm::Context, }; #[pyclass] pub trait PyStructSequence: StaticType + PyClassImpl + Sized + 'static { - const FIELD_NAMES: &'static [&'static str]; + const REQUIRED_FIELD_NAMES: &'static [&'static str]; + const OPTIONAL_FIELD_NAMES: &'static [&'static str]; fn into_tuple(self, vm: &VirtualMachine) -> PyTuple; @@ -17,10 +18,16 @@ pub trait PyStructSequence: StaticType + PyClassImpl + Sized + 'static { .unwrap() } - fn try_elements_from( - obj: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<[PyObjectRef; FIELD_LEN]> { + fn try_elements_from(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult> { + #[cold] + fn sequence_length_error( + name: &str, + len: usize, + vm: &VirtualMachine, + ) -> PyBaseExceptionRef { + vm.new_type_error(format!("{name} takes a sequence of length {len}")) + } + let typ = Self::static_type(); // if !obj.fast_isinstance(typ) { // return Err(vm.new_type_error(format!( @@ -30,13 +37,13 @@ pub trait PyStructSequence: StaticType + PyClassImpl + Sized + 'static { // ))); // } let seq: Vec = obj.try_into_value(vm)?; - let seq: [PyObjectRef; FIELD_LEN] = seq.try_into().map_err(|_| { - vm.new_type_error(format!( - "{} takes a sequence of length {}", - typ.name(), - FIELD_LEN - )) - })?; + if seq.len() < Self::REQUIRED_FIELD_NAMES.len() { + return Err(sequence_length_error( + &typ.name(), + Self::REQUIRED_FIELD_NAMES.len(), + vm, + )); + } Ok(seq) } @@ -49,14 +56,14 @@ pub trait PyStructSequence: StaticType + PyClassImpl + Sized + 'static { let (body, suffix) = if let Some(_guard) = rustpython_vm::recursion::ReprGuard::enter(vm, zelf.as_object()) { - if Self::FIELD_NAMES.len() == 1 { + if Self::REQUIRED_FIELD_NAMES.len() == 1 { let value = zelf.first().unwrap(); - let formatted = format_field((value, Self::FIELD_NAMES[0]))?; + let formatted = format_field((value, Self::REQUIRED_FIELD_NAMES[0]))?; (formatted, ",") } else { let fields: PyResult> = zelf .iter() - .zip(Self::FIELD_NAMES.iter().copied()) + .zip(Self::REQUIRED_FIELD_NAMES.iter().copied()) .map(format_field) .collect(); (fields?.join(", "), "") @@ -74,7 +81,7 @@ pub trait PyStructSequence: StaticType + PyClassImpl + Sized + 'static { #[extend_class] fn extend_pyclass(ctx: &Context, class: &'static Py) { - for (i, &name) in Self::FIELD_NAMES.iter().enumerate() { + for (i, &name) in Self::REQUIRED_FIELD_NAMES.iter().enumerate() { // cast i to a u8 so there's less to store in the getter closure. // Hopefully there's not struct sequences with >=256 elements :P let i = i as u8; @@ -90,7 +97,7 @@ pub trait PyStructSequence: StaticType + PyClassImpl + Sized + 'static { class.set_attr( identifier!(ctx, __match_args__), ctx.new_tuple( - Self::FIELD_NAMES + Self::REQUIRED_FIELD_NAMES .iter() .map(|&name| ctx.new_str(name).into()) .collect::>(), From bb8606dbede6e1a31381cae869f94e9dba962970 Mon Sep 17 00:00:00 2001 From: Christopher Gambrell Date: Tue, 13 Aug 2024 11:18:05 -0400 Subject: [PATCH 143/295] implement tm_gmtoff and tm_zone --- vm/src/stdlib/time.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/vm/src/stdlib/time.rs b/vm/src/stdlib/time.rs index f98530e845..0ed36d2c3e 100644 --- a/vm/src/stdlib/time.rs +++ b/vm/src/stdlib/time.rs @@ -39,7 +39,7 @@ mod decl { types::PyStructSequence, }; use chrono::{ - DateTime, Datelike, Timelike, + DateTime, Datelike, TimeZone, Timelike, naive::{NaiveDate, NaiveDateTime, NaiveTime}, }; use std::time::Duration; @@ -451,6 +451,8 @@ mod decl { tm_wday: PyObjectRef, tm_yday: PyObjectRef, tm_isdst: PyObjectRef, + tm_gmtoff: PyObjectRef, + tm_zone: PyObjectRef, } impl std::fmt::Debug for PyStructTime { @@ -462,6 +464,11 @@ mod decl { #[pyclass(with(PyStructSequence))] impl PyStructTime { fn new(vm: &VirtualMachine, tm: NaiveDateTime, isdst: i32) -> Self { + let local_time = chrono::Local.from_local_datetime(&tm).unwrap(); + let offset_seconds = + local_time.offset().local_minus_utc() + if isdst == 1 { 3600 } else { 0 }; + let tz_abbr = local_time.format("%Z").to_string(); + PyStructTime { tm_year: vm.ctx.new_int(tm.year()).into(), tm_mon: vm.ctx.new_int(tm.month()).into(), @@ -472,6 +479,8 @@ mod decl { tm_wday: vm.ctx.new_int(tm.weekday().num_days_from_monday()).into(), tm_yday: vm.ctx.new_int(tm.ordinal()).into(), tm_isdst: vm.ctx.new_int(isdst).into(), + tm_gmtoff: vm.ctx.new_int(offset_seconds).into(), + tm_zone: vm.ctx.new_str(tz_abbr).into(), } } From b99e7f62b2fb63eab4ee0fe822a9a5366f2bd1cd Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Tue, 17 Sep 2024 21:32:23 +0900 Subject: [PATCH 144/295] Add pystruct(skip) --- vm/src/stdlib/time.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/vm/src/stdlib/time.rs b/vm/src/stdlib/time.rs index 0ed36d2c3e..86b6927fd1 100644 --- a/vm/src/stdlib/time.rs +++ b/vm/src/stdlib/time.rs @@ -451,7 +451,9 @@ mod decl { tm_wday: PyObjectRef, tm_yday: PyObjectRef, tm_isdst: PyObjectRef, + #[pystruct(skip)] tm_gmtoff: PyObjectRef, + #[pystruct(skip)] tm_zone: PyObjectRef, } From fd270775a3795d4300e0bd29709c9b455ef17a09 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 27 Mar 2025 20:36:47 +0900 Subject: [PATCH 145/295] time._STRUCT_TM_ITEMS --- vm/src/stdlib/time.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/vm/src/stdlib/time.rs b/vm/src/stdlib/time.rs index 86b6927fd1..10d51bd39a 100644 --- a/vm/src/stdlib/time.rs +++ b/vm/src/stdlib/time.rs @@ -72,6 +72,9 @@ mod decl { .map_err(|e| vm.new_value_error(format!("Time error: {e:?}"))) } + #[pyattr] + pub const _STRUCT_TM_ITEMS: usize = 11; + // TODO: implement proper monotonic time for wasm/wasi. #[cfg(not(any(unix, windows)))] fn get_monotonic_time(vm: &VirtualMachine) -> PyResult { From 763ba9fd6ac4093fb79f24032645d8596e09fcea Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 27 Mar 2025 20:36:54 +0900 Subject: [PATCH 146/295] edit test_time --- Lib/test/test_time.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 4e618d1d90..3af18efae2 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -718,9 +718,10 @@ class TestAsctime4dyear(_TestAsctimeYear, _Test4dYear, unittest.TestCase): class TestPytime(unittest.TestCase): + # TODO: RUSTPYTHON + @unittest.expectedFailure @skip_if_buggy_ucrt_strfptime - @unittest.skip("TODO: RUSTPYTHON, AttributeError: module 'time' has no attribute '_STRUCT_TM_ITEMS'") - # @unittest.skipUnless(time._STRUCT_TM_ITEMS == 11, "needs tm_zone support") + @unittest.skipUnless(time._STRUCT_TM_ITEMS == 11, "needs tm_zone support") def test_localtime_timezone(self): # Get the localtime and examine it for the offset and zone. @@ -755,16 +756,18 @@ def test_localtime_timezone(self): self.assertEqual(new_lt.tm_gmtoff, lt.tm_gmtoff) self.assertEqual(new_lt9.tm_zone, lt.tm_zone) - @unittest.skip("TODO: RUSTPYTHON, AttributeError: module 'time' has no attribute '_STRUCT_TM_ITEMS'") - # @unittest.skipUnless(time._STRUCT_TM_ITEMS == 11, "needs tm_zone support") + # TODO: RUSTPYTHON + @unittest.expectedFailure + @unittest.skipUnless(time._STRUCT_TM_ITEMS == 11, "needs tm_zone support") def test_strptime_timezone(self): t = time.strptime("UTC", "%Z") self.assertEqual(t.tm_zone, 'UTC') t = time.strptime("+0500", "%z") self.assertEqual(t.tm_gmtoff, 5 * 3600) - @unittest.skip("TODO: RUSTPYTHON, AttributeError: module 'time' has no attribute '_STRUCT_TM_ITEMS'") - # @unittest.skipUnless(time._STRUCT_TM_ITEMS == 11, "needs tm_zone support") + # TODO: RUSTPYTHON + @unittest.expectedFailure + @unittest.skipUnless(time._STRUCT_TM_ITEMS == 11, "needs tm_zone support") def test_short_times(self): import pickle From b0679865769bbd81133b8eec328d6867b5078b0e Mon Sep 17 00:00:00 2001 From: Noa Date: Fri, 28 Mar 2025 12:24:10 -0500 Subject: [PATCH 147/295] Bump cranelift to 0.118 --- Cargo.lock | 81 ++++++++++++++++++++++++++++---------------------- jit/Cargo.toml | 8 ++--- 2 files changed, 50 insertions(+), 39 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5de6ad43ae..a71ceb53b3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -429,37 +429,53 @@ dependencies = [ [[package]] name = "cranelift" -version = "0.116.1" +version = "0.118.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a71de5e59f616d79d14d2c71aa2799ce898241d7f10f7e64a4997014b4000a28" +checksum = "e482b051275b415cf7627bb6b26e9902ce6aec058b443266c2a1e7a0de148960" dependencies = [ "cranelift-codegen", "cranelift-frontend", "cranelift-module", ] +[[package]] +name = "cranelift-assembler-x64" +version = "0.118.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e4b56ebe316895d3fa37775d0a87b0c889cc933f5c8b253dbcc7c7bcb7fe7e4" +dependencies = [ + "cranelift-assembler-x64-meta", +] + +[[package]] +name = "cranelift-assembler-x64-meta" +version = "0.118.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95cabbc01dfbd7dcd6c329ca44f0212910309c221797ac736a67a5bc8857fe1b" + [[package]] name = "cranelift-bforest" -version = "0.116.1" +version = "0.118.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e15d04a0ce86cb36ead88ad68cf693ffd6cda47052b9e0ac114bc47fd9cd23c4" +checksum = "76ffe46df300a45f1dc6f609dc808ce963f0e3a2e971682c479a2d13e3b9b8ef" dependencies = [ "cranelift-entity", ] [[package]] name = "cranelift-bitset" -version = "0.116.1" +version = "0.118.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c6e3969a7ce267259ce244b7867c5d3bc9e65b0a87e81039588dfdeaede9f34" +checksum = "b265bed7c51e1921fdae6419791d31af77d33662ee56d7b0fa0704dc8d231cab" [[package]] name = "cranelift-codegen" -version = "0.116.1" +version = "0.118.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c22032c4cb42558371cf516bb47f26cdad1819d3475c133e93c49f50ebf304e" +checksum = "e606230a7e3a6897d603761baee0d19f88d077f17b996bb5089488a29ae96e41" dependencies = [ "bumpalo", + "cranelift-assembler-x64", "cranelift-bforest", "cranelift-bitset", "cranelift-codegen-meta", @@ -468,7 +484,7 @@ dependencies = [ "cranelift-entity", "cranelift-isle", "gimli", - "hashbrown 0.14.5", + "hashbrown 0.15.2", "log", "regalloc2", "rustc-hash 2.1.1", @@ -479,42 +495,43 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" -version = "0.116.1" +version = "0.118.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c904bc71c61b27fc57827f4a1379f29de64fe95653b620a3db77d59655eee0b8" +checksum = "8a63bffafc23bc60969ad528e138788495999d935f0adcfd6543cb151ca8637d" dependencies = [ + "cranelift-assembler-x64", "cranelift-codegen-shared", ] [[package]] name = "cranelift-codegen-shared" -version = "0.116.1" +version = "0.118.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40180f5497572f644ce88c255480981ae2ec1d7bb4d8e0c0136a13b87a2f2ceb" +checksum = "af50281b67324b58e843170a6a5943cf6d387c06f7eeacc9f5696e4ab7ae7d7e" [[package]] name = "cranelift-control" -version = "0.116.1" +version = "0.118.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d132c6d0bd8a489563472afc171759da0707804a65ece7ceb15a8c6d7dd5ef" +checksum = "8c20c1b38d1abfbcebb0032e497e71156c0e3b8dcb3f0a92b9863b7bcaec290c" dependencies = [ "arbitrary", ] [[package]] name = "cranelift-entity" -version = "0.116.1" +version = "0.118.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b2d0d9618275474fbf679dd018ac6e009acbd6ae6850f6a67be33fb3b00b323" +checksum = "0c2c67d95507c51b4a1ff3f3555fe4bfec36b9e13c1b684ccc602736f5d5f4a2" dependencies = [ "cranelift-bitset", ] [[package]] name = "cranelift-frontend" -version = "0.116.1" +version = "0.118.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fac41e16729107393174b0c9e3730fb072866100e1e64e80a1a963b2e484d57" +checksum = "4e002691cc69c38b54fc7ec93e5be5b744f627d027031d991cc845d1d512d0ce" dependencies = [ "cranelift-codegen", "log", @@ -524,15 +541,15 @@ dependencies = [ [[package]] name = "cranelift-isle" -version = "0.116.1" +version = "0.118.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ca20d576e5070044d0a72a9effc2deacf4d6aa650403189d8ea50126483944d" +checksum = "e93588ed1796cbcb0e2ad160403509e2c5d330d80dd6e0014ac6774c7ebac496" [[package]] name = "cranelift-jit" -version = "0.116.1" +version = "0.118.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e65c42755a719b09662b00c700daaf76cc35d5ace1f5c002ad404b591ff1978" +checksum = "17f6682f0b193d6b7873cc8e7ed67e8776a8a26f50eeabf88534e9be618b9a03" dependencies = [ "anyhow", "cranelift-codegen", @@ -550,9 +567,9 @@ dependencies = [ [[package]] name = "cranelift-module" -version = "0.116.1" +version = "0.118.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d55612bebcf16ff7306c8a6f5bdb6d45662b8aa1ee058ecce8807ad87db719b" +checksum = "ff19784c6de05116e63e6a34791012bd927b2a4eac56233039c46f1b6a4edac8" dependencies = [ "anyhow", "cranelift-codegen", @@ -561,9 +578,9 @@ dependencies = [ [[package]] name = "cranelift-native" -version = "0.116.1" +version = "0.118.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8dee82f3f1f2c4cba9177f1cc5e350fe98764379bcd29340caa7b01f85076c7" +checksum = "e5b09bdd6407bf5d89661b80cf926ce731c9e8cc184bf49102267a2369a8358e" dependencies = [ "cranelift-codegen", "libc", @@ -1012,12 +1029,6 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" -[[package]] -name = "hashbrown" -version = "0.14.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" - [[package]] name = "hashbrown" version = "0.15.2" @@ -3528,9 +3539,9 @@ dependencies = [ [[package]] name = "wasmtime-jit-icache-coherence" -version = "29.0.1" +version = "31.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec5e8552e01692e6c2e5293171704fed8abdec79d1a6995a0870ab190e5747d1" +checksum = "a54f6c6c7e9d7eeee32dfcc10db7f29d505ee7dd28d00593ea241d5f70698e64" dependencies = [ "anyhow", "cfg-if", diff --git a/jit/Cargo.toml b/jit/Cargo.toml index e153b0eca8..0c7f39af07 100644 --- a/jit/Cargo.toml +++ b/jit/Cargo.toml @@ -17,9 +17,9 @@ num-traits = { workspace = true } thiserror = { workspace = true } libffi = { workspace = true, features = ["system"] } -cranelift = "0.116.1" -cranelift-jit = "0.116.1" -cranelift-module = "0.116.1" +cranelift = "0.118" +cranelift-jit = "0.118" +cranelift-module = "0.118" [dev-dependencies] rustpython-derive = { path = "../derive", version = "0.4.0" } @@ -31,4 +31,4 @@ name = "integration" path = "tests/lib.rs" [lints] -workspace = true \ No newline at end of file +workspace = true From b620f037288bdd93bf1e192d28ce1ce688cd2ff2 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Mon, 22 Apr 2024 10:39:54 +0900 Subject: [PATCH 148/295] Update socket from CPython 3.12.2 --- Lib/socket.py | 42 +++-- Lib/test/test_socket.py | 361 +++++++++++++++++++++++++++++++--------- 2 files changed, 310 insertions(+), 93 deletions(-) diff --git a/Lib/socket.py b/Lib/socket.py index 63ba0acc90..42ee130773 100644 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -13,7 +13,7 @@ socketpair() -- create a pair of new socket objects [*] fromfd() -- create a socket object from an open file descriptor [*] send_fds() -- Send file descriptor to the socket. -recv_fds() -- Recieve file descriptors from the socket. +recv_fds() -- Receive file descriptors from the socket. fromshare() -- create a socket object from data received from socket.share() [*] gethostname() -- return the current hostname gethostbyname() -- map a hostname to its IP number @@ -28,6 +28,7 @@ socket.setdefaulttimeout() -- set the default timeout value create_connection() -- connects to an address, with an optional timeout and optional source address. +create_server() -- create a TCP socket and bind it to a specified address. [*] not available on all platforms! @@ -122,7 +123,7 @@ def _intenum_converter(value, enum_klass): errorTab[10014] = "A fault occurred on the network??" # WSAEFAULT errorTab[10022] = "An invalid operation was attempted." errorTab[10024] = "Too many open files." - errorTab[10035] = "The socket operation would block" + errorTab[10035] = "The socket operation would block." errorTab[10036] = "A blocking operation is already in progress." errorTab[10037] = "Operation already in progress." errorTab[10038] = "Socket operation on nonsocket." @@ -254,17 +255,18 @@ def __repr__(self): self.type, self.proto) if not closed: + # getsockname and getpeername may not be available on WASI. try: laddr = self.getsockname() if laddr: s += ", laddr=%s" % str(laddr) - except error: + except (error, AttributeError): pass try: raddr = self.getpeername() if raddr: s += ", raddr=%s" % str(raddr) - except error: + except (error, AttributeError): pass s += '>' return s @@ -380,7 +382,7 @@ def _sendfile_use_sendfile(self, file, offset=0, count=None): if timeout and not selector_select(timeout): raise TimeoutError('timed out') if count: - blocksize = count - total_sent + blocksize = min(count - total_sent, blocksize) if blocksize <= 0: break try: @@ -783,11 +785,11 @@ def getfqdn(name=''): First the hostname returned by gethostbyaddr() is checked, then possibly existing aliases. In case no FQDN is available and `name` - was given, it is returned unchanged. If `name` was empty or '0.0.0.0', + was given, it is returned unchanged. If `name` was empty, '0.0.0.0' or '::', hostname from gethostname() is returned. """ name = name.strip() - if not name or name == '0.0.0.0': + if not name or name in ('0.0.0.0', '::'): name = gethostname() try: hostname, aliases, ipaddrs = gethostbyaddr(name) @@ -806,7 +808,7 @@ def getfqdn(name=''): _GLOBAL_DEFAULT_TIMEOUT = object() def create_connection(address, timeout=_GLOBAL_DEFAULT_TIMEOUT, - source_address=None): + source_address=None, *, all_errors=False): """Connect to *address* and return the socket object. Convenience function. Connect to *address* (a 2-tuple ``(host, @@ -816,11 +818,13 @@ def create_connection(address, timeout=_GLOBAL_DEFAULT_TIMEOUT, global default timeout setting returned by :func:`getdefaulttimeout` is used. If *source_address* is set it must be a tuple of (host, port) for the socket to bind as a source address before making the connection. - A host of '' or port 0 tells the OS to use the default. + A host of '' or port 0 tells the OS to use the default. When a connection + cannot be created, raises the last error if *all_errors* is False, + and an ExceptionGroup of all errors if *all_errors* is True. """ host, port = address - err = None + exceptions = [] for res in getaddrinfo(host, port, 0, SOCK_STREAM): af, socktype, proto, canonname, sa = res sock = None @@ -832,20 +836,24 @@ def create_connection(address, timeout=_GLOBAL_DEFAULT_TIMEOUT, sock.bind(source_address) sock.connect(sa) # Break explicitly a reference cycle - err = None + exceptions.clear() return sock - except error as _: - err = _ + except error as exc: + if not all_errors: + exceptions.clear() # raise only the last error + exceptions.append(exc) if sock is not None: sock.close() - if err is not None: + if len(exceptions): try: - raise err + if not all_errors: + raise exceptions[0] + raise ExceptionGroup("create_connection failed", exceptions) finally: # Break explicitly a reference cycle - err = None + exceptions.clear() else: raise error("getaddrinfo returns an empty list") @@ -902,7 +910,7 @@ def create_server(address, *, family=AF_INET, backlog=None, reuse_port=False, # address, effectively preventing this one from accepting # connections. Also, it may set the process in a state where # it'll no longer respond to any signals or graceful kills. - # See: msdn2.microsoft.com/en-us/library/ms740621(VS.85).aspx + # See: https://learn.microsoft.com/windows/win32/winsock/using-so-reuseaddr-and-so-exclusiveaddruse if os.name not in ('nt', 'cygwin') and \ hasattr(_socket, 'SO_REUSEADDR'): try: diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 0e3eb08b82..d448101dcd 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -4,30 +4,31 @@ from test.support import socket_helper from test.support import threading_helper +import _thread as thread +import array +import contextlib import errno +import gc import io import itertools -import socket -import select -import tempfile -import time -import traceback -import queue -import sys -import os -import platform -import array -import contextlib -from weakref import proxy -import signal import math +import os import pickle -import struct +import platform +import queue import random -import shutil +import re +import select +import signal +import socket import string -import _thread as thread +import struct +import sys +import tempfile import threading +import time +import traceback +from weakref import proxy try: import multiprocessing except ImportError: @@ -37,12 +38,15 @@ except ImportError: fcntl = None +support.requires_working_socket(module=True) + HOST = socket_helper.HOST # test unicode string and carriage return MSG = 'Michael Gilfix was here\u1234\r\n'.encode('utf-8') VSOCKPORT = 1234 AIX = platform.system() == "AIX" +WSL = "microsoft-standard-WSL" in platform.release() try: import _socket @@ -141,6 +145,17 @@ def _have_socket_bluetooth(): return True +def _have_socket_hyperv(): + """Check whether AF_HYPERV sockets are supported on this host.""" + try: + s = socket.socket(socket.AF_HYPERV, socket.SOCK_STREAM, socket.HV_PROTOCOL_RAW) + except (AttributeError, OSError): + return False + else: + s.close() + return True + + @contextlib.contextmanager def socket_setdefaulttimeout(timeout): old_timeout = socket.getdefaulttimeout() @@ -169,6 +184,8 @@ def socket_setdefaulttimeout(timeout): HAVE_SOCKET_BLUETOOTH = _have_socket_bluetooth() +HAVE_SOCKET_HYPERV = _have_socket_hyperv() + # Size in bytes of the int type SIZEOF_INT = array.array("i").itemsize @@ -199,24 +216,6 @@ def setUp(self): self.serv = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDPLITE) self.port = socket_helper.bind_port(self.serv) -class ThreadSafeCleanupTestCase: - """Subclass of unittest.TestCase with thread-safe cleanup methods. - - This subclass protects the addCleanup() and doCleanups() methods - with a recursive lock. - """ - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self._cleanup_lock = threading.RLock() - - def addCleanup(self, *args, **kwargs): - with self._cleanup_lock: - return super().addCleanup(*args, **kwargs) - - def doCleanups(self, *args, **kwargs): - with self._cleanup_lock: - return super().doCleanups(*args, **kwargs) class SocketCANTest(unittest.TestCase): @@ -336,9 +335,7 @@ def serverExplicitReady(self): self.server_ready.set() def _setUp(self): - self.wait_threads = threading_helper.wait_threads_exit() - self.wait_threads.__enter__() - self.addCleanup(self.wait_threads.__exit__, None, None, None) + self.enterContext(threading_helper.wait_threads_exit()) self.server_ready = threading.Event() self.client_ready = threading.Event() @@ -485,6 +482,7 @@ def clientTearDown(self): ThreadableTest.clientTearDown(self) @unittest.skipIf(fcntl is None, "need fcntl") +@unittest.skipIf(WSL, 'VSOCK does not work on Microsoft WSL') @unittest.skipUnless(HAVE_SOCKET_VSOCK, 'VSOCK sockets required for this test.') @unittest.skipUnless(get_cid() != 2, @@ -501,6 +499,7 @@ def setUp(self): self.serv.bind((socket.VMADDR_CID_ANY, VSOCKPORT)) self.serv.listen() self.serverExplicitReady() + self.serv.settimeout(support.LOOPBACK_TIMEOUT) self.conn, self.connaddr = self.serv.accept() self.addCleanup(self.conn.close) @@ -591,17 +590,18 @@ class SocketTestBase(unittest.TestCase): def setUp(self): self.serv = self.newSocket() + self.addCleanup(self.close_server) self.bindServer() + def close_server(self): + self.serv.close() + self.serv = None + def bindServer(self): """Bind server socket and set self.serv_addr to its address.""" self.bindSock(self.serv) self.serv_addr = self.serv.getsockname() - def tearDown(self): - self.serv.close() - self.serv = None - class SocketListeningTestMixin(SocketTestBase): """Mixin to listen on the server socket.""" @@ -611,8 +611,7 @@ def setUp(self): self.serv.listen() -class ThreadedSocketTestMixin(ThreadSafeCleanupTestCase, SocketTestBase, - ThreadableTest): +class ThreadedSocketTestMixin(SocketTestBase, ThreadableTest): """Mixin to add client socket and allow client/server tests. Client socket is self.cli and its address is self.cli_addr. See @@ -686,15 +685,10 @@ class UnixSocketTestBase(SocketTestBase): # can't send anything that might be problematic for a privileged # user running the tests. - def setUp(self): - self.dir_path = tempfile.mkdtemp() - self.addCleanup(os.rmdir, self.dir_path) - super().setUp() - def bindSock(self, sock): - path = tempfile.mktemp(dir=self.dir_path) - socket_helper.bind_unix_socket(sock, path) + path = socket_helper.create_unix_domain_name() self.addCleanup(os_helper.unlink, path) + socket_helper.bind_unix_socket(sock, path) class UnixStreamBase(UnixSocketTestBase): """Base class for Unix-domain SOCK_STREAM tests.""" @@ -827,6 +821,12 @@ def requireSocket(*args): class GeneralModuleTests(unittest.TestCase): + @unittest.skipUnless(_socket is not None, 'need _socket module') + def test_socket_type(self): + self.assertTrue(gc.is_tracked(_socket.socket)) + with self.assertRaisesRegex(TypeError, "immutable"): + _socket.socket.foo = 1 + def test_SocketType_is_socketobject(self): import _socket self.assertTrue(socket.SocketType is _socket.socket) @@ -958,6 +958,19 @@ def testWindowsSpecificConstants(self): socket.IPPROTO_L2TP socket.IPPROTO_SCTP + @unittest.skipIf(support.is_wasi, "WASI is missing these methods") + def test_socket_methods(self): + # socket methods that depend on a configure HAVE_ check. They should + # be present on all platforms except WASI. + names = [ + "_accept", "bind", "connect", "connect_ex", "getpeername", + "getsockname", "listen", "recvfrom", "recvfrom_into", "sendto", + "setsockopt", "shutdown" + ] + for name in names: + if not hasattr(socket.socket, name): + self.fail(f"socket method {name} is missing") + # TODO: RUSTPYTHON @unittest.expectedFailure @unittest.skipUnless(sys.platform == 'darwin', 'macOS specific test') @@ -1021,8 +1034,10 @@ def test_host_resolution(self): def test_host_resolution_bad_address(self): # These are all malformed IP addresses and expected not to resolve to - # any result. But some ISPs, e.g. AWS, may successfully resolve these - # IPs. + # any result. But some ISPs, e.g. AWS and AT&T, may successfully + # resolve these IPs. In particular, AT&T's DNS Error Assist service + # will break this test. See https://bugs.python.org/issue42092 for a + # workaround. explanation = ( "resolving an invalid IP address did not raise OSError; " "can be caused by a broken DNS server" @@ -1074,7 +1089,20 @@ def testInterfaceNameIndex(self): 'socket.if_indextoname() not available.') def testInvalidInterfaceIndexToName(self): self.assertRaises(OSError, socket.if_indextoname, 0) + self.assertRaises(OverflowError, socket.if_indextoname, -1) + self.assertRaises(OverflowError, socket.if_indextoname, 2**1000) self.assertRaises(TypeError, socket.if_indextoname, '_DEADBEEF') + if hasattr(socket, 'if_nameindex'): + indices = dict(socket.if_nameindex()) + for index in indices: + index2 = index + 2**32 + if index2 not in indices: + with self.assertRaises((OverflowError, OSError)): + socket.if_indextoname(index2) + for index in 2**32-1, 2**64-1: + if index not in indices: + with self.assertRaises((OverflowError, OSError)): + socket.if_indextoname(index) @unittest.skipUnless(hasattr(socket, 'if_nametoindex'), 'socket.if_nametoindex() not available.') @@ -1378,10 +1406,21 @@ def testStringToIPv6(self): def testSockName(self): # Testing getsockname() - port = socket_helper.find_unused_port() sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.addCleanup(sock.close) - sock.bind(("0.0.0.0", port)) + + # Since find_unused_port() is inherently subject to race conditions, we + # call it a couple times if necessary. + for i in itertools.count(): + port = socket_helper.find_unused_port() + try: + sock.bind(("0.0.0.0", port)) + except OSError as e: + if e.errno != errno.EADDRINUSE or i == 5: + raise + else: + break + name = sock.getsockname() # XXX(nnorwitz): http://tinyurl.com/os5jz seems to indicate # it reasonable to get the host's addr in addition to 0.0.0.0. @@ -1525,9 +1564,11 @@ def testGetaddrinfo(self): infos = socket.getaddrinfo(HOST, 80, socket.AF_INET, socket.SOCK_STREAM) for family, type, _, _, _ in infos: self.assertEqual(family, socket.AF_INET) - self.assertEqual(str(family), 'AddressFamily.AF_INET') + self.assertEqual(repr(family), '' % family.value) + self.assertEqual(str(family), str(family.value)) self.assertEqual(type, socket.SOCK_STREAM) - self.assertEqual(str(type), 'SocketKind.SOCK_STREAM') + self.assertEqual(repr(type), '' % type.value) + self.assertEqual(str(type), str(type.value)) infos = socket.getaddrinfo(HOST, None, 0, socket.SOCK_STREAM) for _, socktype, _, _, _ in infos: self.assertEqual(socktype, socket.SOCK_STREAM) @@ -1574,6 +1615,54 @@ def testGetaddrinfo(self): except socket.gaierror: pass + def test_getaddrinfo_int_port_overflow(self): + # gh-74895: Test that getaddrinfo does not raise OverflowError on port. + # + # POSIX getaddrinfo() never specify the valid range for "service" + # decimal port number values. For IPv4 and IPv6 they are technically + # unsigned 16-bit values, but the API is protocol agnostic. Which values + # trigger an error from the C library function varies by platform as + # they do not all perform validation. + + # The key here is that we don't want to produce OverflowError as Python + # prior to 3.12 did for ints outside of a [LONG_MIN, LONG_MAX] range. + # Leave the error up to the underlying string based platform C API. + + from _testcapi import ULONG_MAX, LONG_MAX, LONG_MIN + try: + socket.getaddrinfo(None, ULONG_MAX + 1, type=socket.SOCK_STREAM) + except OverflowError: + # Platforms differ as to what values consitute a getaddrinfo() error + # return. Some fail for LONG_MAX+1, others ULONG_MAX+1, and Windows + # silently accepts such huge "port" aka "service" numeric values. + self.fail("Either no error or socket.gaierror expected.") + except socket.gaierror: + pass + + try: + socket.getaddrinfo(None, LONG_MAX + 1, type=socket.SOCK_STREAM) + except OverflowError: + self.fail("Either no error or socket.gaierror expected.") + except socket.gaierror: + pass + + try: + socket.getaddrinfo(None, LONG_MAX - 0xffff + 1, type=socket.SOCK_STREAM) + except OverflowError: + self.fail("Either no error or socket.gaierror expected.") + except socket.gaierror: + pass + + try: + socket.getaddrinfo(None, LONG_MIN - 1, type=socket.SOCK_STREAM) + except OverflowError: + self.fail("Either no error or socket.gaierror expected.") + except socket.gaierror: + pass + + socket.getaddrinfo(None, 0, type=socket.SOCK_STREAM) # No error expected. + socket.getaddrinfo(None, 0xffff, type=socket.SOCK_STREAM) # No error expected. + def test_getnameinfo(self): # only IP addresses are allowed self.assertRaises(OSError, socket.getnameinfo, ('mail.python.org',0), 0) @@ -1746,6 +1835,10 @@ def test_getaddrinfo_ipv6_basic(self): ) self.assertEqual(sockaddr, ('ff02::1de:c0:face:8d', 1234, 0, 0)) + def test_getfqdn_filter_localhost(self): + self.assertEqual(socket.getfqdn(), socket.getfqdn("0.0.0.0")) + self.assertEqual(socket.getfqdn(), socket.getfqdn("::")) + @unittest.skipUnless(socket_helper.IPV6_ENABLED, 'IPv6 required for this test.') @unittest.skipIf(sys.platform == 'win32', 'does not work on Windows') @unittest.skipIf(AIX, 'Symbolic scope id does not work') @@ -1807,8 +1900,10 @@ def test_str_for_enums(self): # Make sure that the AF_* and SOCK_* constants have enum-like string # reprs. with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - self.assertEqual(str(s.family), 'AddressFamily.AF_INET') - self.assertEqual(str(s.type), 'SocketKind.SOCK_STREAM') + self.assertEqual(repr(s.family), '' % s.family.value) + self.assertEqual(repr(s.type), '' % s.type.value) + self.assertEqual(str(s.family), str(s.family.value)) + self.assertEqual(str(s.type), str(s.type.value)) @unittest.expectedFailureIf(sys.platform.startswith("linux"), "TODO: RUSTPYTHON, AssertionError: 526337 != ") def test_socket_consistent_sock_type(self): @@ -1902,17 +1997,18 @@ def test_socket_fileno(self): self._test_socket_fileno(s, socket.AF_INET6, socket.SOCK_STREAM) if hasattr(socket, "AF_UNIX"): - tmpdir = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, tmpdir) + unix_name = socket_helper.create_unix_domain_name() + self.addCleanup(os_helper.unlink, unix_name) + s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - self.addCleanup(s.close) - try: - s.bind(os.path.join(tmpdir, 'socket')) - except PermissionError: - pass - else: - self._test_socket_fileno(s, socket.AF_UNIX, - socket.SOCK_STREAM) + with s: + try: + s.bind(unix_name) + except PermissionError: + pass + else: + self._test_socket_fileno(s, socket.AF_UNIX, + socket.SOCK_STREAM) def test_socket_fileno_rejects_float(self): with self.assertRaises(TypeError): @@ -1956,6 +2052,41 @@ def test_socket_fileno_requires_socket_fd(self): fileno=afile.fileno()) self.assertEqual(cm.exception.errno, errno.ENOTSOCK) + def test_addressfamily_enum(self): + import _socket, enum + CheckedAddressFamily = enum._old_convert_( + enum.IntEnum, 'AddressFamily', 'socket', + lambda C: C.isupper() and C.startswith('AF_'), + source=_socket, + ) + enum._test_simple_enum(CheckedAddressFamily, socket.AddressFamily) + + def test_socketkind_enum(self): + import _socket, enum + CheckedSocketKind = enum._old_convert_( + enum.IntEnum, 'SocketKind', 'socket', + lambda C: C.isupper() and C.startswith('SOCK_'), + source=_socket, + ) + enum._test_simple_enum(CheckedSocketKind, socket.SocketKind) + + def test_msgflag_enum(self): + import _socket, enum + CheckedMsgFlag = enum._old_convert_( + enum.IntFlag, 'MsgFlag', 'socket', + lambda C: C.isupper() and C.startswith('MSG_'), + source=_socket, + ) + enum._test_simple_enum(CheckedMsgFlag, socket.MsgFlag) + + def test_addressinfo_enum(self): + import _socket, enum + CheckedAddressInfo = enum._old_convert_( + enum.IntFlag, 'AddressInfo', 'socket', + lambda C: C.isupper() and C.startswith('AI_'), + source=_socket) + enum._test_simple_enum(CheckedAddressInfo, socket.AddressInfo) + @unittest.skipUnless(HAVE_SOCKET_CAN, 'SocketCan required for this test.') class BasicCANTest(unittest.TestCase): @@ -2449,6 +2580,58 @@ def testCreateScoSocket(self): pass +@unittest.skipUnless(HAVE_SOCKET_HYPERV, + 'Hyper-V sockets required for this test.') +class BasicHyperVTest(unittest.TestCase): + + def testHyperVConstants(self): + socket.HVSOCKET_CONNECT_TIMEOUT + socket.HVSOCKET_CONNECT_TIMEOUT_MAX + socket.HVSOCKET_CONNECTED_SUSPEND + socket.HVSOCKET_ADDRESS_FLAG_PASSTHRU + socket.HV_GUID_ZERO + socket.HV_GUID_WILDCARD + socket.HV_GUID_BROADCAST + socket.HV_GUID_CHILDREN + socket.HV_GUID_LOOPBACK + socket.HV_GUID_PARENT + + def testCreateHyperVSocketWithUnknownProtoFailure(self): + expected = r"\[WinError 10041\]" + with self.assertRaisesRegex(OSError, expected): + socket.socket(socket.AF_HYPERV, socket.SOCK_STREAM) + + def testCreateHyperVSocketAddrNotTupleFailure(self): + expected = "connect(): AF_HYPERV address must be tuple, not str" + with socket.socket(socket.AF_HYPERV, socket.SOCK_STREAM, socket.HV_PROTOCOL_RAW) as s: + with self.assertRaisesRegex(TypeError, re.escape(expected)): + s.connect(socket.HV_GUID_ZERO) + + def testCreateHyperVSocketAddrNotTupleOf2StrsFailure(self): + expected = "AF_HYPERV address must be a str tuple (vm_id, service_id)" + with socket.socket(socket.AF_HYPERV, socket.SOCK_STREAM, socket.HV_PROTOCOL_RAW) as s: + with self.assertRaisesRegex(TypeError, re.escape(expected)): + s.connect((socket.HV_GUID_ZERO,)) + + def testCreateHyperVSocketAddrNotTupleOfStrsFailure(self): + expected = "AF_HYPERV address must be a str tuple (vm_id, service_id)" + with socket.socket(socket.AF_HYPERV, socket.SOCK_STREAM, socket.HV_PROTOCOL_RAW) as s: + with self.assertRaisesRegex(TypeError, re.escape(expected)): + s.connect((1, 2)) + + def testCreateHyperVSocketAddrVmIdNotValidUUIDFailure(self): + expected = "connect(): AF_HYPERV address vm_id is not a valid UUID string" + with socket.socket(socket.AF_HYPERV, socket.SOCK_STREAM, socket.HV_PROTOCOL_RAW) as s: + with self.assertRaisesRegex(ValueError, re.escape(expected)): + s.connect(("00", socket.HV_GUID_ZERO)) + + def testCreateHyperVSocketAddrServiceIdNotValidUUIDFailure(self): + expected = "connect(): AF_HYPERV address service_id is not a valid UUID string" + with socket.socket(socket.AF_HYPERV, socket.SOCK_STREAM, socket.HV_PROTOCOL_RAW) as s: + with self.assertRaisesRegex(ValueError, re.escape(expected)): + s.connect((socket.HV_GUID_ZERO, "00")) + + class BasicTCPTest(SocketConnectedTest): def __init__(self, methodName='runTest'): @@ -2658,7 +2841,7 @@ def _testRecvFromNegative(self): # here assumes that datagram delivery on the local machine will be # reliable. -class SendrecvmsgBase(ThreadSafeCleanupTestCase): +class SendrecvmsgBase: # Base class for sendmsg()/recvmsg() tests. # Time in seconds to wait before considering a test failed, or @@ -4525,7 +4708,6 @@ def testInterruptedRecvmsgIntoTimeout(self): @unittest.skipUnless(hasattr(signal, "alarm") or hasattr(signal, "setitimer"), "Don't have signal.alarm or signal.setitimer") class InterruptedSendTimeoutTest(InterruptedTimeoutBase, - ThreadSafeCleanupTestCase, SocketListeningTestMixin, TCPTestBase): # Test interrupting the interruptible send*() methods with signals # when a timeout is set. @@ -5136,6 +5318,7 @@ def mocked_socket_module(self): finally: socket.socket = old_socket + @socket_helper.skip_if_tcp_blackhole def test_connect(self): port = socket_helper.find_unused_port() cli = socket.socket(socket.AF_INET, socket.SOCK_STREAM) @@ -5144,6 +5327,7 @@ def test_connect(self): cli.connect((HOST, port)) self.assertEqual(cm.exception.errno, errno.ECONNREFUSED) + @socket_helper.skip_if_tcp_blackhole def test_create_connection(self): # Issue #9792: errors raised by create_connection() should have # a proper errno attribute. @@ -5168,6 +5352,24 @@ def test_create_connection(self): expected_errnos = socket_helper.get_socket_conn_refused_errs() self.assertIn(cm.exception.errno, expected_errnos) + def test_create_connection_all_errors(self): + port = socket_helper.find_unused_port() + try: + socket.create_connection((HOST, port), all_errors=True) + except ExceptionGroup as e: + eg = e + else: + self.fail('expected connection to fail') + + self.assertIsInstance(eg, ExceptionGroup) + for e in eg.exceptions: + self.assertIsInstance(e, OSError) + + addresses = socket.getaddrinfo( + 'localhost', port, 0, socket.SOCK_STREAM) + # assert that we got an exception for each address + self.assertEqual(len(addresses), len(eg.exceptions)) + def test_create_connection_timeout(self): # Issue #9792: create_connection() should not recast timeout errors # as generic socket errors. @@ -5184,6 +5386,7 @@ def test_create_connection_timeout(self): class NetworkConnectionAttributesTest(SocketTCPTest, ThreadableTest): + cli = None def __init__(self, methodName='runTest'): SocketTCPTest.__init__(self, methodName=methodName) @@ -5193,7 +5396,8 @@ def clientSetUp(self): self.source_port = socket_helper.find_unused_port() def clientTearDown(self): - self.cli.close() + if self.cli is not None: + self.cli.close() self.cli = None ThreadableTest.clientTearDown(self) @@ -5328,10 +5532,10 @@ def alarm_handler(signal, frame): self.fail("caught timeout instead of Alarm") except Alarm: pass - except: + except BaseException as e: self.fail("caught other exception instead of Alarm:" " %s(%s):\n%s" % - (sys.exc_info()[:2] + (traceback.format_exc(),))) + (type(e), e, traceback.format_exc())) else: self.fail("nothing caught") finally: @@ -6232,6 +6436,7 @@ def _testWithTimeoutTriggeredSend(self): def testWithTimeoutTriggeredSend(self): conn = self.accept_conn() conn.recv(88192) + # bpo-45212: the wait here needs to be longer than the client-side timeout (0.01s) time.sleep(1) # errors @@ -6312,12 +6517,16 @@ def test_sha256(self): # TODO: RUSTPYTHON, OSError: bind(): bad family @unittest.expectedFailure def test_hmac_sha1(self): - expected = bytes.fromhex("effcdf6ae5eb2fa2d27416d5f184df9c259a7c79") + # gh-109396: In FIPS mode, Linux 6.5 requires a key + # of at least 112 bits. Use a key of 152 bits. + key = b"Python loves AF_ALG" + data = b"what do ya want for nothing?" + expected = bytes.fromhex("193dbb43c6297b47ea6277ec0ce67119a3f3aa66") with self.create_alg('hash', 'hmac(sha1)') as algo: - algo.setsockopt(socket.SOL_ALG, socket.ALG_SET_KEY, b"Jefe") + algo.setsockopt(socket.SOL_ALG, socket.ALG_SET_KEY, key) op, _ = algo.accept() with op: - op.sendall(b"what do ya want for nothing?") + op.sendall(data) self.assertEqual(op.recv(512), expected) # Although it should work with 3.19 and newer the test blocks on From 9e2f6bd1875419fa3add740b89db3cdad988fe15 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Mon, 22 Apr 2024 11:20:38 +0900 Subject: [PATCH 149/295] mark failing tests of test_socket --- Lib/test/test_socket.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index d448101dcd..ea544f6afa 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -821,6 +821,8 @@ def requireSocket(*args): class GeneralModuleTests(unittest.TestCase): + # TODO: RUSTPYTHON + @unittest.expectedFailure @unittest.skipUnless(_socket is not None, 'need _socket module') def test_socket_type(self): self.assertTrue(gc.is_tracked(_socket.socket)) @@ -1532,8 +1534,6 @@ def test_sio_loopback_fast_path(self): raise self.assertRaises(TypeError, s.ioctl, socket.SIO_LOOPBACK_FAST_PATH, None) - # TODO: RUSTPYTHON, AssertionError: '2' != 'AddressFamily.AF_INET' - @unittest.expectedFailure def testGetaddrinfo(self): try: socket.getaddrinfo('localhost', 80) @@ -1615,6 +1615,8 @@ def testGetaddrinfo(self): except socket.gaierror: pass + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_getaddrinfo_int_port_overflow(self): # gh-74895: Test that getaddrinfo does not raise OverflowError on port. # @@ -1894,8 +1896,6 @@ def test_getnameinfo_ipv6_scopeid_numeric(self): nameinfo = socket.getnameinfo(sockaddr, socket.NI_NUMERICHOST | socket.NI_NUMERICSERV) self.assertEqual(nameinfo, ('ff02::1de:c0:face:8d%' + str(ifindex), '1234')) - # TODO: RUSTPYTHON, AssertionError: '2' != 'AddressFamily.AF_INET' - @unittest.expectedFailure def test_str_for_enums(self): # Make sure that the AF_* and SOCK_* constants have enum-like string # reprs. @@ -2052,6 +2052,8 @@ def test_socket_fileno_requires_socket_fd(self): fileno=afile.fileno()) self.assertEqual(cm.exception.errno, errno.ENOTSOCK) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_addressfamily_enum(self): import _socket, enum CheckedAddressFamily = enum._old_convert_( @@ -2061,6 +2063,8 @@ def test_addressfamily_enum(self): ) enum._test_simple_enum(CheckedAddressFamily, socket.AddressFamily) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_socketkind_enum(self): import _socket, enum CheckedSocketKind = enum._old_convert_( @@ -2070,6 +2074,8 @@ def test_socketkind_enum(self): ) enum._test_simple_enum(CheckedSocketKind, socket.SocketKind) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_msgflag_enum(self): import _socket, enum CheckedMsgFlag = enum._old_convert_( @@ -2079,6 +2085,8 @@ def test_msgflag_enum(self): ) enum._test_simple_enum(CheckedMsgFlag, socket.MsgFlag) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_addressinfo_enum(self): import _socket, enum CheckedAddressInfo = enum._old_convert_( @@ -5352,6 +5360,8 @@ def test_create_connection(self): expected_errnos = socket_helper.get_socket_conn_refused_errs() self.assertIn(cm.exception.errno, expected_errnos) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_create_connection_all_errors(self): port = socket_helper.find_unused_port() try: From 8f989e4a677b4b5535948aac72d8f34acb92ce7d Mon Sep 17 00:00:00 2001 From: Hanif Ariffin Date: Sat, 8 Mar 2025 21:18:13 +0800 Subject: [PATCH 150/295] Fixed an expected failure in the behavior of negating a bool argument Signed-off-by: Hanif Ariffin --- Lib/test/test_bool.py | 2 -- vm/src/vm/vm_ops.rs | 12 ++++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_bool.py b/Lib/test/test_bool.py index 09eefd422b..34ecb45f16 100644 --- a/Lib/test/test_bool.py +++ b/Lib/test/test_bool.py @@ -46,8 +46,6 @@ def test_complex(self): self.assertEqual(complex(True), 1+0j) self.assertEqual(complex(True), True) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_math(self): self.assertEqual(+False, 0) self.assertIsNot(+False, False) diff --git a/vm/src/vm/vm_ops.rs b/vm/src/vm/vm_ops.rs index 09f849a1a1..31f7781b4a 100644 --- a/vm/src/vm/vm_ops.rs +++ b/vm/src/vm/vm_ops.rs @@ -1,4 +1,5 @@ use super::VirtualMachine; +use crate::stdlib::warnings; use crate::{ builtins::{PyInt, PyIntRef, PyStr, PyStrRef}, object::{AsObject, PyObject, PyObjectRef, PyResult}, @@ -469,6 +470,17 @@ impl VirtualMachine { } pub fn _invert(&self, a: &PyObject) -> PyResult { + const STR: &'static str = "Bitwise inversion '~' on bool is deprecated and will be removed in Python 3.16. \ + This returns the bitwise inversion of the underlying int object and is usually not what you expect from negating a bool. \ + Use the 'not' operator for boolean negation or ~int(x) if you really want the bitwise inversion of the underlying int."; + if a.fast_isinstance(self.ctx.types.bool_type) { + warnings::warn( + self.ctx.exceptions.deprecation_warning, + STR.to_owned(), + 1, + self, + )?; + } self.get_special_method(a, identifier!(self, __invert__))? .ok_or_else(|| self.new_unsupported_unary_error(a, "unary ~"))? .invoke((), self) From 8e7039405ee56137a6ab114a68cd43c80a967f4e Mon Sep 17 00:00:00 2001 From: Hanif Ariffin Date: Sun, 30 Mar 2025 23:53:20 +0800 Subject: [PATCH 151/295] Fix some clippy issues Signed-off-by: Hanif Ariffin --- vm/src/vm/vm_ops.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/src/vm/vm_ops.rs b/vm/src/vm/vm_ops.rs index 31f7781b4a..5235393a69 100644 --- a/vm/src/vm/vm_ops.rs +++ b/vm/src/vm/vm_ops.rs @@ -470,7 +470,7 @@ impl VirtualMachine { } pub fn _invert(&self, a: &PyObject) -> PyResult { - const STR: &'static str = "Bitwise inversion '~' on bool is deprecated and will be removed in Python 3.16. \ + const STR: &str = "Bitwise inversion '~' on bool is deprecated and will be removed in Python 3.16. \ This returns the bitwise inversion of the underlying int object and is usually not what you expect from negating a bool. \ Use the 'not' operator for boolean negation or ~int(x) if you really want the bitwise inversion of the underlying int."; if a.fast_isinstance(self.ctx.types.bool_type) { From 24d995678f62842c521dcab6de99fd235620ec26 Mon Sep 17 00:00:00 2001 From: Noa Date: Wed, 19 Mar 2025 13:10:00 -0500 Subject: [PATCH 152/295] Remove some unncessary dependencies --- Cargo.lock | 45 +++++++++++--------------------------------- Cargo.toml | 3 ++- common/Cargo.toml | 4 ++-- common/src/hash.rs | 9 --------- common/src/lib.rs | 1 + common/src/rand.rs | 13 +++++++++++++ compiler/Cargo.toml | 3 +++ stdlib/Cargo.toml | 2 +- stdlib/src/random.rs | 2 +- stdlib/src/uuid.rs | 3 ++- vm/Cargo.toml | 2 -- vm/build.rs | 12 +++++++----- vm/src/import.rs | 3 ++- vm/src/version.rs | 6 ++---- vm/src/vm/mod.rs | 3 ++- 15 files changed, 49 insertions(+), 62 deletions(-) create mode 100644 common/src/rand.rs diff --git a/Cargo.lock b/Cargo.lock index a71ceb53b3..2607deff1b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1336,7 +1336,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" dependencies = [ "cfg-if", - "windows-targets 0.52.6", + "windows-targets 0.48.5", ] [[package]] @@ -1546,7 +1546,7 @@ version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df7151a832e54d2d6b2c827a20e5bcdd80359281cd2c354e725d4b82e7c471de" dependencies = [ - "rand_core 0.9.1", + "rand_core 0.9.3", ] [[package]] @@ -1797,7 +1797,7 @@ version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" dependencies = [ - "siphasher 1.0.1", + "siphasher", ] [[package]] @@ -2014,7 +2014,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" dependencies = [ "rand_chacha 0.9.0", - "rand_core 0.9.1", + "rand_core 0.9.3", "zerocopy 0.8.20", ] @@ -2035,7 +2035,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core 0.9.1", + "rand_core 0.9.3", ] [[package]] @@ -2049,12 +2049,11 @@ dependencies = [ [[package]] name = "rand_core" -version = "0.9.1" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a88e0da7a2c97baa202165137c158d0a2e824ac465d13d81046727b34cb247d3" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" dependencies = [ "getrandom 0.3.1", - "zerocopy 0.8.20", ] [[package]] @@ -2252,15 +2251,6 @@ version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" -[[package]] -name = "rustc_version" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" -dependencies = [ - "semver", -] - [[package]] name = "rustix" version = "0.38.44" @@ -2329,6 +2319,7 @@ dependencies = [ "bitflags 2.8.0", "bstr", "cfg-if", + "getrandom 0.3.1", "itertools 0.14.0", "libc", "lock_api", @@ -2340,10 +2331,9 @@ dependencies = [ "once_cell", "parking_lot", "radium", - "rand 0.9.0", "rustpython-literal", "rustpython-wtf8", - "siphasher 0.3.11", + "siphasher", "unicode_names2", "volatile", "widestring", @@ -2354,6 +2344,7 @@ dependencies = [ name = "rustpython-compiler" version = "0.4.0" dependencies = [ + "rand 0.9.0", "ruff_python_ast", "ruff_python_parser", "ruff_source_file", @@ -2511,7 +2502,7 @@ dependencies = [ "parking_lot", "paste", "puruspe", - "rand 0.9.0", + "rand_core 0.9.3", "rustix", "rustpython-common", "rustpython-derive", @@ -2581,13 +2572,11 @@ dependencies = [ "optional", "parking_lot", "paste", - "rand 0.9.0", "result-like", "ruff_python_ast", "ruff_python_parser", "ruff_source_file", "ruff_text_size", - "rustc_version", "rustix", "rustpython-codegen", "rustpython-common", @@ -2708,12 +2697,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "semver" -version = "1.0.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f79dfe2d285b0488816f30e700a7438c5a73d816b5b7d3ac72fbc48b0d185e03" - [[package]] name = "serde" version = "1.0.218" @@ -2821,12 +2804,6 @@ version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" -[[package]] -name = "siphasher" -version = "0.3.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" - [[package]] name = "siphasher" version = "1.0.1" diff --git a/Cargo.toml b/Cargo.toml index ccfec120f2..271c11e782 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -158,7 +158,7 @@ chrono = "0.4.39" criterion = { version = "0.3.5", features = ["html_reports"] } crossbeam-utils = "0.8.21" flame = "0.2.2" -getrandom = "0.3" +getrandom = { version = "0.3", features = ["std"] } glob = "0.3" hex = "0.4.3" indexmap = { version = "2.2.6", features = ["std"] } @@ -185,6 +185,7 @@ paste = "1.0.15" proc-macro2 = "1.0.93" quote = "1.0.38" rand = "0.9" +rand_core = { version = "0.9", features = ["os_rng"] } rustix = { version = "0.38", features = ["event"] } rustyline = "15.0.0" serde = { version = "1.0.133", default-features = false } diff --git a/common/Cargo.toml b/common/Cargo.toml index 1cf637faa4..299c2875b2 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -19,6 +19,7 @@ ascii = { workspace = true } bitflags = { workspace = true } bstr = { workspace = true } cfg-if = { workspace = true } +getrandom = { workspace = true } itertools = { workspace = true } libc = { workspace = true } malachite-bigint = { workspace = true } @@ -28,12 +29,11 @@ memchr = { workspace = true } num-traits = { workspace = true } once_cell = { workspace = true } parking_lot = { workspace = true, optional = true } -rand = { workspace = true } unicode_names2 = { workspace = true } lock_api = "0.4" radium = "0.7" -siphasher = "0.3" +siphasher = "1" volatile = "0.3" [target.'cfg(windows)'.dependencies] diff --git a/common/src/hash.rs b/common/src/hash.rs index bbc30b1fe1..8fef70c8b9 100644 --- a/common/src/hash.rs +++ b/common/src/hash.rs @@ -37,15 +37,6 @@ impl BuildHasher for HashSecret { } } -impl rand::distr::Distribution for rand::distr::StandardUniform { - fn sample(&self, rng: &mut R) -> HashSecret { - HashSecret { - k0: rng.random(), - k1: rng.random(), - } - } -} - impl HashSecret { pub fn new(seed: u32) -> Self { let mut buf = [0u8; 16]; diff --git a/common/src/lib.rs b/common/src/lib.rs index 4b50b31c2b..c75451802a 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -23,6 +23,7 @@ pub mod int; pub mod linked_list; pub mod lock; pub mod os; +pub mod rand; pub mod rc; pub mod refcount; pub mod static_cell; diff --git a/common/src/rand.rs b/common/src/rand.rs new file mode 100644 index 0000000000..334505ac94 --- /dev/null +++ b/common/src/rand.rs @@ -0,0 +1,13 @@ +/// Get `N` bytes of random data. +/// +/// This function is mildly expensive to call, as it fetches random data +/// directly from the OS entropy source. +/// +/// # Panics +/// +/// Panics if the OS entropy source returns an error. +pub fn os_random() -> [u8; N] { + let mut buf = [0u8; N]; + getrandom::fill(&mut buf).unwrap(); + buf +} diff --git a/compiler/Cargo.toml b/compiler/Cargo.toml index 19ece85a16..39b7c54beb 100644 --- a/compiler/Cargo.toml +++ b/compiler/Cargo.toml @@ -19,5 +19,8 @@ ruff_source_file = { workspace = true } ruff_text_size = { workspace = true } thiserror = { workspace = true } +[dev-dependencies] +rand = { workspace = true } + [lints] workspace = true \ No newline at end of file diff --git a/stdlib/Cargo.toml b/stdlib/Cargo.toml index d29dc1474e..0ec23bf132 100644 --- a/stdlib/Cargo.toml +++ b/stdlib/Cargo.toml @@ -50,7 +50,7 @@ puruspe = "0.4.0" xml-rs = "0.8.14" # random -rand = { workspace = true } +rand_core = { workspace = true } mt19937 = "3.1" # Crypto: diff --git a/stdlib/src/random.rs b/stdlib/src/random.rs index 685c0ae8b9..31e523b68b 100644 --- a/stdlib/src/random.rs +++ b/stdlib/src/random.rs @@ -16,7 +16,7 @@ mod _random { use malachite_bigint::{BigInt, BigUint, Sign}; use mt19937::MT19937; use num_traits::{Signed, Zero}; - use rand::{RngCore, SeedableRng}; + use rand_core::{RngCore, SeedableRng}; use rustpython_vm::types::DefaultConstructor; #[pyattr] diff --git a/stdlib/src/uuid.rs b/stdlib/src/uuid.rs index 872f5bde0a..9b0e23a81c 100644 --- a/stdlib/src/uuid.rs +++ b/stdlib/src/uuid.rs @@ -10,7 +10,8 @@ mod _uuid { fn get_node_id() -> [u8; 6] { match get_mac_address() { Ok(Some(_ma)) => get_mac_address().unwrap().unwrap().bytes(), - _ => rand::random::<[u8; 6]>(), + // os_random is expensive, but this is only ever called once + _ => rustpython_common::rand::os_random::<6>(), } } diff --git a/vm/Cargo.toml b/vm/Cargo.toml index a59327420f..125c263da9 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -68,7 +68,6 @@ num_enum = { workspace = true } once_cell = { workspace = true } parking_lot = { workspace = true } paste = { workspace = true } -rand = { workspace = true } serde = { workspace = true, optional = true } static_assertions = { workspace = true } strum = { workspace = true } @@ -160,7 +159,6 @@ getrandom = { workspace = true } [build-dependencies] glob = { workspace = true } itertools = { workspace = true } -rustc_version = "0.4.0" [lints] workspace = true diff --git a/vm/build.rs b/vm/build.rs index 063153f6e7..93d29c3a57 100644 --- a/vm/build.rs +++ b/vm/build.rs @@ -20,10 +20,7 @@ fn main() { ); println!("cargo:rustc-env=RUSTPYTHON_GIT_TAG={}", git_tag()); println!("cargo:rustc-env=RUSTPYTHON_GIT_BRANCH={}", git_branch()); - println!( - "cargo:rustc-env=RUSTC_VERSION={}", - rustc_version::version().unwrap() - ); + println!("cargo:rustc-env=RUSTC_VERSION={}", rustc_version()); println!( "cargo:rustc-env=RUSTPYTHON_TARGET_TRIPLE={}", @@ -61,7 +58,12 @@ fn git(args: &[&str]) -> String { command("git", args) } -fn command(cmd: &str, args: &[&str]) -> String { +fn rustc_version() -> String { + let rustc = env::var_os("RUSTC").unwrap_or_else(|| "rustc".into()); + command(rustc, &["-V"]) +} + +fn command(cmd: impl AsRef, args: &[&str]) -> String { match Command::new(cmd).args(args).output() { Ok(output) => match String::from_utf8(output.stdout) { Ok(s) => s, diff --git a/vm/src/import.rs b/vm/src/import.rs index 2d86e47d08..0ce116d014 100644 --- a/vm/src/import.rs +++ b/vm/src/import.rs @@ -49,7 +49,8 @@ pub(crate) fn init_importlib_package(vm: &VirtualMachine, importlib: PyObjectRef let mut magic = get_git_revision().into_bytes(); magic.truncate(4); if magic.len() != 4 { - magic = rand::random::<[u8; 4]>().to_vec(); + // os_random is expensive, but this is only ever called once + magic = rustpython_common::rand::os_random::<4>().to_vec(); } let magic: PyObjectRef = vm.ctx.new_bytes(magic).into(); importlib_external.set_attr("MAGIC_NUMBER", magic, vm)?; diff --git a/vm/src/version.rs b/vm/src/version.rs index f2ac2354f8..7413f8f139 100644 --- a/vm/src/version.rs +++ b/vm/src/version.rs @@ -21,7 +21,7 @@ pub fn get_version() -> String { get_version_number(), get_build_info(), env!("CARGO_PKG_VERSION"), - get_compiler() + COMPILER, ) } @@ -33,9 +33,7 @@ pub fn get_winver_number() -> String { format!("{MAJOR}.{MINOR}") } -pub fn get_compiler() -> String { - format!("rustc {}", env!("RUSTC_VERSION")) -} +const COMPILER: &str = env!("RUSTC_VERSION"); pub fn get_build_info() -> String { // See: https://reproducible-builds.org/docs/timestamps/ diff --git a/vm/src/vm/mod.rs b/vm/src/vm/mod.rs index 67b160726b..752943319d 100644 --- a/vm/src/vm/mod.rs +++ b/vm/src/vm/mod.rs @@ -109,7 +109,8 @@ pub struct PyGlobalState { pub fn process_hash_secret_seed() -> u32 { use std::sync::OnceLock; static SEED: OnceLock = OnceLock::new(); - *SEED.get_or_init(rand::random) + // os_random is expensive, but this is only ever called once + *SEED.get_or_init(|| u32::from_ne_bytes(rustpython_common::rand::os_random())) } impl VirtualMachine { From 0b3594697216ac18159fcfe06170f9f3cc165c9f Mon Sep 17 00:00:00 2001 From: Noa Date: Sun, 30 Mar 2025 19:52:15 -0500 Subject: [PATCH 153/295] Make FromArgs default field take an expression, not a string literal --- Cargo.lock | 1 + derive-impl/src/from_args.rs | 4 ++-- derive/Cargo.toml | 1 + stdlib/src/binascii.rs | 14 +++++++------- stdlib/src/faulthandler.rs | 6 +++--- stdlib/src/hashlib.rs | 8 ++++---- stdlib/src/mmap.rs | 8 ++++---- stdlib/src/pystruct.rs | 2 +- stdlib/src/select.rs | 6 +++--- stdlib/src/socket.rs | 8 ++++---- stdlib/src/sqlite.rs | 20 ++++++++++---------- stdlib/src/ssl.rs | 2 +- stdlib/src/zlib.rs | 20 ++++++++++---------- vm/src/anystr.rs | 6 +++--- vm/src/builtins/int.rs | 6 +++--- vm/src/builtins/list.rs | 2 +- vm/src/stdlib/builtins.rs | 2 +- vm/src/stdlib/codecs.rs | 2 +- vm/src/stdlib/io.rs | 6 +++--- vm/src/stdlib/os.rs | 4 ++-- vm/src/stdlib/posix.rs | 2 +- vm/src/stdlib/signal.rs | 2 +- vm/src/stdlib/sre.rs | 8 ++++---- vm/src/stdlib/thread.rs | 4 ++-- vm/src/stdlib/winreg.rs | 4 ++-- 25 files changed, 75 insertions(+), 73 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2607deff1b..265320f990 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2381,6 +2381,7 @@ dependencies = [ name = "rustpython-derive" version = "0.4.0" dependencies = [ + "proc-macro2", "rustpython-compiler", "rustpython-derive-impl", "syn 2.0.98", diff --git a/derive-impl/src/from_args.rs b/derive-impl/src/from_args.rs index 2273046ed4..9b06da85e3 100644 --- a/derive-impl/src/from_args.rs +++ b/derive-impl/src/from_args.rs @@ -76,7 +76,6 @@ impl ArgAttribute { return Err(meta.error("Default already set")); } let val = meta.value()?; - let val = val.parse::()?; self.default = Some(Some(val.parse()?)) } else if meta.path.is_ident("default") || meta.path.is_ident("optional") { if self.default.is_none() { @@ -138,9 +137,10 @@ fn generate_field((i, field): (usize, &Field)) -> Result { .map(|x| ::rustpython_vm::convert::TryFromObject::try_from_object(vm, x)).transpose()? }; let ending = if let Some(default) = attr.default { + let ty = &field.ty; let default = default.unwrap_or_else(|| parse_quote!(::std::default::Default::default())); quote! { - .map(::rustpython_vm::function::FromArgOptional::from_inner) + .map(<#ty as ::rustpython_vm::function::FromArgOptional>::from_inner) .unwrap_or_else(|| #default) } } else { diff --git a/derive/Cargo.toml b/derive/Cargo.toml index d38686ce7d..17c0810884 100644 --- a/derive/Cargo.toml +++ b/derive/Cargo.toml @@ -14,6 +14,7 @@ proc-macro = true [dependencies] rustpython-compiler = { workspace = true } rustpython-derive-impl = { workspace = true } +proc-macro2 = { workspace = true } syn = { workspace = true } [lints] diff --git a/stdlib/src/binascii.rs b/stdlib/src/binascii.rs index 65be2e0bdc..f154a2251b 100644 --- a/stdlib/src/binascii.rs +++ b/stdlib/src/binascii.rs @@ -143,7 +143,7 @@ mod decl { #[derive(FromArgs)] struct NewlineArg { - #[pyarg(named, default = "true")] + #[pyarg(named, default = true)] newline: bool, } @@ -151,7 +151,7 @@ mod decl { struct A2bBase64Args { #[pyarg(any)] s: ArgAsciiBuffer, - #[pyarg(named, default = "false")] + #[pyarg(named, default = false)] strict_mode: bool, } @@ -298,7 +298,7 @@ mod decl { struct A2bQpArgs { #[pyarg(any)] data: ArgAsciiBuffer, - #[pyarg(named, default = "false")] + #[pyarg(named, default = false)] header: bool, } #[pyfunction] @@ -366,11 +366,11 @@ mod decl { struct B2aQpArgs { #[pyarg(any)] data: ArgAsciiBuffer, - #[pyarg(named, default = "false")] + #[pyarg(named, default = false)] quotetabs: bool, - #[pyarg(named, default = "true")] + #[pyarg(named, default = true)] istext: bool, - #[pyarg(named, default = "false")] + #[pyarg(named, default = false)] header: bool, } @@ -689,7 +689,7 @@ mod decl { #[derive(FromArgs)] struct BacktickArg { - #[pyarg(named, default = "false")] + #[pyarg(named, default = false)] backtick: bool, } diff --git a/stdlib/src/faulthandler.rs b/stdlib/src/faulthandler.rs index e3cd434731..fcfe423ef5 100644 --- a/stdlib/src/faulthandler.rs +++ b/stdlib/src/faulthandler.rs @@ -34,7 +34,7 @@ mod decl { struct EnableArgs { #[pyarg(any, default)] file: Option, - #[pyarg(any, default = "true")] + #[pyarg(any, default = true)] all_threads: bool, } @@ -50,9 +50,9 @@ mod decl { signum: i64, #[pyarg(any, default)] file: Option, - #[pyarg(any, default = "true")] + #[pyarg(any, default = true)] all_threads: bool, - #[pyarg(any, default = "false")] + #[pyarg(any, default = false)] chain: bool, } diff --git a/stdlib/src/hashlib.rs b/stdlib/src/hashlib.rs index f140515c13..586d825b2c 100644 --- a/stdlib/src/hashlib.rs +++ b/stdlib/src/hashlib.rs @@ -28,7 +28,7 @@ pub mod _hashlib { name: PyStrRef, #[pyarg(any, optional)] data: OptionalArg, - #[pyarg(named, default = "true")] + #[pyarg(named, default = true)] usedforsecurity: bool, } @@ -37,7 +37,7 @@ pub mod _hashlib { pub struct BlakeHashArgs { #[pyarg(positional, optional)] pub data: OptionalArg, - #[pyarg(named, default = "true")] + #[pyarg(named, default = true)] usedforsecurity: bool, } @@ -55,7 +55,7 @@ pub mod _hashlib { pub struct HashArgs { #[pyarg(any, optional)] pub string: OptionalArg, - #[pyarg(named, default = "true")] + #[pyarg(named, default = true)] usedforsecurity: bool, } @@ -331,7 +331,7 @@ pub mod _hashlib { name: PyBuffer, #[pyarg(any, optional)] data: OptionalArg, - #[pyarg(named, default = "true")] + #[pyarg(named, default = true)] digestmod: bool, // TODO: RUSTPYTHON support functions & name functions } diff --git a/stdlib/src/mmap.rs b/stdlib/src/mmap.rs index f254286805..bca367ae4d 100644 --- a/stdlib/src/mmap.rs +++ b/stdlib/src/mmap.rs @@ -191,13 +191,13 @@ mod mmap { fileno: RawFd, #[pyarg(any)] length: isize, - #[pyarg(any, default = "MAP_SHARED")] + #[pyarg(any, default = MAP_SHARED)] flags: libc::c_int, - #[pyarg(any, default = "PROT_WRITE|PROT_READ")] + #[pyarg(any, default = PROT_WRITE|PROT_READ)] prot: libc::c_int, - #[pyarg(any, default = "AccessMode::Default")] + #[pyarg(any, default = AccessMode::Default)] access: AccessMode, - #[pyarg(any, default = "0")] + #[pyarg(any, default = 0)] offset: libc::off_t, } diff --git a/stdlib/src/pystruct.rs b/stdlib/src/pystruct.rs index 2c7aa1ebf7..220970dd20 100644 --- a/stdlib/src/pystruct.rs +++ b/stdlib/src/pystruct.rs @@ -134,7 +134,7 @@ pub(crate) mod _struct { #[derive(FromArgs)] struct UpdateFromArgs { buffer: ArgBytesLike, - #[pyarg(any, default = "0")] + #[pyarg(any, default = 0)] offset: isize, } diff --git a/stdlib/src/select.rs b/stdlib/src/select.rs index d4380b69fa..f89a6c4f03 100644 --- a/stdlib/src/select.rs +++ b/stdlib/src/select.rs @@ -529,9 +529,9 @@ mod decl { #[derive(FromArgs)] pub struct EpollNewArgs { - #[pyarg(any, default = "-1")] + #[pyarg(any, default = -1)] sizehint: i32, - #[pyarg(any, default = "0")] + #[pyarg(any, default = 0)] flags: i32, } @@ -555,7 +555,7 @@ mod decl { struct EpollPollArgs { #[pyarg(any, default)] timeout: poll::TimeoutArg, - #[pyarg(any, default = "-1")] + #[pyarg(any, default = -1)] maxevents: i32, } diff --git a/stdlib/src/socket.rs b/stdlib/src/socket.rs index 988784856f..f4f90a5dc4 100644 --- a/stdlib/src/socket.rs +++ b/stdlib/src/socket.rs @@ -1908,13 +1908,13 @@ mod _socket { #[pyarg(positional)] port: Option>, - #[pyarg(positional, default = "c::AF_UNSPEC")] + #[pyarg(positional, default = c::AF_UNSPEC)] family: i32, - #[pyarg(positional, default = "0")] + #[pyarg(positional, default = 0)] ty: i32, - #[pyarg(positional, default = "0")] + #[pyarg(positional, default = 0)] proto: i32, - #[pyarg(positional, default = "0")] + #[pyarg(positional, default = 0)] flags: i32, } diff --git a/stdlib/src/sqlite.rs b/stdlib/src/sqlite.rs index 97ec193c46..00ebec75a9 100644 --- a/stdlib/src/sqlite.rs +++ b/stdlib/src/sqlite.rs @@ -297,21 +297,21 @@ mod _sqlite { struct ConnectArgs { #[pyarg(any)] database: FsPath, - #[pyarg(any, default = "5.0")] + #[pyarg(any, default = 5.0)] timeout: f64, - #[pyarg(any, default = "0")] + #[pyarg(any, default = 0)] detect_types: c_int, - #[pyarg(any, default = "Some(vm.ctx.empty_str.to_owned())")] + #[pyarg(any, default = Some(vm.ctx.empty_str.to_owned()))] isolation_level: Option, - #[pyarg(any, default = "true")] + #[pyarg(any, default = true)] check_same_thread: bool, - #[pyarg(any, default = "Connection::class(&vm.ctx).to_owned()")] + #[pyarg(any, default = Connection::class(&vm.ctx).to_owned())] factory: PyTypeRef, // TODO: cache statements #[allow(dead_code)] - #[pyarg(any, default = "0")] + #[pyarg(any, default = 0)] cached_statements: c_int, - #[pyarg(any, default = "false")] + #[pyarg(any, default = false)] uri: bool, } @@ -326,13 +326,13 @@ mod _sqlite { struct BackupArgs { #[pyarg(any)] target: PyRef, - #[pyarg(named, default = "-1")] + #[pyarg(named, default = -1)] pages: c_int, #[pyarg(named, optional)] progress: Option, #[pyarg(named, optional)] name: Option, - #[pyarg(named, default = "0.250")] + #[pyarg(named, default = 0.250)] sleep: f64, } @@ -375,7 +375,7 @@ mod _sqlite { row: i64, #[pyarg(named, default)] readonly: bool, - #[pyarg(named, default = "vm.ctx.new_str(stringify!(main))")] + #[pyarg(named, default = vm.ctx.new_str("main"))] name: PyStrRef, } diff --git a/stdlib/src/ssl.rs b/stdlib/src/ssl.rs index c1dcea354f..6cc7f3ed32 100644 --- a/stdlib/src/ssl.rs +++ b/stdlib/src/ssl.rs @@ -351,7 +351,7 @@ mod _ssl { #[derive(FromArgs)] struct Txt2ObjArgs { txt: PyStrRef, - #[pyarg(any, default = "false")] + #[pyarg(any, default = false)] name: bool, } diff --git a/stdlib/src/zlib.rs b/stdlib/src/zlib.rs index 19ed659bbb..40e364f8d4 100644 --- a/stdlib/src/zlib.rs +++ b/stdlib/src/zlib.rs @@ -74,9 +74,9 @@ mod zlib { struct PyFuncCompressArgs { #[pyarg(positional)] data: ArgBytesLike, - #[pyarg(any, default = "Level::new(Z_DEFAULT_COMPRESSION)")] + #[pyarg(any, default = Level::new(Z_DEFAULT_COMPRESSION))] level: Level, - #[pyarg(any, default = "ArgPrimitiveIndex { value: MAX_WBITS }")] + #[pyarg(any, default = ArgPrimitiveIndex { value: MAX_WBITS })] wbits: ArgPrimitiveIndex, } @@ -269,9 +269,9 @@ mod zlib { struct PyFuncDecompressArgs { #[pyarg(positional)] data: ArgBytesLike, - #[pyarg(any, default = "ArgPrimitiveIndex { value: MAX_WBITS }")] + #[pyarg(any, default = ArgPrimitiveIndex { value: MAX_WBITS })] wbits: ArgPrimitiveIndex, - #[pyarg(any, default = "ArgPrimitiveIndex { value: DEF_BUF_SIZE }")] + #[pyarg(any, default = ArgPrimitiveIndex { value: DEF_BUF_SIZE })] bufsize: ArgPrimitiveIndex, } @@ -299,7 +299,7 @@ mod zlib { #[derive(FromArgs)] struct DecompressobjArgs { - #[pyarg(any, default = "ArgPrimitiveIndex { value: MAX_WBITS }")] + #[pyarg(any, default = ArgPrimitiveIndex { value: MAX_WBITS })] wbits: ArgPrimitiveIndex, #[pyarg(any, optional)] zdict: OptionalArg, @@ -450,16 +450,16 @@ mod zlib { #[derive(FromArgs)] #[allow(dead_code)] // FIXME: use args struct CompressobjArgs { - #[pyarg(any, default = "Level::new(Z_DEFAULT_COMPRESSION)")] + #[pyarg(any, default = Level::new(Z_DEFAULT_COMPRESSION))] level: Level, // only DEFLATED is valid right now, it's w/e - #[pyarg(any, default = "DEFLATED")] + #[pyarg(any, default = DEFLATED)] method: i32, - #[pyarg(any, default = "ArgPrimitiveIndex { value: MAX_WBITS }")] + #[pyarg(any, default = ArgPrimitiveIndex { value: MAX_WBITS })] wbits: ArgPrimitiveIndex, - #[pyarg(any, name = "memLevel", default = "DEF_MEM_LEVEL")] + #[pyarg(any, name = "memLevel", default = DEF_MEM_LEVEL)] mem_level: u8, - #[pyarg(any, default = "Z_DEFAULT_STRATEGY")] + #[pyarg(any, default = Z_DEFAULT_STRATEGY)] strategy: i32, #[pyarg(any, optional)] zdict: Option, diff --git a/vm/src/anystr.rs b/vm/src/anystr.rs index efac49d917..6bc8a4dd13 100644 --- a/vm/src/anystr.rs +++ b/vm/src/anystr.rs @@ -10,19 +10,19 @@ use num_traits::{cast::ToPrimitive, sign::Signed}; pub struct SplitArgs { #[pyarg(any, default)] sep: Option, - #[pyarg(any, default = "-1")] + #[pyarg(any, default = -1)] maxsplit: isize, } #[derive(FromArgs)] pub struct SplitLinesArgs { - #[pyarg(any, default = "false")] + #[pyarg(any, default = false)] pub keepends: bool, } #[derive(FromArgs)] pub struct ExpandTabsArgs { - #[pyarg(any, default = "8")] + #[pyarg(any, default = 8)] tabsize: isize, } diff --git a/vm/src/builtins/int.rs b/vm/src/builtins/int.rs index f457bf5ed8..d644343f1c 100644 --- a/vm/src/builtins/int.rs +++ b/vm/src/builtins/int.rs @@ -826,7 +826,7 @@ pub struct IntOptions { #[derive(FromArgs)] struct IntFromByteArgs { bytes: PyBytesInner, - #[pyarg(any, default = "ArgByteOrder::Big")] + #[pyarg(any, default = ArgByteOrder::Big)] byteorder: ArgByteOrder, #[pyarg(named, optional)] signed: OptionalArg, @@ -834,9 +834,9 @@ struct IntFromByteArgs { #[derive(FromArgs)] struct IntToByteArgs { - #[pyarg(any, default = "1")] + #[pyarg(any, default = 1)] length: usize, - #[pyarg(any, default = "ArgByteOrder::Big")] + #[pyarg(any, default = ArgByteOrder::Big)] byteorder: ArgByteOrder, #[pyarg(named, optional)] signed: OptionalArg, diff --git a/vm/src/builtins/list.rs b/vm/src/builtins/list.rs index 1d8b4a30e0..4962ae51e3 100644 --- a/vm/src/builtins/list.rs +++ b/vm/src/builtins/list.rs @@ -91,7 +91,7 @@ pub(crate) struct SortOptions { #[pyarg(named, default)] key: Option, #[pytraverse(skip)] - #[pyarg(named, default = "false")] + #[pyarg(named, default = false)] reverse: bool, } diff --git a/vm/src/stdlib/builtins.rs b/vm/src/stdlib/builtins.rs index f778d02ba3..9dcb35aae9 100644 --- a/vm/src/stdlib/builtins.rs +++ b/vm/src/stdlib/builtins.rs @@ -663,7 +663,7 @@ mod builtins { sep: Option, #[pyarg(named, default)] end: Option, - #[pyarg(named, default = "ArgIntoBool::FALSE")] + #[pyarg(named, default = ArgIntoBool::FALSE)] flush: ArgIntoBool, #[pyarg(named, default)] file: Option, diff --git a/vm/src/stdlib/codecs.rs b/vm/src/stdlib/codecs.rs index 320d839682..c0a091bcf8 100644 --- a/vm/src/stdlib/codecs.rs +++ b/vm/src/stdlib/codecs.rs @@ -109,7 +109,7 @@ mod _codecs { data: ArgBytesLike, #[pyarg(positional, optional)] errors: Option, - #[pyarg(positional, default = "false")] + #[pyarg(positional, default = false)] final_decode: bool, } diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index 4cf3c058df..33ef118acd 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -3822,7 +3822,7 @@ mod _io { #[derive(FromArgs)] pub struct OpenArgs { - #[pyarg(any, default = "-1")] + #[pyarg(any, default = -1)] pub buffering: isize, #[pyarg(any, default)] pub encoding: Option, @@ -3830,7 +3830,7 @@ mod _io { pub errors: Option, #[pyarg(any, default)] pub newline: Option, - #[pyarg(any, default = "true")] + #[pyarg(any, default = true)] pub closefd: bool, #[pyarg(any, default)] pub opener: Option, @@ -4165,7 +4165,7 @@ mod fileio { name: PyObjectRef, #[pyarg(any, default)] mode: Option, - #[pyarg(any, default = "true")] + #[pyarg(any, default = true)] closefd: bool, #[pyarg(any, default)] opener: Option, diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index f53be8b01b..641ba54dea 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -43,7 +43,7 @@ pub fn errno_err(vm: &VirtualMachine) -> PyBaseExceptionRef { #[allow(dead_code)] #[derive(FromArgs, Default)] pub struct TargetIsDirectory { - #[pyarg(any, default = "false")] + #[pyarg(any, default = false)] pub(crate) target_is_directory: bool, } @@ -117,7 +117,7 @@ impl FromArgs for DirFd { #[derive(FromArgs)] pub(super) struct FollowSymlinks( - #[pyarg(named, name = "follow_symlinks", default = "true")] pub bool, + #[pyarg(named, name = "follow_symlinks", default = true)] pub bool, ); fn bytes_as_osstr<'a>(b: &'a [u8], vm: &VirtualMachine) -> PyResult<&'a ffi::OsStr> { diff --git a/vm/src/stdlib/posix.rs b/vm/src/stdlib/posix.rs index dc5c74cef4..d75629745c 100644 --- a/vm/src/stdlib/posix.rs +++ b/vm/src/stdlib/posix.rs @@ -1628,7 +1628,7 @@ pub mod module { fd: i32, #[pyarg(positional)] fd2: i32, - #[pyarg(any, default = "true")] + #[pyarg(any, default = true)] inheritable: bool, } diff --git a/vm/src/stdlib/signal.rs b/vm/src/stdlib/signal.rs index 0df93833b7..1e1e779e34 100644 --- a/vm/src/stdlib/signal.rs +++ b/vm/src/stdlib/signal.rs @@ -224,7 +224,7 @@ pub(crate) mod _signal { #[derive(FromArgs)] struct SetWakeupFdArgs { fd: WakeupFd, - #[pyarg(named, default = "true")] + #[pyarg(named, default = true)] warn_on_full_buffer: bool, } diff --git a/vm/src/stdlib/sre.rs b/vm/src/stdlib/sre.rs index 6a0a618e2f..7b67c038f4 100644 --- a/vm/src/stdlib/sre.rs +++ b/vm/src/stdlib/sre.rs @@ -163,9 +163,9 @@ mod _sre { #[derive(FromArgs)] struct StringArgs { string: PyObjectRef, - #[pyarg(any, default = "0")] + #[pyarg(any, default = 0)] pos: usize, - #[pyarg(any, default = "sys::MAXSIZE as usize")] + #[pyarg(any, default = sys::MAXSIZE as usize)] endpos: usize, } @@ -174,14 +174,14 @@ mod _sre { // repl: Either, repl: PyObjectRef, string: PyObjectRef, - #[pyarg(any, default = "0")] + #[pyarg(any, default = 0)] count: usize, } #[derive(FromArgs)] struct SplitArgs { string: PyObjectRef, - #[pyarg(any, default = "0")] + #[pyarg(any, default = 0)] maxsplit: isize, } diff --git a/vm/src/stdlib/thread.rs b/vm/src/stdlib/thread.rs index a664d6cdc5..ad80f1f1e1 100644 --- a/vm/src/stdlib/thread.rs +++ b/vm/src/stdlib/thread.rs @@ -52,9 +52,9 @@ pub(crate) mod _thread { #[derive(FromArgs)] struct AcquireArgs { - #[pyarg(any, default = "true")] + #[pyarg(any, default = true)] blocking: bool, - #[pyarg(any, default = "Either::A(-1.0)")] + #[pyarg(any, default = Either::A(-1.0))] timeout: Either, } diff --git a/vm/src/stdlib/winreg.rs b/vm/src/stdlib/winreg.rs index 30fe016148..8d1ca89ddd 100644 --- a/vm/src/stdlib/winreg.rs +++ b/vm/src/stdlib/winreg.rs @@ -149,9 +149,9 @@ mod winreg { struct OpenKeyArgs { key: Hkey, sub_key: Option, - #[pyarg(any, default = "0")] + #[pyarg(any, default = 0)] reserved: i32, - #[pyarg(any, default = "::winreg::enums::KEY_READ")] + #[pyarg(any, default = ::winreg::enums::KEY_READ)] access: u32, } From 160363fa46bc3e71ad3bebf5a7d17590dc57b44c Mon Sep 17 00:00:00 2001 From: Noa Date: Mon, 31 Mar 2025 00:37:47 -0500 Subject: [PATCH 154/295] Fix float parsing (#5643) * Fix float parsing * Add rustpython_literal::complex * Don't call .to_string() on a constant --- Cargo.lock | 12 ++--- Lib/test/test_float.py | 2 - common/src/str.rs | 43 +++++++++++++++++ compiler/codegen/src/unparse.rs | 2 +- compiler/literal/Cargo.toml | 2 +- compiler/literal/src/complex.rs | 73 +++++++++++++++++++++++++++++ compiler/literal/src/float.rs | 82 +-------------------------------- compiler/literal/src/lib.rs | 1 + vm/src/builtins/complex.rs | 45 ++---------------- vm/src/builtins/float.rs | 26 ++++++++++- 10 files changed, 155 insertions(+), 133 deletions(-) create mode 100644 compiler/literal/src/complex.rs diff --git a/Cargo.lock b/Cargo.lock index 265320f990..577ef516bc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1270,9 +1270,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "lexical-parse-float" -version = "0.8.5" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "683b3a5ebd0130b8fb52ba0bdc718cc56815b6a097e28ae5a6997d0ad17dc05f" +checksum = "de6f9cb01fb0b08060209a057c048fcbab8717b4c1ecd2eac66ebfe39a65b0f2" dependencies = [ "lexical-parse-integer", "lexical-util", @@ -1281,9 +1281,9 @@ dependencies = [ [[package]] name = "lexical-parse-integer" -version = "0.8.6" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d0994485ed0c312f6d965766754ea177d07f9c00c9b82a5ee62ed5b47945ee9" +checksum = "72207aae22fc0a121ba7b6d479e42cbfea549af1479c3f3a4f12c70dd66df12e" dependencies = [ "lexical-util", "static_assertions", @@ -1291,9 +1291,9 @@ dependencies = [ [[package]] name = "lexical-util" -version = "0.8.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5255b9ff16ff898710eb9eb63cb39248ea8a5bb036bea8085b1a767ff6c4e3fc" +checksum = "5a82e24bf537fd24c177ffbbdc6ebcc8d54732c35b50a3f28cc3f4e4c949a0b3" dependencies = [ "static_assertions", ] diff --git a/Lib/test/test_float.py b/Lib/test/test_float.py index 19c17af596..30d27072fb 100644 --- a/Lib/test/test_float.py +++ b/Lib/test/test_float.py @@ -35,8 +35,6 @@ class OtherFloatSubclass(float): class GeneralFloatCases(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_float(self): self.assertEqual(float(3.14), 3.14) self.assertEqual(float(314), 314.0) diff --git a/common/src/str.rs b/common/src/str.rs index e72f2efb95..ca1723e7ef 100644 --- a/common/src/str.rs +++ b/common/src/str.rs @@ -609,6 +609,49 @@ macro_rules! ascii { } pub use ascii; +// TODO: this should probably live in a crate like unic or unicode-properties +const UNICODE_DECIMAL_VALUES: &[char] = &[ + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '٠', '١', '٢', '٣', '٤', '٥', '٦', '٧', '٨', + '٩', '۰', '۱', '۲', '۳', '۴', '۵', '۶', '۷', '۸', '۹', '߀', '߁', '߂', '߃', '߄', '߅', '߆', '߇', + '߈', '߉', '०', '१', '२', '३', '४', '५', '६', '७', '८', '९', '০', '১', '২', '৩', '৪', '৫', '৬', + '৭', '৮', '৯', '੦', '੧', '੨', '੩', '੪', '੫', '੬', '੭', '੮', '੯', '૦', '૧', '૨', '૩', '૪', '૫', + '૬', '૭', '૮', '૯', '୦', '୧', '୨', '୩', '୪', '୫', '୬', '୭', '୮', '୯', '௦', '௧', '௨', '௩', '௪', + '௫', '௬', '௭', '௮', '௯', '౦', '౧', '౨', '౩', '౪', '౫', '౬', '౭', '౮', '౯', '೦', '೧', '೨', '೩', + '೪', '೫', '೬', '೭', '೮', '೯', '൦', '൧', '൨', '൩', '൪', '൫', '൬', '൭', '൮', '൯', '෦', '෧', '෨', + '෩', '෪', '෫', '෬', '෭', '෮', '෯', '๐', '๑', '๒', '๓', '๔', '๕', '๖', '๗', '๘', '๙', '໐', '໑', + '໒', '໓', '໔', '໕', '໖', '໗', '໘', '໙', '༠', '༡', '༢', '༣', '༤', '༥', '༦', '༧', '༨', '༩', '၀', + '၁', '၂', '၃', '၄', '၅', '၆', '၇', '၈', '၉', '႐', '႑', '႒', '႓', '႔', '႕', '႖', '႗', '႘', '႙', + '០', '១', '២', '៣', '៤', '៥', '៦', '៧', '៨', '៩', '᠐', '᠑', '᠒', '᠓', '᠔', '᠕', '᠖', '᠗', '᠘', + '᠙', '᥆', '᥇', '᥈', '᥉', '᥊', '᥋', '᥌', '᥍', '᥎', '᥏', '᧐', '᧑', '᧒', '᧓', '᧔', '᧕', '᧖', '᧗', + '᧘', '᧙', '᪀', '᪁', '᪂', '᪃', '᪄', '᪅', '᪆', '᪇', '᪈', '᪉', '᪐', '᪑', '᪒', '᪓', '᪔', '᪕', '᪖', + '᪗', '᪘', '᪙', '᭐', '᭑', '᭒', '᭓', '᭔', '᭕', '᭖', '᭗', '᭘', '᭙', '᮰', '᮱', '᮲', '᮳', '᮴', '᮵', + '᮶', '᮷', '᮸', '᮹', '᱀', '᱁', '᱂', '᱃', '᱄', '᱅', '᱆', '᱇', '᱈', '᱉', '᱐', '᱑', '᱒', '᱓', '᱔', + '᱕', '᱖', '᱗', '᱘', '᱙', '꘠', '꘡', '꘢', '꘣', '꘤', '꘥', '꘦', '꘧', '꘨', '꘩', '꣐', '꣑', '꣒', '꣓', + '꣔', '꣕', '꣖', '꣗', '꣘', '꣙', '꤀', '꤁', '꤂', '꤃', '꤄', '꤅', '꤆', '꤇', '꤈', '꤉', '꧐', '꧑', '꧒', + '꧓', '꧔', '꧕', '꧖', '꧗', '꧘', '꧙', '꧰', '꧱', '꧲', '꧳', '꧴', '꧵', '꧶', '꧷', '꧸', '꧹', '꩐', '꩑', + '꩒', '꩓', '꩔', '꩕', '꩖', '꩗', '꩘', '꩙', '꯰', '꯱', '꯲', '꯳', '꯴', '꯵', '꯶', '꯷', '꯸', '꯹', '０', + '１', '２', '３', '４', '５', '６', '７', '８', '９', '𐒠', '𐒡', '𐒢', '𐒣', '𐒤', '𐒥', '𐒦', '𐒧', + '𐒨', '𐒩', '𑁦', '𑁧', '𑁨', '𑁩', '𑁪', '𑁫', '𑁬', '𑁭', '𑁮', '𑁯', '𑃰', '𑃱', '𑃲', '𑃳', '𑃴', '𑃵', '𑃶', + '𑃷', '𑃸', '𑃹', '𑄶', '𑄷', '𑄸', '𑄹', '𑄺', '𑄻', '𑄼', '𑄽', '𑄾', '𑄿', '𑇐', '𑇑', '𑇒', '𑇓', '𑇔', '𑇕', + '𑇖', '𑇗', '𑇘', '𑇙', '𑋰', '𑋱', '𑋲', '𑋳', '𑋴', '𑋵', '𑋶', '𑋷', '𑋸', '𑋹', '𑑐', '𑑑', '𑑒', '𑑓', '𑑔', + '𑑕', '𑑖', '𑑗', '𑑘', '𑑙', '𑓐', '𑓑', '𑓒', '𑓓', '𑓔', '𑓕', '𑓖', '𑓗', '𑓘', '𑓙', '𑙐', '𑙑', '𑙒', '𑙓', + '𑙔', '𑙕', '𑙖', '𑙗', '𑙘', '𑙙', '𑛀', '𑛁', '𑛂', '𑛃', '𑛄', '𑛅', '𑛆', '𑛇', '𑛈', '𑛉', '𑜰', '𑜱', '𑜲', + '𑜳', '𑜴', '𑜵', '𑜶', '𑜷', '𑜸', '𑜹', '𑣠', '𑣡', '𑣢', '𑣣', '𑣤', '𑣥', '𑣦', '𑣧', '𑣨', '𑣩', '𑱐', '𑱑', + '𑱒', '𑱓', '𑱔', '𑱕', '𑱖', '𑱗', '𑱘', '𑱙', '𑵐', '𑵑', '𑵒', '𑵓', '𑵔', '𑵕', '𑵖', '𑵗', '𑵘', '𑵙', '𖩠', + '𖩡', '𖩢', '𖩣', '𖩤', '𖩥', '𖩦', '𖩧', '𖩨', '𖩩', '𖭐', '𖭑', '𖭒', '𖭓', '𖭔', '𖭕', '𖭖', '𖭗', '𖭘', '𖭙', + '𝟎', '𝟏', '𝟐', '𝟑', '𝟒', '𝟓', '𝟔', '𝟕', '𝟖', '𝟗', '𝟘', '𝟙', '𝟚', '𝟛', '𝟜', '𝟝', '𝟞', '𝟟', '𝟠', + '𝟡', '𝟢', '𝟣', '𝟤', '𝟥', '𝟦', '𝟧', '𝟨', '𝟩', '𝟪', '𝟫', '𝟬', '𝟭', '𝟮', '𝟯', '𝟰', '𝟱', '𝟲', '𝟳', + '𝟴', '𝟵', '𝟶', '𝟷', '𝟸', '𝟹', '𝟺', '𝟻', '𝟼', '𝟽', '𝟾', '𝟿', '𞥐', '𞥑', '𞥒', '𞥓', '𞥔', '𞥕', '𞥖', + '𞥗', '𞥘', '𞥙', +]; + +pub fn char_to_decimal(ch: char) -> Option { + UNICODE_DECIMAL_VALUES + .binary_search(&ch) + .ok() + .map(|i| (i % 10) as u8) +} + #[cfg(test)] mod tests { use super::*; diff --git a/compiler/codegen/src/unparse.rs b/compiler/codegen/src/unparse.rs index 458ff76fc7..1ecf1f9334 100644 --- a/compiler/codegen/src/unparse.rs +++ b/compiler/codegen/src/unparse.rs @@ -366,7 +366,7 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { } } &ruff::Number::Complex { real, imag } => self - .p(&rustpython_literal::float::complex_to_string(real, imag) + .p(&rustpython_literal::complex::to_string(real, imag) .replace("inf", inf_str))?, } } diff --git a/compiler/literal/Cargo.toml b/compiler/literal/Cargo.toml index b4fa6229f1..da55d107b3 100644 --- a/compiler/literal/Cargo.toml +++ b/compiler/literal/Cargo.toml @@ -13,7 +13,7 @@ rustpython-wtf8 = { workspace = true } hexf-parse = "0.2.1" is-macro.workspace = true -lexical-parse-float = { version = "0.8.0", features = ["format"] } +lexical-parse-float = { version = "1.0.4", features = ["format"] } num-traits = { workspace = true } unic-ucd-category = { workspace = true } diff --git a/compiler/literal/src/complex.rs b/compiler/literal/src/complex.rs new file mode 100644 index 0000000000..076f2807c9 --- /dev/null +++ b/compiler/literal/src/complex.rs @@ -0,0 +1,73 @@ +use crate::float; + +/// Convert a complex number to a string. +pub fn to_string(re: f64, im: f64) -> String { + // integer => drop ., fractional => float_ops + let mut im_part = if im.fract() == 0.0 { + im.to_string() + } else { + float::to_string(im) + }; + im_part.push('j'); + + // positive empty => return im_part, integer => drop ., fractional => float_ops + let re_part = if re == 0.0 { + if re.is_sign_positive() { + return im_part; + } else { + "-0".to_owned() + } + } else if re.fract() == 0.0 { + re.to_string() + } else { + float::to_string(re) + }; + let mut result = + String::with_capacity(re_part.len() + im_part.len() + 2 + im.is_sign_positive() as usize); + result.push('('); + result.push_str(&re_part); + if im.is_sign_positive() || im.is_nan() { + result.push('+'); + } + result.push_str(&im_part); + result.push(')'); + result +} + +/// Parse a complex number from a string. +/// +/// Returns `Some((re, im))` on success. +pub fn parse_str(s: &str) -> Option<(f64, f64)> { + let s = s.trim(); + // Handle parentheses + let s = match s.strip_prefix('(') { + None => s, + Some(s) => s.strip_suffix(')')?.trim(), + }; + + let value = match s.strip_suffix(|c| c == 'j' || c == 'J') { + None => (float::parse_str(s)?, 0.0), + Some(mut s) => { + let mut real = 0.0; + // Find the central +/- operator. If it exists, parse the real part. + for (i, w) in s.as_bytes().windows(2).enumerate() { + if (w[1] == b'+' || w[1] == b'-') && !(w[0] == b'e' || w[0] == b'E') { + real = float::parse_str(&s[..=i])?; + s = &s[i + 1..]; + break; + } + } + + let imag = match s { + // "j", "+j" + "" | "+" => 1.0, + // "-j" + "-" => -1.0, + s => float::parse_str(s)?, + }; + + (real, imag) + } + }; + Some(value) +} diff --git a/compiler/literal/src/float.rs b/compiler/literal/src/float.rs index e05a105fd4..49771b8184 100644 --- a/compiler/literal/src/float.rs +++ b/compiler/literal/src/float.rs @@ -6,49 +6,8 @@ pub fn parse_str(literal: &str) -> Option { parse_inner(literal.trim().as_bytes()) } -fn strip_underlines(literal: &[u8]) -> Option> { - let mut prev = b'\0'; - let mut dup = Vec::::new(); - for p in literal { - if *p == b'_' { - // Underscores are only allowed after digits. - if !prev.is_ascii_digit() { - return None; - } - } else { - dup.push(*p); - // Underscores are only allowed before digits. - if prev == b'_' && !p.is_ascii_digit() { - return None; - } - } - prev = *p; - } - - // Underscores are not allowed at the end. - if prev == b'_' { - return None; - } - - Some(dup) -} - pub fn parse_bytes(literal: &[u8]) -> Option { - parse_inner(trim_slice(literal, |b| b.is_ascii_whitespace())) -} - -fn trim_slice(v: &[T], mut trim: impl FnMut(&T) -> bool) -> &[T] { - let mut it = v.iter(); - // it.take_while_ref(&mut trim).for_each(drop); - // hmm.. `&mut slice::Iter<_>` is not `Clone` - // it.by_ref().rev().take_while_ref(&mut trim).for_each(drop); - while it.clone().next().is_some_and(&mut trim) { - it.next(); - } - while it.clone().next_back().is_some_and(&mut trim) { - it.next_back(); - } - it.as_slice() + parse_inner(literal.trim_ascii()) } fn parse_inner(literal: &[u8]) -> Option { @@ -56,15 +15,11 @@ fn parse_inner(literal: &[u8]) -> Option { FromLexicalWithOptions, NumberFormatBuilder, Options, format::PYTHON3_LITERAL, }; - // Use custom function for underline handling for now. - // For further information see https://github.com/Alexhuszagh/rust-lexical/issues/96. - let stripped = strip_underlines(literal)?; - // lexical-core's format::PYTHON_STRING is inaccurate const PYTHON_STRING: u128 = NumberFormatBuilder::rebuild(PYTHON3_LITERAL) .no_special(false) .build(); - f64::from_lexical_with_options::(&stripped, &Options::new()).ok() + f64::from_lexical_with_options::(literal, &Options::new()).ok() } pub fn is_integer(v: f64) -> bool { @@ -223,39 +178,6 @@ pub fn to_string(value: f64) -> String { } } -pub fn complex_to_string(re: f64, im: f64) -> String { - // integer => drop ., fractional => float_ops - let mut im_part = if im.fract() == 0.0 { - im.to_string() - } else { - to_string(im) - }; - im_part.push('j'); - - // positive empty => return im_part, integer => drop ., fractional => float_ops - let re_part = if re == 0.0 { - if re.is_sign_positive() { - return im_part; - } else { - re.to_string() - } - } else if re.fract() == 0.0 { - re.to_string() - } else { - to_string(re) - }; - let mut result = - String::with_capacity(re_part.len() + im_part.len() + 2 + im.is_sign_positive() as usize); - result.push('('); - result.push_str(&re_part); - if im.is_sign_positive() || im.is_nan() { - result.push('+'); - } - result.push_str(&im_part); - result.push(')'); - result -} - pub fn from_hex(s: &str) -> Option { if let Ok(f) = hexf_parse::parse_hexf64(s, false) { return Some(f); diff --git a/compiler/literal/src/lib.rs b/compiler/literal/src/lib.rs index 9b9620573d..2997107012 100644 --- a/compiler/literal/src/lib.rs +++ b/compiler/literal/src/lib.rs @@ -1,4 +1,5 @@ pub mod char; +pub mod complex; pub mod escape; pub mod float; pub mod format; diff --git a/vm/src/builtins/complex.rs b/vm/src/builtins/complex.rs index a3a6d4d681..d48707261c 100644 --- a/vm/src/builtins/complex.rs +++ b/vm/src/builtins/complex.rs @@ -179,13 +179,13 @@ impl Constructor for PyComplex { "complex() can't take second arg if first is a string".to_owned(), )); } - let value = s + let (re, im) = s .to_str() - .and_then(|s| parse_str(s.trim())) + .and_then(rustpython_literal::complex::parse_str) .ok_or_else(|| { vm.new_value_error("complex() arg is a malformed string".to_owned()) })?; - return Self::from(value) + return Self::from(Complex64 { re, im }) .into_ref_with_type(vm, cls) .map(Into::into); } else { @@ -494,7 +494,7 @@ impl Representable for PyComplex { // TODO: when you fix this, move it to rustpython_common::complex::repr and update // ast/src/unparse.rs + impl Display for Constant in ast/src/constant.rs let Complex64 { re, im } = zelf.value; - Ok(rustpython_literal::float::complex_to_string(re, im)) + Ok(rustpython_literal::complex::to_string(re, im)) } } @@ -519,40 +519,3 @@ pub struct ComplexArgs { #[pyarg(any, optional)] imag: OptionalArg, } - -fn parse_str(s: &str) -> Option { - // Handle parentheses - let s = match s.strip_prefix('(') { - None => s, - Some(s) => match s.strip_suffix(')') { - None => return None, - Some(s) => s.trim(), - }, - }; - - let value = match s.strip_suffix(|c| c == 'j' || c == 'J') { - None => Complex64::new(crate::literal::float::parse_str(s)?, 0.0), - Some(mut s) => { - let mut real = 0.0; - // Find the central +/- operator. If it exists, parse the real part. - for (i, w) in s.as_bytes().windows(2).enumerate() { - if (w[1] == b'+' || w[1] == b'-') && !(w[0] == b'e' || w[0] == b'E') { - real = crate::literal::float::parse_str(&s[..=i])?; - s = &s[i + 1..]; - break; - } - } - - let imag = match s { - // "j", "+j" - "" | "+" => 1.0, - // "-j" - "-" => -1.0, - s => crate::literal::float::parse_str(s)?, - }; - - Complex64::new(real, imag) - } - }; - Some(value) -} diff --git a/vm/src/builtins/float.rs b/vm/src/builtins/float.rs index 48ccd2c437..27f1f3273f 100644 --- a/vm/src/builtins/float.rs +++ b/vm/src/builtins/float.rs @@ -159,9 +159,31 @@ impl Constructor for PyFloat { } fn float_from_string(val: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let (bytearray, buffer, buffer_lock); + let (bytearray, buffer, buffer_lock, mapped_string); let b = if let Some(s) = val.payload_if_subclass::(vm) { - s.as_wtf8().trim().as_bytes() + use crate::common::str::PyKindStr; + match s.as_str_kind() { + PyKindStr::Ascii(s) => s.trim().as_bytes(), + PyKindStr::Utf8(s) => { + mapped_string = s + .trim() + .chars() + .map(|c| { + if let Some(n) = rustpython_common::str::char_to_decimal(c) { + char::from_digit(n.into(), 10).unwrap() + } else if c.is_whitespace() { + ' ' + } else { + c + } + }) + .collect::(); + mapped_string.as_bytes() + } + // if there are surrogates, it's not gonna parse anyway, + // so we can just choose a known bad value + PyKindStr::Wtf8(_) => b"", + } } else if let Some(bytes) = val.payload_if_subclass::(vm) { bytes.as_bytes() } else if let Some(buf) = val.payload_if_subclass::(vm) { From 3ad8fd711fc496526cc2a5bd320bbdfccfc99574 Mon Sep 17 00:00:00 2001 From: ivan-shrimp <70307174+ivan-shrimp@users.noreply.github.com> Date: Sun, 2 Mar 2025 23:50:36 +0800 Subject: [PATCH 155/295] fix expression list order don't emit a no-op when unpacking a single element assume positional args stored as tuple in extended call --- compiler/codegen/src/compile.rs | 57 ++++++++++++++-------------- compiler/core/src/bytecode.rs | 23 ++++++----- extra_tests/snippets/builtin_list.py | 21 ++++++++++ vm/src/frame.rs | 54 ++++++++++++++++---------- 4 files changed, 95 insertions(+), 60 deletions(-) diff --git a/compiler/codegen/src/compile.rs b/compiler/codegen/src/compile.rs index 38d5b8fb12..83e2f5cf44 100644 --- a/compiler/codegen/src/compile.rs +++ b/compiler/codegen/src/compile.rs @@ -2430,7 +2430,7 @@ impl Compiler<'_> { Expr::List(ExprList { elts, .. }) => { let (size, unpack) = self.gather_elements(0, elts)?; if unpack { - emit!(self, Instruction::BuildListUnpack { size }); + emit!(self, Instruction::BuildListFromTuples { size }); } else { emit!(self, Instruction::BuildList { size }); } @@ -2438,7 +2438,9 @@ impl Compiler<'_> { Expr::Tuple(ExprTuple { elts, .. }) => { let (size, unpack) = self.gather_elements(0, elts)?; if unpack { - emit!(self, Instruction::BuildTupleUnpack { size }); + if size > 1 { + emit!(self, Instruction::BuildTupleFromTuples { size }); + } } else { emit!(self, Instruction::BuildTuple { size }); } @@ -2446,7 +2448,7 @@ impl Compiler<'_> { Expr::Set(ExprSet { elts, .. }) => { let (size, unpack) = self.gather_elements(0, elts)?; if unpack { - emit!(self, Instruction::BuildSetUnpack { size }); + emit!(self, Instruction::BuildSetFromTuples { size }); } else { emit!(self, Instruction::BuildSet { size }); } @@ -2819,7 +2821,7 @@ impl Compiler<'_> { let call = if unpack || has_double_star { // Create a tuple with positional args: if unpack { - emit!(self, Instruction::BuildTupleUnpack { size }); + emit!(self, Instruction::BuildTupleFromTuples { size }); } else { emit!(self, Instruction::BuildTuple { size }); } @@ -2863,34 +2865,31 @@ impl Compiler<'_> { let size = if has_stars { let mut size = 0; + let mut iter = elements.iter().peekable(); + let mut run_size = before; - if before > 0 { - emit!(self, Instruction::BuildTuple { size: before }); - size += 1; - } + loop { + if iter.peek().is_none_or(|e| matches!(e, Expr::Starred(_))) { + emit!(self, Instruction::BuildTuple { size: run_size }); + run_size = 0; + size += 1; + } - let groups = elements - .iter() - .map(|element| { - if let Expr::Starred(ExprStarred { value, .. }) = &element { - (true, value.as_ref()) - } else { - (false, element) + match iter.next() { + Some(Expr::Starred(ExprStarred { value, .. })) => { + self.compile_expression(value)?; + // We need to collect each unpacked element into a + // tuple, since any side-effects during the conversion + // should be made visible before evaluating remaining + // expressions. + emit!(self, Instruction::BuildTupleFromIter); + size += 1; } - }) - .chunk_by(|(starred, _)| *starred); - - for (starred, run) in &groups { - let mut run_size = 0; - for (_, value) in run { - self.compile_expression(value)?; - run_size += 1 - } - if starred { - size += run_size - } else { - emit!(self, Instruction::BuildTuple { size: run_size }); - size += 1 + Some(element) => { + self.compile_expression(element)?; + run_size += 1; + } + None => break, } } diff --git a/compiler/core/src/bytecode.rs b/compiler/core/src/bytecode.rs index 2e8ff29014..94d080ace4 100644 --- a/compiler/core/src/bytecode.rs +++ b/compiler/core/src/bytecode.rs @@ -542,19 +542,20 @@ pub enum Instruction { BuildTuple { size: Arg, }, - BuildTupleUnpack { + BuildTupleFromTuples { size: Arg, }, + BuildTupleFromIter, BuildList { size: Arg, }, - BuildListUnpack { + BuildListFromTuples { size: Arg, }, BuildSet { size: Arg, }, - BuildSetUnpack { + BuildSetFromTuples { size: Arg, }, BuildMap { @@ -1260,11 +1261,12 @@ impl Instruction { Raise { kind } => -(kind.get(arg) as u8 as i32), BuildString { size } | BuildTuple { size, .. } - | BuildTupleUnpack { size, .. } + | BuildTupleFromTuples { size, .. } | BuildList { size, .. } - | BuildListUnpack { size, .. } + | BuildListFromTuples { size, .. } | BuildSet { size, .. } - | BuildSetUnpack { size, .. } => -(size.get(arg) as i32) + 1, + | BuildSetFromTuples { size, .. } => -(size.get(arg) as i32) + 1, + BuildTupleFromIter => 0, BuildMap { size } => { let nargs = size.get(arg) * 2; -(nargs as i32) + 1 @@ -1447,13 +1449,14 @@ impl Instruction { Raise { kind } => w!(Raise, ?kind), BuildString { size } => w!(BuildString, size), BuildTuple { size } => w!(BuildTuple, size), - BuildTupleUnpack { size } => w!(BuildTupleUnpack, size), + BuildTupleFromTuples { size } => w!(BuildTupleFromTuples, size), + BuildTupleFromIter => w!(BuildTupleFromIter), BuildList { size } => w!(BuildList, size), - BuildListUnpack { size } => w!(BuildListUnpack, size), + BuildListFromTuples { size } => w!(BuildListFromTuples, size), BuildSet { size } => w!(BuildSet, size), - BuildSetUnpack { size } => w!(BuildSetUnpack, size), + BuildSetFromTuples { size } => w!(BuildSetFromTuples, size), BuildMap { size } => w!(BuildMap, size), - BuildMapForCall { size } => w!(BuildMap, size), + BuildMapForCall { size } => w!(BuildMapForCall, size), DictUpdate => w!(DictUpdate), BuildSlice { step } => w!(BuildSlice, step), ListAppend { i } => w!(ListAppend, i), diff --git a/extra_tests/snippets/builtin_list.py b/extra_tests/snippets/builtin_list.py index 6fde1011ea..b5c08796ba 100644 --- a/extra_tests/snippets/builtin_list.py +++ b/extra_tests/snippets/builtin_list.py @@ -636,6 +636,27 @@ def iadd_slice(): a = [*[1, 2], 3, *[4, 5]] assert a == [1, 2, 3, 4, 5] +# Test for list unpacking evaluation order (https://github.com/RustPython/RustPython/issues/5566) +a = [1, 2] +b = [a.append(3), *a, a.append(4), *a] +assert a == [1, 2, 3, 4] +assert b == [None, 1, 2, 3, None, 1, 2, 3, 4] + +for base in object, list, tuple: + # do not assume that a type inherited from some sequence type behaves like + # that sequence type + class C(base): + def __iter__(self): + a.append(2) + def inner(): + yield 3 + a.append(4) + return inner() + + a = [1] + b = [*a, *C(), *a.copy()] + assert b == [1, 3, 1, 2, 4] + # Test for list entering daedlock or not (https://github.com/RustPython/RustPython/pull/2933) class MutatingCompare: def __eq__(self, other): diff --git a/vm/src/frame.rs b/vm/src/frame.rs index b148934467..78f03a04d8 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -710,27 +710,28 @@ impl ExecutingFrame<'_> { self.push_value(list_obj.into()); Ok(None) } - bytecode::Instruction::BuildListUnpack { size } => { - let elements = self.unpack_elements(vm, size.get(arg) as usize)?; + bytecode::Instruction::BuildListFromTuples { size } => { + // SAFETY: compiler guarantees `size` tuples are on the stack + let elements = unsafe { self.flatten_tuples(size.get(arg) as usize) }; let list_obj = vm.ctx.new_list(elements); self.push_value(list_obj.into()); Ok(None) } bytecode::Instruction::BuildSet { size } => { let set = PySet::new_ref(&vm.ctx); - { - for element in self.pop_multiple(size.get(arg) as usize) { - set.add(element, vm)?; - } + for element in self.pop_multiple(size.get(arg) as usize) { + set.add(element, vm)?; } self.push_value(set.into()); Ok(None) } - bytecode::Instruction::BuildSetUnpack { size } => { + bytecode::Instruction::BuildSetFromTuples { size } => { let set = PySet::new_ref(&vm.ctx); - { - for element in self.pop_multiple(size.get(arg) as usize) { - vm.map_iterable_object(&element, |x| set.add(x, vm))??; + for element in self.pop_multiple(size.get(arg) as usize) { + // SAFETY: trust compiler + let tup = unsafe { element.downcast_unchecked::() }; + for item in tup.iter() { + set.add(item.clone(), vm)?; } } self.push_value(set.into()); @@ -742,12 +743,21 @@ impl ExecutingFrame<'_> { self.push_value(list_obj.into()); Ok(None) } - bytecode::Instruction::BuildTupleUnpack { size } => { - let elements = self.unpack_elements(vm, size.get(arg) as usize)?; + bytecode::Instruction::BuildTupleFromTuples { size } => { + // SAFETY: compiler guarantees `size` tuples are on the stack + let elements = unsafe { self.flatten_tuples(size.get(arg) as usize) }; let list_obj = vm.ctx.new_tuple(elements); self.push_value(list_obj.into()); Ok(None) } + bytecode::Instruction::BuildTupleFromIter => { + if !self.top_value().class().is(vm.ctx.types.tuple_type) { + let elements: Vec<_> = self.pop_value().try_to_value(vm)?; + let list_obj = vm.ctx.new_tuple(elements); + self.push_value(list_obj.into()); + } + Ok(None) + } bytecode::Instruction::BuildMap { size } => self.execute_build_map(vm, size.get(arg)), bytecode::Instruction::BuildMapForCall { size } => { self.execute_build_map_for_call(vm, size.get(arg)) @@ -1234,14 +1244,14 @@ impl ExecutingFrame<'_> { }) } - #[cfg_attr(feature = "flame-it", flame("Frame"))] - fn unpack_elements(&mut self, vm: &VirtualMachine, size: usize) -> PyResult> { - let mut result = Vec::::new(); - for element in self.pop_multiple(size) { - let items: Vec<_> = element.try_to_value(vm)?; - result.extend(items); + unsafe fn flatten_tuples(&mut self, size: usize) -> Vec { + let mut elements = Vec::new(); + for tup in self.pop_multiple(size) { + // SAFETY: caller ensures that the elements are tuples + let tup = unsafe { tup.downcast_unchecked::() }; + elements.extend(tup.iter().cloned()); } - Ok(result) + elements } #[cfg_attr(feature = "flame-it", flame("Frame"))] @@ -1490,8 +1500,10 @@ impl ExecutingFrame<'_> { } else { IndexMap::new() }; - let args = self.pop_value(); - let args = args.try_to_value(vm)?; + // SAFETY: trust compiler + let args = unsafe { self.pop_value().downcast_unchecked::() } + .as_slice() + .to_vec(); Ok(FuncArgs { args, kwargs }) } From 57a83db69de3a93b465762aeb72553a3043d7dbf Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Mon, 31 Mar 2025 17:14:59 +0900 Subject: [PATCH 156/295] try IncrementalNewlineDecoder in doctest --- Lib/doctest.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Lib/doctest.py b/Lib/doctest.py index 65466b4983..387f71b184 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -102,7 +102,7 @@ def _test(): import sys import traceback import unittest -from io import StringIO # XXX: RUSTPYTHON; , IncrementalNewlineDecoder +from io import StringIO, IncrementalNewlineDecoder from collections import namedtuple TestResults = namedtuple('TestResults', 'failed attempted') @@ -230,9 +230,7 @@ def _load_testfile(filename, package, module_relative, encoding): # get_data() opens files as 'rb', so one must do the equivalent # conversion as universal newlines would do. - # TODO: RUSTPYTHON; use _newline_convert once io.IncrementalNewlineDecoder is implemented - return file_contents.replace(os.linesep, '\n'), filename - # return _newline_convert(file_contents), filename + return _newline_convert(file_contents), filename with open(filename, encoding=encoding) as f: return f.read(), filename From f0bcad711657808dd30836d6630ec8b2f0851b5d Mon Sep 17 00:00:00 2001 From: Hanif Ariffin Date: Mon, 31 Mar 2025 00:05:31 +0800 Subject: [PATCH 157/295] Added test_audit This test is currently noop because RustPython does not have sys.audit? Signed-off-by: Hanif Ariffin --- Lib/test/test_audit.py | 318 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 318 insertions(+) create mode 100644 Lib/test/test_audit.py diff --git a/Lib/test/test_audit.py b/Lib/test/test_audit.py new file mode 100644 index 0000000000..ddd9f95114 --- /dev/null +++ b/Lib/test/test_audit.py @@ -0,0 +1,318 @@ +"""Tests for sys.audit and sys.addaudithook +""" + +import subprocess +import sys +import unittest +from test import support +from test.support import import_helper +from test.support import os_helper + + +if not hasattr(sys, "addaudithook") or not hasattr(sys, "audit"): + raise unittest.SkipTest("test only relevant when sys.audit is available") + +AUDIT_TESTS_PY = support.findfile("audit-tests.py") + + +class AuditTest(unittest.TestCase): + maxDiff = None + + @support.requires_subprocess() + def run_test_in_subprocess(self, *args): + with subprocess.Popen( + [sys.executable, "-X utf8", AUDIT_TESTS_PY, *args], + encoding="utf-8", + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) as p: + p.wait() + return p, p.stdout.read(), p.stderr.read() + + def do_test(self, *args): + proc, stdout, stderr = self.run_test_in_subprocess(*args) + + sys.stdout.write(stdout) + sys.stderr.write(stderr) + if proc.returncode: + self.fail(stderr) + + def run_python(self, *args, expect_stderr=False): + events = [] + proc, stdout, stderr = self.run_test_in_subprocess(*args) + if not expect_stderr or support.verbose: + sys.stderr.write(stderr) + return ( + proc.returncode, + [line.strip().partition(" ") for line in stdout.splitlines()], + stderr, + ) + + def test_basic(self): + self.do_test("test_basic") + + def test_block_add_hook(self): + self.do_test("test_block_add_hook") + + def test_block_add_hook_baseexception(self): + self.do_test("test_block_add_hook_baseexception") + + def test_marshal(self): + import_helper.import_module("marshal") + + self.do_test("test_marshal") + + def test_pickle(self): + import_helper.import_module("pickle") + + self.do_test("test_pickle") + + def test_monkeypatch(self): + self.do_test("test_monkeypatch") + + def test_open(self): + self.do_test("test_open", os_helper.TESTFN) + + def test_cantrace(self): + self.do_test("test_cantrace") + + def test_mmap(self): + self.do_test("test_mmap") + + def test_excepthook(self): + returncode, events, stderr = self.run_python("test_excepthook") + if not returncode: + self.fail(f"Expected fatal exception\n{stderr}") + + self.assertSequenceEqual( + [("sys.excepthook", " ", "RuntimeError('fatal-error')")], events + ) + + def test_unraisablehook(self): + import_helper.import_module("_testcapi") + returncode, events, stderr = self.run_python("test_unraisablehook") + if returncode: + self.fail(stderr) + + self.assertEqual(events[0][0], "sys.unraisablehook") + self.assertEqual( + events[0][2], + "RuntimeError('nonfatal-error') Exception ignored for audit hook test", + ) + + def test_winreg(self): + import_helper.import_module("winreg") + returncode, events, stderr = self.run_python("test_winreg") + if returncode: + self.fail(stderr) + + self.assertEqual(events[0][0], "winreg.OpenKey") + self.assertEqual(events[1][0], "winreg.OpenKey/result") + expected = events[1][2] + self.assertTrue(expected) + self.assertSequenceEqual(["winreg.EnumKey", " ", f"{expected} 0"], events[2]) + self.assertSequenceEqual(["winreg.EnumKey", " ", f"{expected} 10000"], events[3]) + self.assertSequenceEqual(["winreg.PyHKEY.Detach", " ", expected], events[4]) + + def test_socket(self): + import_helper.import_module("socket") + returncode, events, stderr = self.run_python("test_socket") + if returncode: + self.fail(stderr) + + if support.verbose: + print(*events, sep='\n') + self.assertEqual(events[0][0], "socket.gethostname") + self.assertEqual(events[1][0], "socket.__new__") + self.assertEqual(events[2][0], "socket.bind") + self.assertTrue(events[2][2].endswith("('127.0.0.1', 8080)")) + + def test_gc(self): + returncode, events, stderr = self.run_python("test_gc") + if returncode: + self.fail(stderr) + + if support.verbose: + print(*events, sep='\n') + self.assertEqual( + [event[0] for event in events], + ["gc.get_objects", "gc.get_referrers", "gc.get_referents"] + ) + + + @support.requires_resource('network') + def test_http(self): + import_helper.import_module("http.client") + returncode, events, stderr = self.run_python("test_http_client") + if returncode: + self.fail(stderr) + + if support.verbose: + print(*events, sep='\n') + self.assertEqual(events[0][0], "http.client.connect") + self.assertEqual(events[0][2], "www.python.org 80") + self.assertEqual(events[1][0], "http.client.send") + if events[1][2] != '[cannot send]': + self.assertIn('HTTP', events[1][2]) + + + def test_sqlite3(self): + sqlite3 = import_helper.import_module("sqlite3") + returncode, events, stderr = self.run_python("test_sqlite3") + if returncode: + self.fail(stderr) + + if support.verbose: + print(*events, sep='\n') + actual = [ev[0] for ev in events] + expected = ["sqlite3.connect", "sqlite3.connect/handle"] * 2 + + if hasattr(sqlite3.Connection, "enable_load_extension"): + expected += [ + "sqlite3.enable_load_extension", + "sqlite3.load_extension", + ] + self.assertEqual(actual, expected) + + + def test_sys_getframe(self): + returncode, events, stderr = self.run_python("test_sys_getframe") + if returncode: + self.fail(stderr) + + if support.verbose: + print(*events, sep='\n') + actual = [(ev[0], ev[2]) for ev in events] + expected = [("sys._getframe", "test_sys_getframe")] + + self.assertEqual(actual, expected) + + def test_sys_getframemodulename(self): + returncode, events, stderr = self.run_python("test_sys_getframemodulename") + if returncode: + self.fail(stderr) + + if support.verbose: + print(*events, sep='\n') + actual = [(ev[0], ev[2]) for ev in events] + expected = [("sys._getframemodulename", "0")] + + self.assertEqual(actual, expected) + + + def test_threading(self): + returncode, events, stderr = self.run_python("test_threading") + if returncode: + self.fail(stderr) + + if support.verbose: + print(*events, sep='\n') + actual = [(ev[0], ev[2]) for ev in events] + expected = [ + ("_thread.start_new_thread", "(, (), None)"), + ("test.test_func", "()"), + ("_thread.start_joinable_thread", "(, 1, None)"), + ("test.test_func", "()"), + ] + + self.assertEqual(actual, expected) + + + def test_wmi_exec_query(self): + import_helper.import_module("_wmi") + returncode, events, stderr = self.run_python("test_wmi_exec_query") + if returncode: + self.fail(stderr) + + if support.verbose: + print(*events, sep='\n') + actual = [(ev[0], ev[2]) for ev in events] + expected = [("_wmi.exec_query", "SELECT * FROM Win32_OperatingSystem")] + + self.assertEqual(actual, expected) + + def test_syslog(self): + syslog = import_helper.import_module("syslog") + + returncode, events, stderr = self.run_python("test_syslog") + if returncode: + self.fail(stderr) + + if support.verbose: + print('Events:', *events, sep='\n ') + + self.assertSequenceEqual( + events, + [('syslog.openlog', ' ', f'python 0 {syslog.LOG_USER}'), + ('syslog.syslog', ' ', f'{syslog.LOG_INFO} test'), + ('syslog.setlogmask', ' ', f'{syslog.LOG_DEBUG}'), + ('syslog.closelog', '', ''), + ('syslog.syslog', ' ', f'{syslog.LOG_INFO} test2'), + ('syslog.openlog', ' ', f'audit-tests.py 0 {syslog.LOG_USER}'), + ('syslog.openlog', ' ', f'audit-tests.py {syslog.LOG_NDELAY} {syslog.LOG_LOCAL0}'), + ('syslog.openlog', ' ', f'None 0 {syslog.LOG_USER}'), + ('syslog.closelog', '', '')] + ) + + def test_not_in_gc(self): + returncode, _, stderr = self.run_python("test_not_in_gc") + if returncode: + self.fail(stderr) + + def test_time(self): + returncode, events, stderr = self.run_python("test_time", "print") + if returncode: + self.fail(stderr) + + if support.verbose: + print(*events, sep='\n') + + actual = [(ev[0], ev[2]) for ev in events] + expected = [("time.sleep", "0"), + ("time.sleep", "0.0625"), + ("time.sleep", "-1")] + + self.assertEqual(actual, expected) + + def test_time_fail(self): + returncode, events, stderr = self.run_python("test_time", "fail", + expect_stderr=True) + self.assertNotEqual(returncode, 0) + self.assertIn('hook failed', stderr.splitlines()[-1]) + + def test_sys_monitoring_register_callback(self): + returncode, events, stderr = self.run_python("test_sys_monitoring_register_callback") + if returncode: + self.fail(stderr) + + if support.verbose: + print(*events, sep='\n') + actual = [(ev[0], ev[2]) for ev in events] + expected = [("sys.monitoring.register_callback", "(None,)")] + + self.assertEqual(actual, expected) + + def test_winapi_createnamedpipe(self): + winapi = import_helper.import_module("_winapi") + + pipe_name = r"\\.\pipe\LOCAL\test_winapi_createnamed_pipe" + returncode, events, stderr = self.run_python("test_winapi_createnamedpipe", pipe_name) + if returncode: + self.fail(stderr) + + if support.verbose: + print(*events, sep='\n') + actual = [(ev[0], ev[2]) for ev in events] + expected = [("_winapi.CreateNamedPipe", f"({pipe_name!r}, 3, 8)")] + + self.assertEqual(actual, expected) + + def test_assert_unicode(self): + # See gh-126018 + returncode, _, stderr = self.run_python("test_assert_unicode") + if returncode: + self.fail(stderr) + + +if __name__ == "__main__": + unittest.main() From cb967c697b1f627b96c884763ff882a55fbd56cc Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Mon, 31 Mar 2025 18:17:29 -0700 Subject: [PATCH 158/295] Fix release CI (#5647) --- .github/workflows/release.yml | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b4e209d0b0..3b0a797e0c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -58,6 +58,7 @@ jobs: - name: Set up Windows Environment shell: bash run: | + git config --global core.longpaths true cargo install --target-dir=target -v cargo-vcpkg cargo vcpkg -v build if: runner.os == 'Windows' @@ -93,17 +94,6 @@ jobs: name: rustpython-installer-msi-${{ runner.os }}-${{ matrix.platform.target }} path: installer/*.msi - - name: Generate DMG - if: runner.os == 'macOS' - run: cargo packager -f dmg --release -o installer - - - name: Upload DMG - if: runner.os == 'macOS' - uses: actions/upload-artifact@v4 - with: - name: rustpython-installer-dmg-${{ runner.os }}-${{ matrix.platform.target }} - path: installer/*.dmg - - name: Upload Binary Artifacts uses: actions/upload-artifact@v4 with: From 264f3d792ac11fe6469d7250298e7e264a55a238 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Mon, 31 Mar 2025 18:18:42 -0700 Subject: [PATCH 159/295] remove xdrlib deprecated in 3.13 (#5648) --- Lib/test/test_xdrlib.py | 81 -------------- Lib/xdrlib.py | 241 ---------------------------------------- 2 files changed, 322 deletions(-) delete mode 100644 Lib/test/test_xdrlib.py delete mode 100644 Lib/xdrlib.py diff --git a/Lib/test/test_xdrlib.py b/Lib/test/test_xdrlib.py deleted file mode 100644 index 5f62649f6e..0000000000 --- a/Lib/test/test_xdrlib.py +++ /dev/null @@ -1,81 +0,0 @@ -import unittest - -import xdrlib - -class XDRTest(unittest.TestCase): - - def test_xdr(self): - p = xdrlib.Packer() - - s = b'hello world' - a = [b'what', b'is', b'hapnin', b'doctor'] - - p.pack_int(42) - p.pack_int(-17) - p.pack_uint(9) - p.pack_bool(True) - p.pack_bool(False) - p.pack_uhyper(45) - p.pack_float(1.9) - p.pack_double(1.9) - p.pack_string(s) - p.pack_list(range(5), p.pack_uint) - p.pack_array(a, p.pack_string) - - # now verify - data = p.get_buffer() - up = xdrlib.Unpacker(data) - - self.assertEqual(up.get_position(), 0) - - self.assertEqual(up.unpack_int(), 42) - self.assertEqual(up.unpack_int(), -17) - self.assertEqual(up.unpack_uint(), 9) - self.assertTrue(up.unpack_bool() is True) - - # remember position - pos = up.get_position() - self.assertTrue(up.unpack_bool() is False) - - # rewind and unpack again - up.set_position(pos) - self.assertTrue(up.unpack_bool() is False) - - self.assertEqual(up.unpack_uhyper(), 45) - self.assertAlmostEqual(up.unpack_float(), 1.9) - self.assertAlmostEqual(up.unpack_double(), 1.9) - self.assertEqual(up.unpack_string(), s) - self.assertEqual(up.unpack_list(up.unpack_uint), list(range(5))) - self.assertEqual(up.unpack_array(up.unpack_string), a) - up.done() - self.assertRaises(EOFError, up.unpack_uint) - -class ConversionErrorTest(unittest.TestCase): - - def setUp(self): - self.packer = xdrlib.Packer() - - def assertRaisesConversion(self, *args): - self.assertRaises(xdrlib.ConversionError, *args) - - def test_pack_int(self): - self.assertRaisesConversion(self.packer.pack_int, 'string') - - def test_pack_uint(self): - self.assertRaisesConversion(self.packer.pack_uint, 'string') - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_float(self): - self.assertRaisesConversion(self.packer.pack_float, 'string') - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_double(self): - self.assertRaisesConversion(self.packer.pack_double, 'string') - - def test_uhyper(self): - self.assertRaisesConversion(self.packer.pack_uhyper, 'string') - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/xdrlib.py b/Lib/xdrlib.py deleted file mode 100644 index d6e1aeb527..0000000000 --- a/Lib/xdrlib.py +++ /dev/null @@ -1,241 +0,0 @@ -"""Implements (a subset of) Sun XDR -- eXternal Data Representation. - -See: RFC 1014 - -""" - -import struct -from io import BytesIO -from functools import wraps - -__all__ = ["Error", "Packer", "Unpacker", "ConversionError"] - -# exceptions -class Error(Exception): - """Exception class for this module. Use: - - except xdrlib.Error as var: - # var has the Error instance for the exception - - Public ivars: - msg -- contains the message - - """ - def __init__(self, msg): - self.msg = msg - def __repr__(self): - return repr(self.msg) - def __str__(self): - return str(self.msg) - - -class ConversionError(Error): - pass - -def raise_conversion_error(function): - """ Wrap any raised struct.errors in a ConversionError. """ - - @wraps(function) - def result(self, value): - try: - return function(self, value) - except struct.error as e: - raise ConversionError(e.args[0]) from None - return result - - -class Packer: - """Pack various data representations into a buffer.""" - - def __init__(self): - self.reset() - - def reset(self): - self.__buf = BytesIO() - - def get_buffer(self): - return self.__buf.getvalue() - # backwards compatibility - get_buf = get_buffer - - @raise_conversion_error - def pack_uint(self, x): - self.__buf.write(struct.pack('>L', x)) - - @raise_conversion_error - def pack_int(self, x): - self.__buf.write(struct.pack('>l', x)) - - pack_enum = pack_int - - def pack_bool(self, x): - if x: self.__buf.write(b'\0\0\0\1') - else: self.__buf.write(b'\0\0\0\0') - - def pack_uhyper(self, x): - try: - self.pack_uint(x>>32 & 0xffffffff) - except (TypeError, struct.error) as e: - raise ConversionError(e.args[0]) from None - try: - self.pack_uint(x & 0xffffffff) - except (TypeError, struct.error) as e: - raise ConversionError(e.args[0]) from None - - pack_hyper = pack_uhyper - - @raise_conversion_error - def pack_float(self, x): - self.__buf.write(struct.pack('>f', x)) - - @raise_conversion_error - def pack_double(self, x): - self.__buf.write(struct.pack('>d', x)) - - def pack_fstring(self, n, s): - if n < 0: - raise ValueError('fstring size must be nonnegative') - data = s[:n] - n = ((n+3)//4)*4 - data = data + (n - len(data)) * b'\0' - self.__buf.write(data) - - pack_fopaque = pack_fstring - - def pack_string(self, s): - n = len(s) - self.pack_uint(n) - self.pack_fstring(n, s) - - pack_opaque = pack_string - pack_bytes = pack_string - - def pack_list(self, list, pack_item): - for item in list: - self.pack_uint(1) - pack_item(item) - self.pack_uint(0) - - def pack_farray(self, n, list, pack_item): - if len(list) != n: - raise ValueError('wrong array size') - for item in list: - pack_item(item) - - def pack_array(self, list, pack_item): - n = len(list) - self.pack_uint(n) - self.pack_farray(n, list, pack_item) - - - -class Unpacker: - """Unpacks various data representations from the given buffer.""" - - def __init__(self, data): - self.reset(data) - - def reset(self, data): - self.__buf = data - self.__pos = 0 - - def get_position(self): - return self.__pos - - def set_position(self, position): - self.__pos = position - - def get_buffer(self): - return self.__buf - - def done(self): - if self.__pos < len(self.__buf): - raise Error('unextracted data remains') - - def unpack_uint(self): - i = self.__pos - self.__pos = j = i+4 - data = self.__buf[i:j] - if len(data) < 4: - raise EOFError - return struct.unpack('>L', data)[0] - - def unpack_int(self): - i = self.__pos - self.__pos = j = i+4 - data = self.__buf[i:j] - if len(data) < 4: - raise EOFError - return struct.unpack('>l', data)[0] - - unpack_enum = unpack_int - - def unpack_bool(self): - return bool(self.unpack_int()) - - def unpack_uhyper(self): - hi = self.unpack_uint() - lo = self.unpack_uint() - return int(hi)<<32 | lo - - def unpack_hyper(self): - x = self.unpack_uhyper() - if x >= 0x8000000000000000: - x = x - 0x10000000000000000 - return x - - def unpack_float(self): - i = self.__pos - self.__pos = j = i+4 - data = self.__buf[i:j] - if len(data) < 4: - raise EOFError - return struct.unpack('>f', data)[0] - - def unpack_double(self): - i = self.__pos - self.__pos = j = i+8 - data = self.__buf[i:j] - if len(data) < 8: - raise EOFError - return struct.unpack('>d', data)[0] - - def unpack_fstring(self, n): - if n < 0: - raise ValueError('fstring size must be nonnegative') - i = self.__pos - j = i + (n+3)//4*4 - if j > len(self.__buf): - raise EOFError - self.__pos = j - return self.__buf[i:i+n] - - unpack_fopaque = unpack_fstring - - def unpack_string(self): - n = self.unpack_uint() - return self.unpack_fstring(n) - - unpack_opaque = unpack_string - unpack_bytes = unpack_string - - def unpack_list(self, unpack_item): - list = [] - while 1: - x = self.unpack_uint() - if x == 0: break - if x != 1: - raise ConversionError('0 or 1 expected, got %r' % (x,)) - item = unpack_item() - list.append(item) - return list - - def unpack_farray(self, n, unpack_item): - list = [] - for i in range(n): - list.append(unpack_item()) - return list - - def unpack_array(self, unpack_item): - n = self.unpack_uint() - return self.unpack_farray(n, unpack_item) From f6a9754b4ea4b4257658822b2ce8cc1a9b652696 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Mon, 31 Mar 2025 18:19:39 -0700 Subject: [PATCH 160/295] Remove telnetlib deprecated in 3.13 (#5649) --- Lib/telnetlib.py | 677 ------------------------------------- Lib/test/test_telnetlib.py | 402 ---------------------- 2 files changed, 1079 deletions(-) delete mode 100644 Lib/telnetlib.py delete mode 100644 Lib/test/test_telnetlib.py diff --git a/Lib/telnetlib.py b/Lib/telnetlib.py deleted file mode 100644 index 8ce053e881..0000000000 --- a/Lib/telnetlib.py +++ /dev/null @@ -1,677 +0,0 @@ -r"""TELNET client class. - -Based on RFC 854: TELNET Protocol Specification, by J. Postel and -J. Reynolds - -Example: - ->>> from telnetlib import Telnet ->>> tn = Telnet('www.python.org', 79) # connect to finger port ->>> tn.write(b'guido\r\n') ->>> print(tn.read_all()) -Login Name TTY Idle When Where -guido Guido van Rossum pts/2 snag.cnri.reston.. - ->>> - -Note that read_all() won't read until eof -- it just reads some data --- but it guarantees to read at least one byte unless EOF is hit. - -It is possible to pass a Telnet object to a selector in order to wait until -more data is available. Note that in this case, read_eager() may return b'' -even if there was data on the socket, because the protocol negotiation may have -eaten the data. This is why EOFError is needed in some cases to distinguish -between "no data" and "connection closed" (since the socket also appears ready -for reading when it is closed). - -To do: -- option negotiation -- timeout should be intrinsic to the connection object instead of an - option on one of the read calls only - -""" - - -# Imported modules -import sys -import socket -import selectors -from time import monotonic as _time - -__all__ = ["Telnet"] - -# Tunable parameters -DEBUGLEVEL = 0 - -# Telnet protocol defaults -TELNET_PORT = 23 - -# Telnet protocol characters (don't change) -IAC = bytes([255]) # "Interpret As Command" -DONT = bytes([254]) -DO = bytes([253]) -WONT = bytes([252]) -WILL = bytes([251]) -theNULL = bytes([0]) - -SE = bytes([240]) # Subnegotiation End -NOP = bytes([241]) # No Operation -DM = bytes([242]) # Data Mark -BRK = bytes([243]) # Break -IP = bytes([244]) # Interrupt process -AO = bytes([245]) # Abort output -AYT = bytes([246]) # Are You There -EC = bytes([247]) # Erase Character -EL = bytes([248]) # Erase Line -GA = bytes([249]) # Go Ahead -SB = bytes([250]) # Subnegotiation Begin - - -# Telnet protocol options code (don't change) -# These ones all come from arpa/telnet.h -BINARY = bytes([0]) # 8-bit data path -ECHO = bytes([1]) # echo -RCP = bytes([2]) # prepare to reconnect -SGA = bytes([3]) # suppress go ahead -NAMS = bytes([4]) # approximate message size -STATUS = bytes([5]) # give status -TM = bytes([6]) # timing mark -RCTE = bytes([7]) # remote controlled transmission and echo -NAOL = bytes([8]) # negotiate about output line width -NAOP = bytes([9]) # negotiate about output page size -NAOCRD = bytes([10]) # negotiate about CR disposition -NAOHTS = bytes([11]) # negotiate about horizontal tabstops -NAOHTD = bytes([12]) # negotiate about horizontal tab disposition -NAOFFD = bytes([13]) # negotiate about formfeed disposition -NAOVTS = bytes([14]) # negotiate about vertical tab stops -NAOVTD = bytes([15]) # negotiate about vertical tab disposition -NAOLFD = bytes([16]) # negotiate about output LF disposition -XASCII = bytes([17]) # extended ascii character set -LOGOUT = bytes([18]) # force logout -BM = bytes([19]) # byte macro -DET = bytes([20]) # data entry terminal -SUPDUP = bytes([21]) # supdup protocol -SUPDUPOUTPUT = bytes([22]) # supdup output -SNDLOC = bytes([23]) # send location -TTYPE = bytes([24]) # terminal type -EOR = bytes([25]) # end or record -TUID = bytes([26]) # TACACS user identification -OUTMRK = bytes([27]) # output marking -TTYLOC = bytes([28]) # terminal location number -VT3270REGIME = bytes([29]) # 3270 regime -X3PAD = bytes([30]) # X.3 PAD -NAWS = bytes([31]) # window size -TSPEED = bytes([32]) # terminal speed -LFLOW = bytes([33]) # remote flow control -LINEMODE = bytes([34]) # Linemode option -XDISPLOC = bytes([35]) # X Display Location -OLD_ENVIRON = bytes([36]) # Old - Environment variables -AUTHENTICATION = bytes([37]) # Authenticate -ENCRYPT = bytes([38]) # Encryption option -NEW_ENVIRON = bytes([39]) # New - Environment variables -# the following ones come from -# http://www.iana.org/assignments/telnet-options -# Unfortunately, that document does not assign identifiers -# to all of them, so we are making them up -TN3270E = bytes([40]) # TN3270E -XAUTH = bytes([41]) # XAUTH -CHARSET = bytes([42]) # CHARSET -RSP = bytes([43]) # Telnet Remote Serial Port -COM_PORT_OPTION = bytes([44]) # Com Port Control Option -SUPPRESS_LOCAL_ECHO = bytes([45]) # Telnet Suppress Local Echo -TLS = bytes([46]) # Telnet Start TLS -KERMIT = bytes([47]) # KERMIT -SEND_URL = bytes([48]) # SEND-URL -FORWARD_X = bytes([49]) # FORWARD_X -PRAGMA_LOGON = bytes([138]) # TELOPT PRAGMA LOGON -SSPI_LOGON = bytes([139]) # TELOPT SSPI LOGON -PRAGMA_HEARTBEAT = bytes([140]) # TELOPT PRAGMA HEARTBEAT -EXOPL = bytes([255]) # Extended-Options-List -NOOPT = bytes([0]) - - -# poll/select have the advantage of not requiring any extra file descriptor, -# contrarily to epoll/kqueue (also, they require a single syscall). -if hasattr(selectors, 'PollSelector'): - _TelnetSelector = selectors.PollSelector -else: - _TelnetSelector = selectors.SelectSelector - - -class Telnet: - - """Telnet interface class. - - An instance of this class represents a connection to a telnet - server. The instance is initially not connected; the open() - method must be used to establish a connection. Alternatively, the - host name and optional port number can be passed to the - constructor, too. - - Don't try to reopen an already connected instance. - - This class has many read_*() methods. Note that some of them - raise EOFError when the end of the connection is read, because - they can return an empty string for other reasons. See the - individual doc strings. - - read_until(expected, [timeout]) - Read until the expected string has been seen, or a timeout is - hit (default is no timeout); may block. - - read_all() - Read all data until EOF; may block. - - read_some() - Read at least one byte or EOF; may block. - - read_very_eager() - Read all data available already queued or on the socket, - without blocking. - - read_eager() - Read either data already queued or some data available on the - socket, without blocking. - - read_lazy() - Read all data in the raw queue (processing it first), without - doing any socket I/O. - - read_very_lazy() - Reads all data in the cooked queue, without doing any socket - I/O. - - read_sb_data() - Reads available data between SB ... SE sequence. Don't block. - - set_option_negotiation_callback(callback) - Each time a telnet option is read on the input flow, this callback - (if set) is called with the following parameters : - callback(telnet socket, command, option) - option will be chr(0) when there is no option. - No other action is done afterwards by telnetlib. - - """ - - def __init__(self, host=None, port=0, - timeout=socket._GLOBAL_DEFAULT_TIMEOUT): - """Constructor. - - When called without arguments, create an unconnected instance. - With a hostname argument, it connects the instance; port number - and timeout are optional. - """ - self.debuglevel = DEBUGLEVEL - self.host = host - self.port = port - self.timeout = timeout - self.sock = None - self.rawq = b'' - self.irawq = 0 - self.cookedq = b'' - self.eof = 0 - self.iacseq = b'' # Buffer for IAC sequence. - self.sb = 0 # flag for SB and SE sequence. - self.sbdataq = b'' - self.option_callback = None - if host is not None: - self.open(host, port, timeout) - - def open(self, host, port=0, timeout=socket._GLOBAL_DEFAULT_TIMEOUT): - """Connect to a host. - - The optional second argument is the port number, which - defaults to the standard telnet port (23). - - Don't try to reopen an already connected instance. - """ - self.eof = 0 - if not port: - port = TELNET_PORT - self.host = host - self.port = port - self.timeout = timeout - sys.audit("telnetlib.Telnet.open", self, host, port) - self.sock = socket.create_connection((host, port), timeout) - - def __del__(self): - """Destructor -- close the connection.""" - self.close() - - def msg(self, msg, *args): - """Print a debug message, when the debug level is > 0. - - If extra arguments are present, they are substituted in the - message using the standard string formatting operator. - - """ - if self.debuglevel > 0: - print('Telnet(%s,%s):' % (self.host, self.port), end=' ') - if args: - print(msg % args) - else: - print(msg) - - def set_debuglevel(self, debuglevel): - """Set the debug level. - - The higher it is, the more debug output you get (on sys.stdout). - - """ - self.debuglevel = debuglevel - - def close(self): - """Close the connection.""" - sock = self.sock - self.sock = None - self.eof = True - self.iacseq = b'' - self.sb = 0 - if sock: - sock.close() - - def get_socket(self): - """Return the socket object used internally.""" - return self.sock - - def fileno(self): - """Return the fileno() of the socket object used internally.""" - return self.sock.fileno() - - def write(self, buffer): - """Write a string to the socket, doubling any IAC characters. - - Can block if the connection is blocked. May raise - OSError if the connection is closed. - - """ - if IAC in buffer: - buffer = buffer.replace(IAC, IAC+IAC) - sys.audit("telnetlib.Telnet.write", self, buffer) - self.msg("send %r", buffer) - self.sock.sendall(buffer) - - def read_until(self, match, timeout=None): - """Read until a given string is encountered or until timeout. - - When no match is found, return whatever is available instead, - possibly the empty string. Raise EOFError if the connection - is closed and no cooked data is available. - - """ - n = len(match) - self.process_rawq() - i = self.cookedq.find(match) - if i >= 0: - i = i+n - buf = self.cookedq[:i] - self.cookedq = self.cookedq[i:] - return buf - if timeout is not None: - deadline = _time() + timeout - with _TelnetSelector() as selector: - selector.register(self, selectors.EVENT_READ) - while not self.eof: - if selector.select(timeout): - i = max(0, len(self.cookedq)-n) - self.fill_rawq() - self.process_rawq() - i = self.cookedq.find(match, i) - if i >= 0: - i = i+n - buf = self.cookedq[:i] - self.cookedq = self.cookedq[i:] - return buf - if timeout is not None: - timeout = deadline - _time() - if timeout < 0: - break - return self.read_very_lazy() - - def read_all(self): - """Read all data until EOF; block until connection closed.""" - self.process_rawq() - while not self.eof: - self.fill_rawq() - self.process_rawq() - buf = self.cookedq - self.cookedq = b'' - return buf - - def read_some(self): - """Read at least one byte of cooked data unless EOF is hit. - - Return b'' if EOF is hit. Block if no data is immediately - available. - - """ - self.process_rawq() - while not self.cookedq and not self.eof: - self.fill_rawq() - self.process_rawq() - buf = self.cookedq - self.cookedq = b'' - return buf - - def read_very_eager(self): - """Read everything that's possible without blocking in I/O (eager). - - Raise EOFError if connection closed and no cooked data - available. Return b'' if no cooked data available otherwise. - Don't block unless in the midst of an IAC sequence. - - """ - self.process_rawq() - while not self.eof and self.sock_avail(): - self.fill_rawq() - self.process_rawq() - return self.read_very_lazy() - - def read_eager(self): - """Read readily available data. - - Raise EOFError if connection closed and no cooked data - available. Return b'' if no cooked data available otherwise. - Don't block unless in the midst of an IAC sequence. - - """ - self.process_rawq() - while not self.cookedq and not self.eof and self.sock_avail(): - self.fill_rawq() - self.process_rawq() - return self.read_very_lazy() - - def read_lazy(self): - """Process and return data that's already in the queues (lazy). - - Raise EOFError if connection closed and no data available. - Return b'' if no cooked data available otherwise. Don't block - unless in the midst of an IAC sequence. - - """ - self.process_rawq() - return self.read_very_lazy() - - def read_very_lazy(self): - """Return any data available in the cooked queue (very lazy). - - Raise EOFError if connection closed and no data available. - Return b'' if no cooked data available otherwise. Don't block. - - """ - buf = self.cookedq - self.cookedq = b'' - if not buf and self.eof and not self.rawq: - raise EOFError('telnet connection closed') - return buf - - def read_sb_data(self): - """Return any data available in the SB ... SE queue. - - Return b'' if no SB ... SE available. Should only be called - after seeing a SB or SE command. When a new SB command is - found, old unread SB data will be discarded. Don't block. - - """ - buf = self.sbdataq - self.sbdataq = b'' - return buf - - def set_option_negotiation_callback(self, callback): - """Provide a callback function called after each receipt of a telnet option.""" - self.option_callback = callback - - def process_rawq(self): - """Transfer from raw queue to cooked queue. - - Set self.eof when connection is closed. Don't block unless in - the midst of an IAC sequence. - - """ - buf = [b'', b''] - try: - while self.rawq: - c = self.rawq_getchar() - if not self.iacseq: - if c == theNULL: - continue - if c == b"\021": - continue - if c != IAC: - buf[self.sb] = buf[self.sb] + c - continue - else: - self.iacseq += c - elif len(self.iacseq) == 1: - # 'IAC: IAC CMD [OPTION only for WILL/WONT/DO/DONT]' - if c in (DO, DONT, WILL, WONT): - self.iacseq += c - continue - - self.iacseq = b'' - if c == IAC: - buf[self.sb] = buf[self.sb] + c - else: - if c == SB: # SB ... SE start. - self.sb = 1 - self.sbdataq = b'' - elif c == SE: - self.sb = 0 - self.sbdataq = self.sbdataq + buf[1] - buf[1] = b'' - if self.option_callback: - # Callback is supposed to look into - # the sbdataq - self.option_callback(self.sock, c, NOOPT) - else: - # We can't offer automatic processing of - # suboptions. Alas, we should not get any - # unless we did a WILL/DO before. - self.msg('IAC %d not recognized' % ord(c)) - elif len(self.iacseq) == 2: - cmd = self.iacseq[1:2] - self.iacseq = b'' - opt = c - if cmd in (DO, DONT): - self.msg('IAC %s %d', - cmd == DO and 'DO' or 'DONT', ord(opt)) - if self.option_callback: - self.option_callback(self.sock, cmd, opt) - else: - self.sock.sendall(IAC + WONT + opt) - elif cmd in (WILL, WONT): - self.msg('IAC %s %d', - cmd == WILL and 'WILL' or 'WONT', ord(opt)) - if self.option_callback: - self.option_callback(self.sock, cmd, opt) - else: - self.sock.sendall(IAC + DONT + opt) - except EOFError: # raised by self.rawq_getchar() - self.iacseq = b'' # Reset on EOF - self.sb = 0 - pass - self.cookedq = self.cookedq + buf[0] - self.sbdataq = self.sbdataq + buf[1] - - def rawq_getchar(self): - """Get next char from raw queue. - - Block if no data is immediately available. Raise EOFError - when connection is closed. - - """ - if not self.rawq: - self.fill_rawq() - if self.eof: - raise EOFError - c = self.rawq[self.irawq:self.irawq+1] - self.irawq = self.irawq + 1 - if self.irawq >= len(self.rawq): - self.rawq = b'' - self.irawq = 0 - return c - - def fill_rawq(self): - """Fill raw queue from exactly one recv() system call. - - Block if no data is immediately available. Set self.eof when - connection is closed. - - """ - if self.irawq >= len(self.rawq): - self.rawq = b'' - self.irawq = 0 - # The buffer size should be fairly small so as to avoid quadratic - # behavior in process_rawq() above - buf = self.sock.recv(50) - self.msg("recv %r", buf) - self.eof = (not buf) - self.rawq = self.rawq + buf - - def sock_avail(self): - """Test whether data is available on the socket.""" - with _TelnetSelector() as selector: - selector.register(self, selectors.EVENT_READ) - return bool(selector.select(0)) - - def interact(self): - """Interaction function, emulates a very dumb telnet client.""" - if sys.platform == "win32": - self.mt_interact() - return - with _TelnetSelector() as selector: - selector.register(self, selectors.EVENT_READ) - selector.register(sys.stdin, selectors.EVENT_READ) - - while True: - for key, events in selector.select(): - if key.fileobj is self: - try: - text = self.read_eager() - except EOFError: - print('*** Connection closed by remote host ***') - return - if text: - sys.stdout.write(text.decode('ascii')) - sys.stdout.flush() - elif key.fileobj is sys.stdin: - line = sys.stdin.readline().encode('ascii') - if not line: - return - self.write(line) - - def mt_interact(self): - """Multithreaded version of interact().""" - import _thread - _thread.start_new_thread(self.listener, ()) - while 1: - line = sys.stdin.readline() - if not line: - break - self.write(line.encode('ascii')) - - def listener(self): - """Helper for mt_interact() -- this executes in the other thread.""" - while 1: - try: - data = self.read_eager() - except EOFError: - print('*** Connection closed by remote host ***') - return - if data: - sys.stdout.write(data.decode('ascii')) - else: - sys.stdout.flush() - - def expect(self, list, timeout=None): - """Read until one from a list of a regular expressions matches. - - The first argument is a list of regular expressions, either - compiled (re.Pattern instances) or uncompiled (strings). - The optional second argument is a timeout, in seconds; default - is no timeout. - - Return a tuple of three items: the index in the list of the - first regular expression that matches; the re.Match object - returned; and the text read up till and including the match. - - If EOF is read and no text was read, raise EOFError. - Otherwise, when nothing matches, return (-1, None, text) where - text is the text received so far (may be the empty string if a - timeout happened). - - If a regular expression ends with a greedy match (e.g. '.*') - or if more than one expression can match the same input, the - results are undeterministic, and may depend on the I/O timing. - - """ - re = None - list = list[:] - indices = range(len(list)) - for i in indices: - if not hasattr(list[i], "search"): - if not re: import re - list[i] = re.compile(list[i]) - if timeout is not None: - deadline = _time() + timeout - with _TelnetSelector() as selector: - selector.register(self, selectors.EVENT_READ) - while not self.eof: - self.process_rawq() - for i in indices: - m = list[i].search(self.cookedq) - if m: - e = m.end() - text = self.cookedq[:e] - self.cookedq = self.cookedq[e:] - return (i, m, text) - if timeout is not None: - ready = selector.select(timeout) - timeout = deadline - _time() - if not ready: - if timeout < 0: - break - else: - continue - self.fill_rawq() - text = self.read_very_lazy() - if not text and self.eof: - raise EOFError - return (-1, None, text) - - def __enter__(self): - return self - - def __exit__(self, type, value, traceback): - self.close() - - -def test(): - """Test program for telnetlib. - - Usage: python telnetlib.py [-d] ... [host [port]] - - Default host is localhost; default port is 23. - - """ - debuglevel = 0 - while sys.argv[1:] and sys.argv[1] == '-d': - debuglevel = debuglevel+1 - del sys.argv[1] - host = 'localhost' - if sys.argv[1:]: - host = sys.argv[1] - port = 0 - if sys.argv[2:]: - portstr = sys.argv[2] - try: - port = int(portstr) - except ValueError: - port = socket.getservbyname(portstr, 'tcp') - with Telnet() as tn: - tn.set_debuglevel(debuglevel) - tn.open(host, port, timeout=0.5) - tn.interact() - -if __name__ == '__main__': - test() diff --git a/Lib/test/test_telnetlib.py b/Lib/test/test_telnetlib.py deleted file mode 100644 index 41c4fcd419..0000000000 --- a/Lib/test/test_telnetlib.py +++ /dev/null @@ -1,402 +0,0 @@ -import socket -import selectors -import telnetlib -import threading -import contextlib - -from test import support -from test.support import socket_helper -import unittest - -HOST = socket_helper.HOST - -def server(evt, serv): - serv.listen() - evt.set() - try: - conn, addr = serv.accept() - conn.close() - except TimeoutError: - pass - finally: - serv.close() - -class GeneralTests(unittest.TestCase): - - def setUp(self): - self.evt = threading.Event() - self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - self.sock.settimeout(60) # Safety net. Look issue 11812 - self.port = socket_helper.bind_port(self.sock) - self.thread = threading.Thread(target=server, args=(self.evt,self.sock)) - self.thread.daemon = True - self.thread.start() - self.evt.wait() - - def tearDown(self): - self.thread.join() - del self.thread # Clear out any dangling Thread objects. - - def testBasic(self): - # connects - telnet = telnetlib.Telnet(HOST, self.port) - telnet.sock.close() - - def testContextManager(self): - with telnetlib.Telnet(HOST, self.port) as tn: - self.assertIsNotNone(tn.get_socket()) - self.assertIsNone(tn.get_socket()) - - def testTimeoutDefault(self): - self.assertTrue(socket.getdefaulttimeout() is None) - socket.setdefaulttimeout(30) - try: - telnet = telnetlib.Telnet(HOST, self.port) - finally: - socket.setdefaulttimeout(None) - self.assertEqual(telnet.sock.gettimeout(), 30) - telnet.sock.close() - - def testTimeoutNone(self): - # None, having other default - self.assertTrue(socket.getdefaulttimeout() is None) - socket.setdefaulttimeout(30) - try: - telnet = telnetlib.Telnet(HOST, self.port, timeout=None) - finally: - socket.setdefaulttimeout(None) - self.assertTrue(telnet.sock.gettimeout() is None) - telnet.sock.close() - - def testTimeoutValue(self): - telnet = telnetlib.Telnet(HOST, self.port, timeout=30) - self.assertEqual(telnet.sock.gettimeout(), 30) - telnet.sock.close() - - def testTimeoutOpen(self): - telnet = telnetlib.Telnet() - telnet.open(HOST, self.port, timeout=30) - self.assertEqual(telnet.sock.gettimeout(), 30) - telnet.sock.close() - - def testGetters(self): - # Test telnet getter methods - telnet = telnetlib.Telnet(HOST, self.port, timeout=30) - t_sock = telnet.sock - self.assertEqual(telnet.get_socket(), t_sock) - self.assertEqual(telnet.fileno(), t_sock.fileno()) - telnet.sock.close() - -class SocketStub(object): - ''' a socket proxy that re-defines sendall() ''' - def __init__(self, reads=()): - self.reads = list(reads) # Intentionally make a copy. - self.writes = [] - self.block = False - def sendall(self, data): - self.writes.append(data) - def recv(self, size): - out = b'' - while self.reads and len(out) < size: - out += self.reads.pop(0) - if len(out) > size: - self.reads.insert(0, out[size:]) - out = out[:size] - return out - -class TelnetAlike(telnetlib.Telnet): - def fileno(self): - raise NotImplementedError() - def close(self): pass - def sock_avail(self): - return (not self.sock.block) - def msg(self, msg, *args): - with support.captured_stdout() as out: - telnetlib.Telnet.msg(self, msg, *args) - self._messages += out.getvalue() - return - -class MockSelector(selectors.BaseSelector): - - def __init__(self): - self.keys = {} - - @property - def resolution(self): - return 1e-3 - - def register(self, fileobj, events, data=None): - key = selectors.SelectorKey(fileobj, 0, events, data) - self.keys[fileobj] = key - return key - - def unregister(self, fileobj): - return self.keys.pop(fileobj) - - def select(self, timeout=None): - block = False - for fileobj in self.keys: - if isinstance(fileobj, TelnetAlike): - block = fileobj.sock.block - break - if block: - return [] - else: - return [(key, key.events) for key in self.keys.values()] - - def get_map(self): - return self.keys - - -@contextlib.contextmanager -def test_socket(reads): - def new_conn(*ignored): - return SocketStub(reads) - try: - old_conn = socket.create_connection - socket.create_connection = new_conn - yield None - finally: - socket.create_connection = old_conn - return - -def test_telnet(reads=(), cls=TelnetAlike): - ''' return a telnetlib.Telnet object that uses a SocketStub with - reads queued up to be read ''' - for x in reads: - assert type(x) is bytes, x - with test_socket(reads): - telnet = cls('dummy', 0) - telnet._messages = '' # debuglevel output - return telnet - -class ExpectAndReadTestCase(unittest.TestCase): - def setUp(self): - self.old_selector = telnetlib._TelnetSelector - telnetlib._TelnetSelector = MockSelector - def tearDown(self): - telnetlib._TelnetSelector = self.old_selector - -class ReadTests(ExpectAndReadTestCase): - def test_read_until(self): - """ - read_until(expected, timeout=None) - test the blocking version of read_util - """ - want = [b'xxxmatchyyy'] - telnet = test_telnet(want) - data = telnet.read_until(b'match') - self.assertEqual(data, b'xxxmatch', msg=(telnet.cookedq, telnet.rawq, telnet.sock.reads)) - - reads = [b'x' * 50, b'match', b'y' * 50] - expect = b''.join(reads[:-1]) - telnet = test_telnet(reads) - data = telnet.read_until(b'match') - self.assertEqual(data, expect) - - - def test_read_all(self): - """ - read_all() - Read all data until EOF; may block. - """ - reads = [b'x' * 500, b'y' * 500, b'z' * 500] - expect = b''.join(reads) - telnet = test_telnet(reads) - data = telnet.read_all() - self.assertEqual(data, expect) - return - - def test_read_some(self): - """ - read_some() - Read at least one byte or EOF; may block. - """ - # test 'at least one byte' - telnet = test_telnet([b'x' * 500]) - data = telnet.read_some() - self.assertTrue(len(data) >= 1) - # test EOF - telnet = test_telnet() - data = telnet.read_some() - self.assertEqual(b'', data) - - def _read_eager(self, func_name): - """ - read_*_eager() - Read all data available already queued or on the socket, - without blocking. - """ - want = b'x' * 100 - telnet = test_telnet([want]) - func = getattr(telnet, func_name) - telnet.sock.block = True - self.assertEqual(b'', func()) - telnet.sock.block = False - data = b'' - while True: - try: - data += func() - except EOFError: - break - self.assertEqual(data, want) - - def test_read_eager(self): - # read_eager and read_very_eager make the same guarantees - # (they behave differently but we only test the guarantees) - self._read_eager('read_eager') - self._read_eager('read_very_eager') - # NB -- we need to test the IAC block which is mentioned in the - # docstring but not in the module docs - - def read_very_lazy(self): - want = b'x' * 100 - telnet = test_telnet([want]) - self.assertEqual(b'', telnet.read_very_lazy()) - while telnet.sock.reads: - telnet.fill_rawq() - data = telnet.read_very_lazy() - self.assertEqual(want, data) - self.assertRaises(EOFError, telnet.read_very_lazy) - - def test_read_lazy(self): - want = b'x' * 100 - telnet = test_telnet([want]) - self.assertEqual(b'', telnet.read_lazy()) - data = b'' - while True: - try: - read_data = telnet.read_lazy() - data += read_data - if not read_data: - telnet.fill_rawq() - except EOFError: - break - self.assertTrue(want.startswith(data)) - self.assertEqual(data, want) - -class nego_collector(object): - def __init__(self, sb_getter=None): - self.seen = b'' - self.sb_getter = sb_getter - self.sb_seen = b'' - - def do_nego(self, sock, cmd, opt): - self.seen += cmd + opt - if cmd == tl.SE and self.sb_getter: - sb_data = self.sb_getter() - self.sb_seen += sb_data - -tl = telnetlib - -class WriteTests(unittest.TestCase): - '''The only thing that write does is replace each tl.IAC for - tl.IAC+tl.IAC''' - - def test_write(self): - data_sample = [b'data sample without IAC', - b'data sample with' + tl.IAC + b' one IAC', - b'a few' + tl.IAC + tl.IAC + b' iacs' + tl.IAC, - tl.IAC, - b''] - for data in data_sample: - telnet = test_telnet() - telnet.write(data) - written = b''.join(telnet.sock.writes) - self.assertEqual(data.replace(tl.IAC,tl.IAC+tl.IAC), written) - -class OptionTests(unittest.TestCase): - # RFC 854 commands - cmds = [tl.AO, tl.AYT, tl.BRK, tl.EC, tl.EL, tl.GA, tl.IP, tl.NOP] - - def _test_command(self, data): - """ helper for testing IAC + cmd """ - telnet = test_telnet(data) - data_len = len(b''.join(data)) - nego = nego_collector() - telnet.set_option_negotiation_callback(nego.do_nego) - txt = telnet.read_all() - cmd = nego.seen - self.assertTrue(len(cmd) > 0) # we expect at least one command - self.assertIn(cmd[:1], self.cmds) - self.assertEqual(cmd[1:2], tl.NOOPT) - self.assertEqual(data_len, len(txt + cmd)) - nego.sb_getter = None # break the nego => telnet cycle - - def test_IAC_commands(self): - for cmd in self.cmds: - self._test_command([tl.IAC, cmd]) - self._test_command([b'x' * 100, tl.IAC, cmd, b'y'*100]) - self._test_command([b'x' * 10, tl.IAC, cmd, b'y'*10]) - # all at once - self._test_command([tl.IAC + cmd for (cmd) in self.cmds]) - - def test_SB_commands(self): - # RFC 855, subnegotiations portion - send = [tl.IAC + tl.SB + tl.IAC + tl.SE, - tl.IAC + tl.SB + tl.IAC + tl.IAC + tl.IAC + tl.SE, - tl.IAC + tl.SB + tl.IAC + tl.IAC + b'aa' + tl.IAC + tl.SE, - tl.IAC + tl.SB + b'bb' + tl.IAC + tl.IAC + tl.IAC + tl.SE, - tl.IAC + tl.SB + b'cc' + tl.IAC + tl.IAC + b'dd' + tl.IAC + tl.SE, - ] - telnet = test_telnet(send) - nego = nego_collector(telnet.read_sb_data) - telnet.set_option_negotiation_callback(nego.do_nego) - txt = telnet.read_all() - self.assertEqual(txt, b'') - want_sb_data = tl.IAC + tl.IAC + b'aabb' + tl.IAC + b'cc' + tl.IAC + b'dd' - self.assertEqual(nego.sb_seen, want_sb_data) - self.assertEqual(b'', telnet.read_sb_data()) - nego.sb_getter = None # break the nego => telnet cycle - - def test_debuglevel_reads(self): - # test all the various places that self.msg(...) is called - given_a_expect_b = [ - # Telnet.fill_rawq - (b'a', ": recv b''\n"), - # Telnet.process_rawq - (tl.IAC + bytes([88]), ": IAC 88 not recognized\n"), - (tl.IAC + tl.DO + bytes([1]), ": IAC DO 1\n"), - (tl.IAC + tl.DONT + bytes([1]), ": IAC DONT 1\n"), - (tl.IAC + tl.WILL + bytes([1]), ": IAC WILL 1\n"), - (tl.IAC + tl.WONT + bytes([1]), ": IAC WONT 1\n"), - ] - for a, b in given_a_expect_b: - telnet = test_telnet([a]) - telnet.set_debuglevel(1) - txt = telnet.read_all() - self.assertIn(b, telnet._messages) - return - - def test_debuglevel_write(self): - telnet = test_telnet() - telnet.set_debuglevel(1) - telnet.write(b'xxx') - expected = "send b'xxx'\n" - self.assertIn(expected, telnet._messages) - - def test_debug_accepts_str_port(self): - # Issue 10695 - with test_socket([]): - telnet = TelnetAlike('dummy', '0') - telnet._messages = '' - telnet.set_debuglevel(1) - telnet.msg('test') - self.assertRegex(telnet._messages, r'0.*test') - - -class ExpectTests(ExpectAndReadTestCase): - def test_expect(self): - """ - expect(expected, [timeout]) - Read until the expected string has been seen, or a timeout is - hit (default is no timeout); may block. - """ - want = [b'x' * 10, b'match', b'y' * 10] - telnet = test_telnet(want) - (_,_,data) = telnet.expect([b'match']) - self.assertEqual(data, b''.join(want[:-1])) - - -if __name__ == '__main__': - unittest.main() From c3ed002b1204d9ff156b5192b634a4056101b255 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Apr 2025 10:31:07 +0900 Subject: [PATCH 161/295] Bump streetsidesoftware/cspell-action from 2 to 6 in the github-actions group (#5646) * Bump streetsidesoftware/cspell-action in the github-actions group Bumps the github-actions group with 1 update: [streetsidesoftware/cspell-action](https://github.com/streetsidesoftware/cspell-action). Updates `streetsidesoftware/cspell-action` from 2 to 6 - [Release notes](https://github.com/streetsidesoftware/cspell-action/releases) - [Changelog](https://github.com/streetsidesoftware/cspell-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/streetsidesoftware/cspell-action/compare/v2...v6) --- updated-dependencies: - dependency-name: streetsidesoftware/cspell-action dependency-type: direct:production update-type: version-update:semver-major dependency-group: github-actions ... Signed-off-by: dependabot[bot] * Update .github/workflows/ci.yaml --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Jeong, YunWon <69878+youknowone@users.noreply.github.com> --- .github/workflows/ci.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 869ccb530a..586e00be26 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -313,8 +313,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - name: install extra dictionaries + run: npm install @cspell/dict-en_us @cspell/dict-cpp @cspell/dict-python @cspell/dict-rust @cspell/dict-win32 @cspell/dict-shell - name: spell checker - uses: streetsidesoftware/cspell-action@v2 + uses: streetsidesoftware/cspell-action@v6 with: files: '**/*.rs' incremental_files_only: true From 64a07216160910015e2f3afb91cae0201ff40bcc Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Tue, 1 Apr 2025 08:08:16 -0700 Subject: [PATCH 162/295] Remove broken badge from readme --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 38e4d8fa8c..e472bd9f6a 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,6 @@ A Python-3 (CPython >= 3.13.0) Interpreter written in Rust :snake: :scream: [![docs.rs](https://docs.rs/rustpython/badge.svg)](https://docs.rs/rustpython/) [![Crates.io](https://img.shields.io/crates/v/rustpython)](https://crates.io/crates/rustpython) [![dependency status](https://deps.rs/crate/rustpython/0.1.1/status.svg)](https://deps.rs/crate/rustpython/0.1.1) -[![WAPM package](https://wapm.io/package/rustpython/badge.svg?style=flat)](https://wapm.io/package/rustpython) [![Open in Gitpod](https://img.shields.io/static/v1?label=Open%20in&message=Gitpod&color=1aa6e4&logo=gitpod)](https://gitpod.io#https://github.com/RustPython/RustPython) ## Usage From 2bf233280684bddaa001f6e17b1bd48d7ebc4996 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Wed, 2 Apr 2025 00:31:25 -0700 Subject: [PATCH 163/295] Cleanup whats_left.py (#5654) * cleanup whats_left.py * add features flag --- whats_left.py | 45 +++++++++------------------------------------ 1 file changed, 9 insertions(+), 36 deletions(-) diff --git a/whats_left.py b/whats_left.py index 7c1c30ba6c..82df5cabe1 100755 --- a/whats_left.py +++ b/whats_left.py @@ -55,6 +55,12 @@ def parse_args(): action="store_true", help="print output as JSON (instead of line by line)", ) + parser.add_argument( + "--features", + action="store", + help="which features to enable when building RustPython (default: ssl)", + default="ssl", + ) args = parser.parse_args() return args @@ -62,46 +68,13 @@ def parse_args(): args = parse_args() - -# modules suggested for deprecation by PEP 594 (www.python.org/dev/peps/pep-0594/) -# some of these might be implemented, but they are not a priority -PEP_594_MODULES = { - "aifc", - "asynchat", - "asyncore", - "audioop", - "binhex", - "cgi", - "cgitb", - "chunk", - "crypt", - "formatter", - "fpectl", - "imghdr", - "imp", - "macpath", - "msilib", - "nntplib", - "nis", - "ossaudiodev", - "parser", - "pipes", - "smtpd", - "sndhdr", - "spwd", - "sunau", - "telnetlib", - "uu", - "xdrlib", -} - # CPython specific modules (mostly consisting of templates/tests) CPYTHON_SPECIFIC_MODS = { 'xxmodule', 'xxsubtype', 'xxlimited', '_xxtestfuzz', '_testbuffer', '_testcapi', '_testimportmultiple', '_testinternalcapi', '_testmultiphase', '_testlimitedcapi' } -IGNORED_MODULES = {"this", "antigravity"} | PEP_594_MODULES | CPYTHON_SPECIFIC_MODS +IGNORED_MODULES = {"this", "antigravity"} | CPYTHON_SPECIFIC_MODS sys.path = [ path @@ -446,9 +419,9 @@ def remove_one_indent(s): f.write(output + "\n") -subprocess.run(["cargo", "build", "--release", "--features=ssl"], check=True) +subprocess.run(["cargo", "build", "--release", f"--features={args.features}"], check=True) result = subprocess.run( - ["cargo", "run", "--release", "--features=ssl", "-q", "--", GENERATED_FILE], + ["cargo", "run", "--release", f"--features={args.features}", "-q", "--", GENERATED_FILE], env={**os.environ.copy(), "RUSTPYTHONPATH": "Lib"}, text=True, capture_output=True, From 8063148598be283b7f5aa0972d8e29217f47c9f8 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Thu, 3 Apr 2025 22:04:13 -0700 Subject: [PATCH 164/295] Fix clippy lints from rust 1.86 update (#5665) * handle rust 1.86 update * fix windows clippy lint * disable cspell under jit/instruction --------- Co-authored-by: Jeong YunWon --- common/src/fileutils.rs | 2 +- jit/src/instructions.rs | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/common/src/fileutils.rs b/common/src/fileutils.rs index 67713c0148..5a0d380e20 100644 --- a/common/src/fileutils.rs +++ b/common/src/fileutils.rs @@ -78,7 +78,7 @@ pub mod windows { .encode_wide() .collect::>() .split(|&c| c == '.' as u16) - .last() + .next_back() .and_then(|s| String::from_utf16(s).ok()); if let Some(file_extension) = file_extension { diff --git a/jit/src/instructions.rs b/jit/src/instructions.rs index bf30e51d74..830a578562 100644 --- a/jit/src/instructions.rs +++ b/jit/src/instructions.rs @@ -1,3 +1,4 @@ +// cspell: disable use super::{JitCompileError, JitSig, JitType}; use cranelift::codegen::ir::FuncRef; use cranelift::prelude::*; @@ -559,7 +560,7 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> { Ok(()) } - Instruction::SetupLoop { .. } => { + Instruction::SetupLoop => { let loop_head = self.builder.create_block(); self.builder.ins().jump(loop_head, &[]); self.builder.switch_to_block(loop_head); From 6620aa07af4b1c296d1e60c18b3f70463bec43ba Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Thu, 3 Apr 2025 09:53:45 -0700 Subject: [PATCH 165/295] update email to 3.13.2 --- Lib/email/__init__.py | 1 - Lib/email/_encoded_words.py | 60 +- Lib/email/_header_value_parser.py | 1086 ++++++++++++++++------------- Lib/email/_parseaddr.py | 26 +- Lib/email/_policybase.py | 22 +- Lib/email/architecture.rst | 2 +- Lib/email/base64mime.py | 6 +- Lib/email/charset.py | 20 +- Lib/email/contentmanager.py | 23 +- Lib/email/encoders.py | 4 - Lib/email/errors.py | 10 + Lib/email/feedparser.py | 23 +- Lib/email/generator.py | 28 +- Lib/email/header.py | 11 +- Lib/email/headerregistry.py | 76 +- Lib/email/iterators.py | 3 - Lib/email/message.py | 70 +- Lib/email/mime/application.py | 2 +- Lib/email/mime/audio.py | 87 ++- Lib/email/mime/base.py | 1 - Lib/email/mime/image.py | 125 +++- Lib/email/mime/message.py | 1 - Lib/email/mime/multipart.py | 1 - Lib/email/mime/nonmultipart.py | 1 - Lib/email/mime/text.py | 4 +- Lib/email/parser.py | 9 +- Lib/email/policy.py | 21 +- Lib/email/quoprimime.py | 3 +- Lib/email/utils.py | 250 +++++-- 29 files changed, 1197 insertions(+), 779 deletions(-) diff --git a/Lib/email/__init__.py b/Lib/email/__init__.py index fae872439e..9fa4778300 100644 --- a/Lib/email/__init__.py +++ b/Lib/email/__init__.py @@ -25,7 +25,6 @@ ] - # Some convenience routines. Don't import Parser and Message as side-effects # of importing email since those cascadingly import most of the rest of the # email package. diff --git a/Lib/email/_encoded_words.py b/Lib/email/_encoded_words.py index 5eaab36ed0..6795a606de 100644 --- a/Lib/email/_encoded_words.py +++ b/Lib/email/_encoded_words.py @@ -62,7 +62,7 @@ # regex based decoder. _q_byte_subber = functools.partial(re.compile(br'=([a-fA-F0-9]{2})').sub, - lambda m: bytes([int(m.group(1), 16)])) + lambda m: bytes.fromhex(m.group(1).decode())) def decode_q(encoded): encoded = encoded.replace(b'_', b' ') @@ -98,30 +98,42 @@ def len_q(bstring): # def decode_b(encoded): - defects = [] + # First try encoding with validate=True, fixing the padding if needed. + # This will succeed only if encoded includes no invalid characters. pad_err = len(encoded) % 4 - if pad_err: - defects.append(errors.InvalidBase64PaddingDefect()) - padded_encoded = encoded + b'==='[:4-pad_err] - else: - padded_encoded = encoded + missing_padding = b'==='[:4-pad_err] if pad_err else b'' try: - return base64.b64decode(padded_encoded, validate=True), defects + return ( + base64.b64decode(encoded + missing_padding, validate=True), + [errors.InvalidBase64PaddingDefect()] if pad_err else [], + ) except binascii.Error: - # Since we had correct padding, this must an invalid char error. - defects = [errors.InvalidBase64CharactersDefect()] + # Since we had correct padding, this is likely an invalid char error. + # # The non-alphabet characters are ignored as far as padding - # goes, but we don't know how many there are. So we'll just - # try various padding lengths until something works. - for i in 0, 1, 2, 3: + # goes, but we don't know how many there are. So try without adding + # padding to see if it works. + try: + return ( + base64.b64decode(encoded, validate=False), + [errors.InvalidBase64CharactersDefect()], + ) + except binascii.Error: + # Add as much padding as could possibly be necessary (extra padding + # is ignored). try: - return base64.b64decode(encoded+b'='*i, validate=False), defects + return ( + base64.b64decode(encoded + b'==', validate=False), + [errors.InvalidBase64CharactersDefect(), + errors.InvalidBase64PaddingDefect()], + ) except binascii.Error: - if i==0: - defects.append(errors.InvalidBase64PaddingDefect()) - else: - # This should never happen. - raise AssertionError("unexpected binascii.Error") + # This only happens when the encoded string's length is 1 more + # than a multiple of 4, which is invalid. + # + # bpo-27397: Just return the encoded string since there's no + # way to decode. + return encoded, [errors.InvalidBase64LengthDefect()] def encode_b(bstring): return base64.b64encode(bstring).decode('ascii') @@ -167,15 +179,15 @@ def decode(ew): # Turn the CTE decoded bytes into unicode. try: string = bstring.decode(charset) - except UnicodeError: + except UnicodeDecodeError: defects.append(errors.UndecodableBytesDefect("Encoded word " - "contains bytes not decodable using {} charset".format(charset))) + f"contains bytes not decodable using {charset!r} charset")) string = bstring.decode(charset, 'surrogateescape') - except LookupError: + except (LookupError, UnicodeEncodeError): string = bstring.decode('ascii', 'surrogateescape') if charset.lower() != 'unknown-8bit': - defects.append(errors.CharsetError("Unknown charset {} " - "in encoded word; decoded as unknown bytes".format(charset))) + defects.append(errors.CharsetError(f"Unknown charset {charset!r} " + f"in encoded word; decoded as unknown bytes")) return string, charset, lang, defects diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index 57d01fbcb0..ec2215a5e5 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -68,9 +68,9 @@ """ import re +import sys import urllib # For urllib.parse.unquote from string import hexdigits -from collections import OrderedDict from operator import itemgetter from email import _encoded_words as _ew from email import errors @@ -92,93 +92,23 @@ ASPECIALS = TSPECIALS | set("*'%") ATTRIBUTE_ENDS = ASPECIALS | WSP EXTENDED_ATTRIBUTE_ENDS = ATTRIBUTE_ENDS - set('%') +NLSET = {'\n', '\r'} +SPECIALSNL = SPECIALS | NLSET def quote_string(value): return '"'+str(value).replace('\\', '\\\\').replace('"', r'\"')+'"' -# -# Accumulator for header folding -# - -class _Folded: - - def __init__(self, maxlen, policy): - self.maxlen = maxlen - self.policy = policy - self.lastlen = 0 - self.stickyspace = None - self.firstline = True - self.done = [] - self.current = [] +# Match a RFC 2047 word, looks like =?utf-8?q?someword?= +rfc2047_matcher = re.compile(r''' + =\? # literal =? + [^?]* # charset + \? # literal ? + [qQbB] # literal 'q' or 'b', case insensitive + \? # literal ? + .*? # encoded word + \?= # literal ?= +''', re.VERBOSE | re.MULTILINE) - def newline(self): - self.done.extend(self.current) - self.done.append(self.policy.linesep) - self.current.clear() - self.lastlen = 0 - - def finalize(self): - if self.current: - self.newline() - - def __str__(self): - return ''.join(self.done) - - def append(self, stoken): - self.current.append(stoken) - - def append_if_fits(self, token, stoken=None): - if stoken is None: - stoken = str(token) - l = len(stoken) - if self.stickyspace is not None: - stickyspace_len = len(self.stickyspace) - if self.lastlen + stickyspace_len + l <= self.maxlen: - self.current.append(self.stickyspace) - self.lastlen += stickyspace_len - self.current.append(stoken) - self.lastlen += l - self.stickyspace = None - self.firstline = False - return True - if token.has_fws: - ws = token.pop_leading_fws() - if ws is not None: - self.stickyspace += str(ws) - stickyspace_len += len(ws) - token._fold(self) - return True - if stickyspace_len and l + 1 <= self.maxlen: - margin = self.maxlen - l - if 0 < margin < stickyspace_len: - trim = stickyspace_len - margin - self.current.append(self.stickyspace[:trim]) - self.stickyspace = self.stickyspace[trim:] - stickyspace_len = trim - self.newline() - self.current.append(self.stickyspace) - self.current.append(stoken) - self.lastlen = l + stickyspace_len - self.stickyspace = None - self.firstline = False - return True - if not self.firstline: - self.newline() - self.current.append(self.stickyspace) - self.current.append(stoken) - self.stickyspace = None - self.firstline = False - return True - if self.lastlen + l <= self.maxlen: - self.current.append(stoken) - self.lastlen += l - return True - if l < self.maxlen: - self.newline() - self.current.append(stoken) - self.lastlen = l - return True - return False # # TokenList and its subclasses @@ -187,6 +117,8 @@ def append_if_fits(self, token, stoken=None): class TokenList(list): token_type = None + syntactic_break = True + ew_combine_allowed = True def __init__(self, *args, **kw): super().__init__(*args, **kw) @@ -207,84 +139,13 @@ def value(self): def all_defects(self): return sum((x.all_defects for x in self), self.defects) - # - # Folding API - # - # parts(): - # - # return a list of objects that constitute the "higher level syntactic - # objects" specified by the RFC as the best places to fold a header line. - # The returned objects must include leading folding white space, even if - # this means mutating the underlying parse tree of the object. Each object - # is only responsible for returning *its* parts, and should not drill down - # to any lower level except as required to meet the leading folding white - # space constraint. - # - # _fold(folded): - # - # folded: the result accumulator. This is an instance of _Folded. - # (XXX: I haven't finished factoring this out yet, the folding code - # pretty much uses this as a state object.) When the folded.current - # contains as much text as will fit, the _fold method should call - # folded.newline. - # folded.lastlen: the current length of the test stored in folded.current. - # folded.maxlen: The maximum number of characters that may appear on a - # folded line. Differs from the policy setting in that "no limit" is - # represented by +inf, which means it can be used in the trivially - # logical fashion in comparisons. - # - # Currently no subclasses implement parts, and I think this will remain - # true. A subclass only needs to implement _fold when the generic version - # isn't sufficient. _fold will need to be implemented primarily when it is - # possible for encoded words to appear in the specialized token-list, since - # there is no generic algorithm that can know where exactly the encoded - # words are allowed. A _fold implementation is responsible for filling - # lines in the same general way that the top level _fold does. It may, and - # should, call the _fold method of sub-objects in a similar fashion to that - # of the top level _fold. - # - # XXX: I'm hoping it will be possible to factor the existing code further - # to reduce redundancy and make the logic clearer. - - @property - def parts(self): - klass = self.__class__ - this = [] - for token in self: - if token.startswith_fws(): - if this: - yield this[0] if len(this)==1 else klass(this) - this.clear() - end_ws = token.pop_trailing_ws() - this.append(token) - if end_ws: - yield klass(this) - this = [end_ws] - if this: - yield this[0] if len(this)==1 else klass(this) - def startswith_fws(self): return self[0].startswith_fws() - def pop_leading_fws(self): - if self[0].token_type == 'fws': - return self.pop(0) - return self[0].pop_leading_fws() - - def pop_trailing_ws(self): - if self[-1].token_type == 'cfws': - return self.pop(-1) - return self[-1].pop_trailing_ws() - @property - def has_fws(self): - for part in self: - if part.has_fws: - return True - return False - - def has_leading_comment(self): - return self[0].has_leading_comment() + def as_ew_allowed(self): + """True if all top level tokens of this part may be RFC2047 encoded.""" + return all(part.as_ew_allowed for part in self) @property def comments(self): @@ -294,71 +155,13 @@ def comments(self): return comments def fold(self, *, policy): - # max_line_length 0/None means no limit, ie: infinitely long. - maxlen = policy.max_line_length or float("+inf") - folded = _Folded(maxlen, policy) - self._fold(folded) - folded.finalize() - return str(folded) - - def as_encoded_word(self, charset): - # This works only for things returned by 'parts', which include - # the leading fws, if any, that should be used. - res = [] - ws = self.pop_leading_fws() - if ws: - res.append(ws) - trailer = self.pop(-1) if self[-1].token_type=='fws' else '' - res.append(_ew.encode(str(self), charset)) - res.append(trailer) - return ''.join(res) - - def cte_encode(self, charset, policy): - res = [] - for part in self: - res.append(part.cte_encode(charset, policy)) - return ''.join(res) - - def _fold(self, folded): - encoding = 'utf-8' if folded.policy.utf8 else 'ascii' - for part in self.parts: - tstr = str(part) - tlen = len(tstr) - try: - str(part).encode(encoding) - except UnicodeEncodeError: - if any(isinstance(x, errors.UndecodableBytesDefect) - for x in part.all_defects): - charset = 'unknown-8bit' - else: - # XXX: this should be a policy setting when utf8 is False. - charset = 'utf-8' - tstr = part.cte_encode(charset, folded.policy) - tlen = len(tstr) - if folded.append_if_fits(part, tstr): - continue - # Peel off the leading whitespace if any and make it sticky, to - # avoid infinite recursion. - ws = part.pop_leading_fws() - if ws is not None: - # Peel off the leading whitespace and make it sticky, to - # avoid infinite recursion. - folded.stickyspace = str(part.pop(0)) - if folded.append_if_fits(part): - continue - if part.has_fws: - part._fold(folded) - continue - # There are no fold points in this one; it is too long for a single - # line and can't be split...we just have to put it on its own line. - folded.append(tstr) - folded.newline() + return _refold_parse_tree(self, policy=policy) def pprint(self, indent=''): - print('\n'.join(self._pp(indent=''))) + print(self.ppstr(indent=indent)) def ppstr(self, indent=''): - return '\n'.join(self._pp(indent='')) + return '\n'.join(self._pp(indent=indent)) def _pp(self, indent=''): yield '{}{}/{}('.format( @@ -390,213 +193,35 @@ def comments(self): class UnstructuredTokenList(TokenList): - token_type = 'unstructured' - def _fold(self, folded): - last_ew = None - encoding = 'utf-8' if folded.policy.utf8 else 'ascii' - for part in self.parts: - tstr = str(part) - is_ew = False - try: - str(part).encode(encoding) - except UnicodeEncodeError: - if any(isinstance(x, errors.UndecodableBytesDefect) - for x in part.all_defects): - charset = 'unknown-8bit' - else: - charset = 'utf-8' - if last_ew is not None: - # We've already done an EW, combine this one with it - # if there's room. - chunk = get_unstructured( - ''.join(folded.current[last_ew:]+[tstr])).as_encoded_word(charset) - oldlastlen = sum(len(x) for x in folded.current[:last_ew]) - schunk = str(chunk) - lchunk = len(schunk) - if oldlastlen + lchunk <= folded.maxlen: - del folded.current[last_ew:] - folded.append(schunk) - folded.lastlen = oldlastlen + lchunk - continue - tstr = part.as_encoded_word(charset) - is_ew = True - if folded.append_if_fits(part, tstr): - if is_ew: - last_ew = len(folded.current) - 1 - continue - if is_ew or last_ew: - # It's too big to fit on the line, but since we've - # got encoded words we can use encoded word folding. - part._fold_as_ew(folded) - continue - # Peel off the leading whitespace if any and make it sticky, to - # avoid infinite recursion. - ws = part.pop_leading_fws() - if ws is not None: - folded.stickyspace = str(ws) - if folded.append_if_fits(part): - continue - if part.has_fws: - part._fold(folded) - continue - # It can't be split...we just have to put it on its own line. - folded.append(tstr) - folded.newline() - last_ew = None - - def cte_encode(self, charset, policy): - res = [] - last_ew = None - for part in self: - spart = str(part) - try: - spart.encode('us-ascii') - res.append(spart) - except UnicodeEncodeError: - if last_ew is None: - res.append(part.cte_encode(charset, policy)) - last_ew = len(res) - else: - tl = get_unstructured(''.join(res[last_ew:] + [spart])) - res.append(tl.as_encoded_word(charset)) - return ''.join(res) - class Phrase(TokenList): - token_type = 'phrase' - def _fold(self, folded): - # As with Unstructured, we can have pure ASCII with or without - # surrogateescape encoded bytes, or we could have unicode. But this - # case is more complicated, since we have to deal with the various - # sub-token types and how they can be composed in the face of - # unicode-that-needs-CTE-encoding, and the fact that if a token a - # comment that becomes a barrier across which we can't compose encoded - # words. - last_ew = None - encoding = 'utf-8' if folded.policy.utf8 else 'ascii' - for part in self.parts: - tstr = str(part) - tlen = len(tstr) - has_ew = False - try: - str(part).encode(encoding) - except UnicodeEncodeError: - if any(isinstance(x, errors.UndecodableBytesDefect) - for x in part.all_defects): - charset = 'unknown-8bit' - else: - charset = 'utf-8' - if last_ew is not None and not part.has_leading_comment(): - # We've already done an EW, let's see if we can combine - # this one with it. The last_ew logic ensures that all we - # have at this point is atoms, no comments or quoted - # strings. So we can treat the text between the last - # encoded word and the content of this token as - # unstructured text, and things will work correctly. But - # we have to strip off any trailing comment on this token - # first, and if it is a quoted string we have to pull out - # the content (we're encoding it, so it no longer needs to - # be quoted). - if part[-1].token_type == 'cfws' and part.comments: - remainder = part.pop(-1) - else: - remainder = '' - for i, token in enumerate(part): - if token.token_type == 'bare-quoted-string': - part[i] = UnstructuredTokenList(token[:]) - chunk = get_unstructured( - ''.join(folded.current[last_ew:]+[tstr])).as_encoded_word(charset) - schunk = str(chunk) - lchunk = len(schunk) - if last_ew + lchunk <= folded.maxlen: - del folded.current[last_ew:] - folded.append(schunk) - folded.lastlen = sum(len(x) for x in folded.current) - continue - tstr = part.as_encoded_word(charset) - tlen = len(tstr) - has_ew = True - if folded.append_if_fits(part, tstr): - if has_ew and not part.comments: - last_ew = len(folded.current) - 1 - elif part.comments or part.token_type == 'quoted-string': - # If a comment is involved we can't combine EWs. And if a - # quoted string is involved, it's not worth the effort to - # try to combine them. - last_ew = None - continue - part._fold(folded) - - def cte_encode(self, charset, policy): - res = [] - last_ew = None - is_ew = False - for part in self: - spart = str(part) - try: - spart.encode('us-ascii') - res.append(spart) - except UnicodeEncodeError: - is_ew = True - if last_ew is None: - if not part.comments: - last_ew = len(res) - res.append(part.cte_encode(charset, policy)) - elif not part.has_leading_comment(): - if part[-1].token_type == 'cfws' and part.comments: - remainder = part.pop(-1) - else: - remainder = '' - for i, token in enumerate(part): - if token.token_type == 'bare-quoted-string': - part[i] = UnstructuredTokenList(token[:]) - tl = get_unstructured(''.join(res[last_ew:] + [spart])) - res[last_ew:] = [tl.as_encoded_word(charset)] - if part.comments or (not is_ew and part.token_type == 'quoted-string'): - last_ew = None - return ''.join(res) - class Word(TokenList): - token_type = 'word' class CFWSList(WhiteSpaceTokenList): - token_type = 'cfws' - def has_leading_comment(self): - return bool(self.comments) - class Atom(TokenList): - token_type = 'atom' class Token(TokenList): - token_type = 'token' + encode_as_ew = False class EncodedWord(TokenList): - token_type = 'encoded-word' cte = None charset = None lang = None - @property - def encoded(self): - if self.cte is not None: - return self.cte - _ew.encode(str(self), self.charset) - - class QuotedString(TokenList): @@ -812,7 +437,10 @@ def route(self): def addr_spec(self): for x in self: if x.token_type == 'addr-spec': - return x.addr_spec + if x.local_part: + return x.addr_spec + else: + return quote_string(x.local_part) + x.addr_spec else: return '<>' @@ -867,6 +495,7 @@ def display_name(self): class Domain(TokenList): token_type = 'domain' + as_ew_allowed = False @property def domain(self): @@ -874,18 +503,23 @@ def domain(self): class DotAtom(TokenList): - token_type = 'dot-atom' class DotAtomText(TokenList): - token_type = 'dot-atom-text' + as_ew_allowed = True + + +class NoFoldLiteral(TokenList): + token_type = 'no-fold-literal' + as_ew_allowed = False class AddrSpec(TokenList): token_type = 'addr-spec' + as_ew_allowed = False @property def local_part(self): @@ -918,24 +552,30 @@ def addr_spec(self): class ObsLocalPart(TokenList): token_type = 'obs-local-part' + as_ew_allowed = False class DisplayName(Phrase): token_type = 'display-name' + ew_combine_allowed = False @property def display_name(self): res = TokenList(self) + if len(res) == 0: + return res.value if res[0].token_type == 'cfws': res.pop(0) else: - if res[0][0].token_type == 'cfws': + if (isinstance(res[0], TokenList) and + res[0][0].token_type == 'cfws'): res[0] = TokenList(res[0][1:]) if res[-1].token_type == 'cfws': res.pop() else: - if res[-1][-1].token_type == 'cfws': + if (isinstance(res[-1], TokenList) and + res[-1][-1].token_type == 'cfws'): res[-1] = TokenList(res[-1][:-1]) return res.value @@ -948,11 +588,15 @@ def value(self): for x in self: if x.token_type == 'quoted-string': quote = True - if quote: + if len(self) != 0 and quote: pre = post = '' - if self[0].token_type=='cfws' or self[0][0].token_type=='cfws': + if (self[0].token_type == 'cfws' or + isinstance(self[0], TokenList) and + self[0][0].token_type == 'cfws'): pre = ' ' - if self[-1].token_type=='cfws' or self[-1][-1].token_type=='cfws': + if (self[-1].token_type == 'cfws' or + isinstance(self[-1], TokenList) and + self[-1][-1].token_type == 'cfws'): post = ' ' return pre+quote_string(self.display_name)+post else: @@ -962,6 +606,7 @@ def value(self): class LocalPart(TokenList): token_type = 'local-part' + as_ew_allowed = False @property def value(self): @@ -997,6 +642,7 @@ def local_part(self): class DomainLiteral(TokenList): token_type = 'domain-literal' + as_ew_allowed = False @property def domain(self): @@ -1083,6 +729,7 @@ def stripped_value(self): class MimeParameters(TokenList): token_type = 'mime-parameters' + syntactic_break = False @property def params(self): @@ -1091,7 +738,7 @@ def params(self): # to assume the RFC 2231 pieces can come in any order. However, we # output them in the order that we first see a given name, which gives # us a stable __str__. - params = OrderedDict() + params = {} # Using order preserving dict from Python 3.7+ for token in self: if not token.token_type.endswith('parameter'): continue @@ -1142,7 +789,7 @@ def params(self): else: try: value = value.decode(charset, 'surrogateescape') - except LookupError: + except (LookupError, UnicodeEncodeError): # XXX: there should really be a custom defect for # unknown character set to make it easy to find, # because otherwise unknown charset is a silent @@ -1167,6 +814,10 @@ def __str__(self): class ParameterizedHeaderValue(TokenList): + # Set this false so that the value doesn't wind up on a new line even + # if it and the parameters would fit there but not on the first line. + syntactic_break = False + @property def params(self): for token in reversed(self): @@ -1174,58 +825,50 @@ def params(self): return token.params return {} - @property - def parts(self): - if self and self[-1].token_type == 'mime-parameters': - # We don't want to start a new line if all of the params don't fit - # after the value, so unwrap the parameter list. - return TokenList(self[:-1] + self[-1]) - return TokenList(self).parts - class ContentType(ParameterizedHeaderValue): - token_type = 'content-type' + as_ew_allowed = False maintype = 'text' subtype = 'plain' class ContentDisposition(ParameterizedHeaderValue): - token_type = 'content-disposition' + as_ew_allowed = False content_disposition = None class ContentTransferEncoding(TokenList): - token_type = 'content-transfer-encoding' + as_ew_allowed = False cte = '7bit' class HeaderLabel(TokenList): - token_type = 'header-label' + as_ew_allowed = False -class Header(TokenList): +class MsgID(TokenList): + token_type = 'msg-id' + as_ew_allowed = False - token_type = 'header' + def fold(self, policy): + # message-id tokens may not be folded. + return str(self) + policy.linesep + + +class MessageID(MsgID): + token_type = 'message-id' - def _fold(self, folded): - folded.append(str(self.pop(0))) - folded.lastlen = len(folded.current[0]) - # The first line of the header is different from all others: we don't - # want to start a new object on a new line if it has any fold points in - # it that would allow part of it to be on the first header line. - # Further, if the first fold point would fit on the new line, we want - # to do that, but if it doesn't we want to put it on the first line. - # Folded supports this via the stickyspace attribute. If this - # attribute is not None, it does the special handling. - folded.stickyspace = str(self.pop(0)) if self[0].token_type == 'cfws' else '' - rest = self.pop(0) - if self: - raise ValueError("Malformed Header token list") - rest._fold(folded) + +class InvalidMessageID(MessageID): + token_type = 'invalid-message-id' + + +class Header(TokenList): + token_type = 'header' # @@ -1234,6 +877,10 @@ def _fold(self, folded): class Terminal(str): + as_ew_allowed = True + ew_combine_allowed = True + syntactic_break = True + def __new__(cls, value, token_type): self = super().__new__(cls, value) self.token_type = token_type @@ -1243,6 +890,9 @@ def __new__(cls, value, token_type): def __repr__(self): return "{}({})".format(self.__class__.__name__, super().__repr__()) + def pprint(self): + print(self.__class__.__name__ + '/' + self.token_type) + @property def all_defects(self): return list(self.defects) @@ -1256,29 +906,14 @@ def _pp(self, indent=''): '' if not self.defects else ' {}'.format(self.defects), )] - def cte_encode(self, charset, policy): - value = str(self) - try: - value.encode('us-ascii') - return value - except UnicodeEncodeError: - return _ew.encode(value, charset) - def pop_trailing_ws(self): # This terminates the recursion. return None - def pop_leading_fws(self): - # This terminates the recursion. - return None - @property def comments(self): return [] - def has_leading_comment(self): - return False - def __getnewargs__(self): return(str(self), self.token_type) @@ -1292,8 +927,6 @@ def value(self): def startswith_fws(self): return True - has_fws = True - class ValueTerminal(Terminal): @@ -1304,11 +937,6 @@ def value(self): def startswith_fws(self): return False - has_fws = False - - def as_encoded_word(self, charset): - return _ew.encode(str(self), charset) - class EWWhiteSpaceTerminal(WhiteSpaceTerminal): @@ -1316,14 +944,12 @@ class EWWhiteSpaceTerminal(WhiteSpaceTerminal): def value(self): return '' - @property - def encoded(self): - return self[:] - def __str__(self): return '' - has_fws = True + +class _InvalidEwError(errors.HeaderParseError): + """Invalid encoded word found while parsing headers.""" # XXX these need to become classes and used as instances so @@ -1331,6 +957,8 @@ def __str__(self): # up other parse trees. Maybe should have tests for that, too. DOT = ValueTerminal('.', 'dot') ListSeparator = ValueTerminal(',', 'list-separator') +ListSeparator.as_ew_allowed = False +ListSeparator.syntactic_break = False RouteComponentMarker = ValueTerminal('@', 'route-component-marker') # @@ -1356,15 +984,14 @@ def __str__(self): _wsp_splitter = re.compile(r'([{}]+)'.format(''.join(WSP))).split _non_atom_end_matcher = re.compile(r"[^{}]+".format( - ''.join(ATOM_ENDS).replace('\\','\\\\').replace(']',r'\]'))).match + re.escape(''.join(ATOM_ENDS)))).match _non_printable_finder = re.compile(r"[\x00-\x20\x7F]").findall _non_token_end_matcher = re.compile(r"[^{}]+".format( - ''.join(TOKEN_ENDS).replace('\\','\\\\').replace(']',r'\]'))).match + re.escape(''.join(TOKEN_ENDS)))).match _non_attribute_end_matcher = re.compile(r"[^{}]+".format( - ''.join(ATTRIBUTE_ENDS).replace('\\','\\\\').replace(']',r'\]'))).match + re.escape(''.join(ATTRIBUTE_ENDS)))).match _non_extended_attribute_end_matcher = re.compile(r"[^{}]+".format( - ''.join(EXTENDED_ATTRIBUTE_ENDS).replace( - '\\','\\\\').replace(']',r'\]'))).match + re.escape(''.join(EXTENDED_ATTRIBUTE_ENDS)))).match def _validate_xtext(xtext): """If input token contains ASCII non-printables, register a defect.""" @@ -1431,7 +1058,10 @@ def get_encoded_word(value): raise errors.HeaderParseError( "expected encoded word but found {}".format(value)) remstr = ''.join(remainder) - if len(remstr) > 1 and remstr[0] in hexdigits and remstr[1] in hexdigits: + if (len(remstr) > 1 and + remstr[0] in hexdigits and + remstr[1] in hexdigits and + tok.count('?') < 2): # The ? after the CTE was followed by an encoded word escape (=XX). rest, *remainder = remstr.split('?=', 1) tok = tok + '?=' + rest @@ -1442,8 +1072,8 @@ def get_encoded_word(value): value = ''.join(remainder) try: text, charset, lang, defects = _ew.decode('=?' + tok + '?=') - except ValueError: - raise errors.HeaderParseError( + except (ValueError, KeyError): + raise _InvalidEwError( "encoded word format invalid: '{}'".format(ew.cte)) ew.charset = charset ew.lang = lang @@ -1458,6 +1088,10 @@ def get_encoded_word(value): _validate_xtext(vtext) ew.append(vtext) text = ''.join(remainder) + # Encoded words should be followed by a WS + if value and value[0] not in WSP: + ew.defects.append(errors.InvalidHeaderDefect( + "missing trailing whitespace after encoded-word")) return ew, value def get_unstructured(value): @@ -1489,9 +1123,12 @@ def get_unstructured(value): token, value = get_fws(value) unstructured.append(token) continue + valid_ew = True if value.startswith('=?'): try: token, value = get_encoded_word(value) + except _InvalidEwError: + valid_ew = False except errors.HeaderParseError: # XXX: Need to figure out how to register defects when # appropriate here. @@ -1510,6 +1147,14 @@ def get_unstructured(value): unstructured.append(token) continue tok, *remainder = _wsp_splitter(value, 1) + # Split in the middle of an atom if there is a rfc2047 encoded word + # which does not have WSP on both sides. The defect will be registered + # the next time through the loop. + # This needs to only be performed when the encoded word is valid; + # otherwise, performing it on an invalid encoded word can cause + # the parser to go in an infinite loop. + if valid_ew and rfc2047_matcher.search(tok): + tok, *remainder = value.partition('=?') vtext = ValueTerminal(tok, 'vtext') _validate_xtext(vtext) unstructured.append(vtext) @@ -1571,21 +1216,33 @@ def get_bare_quoted_string(value): value is the text between the quote marks, with whitespace preserved and quoted pairs decoded. """ - if value[0] != '"': + if not value or value[0] != '"': raise errors.HeaderParseError( "expected '\"' but found '{}'".format(value)) bare_quoted_string = BareQuotedString() value = value[1:] + if value and value[0] == '"': + token, value = get_qcontent(value) + bare_quoted_string.append(token) while value and value[0] != '"': if value[0] in WSP: token, value = get_fws(value) elif value[:2] == '=?': + valid_ew = False try: token, value = get_encoded_word(value) bare_quoted_string.defects.append(errors.InvalidHeaderDefect( "encoded word inside quoted string")) + valid_ew = True except errors.HeaderParseError: token, value = get_qcontent(value) + # Collapse the whitespace between two encoded words that occur in a + # bare-quoted-string. + if valid_ew and len(bare_quoted_string) > 1: + if (bare_quoted_string[-1].token_type == 'fws' and + bare_quoted_string[-2].token_type == 'encoded-word'): + bare_quoted_string[-1] = EWWhiteSpaceTerminal( + bare_quoted_string[-1], 'fws') else: token, value = get_qcontent(value) bare_quoted_string.append(token) @@ -1742,6 +1399,9 @@ def get_word(value): leader, value = get_cfws(value) else: leader = None + if not value: + raise errors.HeaderParseError( + "Expected 'atom' or 'quoted-string' but found nothing.") if value[0]=='"': token, value = get_quoted_string(value) elif value[0] in SPECIALS: @@ -1797,7 +1457,7 @@ def get_local_part(value): """ local_part = LocalPart() leader = None - if value[0] in CFWS_LEADER: + if value and value[0] in CFWS_LEADER: leader, value = get_cfws(value) if not value: raise errors.HeaderParseError( @@ -1863,13 +1523,18 @@ def get_obs_local_part(value): raise token, value = get_cfws(value) obs_local_part.append(token) + if not obs_local_part: + raise errors.HeaderParseError( + "expected obs-local-part but found '{}'".format(value)) if (obs_local_part[0].token_type == 'dot' or obs_local_part[0].token_type=='cfws' and + len(obs_local_part) > 1 and obs_local_part[1].token_type=='dot'): obs_local_part.defects.append(errors.InvalidHeaderDefect( "Invalid leading '.' in local part")) if (obs_local_part[-1].token_type == 'dot' or obs_local_part[-1].token_type=='cfws' and + len(obs_local_part) > 1 and obs_local_part[-2].token_type=='dot'): obs_local_part.defects.append(errors.InvalidHeaderDefect( "Invalid trailing '.' in local part")) @@ -1951,7 +1616,7 @@ def get_domain(value): """ domain = Domain() leader = None - if value[0] in CFWS_LEADER: + if value and value[0] in CFWS_LEADER: leader, value = get_cfws(value) if not value: raise errors.HeaderParseError( @@ -1966,6 +1631,8 @@ def get_domain(value): token, value = get_dot_atom(value) except errors.HeaderParseError: token, value = get_atom(value) + if value and value[0] == '@': + raise errors.HeaderParseError('Invalid Domain') if leader is not None: token[:0] = [leader] domain.append(token) @@ -1989,7 +1656,7 @@ def get_addr_spec(value): addr_spec.append(token) if not value or value[0] != '@': addr_spec.defects.append(errors.InvalidHeaderDefect( - "add-spec local part with no domain")) + "addr-spec local part with no domain")) return addr_spec, value addr_spec.append(ValueTerminal('@', 'address-at-symbol')) token, value = get_domain(value[1:]) @@ -2025,6 +1692,8 @@ def get_obs_route(value): if value[0] in CFWS_LEADER: token, value = get_cfws(value) obs_route.append(token) + if not value: + break if value[0] == '@': obs_route.append(RouteComponentMarker) token, value = get_domain(value[1:]) @@ -2043,7 +1712,7 @@ def get_angle_addr(value): """ angle_addr = AngleAddr() - if value[0] in CFWS_LEADER: + if value and value[0] in CFWS_LEADER: token, value = get_cfws(value) angle_addr.append(token) if not value or value[0] != '<': @@ -2053,7 +1722,7 @@ def get_angle_addr(value): value = value[1:] # Although it is not legal per RFC5322, SMTP uses '<>' in certain # circumstances. - if value[0] == '>': + if value and value[0] == '>': angle_addr.append(ValueTerminal('>', 'angle-addr-end')) angle_addr.defects.append(errors.InvalidHeaderDefect( "null addr-spec in angle-addr")) @@ -2105,6 +1774,9 @@ def get_name_addr(value): name_addr = NameAddr() # Both the optional display name and the angle-addr can start with cfws. leader = None + if not value: + raise errors.HeaderParseError( + "expected name-addr but found '{}'".format(value)) if value[0] in CFWS_LEADER: leader, value = get_cfws(value) if not value: @@ -2119,7 +1791,10 @@ def get_name_addr(value): raise errors.HeaderParseError( "expected name-addr but found '{}'".format(token)) if leader is not None: - token[0][:0] = [leader] + if isinstance(token[0], TokenList): + token[0][:0] = [leader] + else: + token[:0] = [leader] leader = None name_addr.append(token) token, value = get_angle_addr(value) @@ -2281,7 +1956,7 @@ def get_group(value): if not value: group.defects.append(errors.InvalidHeaderDefect( "end of header in group")) - if value[0] != ';': + elif value[0] != ';': raise errors.HeaderParseError( "expected ';' at end of group but found {}".format(value)) group.append(ValueTerminal(';', 'group-terminator')) @@ -2335,7 +2010,7 @@ def get_address_list(value): try: token, value = get_address(value) address_list.append(token) - except errors.HeaderParseError as err: + except errors.HeaderParseError: leader = None if value[0] in CFWS_LEADER: leader, value = get_cfws(value) @@ -2370,10 +2045,122 @@ def get_address_list(value): address_list.defects.append(errors.InvalidHeaderDefect( "invalid address in address-list")) if value: # Must be a , at this point. - address_list.append(ValueTerminal(',', 'list-separator')) + address_list.append(ListSeparator) value = value[1:] return address_list, value + +def get_no_fold_literal(value): + """ no-fold-literal = "[" *dtext "]" + """ + no_fold_literal = NoFoldLiteral() + if not value: + raise errors.HeaderParseError( + "expected no-fold-literal but found '{}'".format(value)) + if value[0] != '[': + raise errors.HeaderParseError( + "expected '[' at the start of no-fold-literal " + "but found '{}'".format(value)) + no_fold_literal.append(ValueTerminal('[', 'no-fold-literal-start')) + value = value[1:] + token, value = get_dtext(value) + no_fold_literal.append(token) + if not value or value[0] != ']': + raise errors.HeaderParseError( + "expected ']' at the end of no-fold-literal " + "but found '{}'".format(value)) + no_fold_literal.append(ValueTerminal(']', 'no-fold-literal-end')) + return no_fold_literal, value[1:] + +def get_msg_id(value): + """msg-id = [CFWS] "<" id-left '@' id-right ">" [CFWS] + id-left = dot-atom-text / obs-id-left + id-right = dot-atom-text / no-fold-literal / obs-id-right + no-fold-literal = "[" *dtext "]" + """ + msg_id = MsgID() + if value and value[0] in CFWS_LEADER: + token, value = get_cfws(value) + msg_id.append(token) + if not value or value[0] != '<': + raise errors.HeaderParseError( + "expected msg-id but found '{}'".format(value)) + msg_id.append(ValueTerminal('<', 'msg-id-start')) + value = value[1:] + # Parse id-left. + try: + token, value = get_dot_atom_text(value) + except errors.HeaderParseError: + try: + # obs-id-left is same as local-part of add-spec. + token, value = get_obs_local_part(value) + msg_id.defects.append(errors.ObsoleteHeaderDefect( + "obsolete id-left in msg-id")) + except errors.HeaderParseError: + raise errors.HeaderParseError( + "expected dot-atom-text or obs-id-left" + " but found '{}'".format(value)) + msg_id.append(token) + if not value or value[0] != '@': + msg_id.defects.append(errors.InvalidHeaderDefect( + "msg-id with no id-right")) + # Even though there is no id-right, if the local part + # ends with `>` let's just parse it too and return + # along with the defect. + if value and value[0] == '>': + msg_id.append(ValueTerminal('>', 'msg-id-end')) + value = value[1:] + return msg_id, value + msg_id.append(ValueTerminal('@', 'address-at-symbol')) + value = value[1:] + # Parse id-right. + try: + token, value = get_dot_atom_text(value) + except errors.HeaderParseError: + try: + token, value = get_no_fold_literal(value) + except errors.HeaderParseError: + try: + token, value = get_domain(value) + msg_id.defects.append(errors.ObsoleteHeaderDefect( + "obsolete id-right in msg-id")) + except errors.HeaderParseError: + raise errors.HeaderParseError( + "expected dot-atom-text, no-fold-literal or obs-id-right" + " but found '{}'".format(value)) + msg_id.append(token) + if value and value[0] == '>': + value = value[1:] + else: + msg_id.defects.append(errors.InvalidHeaderDefect( + "missing trailing '>' on msg-id")) + msg_id.append(ValueTerminal('>', 'msg-id-end')) + if value and value[0] in CFWS_LEADER: + token, value = get_cfws(value) + msg_id.append(token) + return msg_id, value + + +def parse_message_id(value): + """message-id = "Message-ID:" msg-id CRLF + """ + message_id = MessageID() + try: + token, value = get_msg_id(value) + message_id.append(token) + except errors.HeaderParseError as ex: + token = get_unstructured(value) + message_id = InvalidMessageID(token) + message_id.defects.append( + errors.InvalidHeaderDefect("Invalid msg-id: {!r}".format(ex))) + else: + # Value after parsing a valid msg_id should be None. + if value: + message_id.defects.append(errors.InvalidHeaderDefect( + "Unexpected {!r}".format(value))) + + return message_id + # # XXX: As I begin to add additional header parsers, I'm realizing we probably # have two level of parser routines: the get_XXX methods that get a token in @@ -2615,8 +2402,8 @@ def get_section(value): digits += value[0] value = value[1:] if digits[0] == '0' and digits != '0': - section.defects.append(errors.InvalidHeaderError("section number" - "has an invalid leading 0")) + section.defects.append(errors.InvalidHeaderDefect( + "section number has an invalid leading 0")) section.number = int(digits) section.append(ValueTerminal(digits, 'digits')) return section, value @@ -2679,7 +2466,6 @@ def get_parameter(value): raise errors.HeaderParseError("Parameter not followed by '='") param.append(ValueTerminal('=', 'parameter-separator')) value = value[1:] - leader = None if value and value[0] in CFWS_LEADER: token, value = get_cfws(value) param.append(token) @@ -2754,7 +2540,7 @@ def get_parameter(value): if value[0] != "'": raise errors.HeaderParseError("Expected RFC2231 char/lang encoding " "delimiter, but found {!r}".format(value)) - appendto.append(ValueTerminal("'", 'RFC2231 delimiter')) + appendto.append(ValueTerminal("'", 'RFC2231-delimiter')) value = value[1:] if value and value[0] != "'": token, value = get_attrtext(value) @@ -2763,7 +2549,7 @@ def get_parameter(value): if not value or value[0] != "'": raise errors.HeaderParseError("Expected RFC2231 char/lang encoding " "delimiter, but found {}".format(value)) - appendto.append(ValueTerminal("'", 'RFC2231 delimiter')) + appendto.append(ValueTerminal("'", 'RFC2231-delimiter')) value = value[1:] if remainder is not None: # Treat the rest of value as bare quoted string content. @@ -2771,6 +2557,9 @@ def get_parameter(value): while value: if value[0] in WSP: token, value = get_fws(value) + elif value[0] == '"': + token = ValueTerminal('"', 'DQUOTE') + value = value[1:] else: token, value = get_qcontent(value) v.append(token) @@ -2791,7 +2580,7 @@ def parse_mime_parameters(value): the formal RFC grammar, but it is more convenient for us for the set of parameters to be treated as its own TokenList. - This is 'parse' routine because it consumes the reminaing value, but it + This is 'parse' routine because it consumes the remaining value, but it would never be called to parse a full header. Instead it is called to parse everything after the non-parameter value of a specific MIME header. @@ -2801,7 +2590,7 @@ def parse_mime_parameters(value): try: token, value = get_parameter(value) mime_parameters.append(token) - except errors.HeaderParseError as err: + except errors.HeaderParseError: leader = None if value[0] in CFWS_LEADER: leader, value = get_cfws(value) @@ -2859,7 +2648,6 @@ def parse_content_type_header(value): don't do that. """ ctype = ContentType() - recover = False if not value: ctype.defects.append(errors.HeaderMissingRequiredValue( "Missing content type specification")) @@ -2968,3 +2756,323 @@ def parse_content_transfer_encoding_header(value): token, value = get_phrase(value) cte_header.append(token) return cte_header + + +# +# Header folding +# +# Header folding is complex, with lots of rules and corner cases. The +# following code does its best to obey the rules and handle the corner +# cases, but you can be sure there are few bugs:) +# +# This folder generally canonicalizes as it goes, preferring the stringified +# version of each token. The tokens contain information that supports the +# folder, including which tokens can be encoded in which ways. +# +# Folded text is accumulated in a simple list of strings ('lines'), each +# one of which should be less than policy.max_line_length ('maxlen'). +# + +def _steal_trailing_WSP_if_exists(lines): + wsp = '' + if lines and lines[-1] and lines[-1][-1] in WSP: + wsp = lines[-1][-1] + lines[-1] = lines[-1][:-1] + return wsp + +def _refold_parse_tree(parse_tree, *, policy): + """Return string of contents of parse_tree folded according to RFC rules. + + """ + # max_line_length 0/None means no limit, ie: infinitely long. + maxlen = policy.max_line_length or sys.maxsize + encoding = 'utf-8' if policy.utf8 else 'us-ascii' + lines = [''] # Folded lines to be output + leading_whitespace = '' # When we have whitespace between two encoded + # words, we may need to encode the whitespace + # at the beginning of the second word. + last_ew = None # Points to the last encoded character if there's an ew on + # the line + last_charset = None + wrap_as_ew_blocked = 0 + want_encoding = False # This is set to True if we need to encode this part + end_ew_not_allowed = Terminal('', 'wrap_as_ew_blocked') + parts = list(parse_tree) + while parts: + part = parts.pop(0) + if part is end_ew_not_allowed: + wrap_as_ew_blocked -= 1 + continue + tstr = str(part) + if not want_encoding: + if part.token_type == 'ptext': + # Encode if tstr contains special characters. + want_encoding = not SPECIALSNL.isdisjoint(tstr) + else: + # Encode if tstr contains newlines. + want_encoding = not NLSET.isdisjoint(tstr) + try: + tstr.encode(encoding) + charset = encoding + except UnicodeEncodeError: + if any(isinstance(x, errors.UndecodableBytesDefect) + for x in part.all_defects): + charset = 'unknown-8bit' + else: + # If policy.utf8 is false this should really be taken from a + # 'charset' property on the policy. + charset = 'utf-8' + want_encoding = True + + if part.token_type == 'mime-parameters': + # Mime parameter folding (using RFC2231) is extra special. + _fold_mime_parameters(part, lines, maxlen, encoding) + continue + + if want_encoding and not wrap_as_ew_blocked: + if not part.as_ew_allowed: + want_encoding = False + last_ew = None + if part.syntactic_break: + encoded_part = part.fold(policy=policy)[:-len(policy.linesep)] + if policy.linesep not in encoded_part: + # It fits on a single line + if len(encoded_part) > maxlen - len(lines[-1]): + # But not on this one, so start a new one. + newline = _steal_trailing_WSP_if_exists(lines) + # XXX what if encoded_part has no leading FWS? + lines.append(newline) + lines[-1] += encoded_part + continue + # Either this is not a major syntactic break, so we don't + # want it on a line by itself even if it fits, or it + # doesn't fit on a line by itself. Either way, fall through + # to unpacking the subparts and wrapping them. + if not hasattr(part, 'encode'): + # It's not a Terminal, do each piece individually. + parts = list(part) + parts + want_encoding = False + continue + elif part.as_ew_allowed: + # It's a terminal, wrap it as an encoded word, possibly + # combining it with previously encoded words if allowed. + if (last_ew is not None and + charset != last_charset and + (last_charset == 'unknown-8bit' or + last_charset == 'utf-8' and charset != 'us-ascii')): + last_ew = None + last_ew = _fold_as_ew(tstr, lines, maxlen, last_ew, + part.ew_combine_allowed, charset, leading_whitespace) + # This whitespace has been added to the lines in _fold_as_ew() + # so clear it now. + leading_whitespace = '' + last_charset = charset + want_encoding = False + continue + else: + # It's a terminal which should be kept non-encoded + # (e.g. a ListSeparator). + last_ew = None + want_encoding = False + # fall through + + if len(tstr) <= maxlen - len(lines[-1]): + lines[-1] += tstr + continue + + # This part is too long to fit. The RFC wants us to break at + # "major syntactic breaks", so unless we don't consider this + # to be one, check if it will fit on the next line by itself. + leading_whitespace = '' + if (part.syntactic_break and + len(tstr) + 1 <= maxlen): + newline = _steal_trailing_WSP_if_exists(lines) + if newline or part.startswith_fws(): + # We're going to fold the data onto a new line here. Due to + # the way encoded strings handle continuation lines, we need to + # be prepared to encode any whitespace if the next line turns + # out to start with an encoded word. + lines.append(newline + tstr) + + whitespace_accumulator = [] + for char in lines[-1]: + if char not in WSP: + break + whitespace_accumulator.append(char) + leading_whitespace = ''.join(whitespace_accumulator) + last_ew = None + continue + if not hasattr(part, 'encode'): + # It's not a terminal, try folding the subparts. + newparts = list(part) + if not part.as_ew_allowed: + wrap_as_ew_blocked += 1 + newparts.append(end_ew_not_allowed) + parts = newparts + parts + continue + if part.as_ew_allowed and not wrap_as_ew_blocked: + # It doesn't need CTE encoding, but encode it anyway so we can + # wrap it. + parts.insert(0, part) + want_encoding = True + continue + # We can't figure out how to wrap, it, so give up. + newline = _steal_trailing_WSP_if_exists(lines) + if newline or part.startswith_fws(): + lines.append(newline + tstr) + else: + # We can't fold it onto the next line either... + lines[-1] += tstr + + return policy.linesep.join(lines) + policy.linesep + +def _fold_as_ew(to_encode, lines, maxlen, last_ew, ew_combine_allowed, charset, leading_whitespace): + """Fold string to_encode into lines as encoded word, combining if allowed. + Return the new value for last_ew, or None if ew_combine_allowed is False. + + If there is already an encoded word in the last line of lines (indicated by + a non-None value for last_ew) and ew_combine_allowed is true, decode the + existing ew, combine it with to_encode, and re-encode. Otherwise, encode + to_encode. In either case, split to_encode as necessary so that the + encoded segments fit within maxlen. + + """ + if last_ew is not None and ew_combine_allowed: + to_encode = str( + get_unstructured(lines[-1][last_ew:] + to_encode)) + lines[-1] = lines[-1][:last_ew] + elif to_encode[0] in WSP: + # We're joining this to non-encoded text, so don't encode + # the leading blank. + leading_wsp = to_encode[0] + to_encode = to_encode[1:] + if (len(lines[-1]) == maxlen): + lines.append(_steal_trailing_WSP_if_exists(lines)) + lines[-1] += leading_wsp + + trailing_wsp = '' + if to_encode[-1] in WSP: + # Likewise for the trailing space. + trailing_wsp = to_encode[-1] + to_encode = to_encode[:-1] + new_last_ew = len(lines[-1]) if last_ew is None else last_ew + + encode_as = 'utf-8' if charset == 'us-ascii' else charset + + # The RFC2047 chrome takes up 7 characters plus the length + # of the charset name. + chrome_len = len(encode_as) + 7 + + if (chrome_len + 1) >= maxlen: + raise errors.HeaderParseError( + "max_line_length is too small to fit an encoded word") + + while to_encode: + remaining_space = maxlen - len(lines[-1]) + text_space = remaining_space - chrome_len - len(leading_whitespace) + if text_space <= 0: + lines.append(' ') + continue + + # If we are at the start of a continuation line, prepend whitespace + # (we only want to do this when the line starts with an encoded word + # but if we're folding in this helper function, then we know that we + # are going to be writing out an encoded word.) + if len(lines) > 1 and len(lines[-1]) == 1 and leading_whitespace: + encoded_word = _ew.encode(leading_whitespace, charset=encode_as) + lines[-1] += encoded_word + leading_whitespace = '' + + to_encode_word = to_encode[:text_space] + encoded_word = _ew.encode(to_encode_word, charset=encode_as) + excess = len(encoded_word) - remaining_space + while excess > 0: + # Since the chunk to encode is guaranteed to fit into less than 100 characters, + # shrinking it by one at a time shouldn't take long. + to_encode_word = to_encode_word[:-1] + encoded_word = _ew.encode(to_encode_word, charset=encode_as) + excess = len(encoded_word) - remaining_space + lines[-1] += encoded_word + to_encode = to_encode[len(to_encode_word):] + leading_whitespace = '' + + if to_encode: + lines.append(' ') + new_last_ew = len(lines[-1]) + lines[-1] += trailing_wsp + return new_last_ew if ew_combine_allowed else None + +def _fold_mime_parameters(part, lines, maxlen, encoding): + """Fold TokenList 'part' into the 'lines' list as mime parameters. + + Using the decoded list of parameters and values, format them according to + the RFC rules, including using RFC2231 encoding if the value cannot be + expressed in 'encoding' and/or the parameter+value is too long to fit + within 'maxlen'. + + """ + # Special case for RFC2231 encoding: start from decoded values and use + # RFC2231 encoding iff needed. + # + # Note that the 1 and 2s being added to the length calculations are + # accounting for the possibly-needed spaces and semicolons we'll be adding. + # + for name, value in part.params: + # XXX What if this ';' puts us over maxlen the first time through the + # loop? We should split the header value onto a newline in that case, + # but to do that we need to recognize the need earlier or reparse the + # header, so I'm going to ignore that bug for now. It'll only put us + # one character over. + if not lines[-1].rstrip().endswith(';'): + lines[-1] += ';' + charset = encoding + error_handler = 'strict' + try: + value.encode(encoding) + encoding_required = False + except UnicodeEncodeError: + encoding_required = True + if utils._has_surrogates(value): + charset = 'unknown-8bit' + error_handler = 'surrogateescape' + else: + charset = 'utf-8' + if encoding_required: + encoded_value = urllib.parse.quote( + value, safe='', errors=error_handler) + tstr = "{}*={}''{}".format(name, charset, encoded_value) + else: + tstr = '{}={}'.format(name, quote_string(value)) + if len(lines[-1]) + len(tstr) + 1 < maxlen: + lines[-1] = lines[-1] + ' ' + tstr + continue + elif len(tstr) + 2 <= maxlen: + lines.append(' ' + tstr) + continue + # We need multiple sections. We are allowed to mix encoded and + # non-encoded sections, but we aren't going to. We'll encode them all. + section = 0 + extra_chrome = charset + "''" + while value: + chrome_len = len(name) + len(str(section)) + 3 + len(extra_chrome) + if maxlen <= chrome_len + 3: + # We need room for the leading blank, the trailing semicolon, + # and at least one character of the value. If we don't + # have that, we'd be stuck, so in that case fall back to + # the RFC standard width. + maxlen = 78 + splitpoint = maxchars = maxlen - chrome_len - 2 + while True: + partial = value[:splitpoint] + encoded_value = urllib.parse.quote( + partial, safe='', errors=error_handler) + if len(encoded_value) <= maxchars: + break + splitpoint -= 1 + lines.append(" {}*{}*={}{}".format( + name, section, extra_chrome, encoded_value)) + extra_chrome = '' + section += 1 + value = value[splitpoint:] + if value: + lines[-1] += ';' diff --git a/Lib/email/_parseaddr.py b/Lib/email/_parseaddr.py index cdfa3729ad..0f1bf8e425 100644 --- a/Lib/email/_parseaddr.py +++ b/Lib/email/_parseaddr.py @@ -13,7 +13,7 @@ 'quote', ] -import time, calendar +import time SPACE = ' ' EMPTYSTRING = '' @@ -65,8 +65,10 @@ def _parsedate_tz(data): """ if not data: - return + return None data = data.split() + if not data: # This happens for whitespace-only input. + return None # The FWS after the comma after the day-of-week is optional, so search and # adjust for this. if data[0].endswith(',') or data[0].lower() in _daynames: @@ -93,6 +95,8 @@ def _parsedate_tz(data): return None data = data[:5] [dd, mm, yy, tm, tz] = data + if not (dd and mm and yy): + return None mm = mm.lower() if mm not in _monthnames: dd, mm = mm, dd.lower() @@ -108,6 +112,8 @@ def _parsedate_tz(data): yy, tm = tm, yy if yy[-1] == ',': yy = yy[:-1] + if not yy: + return None if not yy[0].isdigit(): yy, tz = tz, yy if tm[-1] == ',': @@ -126,6 +132,8 @@ def _parsedate_tz(data): tss = 0 elif len(tm) == 3: [thh, tmm, tss] = tm + else: + return None else: return None try: @@ -186,6 +194,9 @@ def mktime_tz(data): # No zone info, so localtime is better assumption than GMT return time.mktime(data[:8] + (-1,)) else: + # Delay the import, since mktime_tz is rarely used + import calendar + t = calendar.timegm(data) return t - data[9] @@ -379,7 +390,12 @@ def getaddrspec(self): aslist.append('@') self.pos += 1 self.gotonext() - return EMPTYSTRING.join(aslist) + self.getdomain() + domain = self.getdomain() + if not domain: + # Invalid domain, return an empty address instead of returning a + # local part to denote failed parsing. + return EMPTYSTRING + return EMPTYSTRING.join(aslist) + domain def getdomain(self): """Get the complete domain name from an address.""" @@ -394,6 +410,10 @@ def getdomain(self): elif self.field[self.pos] == '.': self.pos += 1 sdlist.append('.') + elif self.field[self.pos] == '@': + # bpo-34155: Don't parse domains with two `@` like + # `a@malicious.org@important.com`. + return EMPTYSTRING elif self.field[self.pos] in self.atomends: break else: diff --git a/Lib/email/_policybase.py b/Lib/email/_policybase.py index df4649676a..c9f0d74309 100644 --- a/Lib/email/_policybase.py +++ b/Lib/email/_policybase.py @@ -152,11 +152,18 @@ class Policy(_PolicyBase, metaclass=abc.ABCMeta): mangle_from_ -- a flag that, when True escapes From_ lines in the body of the message by putting a `>' in front of them. This is used when the message is being - serialized by a generator. Default: True. + serialized by a generator. Default: False. message_factory -- the class to use to create new message objects. If the value is None, the default is Message. + verify_generated_headers + -- if true, the generator verifies that each header + they are properly folded, so that a parser won't + treat it as multiple headers, start-of-body, or + part of another header. + This is a check against custom Header & fold() + implementations. """ raise_on_defect = False @@ -165,6 +172,7 @@ class Policy(_PolicyBase, metaclass=abc.ABCMeta): max_line_length = 78 mangle_from_ = False message_factory = None + verify_generated_headers = True def handle_defect(self, obj, defect): """Based on policy, either raise defect or call register_defect. @@ -294,12 +302,12 @@ def header_source_parse(self, sourcelines): """+ The name is parsed as everything up to the ':' and returned unmodified. The value is determined by stripping leading whitespace off the - remainder of the first line, joining all subsequent lines together, and + remainder of the first line joined with all subsequent lines, and stripping any trailing carriage return or linefeed characters. """ name, value = sourcelines[0].split(':', 1) - value = value.lstrip(' \t') + ''.join(sourcelines[1:]) + value = ''.join((value, *sourcelines[1:])).lstrip(' \t\r\n') return (name, value.rstrip('\r\n')) def header_store_parse(self, name, value): @@ -361,8 +369,12 @@ def _fold(self, name, value, sanitize): # Assume it is a Header-like object. h = value if h is not None: - parts.append(h.encode(linesep=self.linesep, - maxlinelen=self.max_line_length)) + # The Header class interprets a value of None for maxlinelen as the + # default value of 78, as recommended by RFC 2822. + maxlinelen = 0 + if self.max_line_length is not None: + maxlinelen = self.max_line_length + parts.append(h.encode(linesep=self.linesep, maxlinelen=maxlinelen)) parts.append(self.linesep) return ''.join(parts) diff --git a/Lib/email/architecture.rst b/Lib/email/architecture.rst index 78572ae63b..fcd10bde13 100644 --- a/Lib/email/architecture.rst +++ b/Lib/email/architecture.rst @@ -66,7 +66,7 @@ data payloads. Message Lifecycle ----------------- -The general lifecyle of a message is: +The general lifecycle of a message is: Creation A `Message` object can be created by a Parser, or it can be diff --git a/Lib/email/base64mime.py b/Lib/email/base64mime.py index 17f0818f6c..4cdf22666e 100644 --- a/Lib/email/base64mime.py +++ b/Lib/email/base64mime.py @@ -45,7 +45,6 @@ MISC_LEN = 7 - # Helpers def header_length(bytearray): """Return the length of s when it is encoded with base64.""" @@ -57,7 +56,6 @@ def header_length(bytearray): return n - def header_encode(header_bytes, charset='iso-8859-1'): """Encode a single header line with Base64 encoding in a given charset. @@ -72,7 +70,6 @@ def header_encode(header_bytes, charset='iso-8859-1'): return '=?%s?b?%s?=' % (charset, encoded) - def body_encode(s, maxlinelen=76, eol=NL): r"""Encode a string with base64. @@ -84,7 +81,7 @@ def body_encode(s, maxlinelen=76, eol=NL): in an email. """ if not s: - return s + return "" encvec = [] max_unencoded = maxlinelen * 3 // 4 @@ -98,7 +95,6 @@ def body_encode(s, maxlinelen=76, eol=NL): return EMPTYSTRING.join(encvec) - def decode(string): """Decode a raw base64 string, returning a bytes object. diff --git a/Lib/email/charset.py b/Lib/email/charset.py index ee564040c6..043801107b 100644 --- a/Lib/email/charset.py +++ b/Lib/email/charset.py @@ -18,7 +18,6 @@ from email.encoders import encode_7or8bit - # Flags for types of header encodings QP = 1 # Quoted-Printable BASE64 = 2 # Base64 @@ -32,7 +31,6 @@ EMPTYSTRING = '' - # Defaults CHARSETS = { # input header enc body enc output conv @@ -104,7 +102,6 @@ } - # Convenience functions for extending the above mappings def add_charset(charset, header_enc=None, body_enc=None, output_charset=None): """Add character set properties to the global registry. @@ -112,8 +109,8 @@ def add_charset(charset, header_enc=None, body_enc=None, output_charset=None): charset is the input character set, and must be the canonical name of a character set. - Optional header_enc and body_enc is either Charset.QP for - quoted-printable, Charset.BASE64 for base64 encoding, Charset.SHORTEST for + Optional header_enc and body_enc is either charset.QP for + quoted-printable, charset.BASE64 for base64 encoding, charset.SHORTEST for the shortest of qp or base64 encoding, or None for no encoding. SHORTEST is only valid for header_enc. It describes how message headers and message bodies in the input charset are to be encoded. Default is no @@ -153,7 +150,6 @@ def add_codec(charset, codecname): CODEC_MAP[charset] = codecname - # Convenience function for encoding strings, taking into account # that they might be unknown-8bit (ie: have surrogate-escaped bytes) def _encode(string, codec): @@ -163,7 +159,6 @@ def _encode(string, codec): return string.encode(codec) - class Charset: """Map character sets to their email properties. @@ -185,13 +180,13 @@ class Charset: header_encoding: If the character set must be encoded before it can be used in an email header, this attribute will be set to - Charset.QP (for quoted-printable), Charset.BASE64 (for - base64 encoding), or Charset.SHORTEST for the shortest of + charset.QP (for quoted-printable), charset.BASE64 (for + base64 encoding), or charset.SHORTEST for the shortest of QP or BASE64 encoding. Otherwise, it will be None. body_encoding: Same as header_encoding, but describes the encoding for the mail message's body, which indeed may be different than the - header encoding. Charset.SHORTEST is not allowed for + header encoding. charset.SHORTEST is not allowed for body_encoding. output_charset: Some character sets must be converted before they can be @@ -241,11 +236,9 @@ def __init__(self, input_charset=DEFAULT_CHARSET): self.output_codec = CODEC_MAP.get(self.output_charset, self.output_charset) - def __str__(self): + def __repr__(self): return self.input_charset.lower() - __repr__ = __str__ - def __eq__(self, other): return str(self) == str(other).lower() @@ -348,7 +341,6 @@ def header_encode_lines(self, string, maxlengths): if not lines and not current_line: lines.append(None) else: - separator = (' ' if lines else '') joined_line = EMPTYSTRING.join(current_line) header_bytes = _encode(joined_line, codec) lines.append(encoder(header_bytes)) diff --git a/Lib/email/contentmanager.py b/Lib/email/contentmanager.py index b904ded94c..b4f5830bea 100644 --- a/Lib/email/contentmanager.py +++ b/Lib/email/contentmanager.py @@ -72,12 +72,14 @@ def get_non_text_content(msg): return msg.get_payload(decode=True) for maintype in 'audio image video application'.split(): raw_data_manager.add_get_handler(maintype, get_non_text_content) +del maintype def get_message_content(msg): return msg.get_payload(0) for subtype in 'rfc822 external-body'.split(): raw_data_manager.add_get_handler('message/'+subtype, get_message_content) +del subtype def get_and_fixup_unknown_message_content(msg): @@ -144,15 +146,15 @@ def _encode_text(string, charset, cte, policy): linesep = policy.linesep.encode('ascii') def embedded_body(lines): return linesep.join(lines) + linesep def normal_body(lines): return b'\n'.join(lines) + b'\n' - if cte==None: + if cte is None: # Use heuristics to decide on the "best" encoding. - try: - return '7bit', normal_body(lines).decode('ascii') - except UnicodeDecodeError: - pass - if (policy.cte_type == '8bit' and - max(len(x) for x in lines) <= policy.max_line_length): - return '8bit', normal_body(lines).decode('ascii', 'surrogateescape') + if max((len(x) for x in lines), default=0) <= policy.max_line_length: + try: + return '7bit', normal_body(lines).decode('ascii') + except UnicodeDecodeError: + pass + if policy.cte_type == '8bit': + return '8bit', normal_body(lines).decode('ascii', 'surrogateescape') sniff = embedded_body(lines[:10]) sniff_qp = quoprimime.body_encode(sniff.decode('latin-1'), policy.max_line_length) @@ -238,9 +240,7 @@ def set_bytes_content(msg, data, maintype, subtype, cte='base64', data = binascii.b2a_qp(data, istext=False, header=False, quotetabs=True) data = data.decode('ascii') elif cte == '7bit': - # Make sure it really is only ASCII. The early warning here seems - # worth the overhead...if you care write your own content manager :). - data.encode('ascii') + data = data.decode('ascii') elif cte in ('8bit', 'binary'): data = data.decode('ascii', 'surrogateescape') msg.set_payload(data) @@ -248,3 +248,4 @@ def set_bytes_content(msg, data, maintype, subtype, cte='base64', _finalize_set(msg, disposition, filename, cid, params) for typ in (bytes, bytearray, memoryview): raw_data_manager.add_set_handler(typ, set_bytes_content) +del typ diff --git a/Lib/email/encoders.py b/Lib/email/encoders.py index 0a66acb624..17bd1ab7b1 100644 --- a/Lib/email/encoders.py +++ b/Lib/email/encoders.py @@ -16,7 +16,6 @@ from quopri import encodestring as _encodestring - def _qencode(s): enc = _encodestring(s, quotetabs=True) # Must encode spaces, which quopri.encodestring() doesn't do @@ -34,7 +33,6 @@ def encode_base64(msg): msg['Content-Transfer-Encoding'] = 'base64' - def encode_quopri(msg): """Encode the message's payload in quoted-printable. @@ -46,7 +44,6 @@ def encode_quopri(msg): msg['Content-Transfer-Encoding'] = 'quoted-printable' - def encode_7or8bit(msg): """Set the Content-Transfer-Encoding header to 7bit or 8bit.""" orig = msg.get_payload(decode=True) @@ -64,6 +61,5 @@ def encode_7or8bit(msg): msg['Content-Transfer-Encoding'] = '7bit' - def encode_noop(msg): """Do nothing.""" diff --git a/Lib/email/errors.py b/Lib/email/errors.py index 791239fa6a..02aa5eced6 100644 --- a/Lib/email/errors.py +++ b/Lib/email/errors.py @@ -29,6 +29,10 @@ class CharsetError(MessageError): """An illegal charset was given.""" +class HeaderWriteError(MessageError): + """Error while writing headers.""" + + # These are parsing defects which the parser was able to work around. class MessageDefect(ValueError): """Base class for a message defect.""" @@ -73,6 +77,9 @@ class InvalidBase64PaddingDefect(MessageDefect): class InvalidBase64CharactersDefect(MessageDefect): """base64 encoded sequence had characters not in base64 alphabet""" +class InvalidBase64LengthDefect(MessageDefect): + """base64 encoded sequence had invalid length (1 mod 4)""" + # These errors are specific to header parsing. class HeaderDefect(MessageDefect): @@ -105,3 +112,6 @@ class NonASCIILocalPartDefect(HeaderDefect): """local_part contains non-ASCII characters""" # This defect only occurs during unicode parsing, not when # parsing messages decoded from binary. + +class InvalidDateDefect(HeaderDefect): + """Header has unparsable or invalid date""" diff --git a/Lib/email/feedparser.py b/Lib/email/feedparser.py index 7c07ca8645..06d6b4a3af 100644 --- a/Lib/email/feedparser.py +++ b/Lib/email/feedparser.py @@ -37,11 +37,12 @@ headerRE = re.compile(r'^(From |[\041-\071\073-\176]*:|[\t ])') EMPTYSTRING = '' NL = '\n' +boundaryendRE = re.compile( + r'(?P--)?(?P[ \t]*)(?P\r\n|\r|\n)?$') NeedMoreData = object() - class BufferedSubFile(object): """A file-ish object that can have new data loaded into it. @@ -132,7 +133,6 @@ def __next__(self): return line - class FeedParser: """A feed-style parser of email.""" @@ -189,7 +189,7 @@ def close(self): assert not self._msgstack # Look for final set of defects if root.get_content_maintype() == 'multipart' \ - and not root.is_multipart(): + and not root.is_multipart() and not self._headersonly: defect = errors.MultipartInvariantViolationDefect() self.policy.handle_defect(root, defect) return root @@ -266,7 +266,7 @@ def _parsegen(self): yield NeedMoreData continue break - msg = self._pop_message() + self._pop_message() # We need to pop the EOF matcher in order to tell if we're at # the end of the current file, not the end of the last block # of message headers. @@ -320,7 +320,7 @@ def _parsegen(self): self._cur.set_payload(EMPTYSTRING.join(lines)) return # Make sure a valid content type was specified per RFC 2045:6.4. - if (self._cur.get('content-transfer-encoding', '8bit').lower() + if (str(self._cur.get('content-transfer-encoding', '8bit')).lower() not in ('7bit', '8bit', 'binary')): defect = errors.InvalidMultipartContentTransferEncodingDefect() self.policy.handle_defect(self._cur, defect) @@ -329,9 +329,10 @@ def _parsegen(self): # this onto the input stream until we've scanned past the # preamble. separator = '--' + boundary - boundaryre = re.compile( - '(?P' + re.escape(separator) + - r')(?P--)?(?P[ \t]*)(?P\r\n|\r|\n)?$') + def boundarymatch(line): + if not line.startswith(separator): + return None + return boundaryendRE.match(line, len(separator)) capturing_preamble = True preamble = [] linesep = False @@ -343,7 +344,7 @@ def _parsegen(self): continue if line == '': break - mo = boundaryre.match(line) + mo = boundarymatch(line) if mo: # If we're looking at the end boundary, we're done with # this multipart. If there was a newline at the end of @@ -375,13 +376,13 @@ def _parsegen(self): if line is NeedMoreData: yield NeedMoreData continue - mo = boundaryre.match(line) + mo = boundarymatch(line) if not mo: self._input.unreadline(line) break # Recurse to parse this subpart; the input stream points # at the subpart's first line. - self._input.push_eof_matcher(boundaryre.match) + self._input.push_eof_matcher(boundarymatch) for retval in self._parsegen(): if retval is NeedMoreData: yield NeedMoreData diff --git a/Lib/email/generator.py b/Lib/email/generator.py index ae670c2353..47b9df8f4e 100644 --- a/Lib/email/generator.py +++ b/Lib/email/generator.py @@ -14,15 +14,16 @@ from copy import deepcopy from io import StringIO, BytesIO from email.utils import _has_surrogates +from email.errors import HeaderWriteError UNDERSCORE = '_' NL = '\n' # XXX: no longer used by the code below. NLCRE = re.compile(r'\r\n|\r|\n') fcre = re.compile(r'^From ', re.MULTILINE) +NEWLINE_WITHOUT_FWSP = re.compile(r'\r\n[^ \t]|\r[^ \n\t]|\n[^ \t]') - class Generator: """Generates output from a Message object tree. @@ -170,7 +171,7 @@ def _write(self, msg): # parameter. # # The way we do this, so as to make the _handle_*() methods simpler, - # is to cache any subpart writes into a buffer. The we write the + # is to cache any subpart writes into a buffer. Then we write the # headers and the buffer contents. That way, subpart handlers can # Do The Right Thing, and can still modify the Content-Type: header if # necessary. @@ -186,7 +187,11 @@ def _write(self, msg): # If we munged the cte, copy the message again and re-fix the CTE. if munge_cte: msg = deepcopy(msg) - msg.replace_header('content-transfer-encoding', munge_cte[0]) + # Preserve the header order if the CTE header already exists. + if msg.get('content-transfer-encoding') is None: + msg['Content-Transfer-Encoding'] = munge_cte[0] + else: + msg.replace_header('content-transfer-encoding', munge_cte[0]) msg.replace_header('content-type', munge_cte[1]) # Write the headers. First we see if the message object wants to # handle that itself. If not, we'll do it generically. @@ -219,7 +224,16 @@ def _dispatch(self, msg): def _write_headers(self, msg): for h, v in msg.raw_items(): - self.write(self.policy.fold(h, v)) + folded = self.policy.fold(h, v) + if self.policy.verify_generated_headers: + linesep = self.policy.linesep + if not folded.endswith(self.policy.linesep): + raise HeaderWriteError( + f'folded header does not end with {linesep!r}: {folded!r}') + if NEWLINE_WITHOUT_FWSP.search(folded.removesuffix(linesep)): + raise HeaderWriteError( + f'folded header contains newline: {folded!r}') + self.write(folded) # A blank line always separates headers from body self.write(self._NL) @@ -240,7 +254,7 @@ def _handle_text(self, msg): # existing message. msg = deepcopy(msg) del msg['content-transfer-encoding'] - msg.set_payload(payload, charset) + msg.set_payload(msg._payload, charset) payload = msg.get_payload() self._munge_cte = (msg['content-transfer-encoding'], msg['content-type']) @@ -388,7 +402,7 @@ def _make_boundary(cls, text=None): def _compile_re(cls, s, flags): return re.compile(s, flags) - + class BytesGenerator(Generator): """Generates a bytes version of a Message object tree. @@ -439,7 +453,6 @@ def _compile_re(cls, s, flags): return re.compile(s.encode('ascii'), flags) - _FMT = '[Non-text (%(type)s) part of message omitted, filename %(filename)s]' class DecodedGenerator(Generator): @@ -499,7 +512,6 @@ def _dispatch(self, msg): }, file=self) - # Helper used by Generator._make_boundary _width = len(repr(sys.maxsize-1)) _fmt = '%%0%dd' % _width diff --git a/Lib/email/header.py b/Lib/email/header.py index c7b2dd9f31..984851a7d9 100644 --- a/Lib/email/header.py +++ b/Lib/email/header.py @@ -36,11 +36,11 @@ =\? # literal =? (?P[^?]*?) # non-greedy up to the next ? is the charset \? # literal ? - (?P[qb]) # either a "q" or a "b", case insensitive + (?P[qQbB]) # either a "q" or a "b", case insensitive \? # literal ? (?P.*?) # non-greedy up to the next ?= is the encoded string \?= # literal ?= - ''', re.VERBOSE | re.IGNORECASE | re.MULTILINE) + ''', re.VERBOSE | re.MULTILINE) # Field name regexp, including trailing colon, but not separating whitespace, # according to RFC 2822. Character range is from tilde to exclamation mark. @@ -52,12 +52,10 @@ _embedded_header = re.compile(r'\n[^ \t]+:') - # Helpers _max_append = email.quoprimime._max_append - def decode_header(header): """Decode a message header value without converting charset. @@ -152,7 +150,6 @@ def decode_header(header): return collapsed - def make_header(decoded_seq, maxlinelen=None, header_name=None, continuation_ws=' '): """Create a Header from a sequence of pairs as returned by decode_header() @@ -175,7 +172,6 @@ def make_header(decoded_seq, maxlinelen=None, header_name=None, return h - class Header: def __init__(self, s=None, charset=None, maxlinelen=None, header_name=None, @@ -409,7 +405,6 @@ def _normalize(self): self._chunks = chunks - class _ValueFormatter: def __init__(self, headerlen, maxlen, continuation_ws, splitchars): self._maxlen = maxlen @@ -431,7 +426,7 @@ def newline(self): if end_of_line != (' ', ''): self._current_line.push(*end_of_line) if len(self._current_line) > 0: - if self._current_line.is_onlyws(): + if self._current_line.is_onlyws() and self._lines: self._lines[-1] += str(self._current_line) else: self._lines.append(str(self._current_line)) diff --git a/Lib/email/headerregistry.py b/Lib/email/headerregistry.py index 0fc2231e5c..543141dc42 100644 --- a/Lib/email/headerregistry.py +++ b/Lib/email/headerregistry.py @@ -2,10 +2,6 @@ This module provides an implementation of the HeaderRegistry API. The implementation is designed to flexibly follow RFC5322 rules. - -Eventually HeaderRegistry will be a public API, but it isn't yet, -and will probably change some before that happens. - """ from types import MappingProxyType @@ -31,6 +27,11 @@ def __init__(self, display_name='', username='', domain='', addr_spec=None): without any Content Transfer Encoding. """ + + inputs = ''.join(filter(None, (display_name, username, domain, addr_spec))) + if '\r' in inputs or '\n' in inputs: + raise ValueError("invalid arguments; address parts cannot contain CR or LF") + # This clause with its potential 'raise' may only happen when an # application program creates an Address object using an addr_spec # keyword. The email library code itself must always supply username @@ -69,11 +70,9 @@ def addr_spec(self): """The addr_spec (username@domain) portion of the address, quoted according to RFC 5322 rules, but with no Content Transfer Encoding. """ - nameset = set(self.username) - if len(nameset) > len(nameset-parser.DOT_ATOM_ENDS): - lp = parser.quote_string(self.username) - else: - lp = self.username + lp = self.username + if not parser.DOT_ATOM_ENDS.isdisjoint(lp): + lp = parser.quote_string(lp) if self.domain: return lp + '@' + self.domain if not lp: @@ -86,19 +85,17 @@ def __repr__(self): self.display_name, self.username, self.domain) def __str__(self): - nameset = set(self.display_name) - if len(nameset) > len(nameset-parser.SPECIALS): - disp = parser.quote_string(self.display_name) - else: - disp = self.display_name + disp = self.display_name + if not parser.SPECIALS.isdisjoint(disp): + disp = parser.quote_string(disp) if disp: addr_spec = '' if self.addr_spec=='<>' else self.addr_spec return "{} <{}>".format(disp, addr_spec) return self.addr_spec def __eq__(self, other): - if type(other) != type(self): - return False + if not isinstance(other, Address): + return NotImplemented return (self.display_name == other.display_name and self.username == other.username and self.domain == other.domain) @@ -141,17 +138,15 @@ def __str__(self): if self.display_name is None and len(self.addresses)==1: return str(self.addresses[0]) disp = self.display_name - if disp is not None: - nameset = set(disp) - if len(nameset) > len(nameset-parser.SPECIALS): - disp = parser.quote_string(disp) + if disp is not None and not parser.SPECIALS.isdisjoint(disp): + disp = parser.quote_string(disp) adrstr = ", ".join(str(x) for x in self.addresses) adrstr = ' ' + adrstr if adrstr else adrstr return "{}:{};".format(disp, adrstr) def __eq__(self, other): - if type(other) != type(self): - return False + if not isinstance(other, Group): + return NotImplemented return (self.display_name == other.display_name and self.addresses == other.addresses) @@ -223,7 +218,7 @@ def __reduce__(self): self.__class__.__bases__, str(self), ), - self.__dict__) + self.__getstate__()) @classmethod def _reconstruct(cls, value): @@ -245,13 +240,16 @@ def fold(self, *, policy): the header name and the ': ' separator. """ - # At some point we need to only put fws here if it was in the source. + # At some point we need to put fws here if it was in the source. header = parser.Header([ parser.HeaderLabel([ parser.ValueTerminal(self.name, 'header-name'), parser.ValueTerminal(':', 'header-sep')]), - parser.CFWSList([parser.WhiteSpaceTerminal(' ', 'fws')]), - self._parse_tree]) + ]) + if self._parse_tree: + header.append( + parser.CFWSList([parser.WhiteSpaceTerminal(' ', 'fws')])) + header.append(self._parse_tree) return header.fold(policy=policy) @@ -300,7 +298,14 @@ def parse(cls, value, kwds): kwds['parse_tree'] = parser.TokenList() return if isinstance(value, str): - value = utils.parsedate_to_datetime(value) + kwds['decoded'] = value + try: + value = utils.parsedate_to_datetime(value) + except ValueError: + kwds['defects'].append(errors.InvalidDateDefect('Invalid date value or format')) + kwds['datetime'] = None + kwds['parse_tree'] = parser.TokenList() + return kwds['datetime'] = value kwds['decoded'] = utils.format_datetime(kwds['datetime']) kwds['parse_tree'] = cls.value_parser(kwds['decoded']) @@ -369,8 +374,8 @@ def groups(self): @property def addresses(self): if self._addresses is None: - self._addresses = tuple([address for group in self._groups - for address in group.addresses]) + self._addresses = tuple(address for group in self._groups + for address in group.addresses) return self._addresses @@ -517,6 +522,18 @@ def cte(self): return self._cte +class MessageIDHeader: + + max_count = 1 + value_parser = staticmethod(parser.parse_message_id) + + @classmethod + def parse(cls, value, kwds): + kwds['parse_tree'] = parse_tree = cls.value_parser(value) + kwds['decoded'] = str(parse_tree) + kwds['defects'].extend(parse_tree.all_defects) + + # The header factory # _default_header_map = { @@ -539,6 +556,7 @@ def cte(self): 'content-type': ContentTypeHeader, 'content-disposition': ContentDispositionHeader, 'content-transfer-encoding': ContentTransferEncodingHeader, + 'message-id': MessageIDHeader, } class HeaderRegistry: diff --git a/Lib/email/iterators.py b/Lib/email/iterators.py index b5502ee975..3410935e38 100644 --- a/Lib/email/iterators.py +++ b/Lib/email/iterators.py @@ -15,7 +15,6 @@ from io import StringIO - # This function will become a method of the Message class def walk(self): """Walk over the message tree, yielding each subpart. @@ -29,7 +28,6 @@ def walk(self): yield from subpart.walk() - # These two functions are imported into the Iterators.py interface module. def body_line_iterator(msg, decode=False): """Iterate over the parts, returning string payloads line-by-line. @@ -55,7 +53,6 @@ def typed_subpart_iterator(msg, maintype='text', subtype=None): yield subpart - def _structure(msg, fp=None, level=0, include_default=False): """A handy debugging aid""" if fp is None: diff --git a/Lib/email/message.py b/Lib/email/message.py index f932186875..46bb8c2194 100644 --- a/Lib/email/message.py +++ b/Lib/email/message.py @@ -6,6 +6,7 @@ __all__ = ['Message', 'EmailMessage'] +import binascii import re import quopri from io import BytesIO, StringIO @@ -13,7 +14,7 @@ # Intrapackage imports from email import utils from email import errors -from email._policybase import Policy, compat32 +from email._policybase import compat32 from email import charset as _charset from email._encoded_words import decode_b Charset = _charset.Charset @@ -34,7 +35,7 @@ def _splitparam(param): if not sep: return a.strip(), None return a.strip(), b.strip() - + def _formatparam(param, value=None, quote=True): """Convenience function to format and return a key=value pair. @@ -129,7 +130,8 @@ def _decode_uu(encoded): decoded_lines.append(decoded_line) return b''.join(decoded_lines) - + + class Message: """Basic message object. @@ -169,7 +171,7 @@ def as_string(self, unixfrom=False, maxheaderlen=0, policy=None): header. For backward compatibility reasons, if maxheaderlen is not specified it defaults to 0, so you must override it explicitly if you want a different maxheaderlen. 'policy' is passed to the - Generator instance used to serialize the mesasge; if it is not + Generator instance used to serialize the message; if it is not specified the policy associated with the message instance is used. If the message object contains binary data that is not encoded @@ -287,25 +289,26 @@ def get_payload(self, i=None, decode=False): # cte might be a Header, so for now stringify it. cte = str(self.get('content-transfer-encoding', '')).lower() # payload may be bytes here. - if isinstance(payload, str): - if utils._has_surrogates(payload): - bpayload = payload.encode('ascii', 'surrogateescape') - if not decode: + if not decode: + if isinstance(payload, str) and utils._has_surrogates(payload): + try: + bpayload = payload.encode('ascii', 'surrogateescape') try: - payload = bpayload.decode(self.get_param('charset', 'ascii'), 'replace') + payload = bpayload.decode(self.get_content_charset('ascii'), 'replace') except LookupError: payload = bpayload.decode('ascii', 'replace') - elif decode: - try: - bpayload = payload.encode('ascii') - except UnicodeError: - # This won't happen for RFC compliant messages (messages - # containing only ASCII code points in the unicode input). - # If it does happen, turn the string into bytes in a way - # guaranteed not to fail. - bpayload = payload.encode('raw-unicode-escape') - if not decode: + except UnicodeEncodeError: + pass return payload + if isinstance(payload, str): + try: + bpayload = payload.encode('ascii', 'surrogateescape') + except UnicodeEncodeError: + # This won't happen for RFC compliant messages (messages + # containing only ASCII code points in the unicode input). + # If it does happen, turn the string into bytes in a way + # guaranteed not to fail. + bpayload = payload.encode('raw-unicode-escape') if cte == 'quoted-printable': return quopri.decodestring(bpayload) elif cte == 'base64': @@ -337,7 +340,7 @@ def set_payload(self, payload, charset=None): return if not isinstance(charset, Charset): charset = Charset(charset) - payload = payload.encode(charset.output_charset) + payload = payload.encode(charset.output_charset, 'surrogateescape') if hasattr(payload, 'decode'): self._payload = payload.decode('ascii', 'surrogateescape') else: @@ -446,7 +449,11 @@ def __delitem__(self, name): self._headers = newheaders def __contains__(self, name): - return name.lower() in [k.lower() for k, v in self._headers] + name_lower = name.lower() + for k, v in self._headers: + if name_lower == k.lower(): + return True + return False def __iter__(self): for field, value in self._headers: @@ -973,7 +980,7 @@ def __init__(self, policy=None): if policy is None: from email.policy import default policy = default - Message.__init__(self, policy) + super().__init__(policy) def as_string(self, unixfrom=False, maxheaderlen=None, policy=None): @@ -983,14 +990,14 @@ def as_string(self, unixfrom=False, maxheaderlen=None, policy=None): header. maxheaderlen is retained for backward compatibility with the base Message class, but defaults to None, meaning that the policy value for max_line_length controls the header maximum length. 'policy' is - passed to the Generator instance used to serialize the mesasge; if it + passed to the Generator instance used to serialize the message; if it is not specified the policy associated with the message instance is used. """ policy = self.policy if policy is None else policy if maxheaderlen is None: maxheaderlen = policy.max_line_length - return super().as_string(maxheaderlen=maxheaderlen, policy=policy) + return super().as_string(unixfrom, maxheaderlen, policy) def __str__(self): return self.as_string(policy=self.policy.clone(utf8=True)) @@ -1007,7 +1014,7 @@ def _find_body(self, part, preferencelist): if subtype in preferencelist: yield (preferencelist.index(subtype), part) return - if maintype != 'multipart': + if maintype != 'multipart' or not self.is_multipart(): return if subtype != 'related': for subpart in part.iter_parts(): @@ -1066,7 +1073,16 @@ def iter_attachments(self): maintype, subtype = self.get_content_type().split('/') if maintype != 'multipart' or subtype == 'alternative': return - parts = self.get_payload().copy() + payload = self.get_payload() + # Certain malformed messages can have content type set to `multipart/*` + # but still have single part body, in which case payload.copy() can + # fail with AttributeError. + try: + parts = payload.copy() + except AttributeError: + # payload is not a list, it is most probably a string. + return + if maintype == 'multipart' and subtype == 'related': # For related, we treat everything but the root as an attachment. # The root may be indicated by 'start'; if there's no start or we @@ -1103,7 +1119,7 @@ def iter_parts(self): Return an empty iterator for a non-multipart. """ - if self.get_content_maintype() == 'multipart': + if self.is_multipart(): yield from self.get_payload() def get_content(self, *args, content_manager=None, **kw): diff --git a/Lib/email/mime/application.py b/Lib/email/mime/application.py index 6877e554e1..f67cbad3f0 100644 --- a/Lib/email/mime/application.py +++ b/Lib/email/mime/application.py @@ -17,7 +17,7 @@ def __init__(self, _data, _subtype='octet-stream', _encoder=encoders.encode_base64, *, policy=None, **_params): """Create an application/* type MIME document. - _data is a string containing the raw application data. + _data contains the bytes for the raw application data. _subtype is the MIME content type subtype, defaulting to 'octet-stream'. diff --git a/Lib/email/mime/audio.py b/Lib/email/mime/audio.py index 4bcd7b224a..aa0c4905cb 100644 --- a/Lib/email/mime/audio.py +++ b/Lib/email/mime/audio.py @@ -6,39 +6,10 @@ __all__ = ['MIMEAudio'] -import sndhdr - -from io import BytesIO from email import encoders from email.mime.nonmultipart import MIMENonMultipart - -_sndhdr_MIMEmap = {'au' : 'basic', - 'wav' :'x-wav', - 'aiff':'x-aiff', - 'aifc':'x-aiff', - } - -# There are others in sndhdr that don't have MIME types. :( -# Additional ones to be added to sndhdr? midi, mp3, realaudio, wma?? -def _whatsnd(data): - """Try to identify a sound file type. - - sndhdr.what() has a pretty cruddy interface, unfortunately. This is why - we re-do it here. It would be easier to reverse engineer the Unix 'file' - command and use the standard 'magic' file, as shipped with a modern Unix. - """ - hdr = data[:512] - fakefile = BytesIO(hdr) - for testfn in sndhdr.tests: - res = testfn(hdr, fakefile) - if res is not None: - return _sndhdr_MIMEmap.get(res[0]) - return None - - - class MIMEAudio(MIMENonMultipart): """Class for generating audio/* MIME documents.""" @@ -46,8 +17,8 @@ def __init__(self, _audiodata, _subtype=None, _encoder=encoders.encode_base64, *, policy=None, **_params): """Create an audio/* type MIME document. - _audiodata is a string containing the raw audio data. If this data - can be decoded by the standard Python `sndhdr' module, then the + _audiodata contains the bytes for the raw audio data. If this data + can be decoded as au, wav, aiff, or aifc, then the subtype will be automatically included in the Content-Type header. Otherwise, you can specify the specific audio subtype via the _subtype parameter. If _subtype is not given, and no subtype can be @@ -65,10 +36,62 @@ def __init__(self, _audiodata, _subtype=None, header. """ if _subtype is None: - _subtype = _whatsnd(_audiodata) + _subtype = _what(_audiodata) if _subtype is None: raise TypeError('Could not find audio MIME subtype') MIMENonMultipart.__init__(self, 'audio', _subtype, policy=policy, **_params) self.set_payload(_audiodata) _encoder(self) + + +_rules = [] + + +# Originally from the sndhdr module. +# +# There are others in sndhdr that don't have MIME types. :( +# Additional ones to be added to sndhdr? midi, mp3, realaudio, wma?? +def _what(data): + # Try to identify a sound file type. + # + # sndhdr.what() had a pretty cruddy interface, unfortunately. This is why + # we re-do it here. It would be easier to reverse engineer the Unix 'file' + # command and use the standard 'magic' file, as shipped with a modern Unix. + for testfn in _rules: + if res := testfn(data): + return res + else: + return None + + +def rule(rulefunc): + _rules.append(rulefunc) + return rulefunc + + +@rule +def _aiff(h): + if not h.startswith(b'FORM'): + return None + if h[8:12] in {b'AIFC', b'AIFF'}: + return 'x-aiff' + else: + return None + + +@rule +def _au(h): + if h.startswith(b'.snd'): + return 'basic' + else: + return None + + +@rule +def _wav(h): + # 'RIFF' 'WAVE' 'fmt ' + if not h.startswith(b'RIFF') or h[8:12] != b'WAVE' or h[12:16] != b'fmt ': + return None + else: + return "x-wav" diff --git a/Lib/email/mime/base.py b/Lib/email/mime/base.py index 1a3f9b51f6..f601f621ce 100644 --- a/Lib/email/mime/base.py +++ b/Lib/email/mime/base.py @@ -11,7 +11,6 @@ from email import message - class MIMEBase(message.Message): """Base class for MIME specializations.""" diff --git a/Lib/email/mime/image.py b/Lib/email/mime/image.py index 92724643cd..4b7f2f9cba 100644 --- a/Lib/email/mime/image.py +++ b/Lib/email/mime/image.py @@ -6,13 +6,10 @@ __all__ = ['MIMEImage'] -import imghdr - from email import encoders from email.mime.nonmultipart import MIMENonMultipart - class MIMEImage(MIMENonMultipart): """Class for generating image/* type MIME documents.""" @@ -20,11 +17,11 @@ def __init__(self, _imagedata, _subtype=None, _encoder=encoders.encode_base64, *, policy=None, **_params): """Create an image/* type MIME document. - _imagedata is a string containing the raw image data. If this data - can be decoded by the standard Python `imghdr' module, then the - subtype will be automatically included in the Content-Type header. - Otherwise, you can specify the specific image subtype via the _subtype - parameter. + _imagedata contains the bytes for the raw image data. If the data + type can be detected (jpeg, png, gif, tiff, rgb, pbm, pgm, ppm, + rast, xbm, bmp, webp, and exr attempted), then the subtype will be + automatically included in the Content-Type header. Otherwise, you can + specify the specific image subtype via the _subtype parameter. _encoder is a function which will perform the actual encoding for transport of the image data. It takes one argument, which is this @@ -37,11 +34,119 @@ def __init__(self, _imagedata, _subtype=None, constructor, which turns them into parameters on the Content-Type header. """ - if _subtype is None: - _subtype = imghdr.what(None, _imagedata) + _subtype = _what(_imagedata) if _subtype is None else _subtype if _subtype is None: raise TypeError('Could not guess image MIME subtype') MIMENonMultipart.__init__(self, 'image', _subtype, policy=policy, **_params) self.set_payload(_imagedata) _encoder(self) + + +_rules = [] + + +# Originally from the imghdr module. +def _what(data): + for rule in _rules: + if res := rule(data): + return res + else: + return None + + +def rule(rulefunc): + _rules.append(rulefunc) + return rulefunc + + +@rule +def _jpeg(h): + """JPEG data with JFIF or Exif markers; and raw JPEG""" + if h[6:10] in (b'JFIF', b'Exif'): + return 'jpeg' + elif h[:4] == b'\xff\xd8\xff\xdb': + return 'jpeg' + + +@rule +def _png(h): + if h.startswith(b'\211PNG\r\n\032\n'): + return 'png' + + +@rule +def _gif(h): + """GIF ('87 and '89 variants)""" + if h[:6] in (b'GIF87a', b'GIF89a'): + return 'gif' + + +@rule +def _tiff(h): + """TIFF (can be in Motorola or Intel byte order)""" + if h[:2] in (b'MM', b'II'): + return 'tiff' + + +@rule +def _rgb(h): + """SGI image library""" + if h.startswith(b'\001\332'): + return 'rgb' + + +@rule +def _pbm(h): + """PBM (portable bitmap)""" + if len(h) >= 3 and \ + h[0] == ord(b'P') and h[1] in b'14' and h[2] in b' \t\n\r': + return 'pbm' + + +@rule +def _pgm(h): + """PGM (portable graymap)""" + if len(h) >= 3 and \ + h[0] == ord(b'P') and h[1] in b'25' and h[2] in b' \t\n\r': + return 'pgm' + + +@rule +def _ppm(h): + """PPM (portable pixmap)""" + if len(h) >= 3 and \ + h[0] == ord(b'P') and h[1] in b'36' and h[2] in b' \t\n\r': + return 'ppm' + + +@rule +def _rast(h): + """Sun raster file""" + if h.startswith(b'\x59\xA6\x6A\x95'): + return 'rast' + + +@rule +def _xbm(h): + """X bitmap (X10 or X11)""" + if h.startswith(b'#define '): + return 'xbm' + + +@rule +def _bmp(h): + if h.startswith(b'BM'): + return 'bmp' + + +@rule +def _webp(h): + if h.startswith(b'RIFF') and h[8:12] == b'WEBP': + return 'webp' + + +@rule +def _exr(h): + if h.startswith(b'\x76\x2f\x31\x01'): + return 'exr' diff --git a/Lib/email/mime/message.py b/Lib/email/mime/message.py index 07e4f2d119..61836b5a78 100644 --- a/Lib/email/mime/message.py +++ b/Lib/email/mime/message.py @@ -10,7 +10,6 @@ from email.mime.nonmultipart import MIMENonMultipart - class MIMEMessage(MIMENonMultipart): """Class representing message/* MIME documents.""" diff --git a/Lib/email/mime/multipart.py b/Lib/email/mime/multipart.py index 2d3f288810..94d81c771a 100644 --- a/Lib/email/mime/multipart.py +++ b/Lib/email/mime/multipart.py @@ -9,7 +9,6 @@ from email.mime.base import MIMEBase - class MIMEMultipart(MIMEBase): """Base class for MIME multipart/* type messages.""" diff --git a/Lib/email/mime/nonmultipart.py b/Lib/email/mime/nonmultipart.py index e1f51968b5..a41386eb14 100644 --- a/Lib/email/mime/nonmultipart.py +++ b/Lib/email/mime/nonmultipart.py @@ -10,7 +10,6 @@ from email.mime.base import MIMEBase - class MIMENonMultipart(MIMEBase): """Base class for MIME non-multipart type messages.""" diff --git a/Lib/email/mime/text.py b/Lib/email/mime/text.py index 35b4423830..7672b78913 100644 --- a/Lib/email/mime/text.py +++ b/Lib/email/mime/text.py @@ -6,11 +6,9 @@ __all__ = ['MIMEText'] -from email.charset import Charset from email.mime.nonmultipart import MIMENonMultipart - class MIMEText(MIMENonMultipart): """Class for generating text/* type MIME documents.""" @@ -37,6 +35,6 @@ def __init__(self, _text, _subtype='plain', _charset=None, *, policy=None): _charset = 'utf-8' MIMENonMultipart.__init__(self, 'text', _subtype, policy=policy, - **{'charset': str(_charset)}) + charset=str(_charset)) self.set_payload(_text, _charset) diff --git a/Lib/email/parser.py b/Lib/email/parser.py index 555b172560..06d99b17f2 100644 --- a/Lib/email/parser.py +++ b/Lib/email/parser.py @@ -13,7 +13,6 @@ from email._policybase import compat32 - class Parser: def __init__(self, _class=None, *, policy=compat32): """Parser of RFC 2822 and MIME email messages. @@ -50,10 +49,7 @@ def parse(self, fp, headersonly=False): feedparser = FeedParser(self._class, policy=self.policy) if headersonly: feedparser._set_headersonly() - while True: - data = fp.read(8192) - if not data: - break + while data := fp.read(8192): feedparser.feed(data) return feedparser.close() @@ -68,7 +64,6 @@ def parsestr(self, text, headersonly=False): return self.parse(StringIO(text), headersonly=headersonly) - class HeaderParser(Parser): def parse(self, fp, headersonly=True): return Parser.parse(self, fp, True) @@ -76,7 +71,7 @@ def parse(self, fp, headersonly=True): def parsestr(self, text, headersonly=True): return Parser.parsestr(self, text, True) - + class BytesParser: def __init__(self, *args, **kw): diff --git a/Lib/email/policy.py b/Lib/email/policy.py index 5131311ac5..6e109b6501 100644 --- a/Lib/email/policy.py +++ b/Lib/email/policy.py @@ -3,6 +3,7 @@ """ import re +import sys from email._policybase import Policy, Compat32, compat32, _extend_docstrings from email.utils import _has_surrogates from email.headerregistry import HeaderRegistry as HeaderRegistry @@ -20,7 +21,7 @@ 'HTTP', ] -linesep_splitter = re.compile(r'\n|\r') +linesep_splitter = re.compile(r'\n|\r\n?') @_extend_docstrings class EmailPolicy(Policy): @@ -118,13 +119,13 @@ def header_source_parse(self, sourcelines): """+ The name is parsed as everything up to the ':' and returned unmodified. The value is determined by stripping leading whitespace off the - remainder of the first line, joining all subsequent lines together, and + remainder of the first line joined with all subsequent lines, and stripping any trailing carriage return or linefeed characters. (This is the same as Compat32). """ name, value = sourcelines[0].split(':', 1) - value = value.lstrip(' \t') + ''.join(sourcelines[1:]) + value = ''.join((value, *sourcelines[1:])).lstrip(' \t\r\n') return (name, value.rstrip('\r\n')) def header_store_parse(self, name, value): @@ -203,14 +204,22 @@ def fold_binary(self, name, value): def _fold(self, name, value, refold_binary=False): if hasattr(value, 'name'): return value.fold(policy=self) - maxlen = self.max_line_length if self.max_line_length else float('inf') - lines = value.splitlines() + maxlen = self.max_line_length if self.max_line_length else sys.maxsize + # We can't use splitlines here because it splits on more than \r and \n. + lines = linesep_splitter.split(value) refold = (self.refold_source == 'all' or self.refold_source == 'long' and (lines and len(lines[0])+len(name)+2 > maxlen or any(len(x) > maxlen for x in lines[1:]))) - if refold or refold_binary and _has_surrogates(value): + + if not refold: + if not self.utf8: + refold = not value.isascii() + elif refold_binary: + refold = _has_surrogates(value) + if refold: return self.header_factory(name, ''.join(lines)).fold(policy=self) + return name + ': ' + self.linesep.join(lines) + self.linesep diff --git a/Lib/email/quoprimime.py b/Lib/email/quoprimime.py index c543eb59ae..27fcbb5a26 100644 --- a/Lib/email/quoprimime.py +++ b/Lib/email/quoprimime.py @@ -148,6 +148,7 @@ def header_encode(header_bytes, charset='iso-8859-1'): _QUOPRI_BODY_ENCODE_MAP = _QUOPRI_BODY_MAP[:] for c in b'\r\n': _QUOPRI_BODY_ENCODE_MAP[c] = chr(c) +del c def body_encode(body, maxlinelen=76, eol=NL): """Encode with quoted-printable, wrapping at maxlinelen characters. @@ -173,7 +174,7 @@ def body_encode(body, maxlinelen=76, eol=NL): if not body: return body - # quote speacial characters + # quote special characters body = body.translate(_QUOPRI_BODY_ENCODE_MAP) soft_break = '=' + eol diff --git a/Lib/email/utils.py b/Lib/email/utils.py index a759d23308..e42674fa4f 100644 --- a/Lib/email/utils.py +++ b/Lib/email/utils.py @@ -25,8 +25,6 @@ import os import re import time -import random -import socket import datetime import urllib.parse @@ -36,9 +34,6 @@ from email._parseaddr import parsedate, parsedate_tz, _parsedate_tz -# Intrapackage imports -from email.charset import Charset - COMMASPACE = ', ' EMPTYSTRING = '' UEMPTYSTRING = '' @@ -48,11 +43,12 @@ specialsre = re.compile(r'[][\\()<>@,:;".]') escapesre = re.compile(r'[\\"]') + def _has_surrogates(s): - """Return True if s contains surrogate-escaped binary data.""" + """Return True if s may contain surrogate-escaped binary data.""" # This check is based on the fact that unless there are surrogates, utf8 # (Python's default encoding) can encode any string. This is the fastest - # way to check for surrogates, see issue 11454 for timings. + # way to check for surrogates, see bpo-11454 (moved to gh-55663) for timings. try: s.encode() return False @@ -81,7 +77,7 @@ def formataddr(pair, charset='utf-8'): If the first element of pair is false, then the second element is returned unmodified. - Optional charset if given is the character set that is used to encode + The optional charset is the character set that is used to encode realname in case realname is not ASCII safe. Can be an instance of str or a Charset-like object which has a header_encode method. Default is 'utf-8'. @@ -94,6 +90,8 @@ def formataddr(pair, charset='utf-8'): name.encode('ascii') except UnicodeEncodeError: if isinstance(charset, str): + # lazy import to improve module import time + from email.charset import Charset charset = Charset(charset) encoded_name = charset.header_encode(name) return "%s <%s>" % (encoded_name, address) @@ -106,24 +104,127 @@ def formataddr(pair, charset='utf-8'): return address +def _iter_escaped_chars(addr): + pos = 0 + escape = False + for pos, ch in enumerate(addr): + if escape: + yield (pos, '\\' + ch) + escape = False + elif ch == '\\': + escape = True + else: + yield (pos, ch) + if escape: + yield (pos, '\\') + + +def _strip_quoted_realnames(addr): + """Strip real names between quotes.""" + if '"' not in addr: + # Fast path + return addr + + start = 0 + open_pos = None + result = [] + for pos, ch in _iter_escaped_chars(addr): + if ch == '"': + if open_pos is None: + open_pos = pos + else: + if start != open_pos: + result.append(addr[start:open_pos]) + start = pos + 1 + open_pos = None + + if start < len(addr): + result.append(addr[start:]) + + return ''.join(result) -def getaddresses(fieldvalues): - """Return a list of (REALNAME, EMAIL) for each fieldvalue.""" - all = COMMASPACE.join(fieldvalues) - a = _AddressList(all) - return a.addresslist +supports_strict_parsing = True +def getaddresses(fieldvalues, *, strict=True): + """Return a list of (REALNAME, EMAIL) or ('','') for each fieldvalue. -ecre = re.compile(r''' - =\? # literal =? - (?P[^?]*?) # non-greedy up to the next ? is the charset - \? # literal ? - (?P[qb]) # either a "q" or a "b", case insensitive - \? # literal ? - (?P.*?) # non-greedy up to the next ?= is the atom - \?= # literal ?= - ''', re.VERBOSE | re.IGNORECASE) + When parsing fails for a fieldvalue, a 2-tuple of ('', '') is returned in + its place. + + If strict is true, use a strict parser which rejects malformed inputs. + """ + + # If strict is true, if the resulting list of parsed addresses is greater + # than the number of fieldvalues in the input list, a parsing error has + # occurred and consequently a list containing a single empty 2-tuple [('', + # '')] is returned in its place. This is done to avoid invalid output. + # + # Malformed input: getaddresses(['alice@example.com ']) + # Invalid output: [('', 'alice@example.com'), ('', 'bob@example.com')] + # Safe output: [('', '')] + + if not strict: + all = COMMASPACE.join(str(v) for v in fieldvalues) + a = _AddressList(all) + return a.addresslist + + fieldvalues = [str(v) for v in fieldvalues] + fieldvalues = _pre_parse_validation(fieldvalues) + addr = COMMASPACE.join(fieldvalues) + a = _AddressList(addr) + result = _post_parse_validation(a.addresslist) + + # Treat output as invalid if the number of addresses is not equal to the + # expected number of addresses. + n = 0 + for v in fieldvalues: + # When a comma is used in the Real Name part it is not a deliminator. + # So strip those out before counting the commas. + v = _strip_quoted_realnames(v) + # Expected number of addresses: 1 + number of commas + n += 1 + v.count(',') + if len(result) != n: + return [('', '')] + + return result + + +def _check_parenthesis(addr): + # Ignore parenthesis in quoted real names. + addr = _strip_quoted_realnames(addr) + + opens = 0 + for pos, ch in _iter_escaped_chars(addr): + if ch == '(': + opens += 1 + elif ch == ')': + opens -= 1 + if opens < 0: + return False + return (opens == 0) + + +def _pre_parse_validation(email_header_fields): + accepted_values = [] + for v in email_header_fields: + if not _check_parenthesis(v): + v = "('', '')" + accepted_values.append(v) + + return accepted_values + + +def _post_parse_validation(parsed_email_header_tuples): + accepted_values = [] + # The parser would have parsed a correctly formatted domain-literal + # The existence of an [ after parsing indicates a parsing failure + for v in parsed_email_header_tuples: + if '[' in v[1]: + v = ('', '') + accepted_values.append(v) + + return accepted_values def _format_timetuple_and_zone(timetuple, zone): @@ -140,7 +241,7 @@ def formatdate(timeval=None, localtime=False, usegmt=False): Fri, 09 Nov 2001 01:08:47 -0000 - Optional timeval if given is a floating point time value as accepted by + Optional timeval if given is a floating-point time value as accepted by gmtime() and localtime(), otherwise the current time is used. Optional localtime is a flag that when True, interprets timeval, and @@ -155,13 +256,13 @@ def formatdate(timeval=None, localtime=False, usegmt=False): # 2822 requires that day and month names be the English abbreviations. if timeval is None: timeval = time.time() - if localtime or usegmt: - dt = datetime.datetime.fromtimestamp(timeval, datetime.timezone.utc) - else: - dt = datetime.datetime.utcfromtimestamp(timeval) + dt = datetime.datetime.fromtimestamp(timeval, datetime.timezone.utc) + if localtime: dt = dt.astimezone() usegmt = False + elif not usegmt: + dt = dt.replace(tzinfo=None) return format_datetime(dt, usegmt) def format_datetime(dt, usegmt=False): @@ -193,6 +294,11 @@ def make_msgid(idstring=None, domain=None): portion of the message id after the '@'. It defaults to the locally defined hostname. """ + # Lazy imports to speedup module import time + # (no other functions in email.utils need these modules) + import random + import socket + timeval = int(time.time()*100) pid = os.getpid() randint = random.getrandbits(64) @@ -207,17 +313,43 @@ def make_msgid(idstring=None, domain=None): def parsedate_to_datetime(data): - *dtuple, tz = _parsedate_tz(data) + parsed_date_tz = _parsedate_tz(data) + if parsed_date_tz is None: + raise ValueError('Invalid date value or format "%s"' % str(data)) + *dtuple, tz = parsed_date_tz if tz is None: return datetime.datetime(*dtuple[:6]) return datetime.datetime(*dtuple[:6], tzinfo=datetime.timezone(datetime.timedelta(seconds=tz))) -def parseaddr(addr): - addrs = _AddressList(addr).addresslist - if not addrs: - return '', '' +def parseaddr(addr, *, strict=True): + """ + Parse addr into its constituent realname and email address parts. + + Return a tuple of realname and email address, unless the parse fails, in + which case return a 2-tuple of ('', ''). + + If strict is True, use a strict parser which rejects malformed inputs. + """ + if not strict: + addrs = _AddressList(addr).addresslist + if not addrs: + return ('', '') + return addrs[0] + + if isinstance(addr, list): + addr = addr[0] + + if not isinstance(addr, str): + return ('', '') + + addr = _pre_parse_validation([addr])[0] + addrs = _post_parse_validation(_AddressList(addr).addresslist) + + if not addrs or len(addrs) > 1: + return ('', '') + return addrs[0] @@ -265,21 +397,13 @@ def decode_params(params): params is a sequence of 2-tuples containing (param name, string value). """ - # Copy params so we don't mess with the original - params = params[:] - new_params = [] + new_params = [params[0]] # Map parameter's name to a list of continuations. The values are a # 3-tuple of the continuation number, the string value, and a flag # specifying whether a particular segment is %-encoded. rfc2231_params = {} - name, value = params.pop(0) - new_params.append((name, value)) - while params: - name, value = params.pop(0) - if name.endswith('*'): - encoded = True - else: - encoded = False + for name, value in params[1:]: + encoded = name.endswith('*') value = unquote(value) mo = rfc2231_continuation.match(name) if mo: @@ -342,41 +466,23 @@ def collapse_rfc2231_value(value, errors='replace', # better than not having it. # -def localtime(dt=None, isdst=-1): +def localtime(dt=None, isdst=None): """Return local time as an aware datetime object. If called without arguments, return current time. Otherwise *dt* argument should be a datetime instance, and it is converted to the local time zone according to the system time zone database. If *dt* is naive (that is, dt.tzinfo is None), it is assumed to be in local time. - In this case, a positive or zero value for *isdst* causes localtime to - presume initially that summer time (for example, Daylight Saving Time) - is or is not (respectively) in effect for the specified time. A - negative value for *isdst* causes the localtime() function to attempt - to divine whether summer time is in effect for the specified time. + The isdst parameter is ignored. """ + if isdst is not None: + import warnings + warnings._deprecated( + "The 'isdst' parameter to 'localtime'", + message='{name} is deprecated and slated for removal in Python {remove}', + remove=(3, 14), + ) if dt is None: - return datetime.datetime.now(datetime.timezone.utc).astimezone() - if dt.tzinfo is not None: - return dt.astimezone() - # We have a naive datetime. Convert to a (localtime) timetuple and pass to - # system mktime together with the isdst hint. System mktime will return - # seconds since epoch. - tm = dt.timetuple()[:-1] + (isdst,) - seconds = time.mktime(tm) - localtm = time.localtime(seconds) - try: - delta = datetime.timedelta(seconds=localtm.tm_gmtoff) - tz = datetime.timezone(delta, localtm.tm_zone) - except AttributeError: - # Compute UTC offset and compare with the value implied by tm_isdst. - # If the values match, use the zone name implied by tm_isdst. - delta = dt - datetime.datetime(*time.gmtime(seconds)[:6]) - dst = time.daylight and localtm.tm_isdst > 0 - gmtoff = -(time.altzone if dst else time.timezone) - if delta == datetime.timedelta(seconds=gmtoff): - tz = datetime.timezone(delta, time.tzname[dst]) - else: - tz = datetime.timezone(delta) - return dt.replace(tzinfo=tz) + dt = datetime.datetime.now() + return dt.astimezone() From f1d45ee5a7a0f3b6c9f91d572e770ddf1a1fb106 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Thu, 3 Apr 2025 10:05:38 -0700 Subject: [PATCH 166/295] remove unused deprecated libraries --- Lib/aifc.py | 984 -------------------------------------------------- Lib/imghdr.py | 175 --------- Lib/sndhdr.py | 257 ------------- Lib/sunau.py | 531 --------------------------- 4 files changed, 1947 deletions(-) delete mode 100644 Lib/aifc.py delete mode 100644 Lib/imghdr.py delete mode 100644 Lib/sndhdr.py delete mode 100644 Lib/sunau.py diff --git a/Lib/aifc.py b/Lib/aifc.py deleted file mode 100644 index 5254987e22..0000000000 --- a/Lib/aifc.py +++ /dev/null @@ -1,984 +0,0 @@ -"""Stuff to parse AIFF-C and AIFF files. - -Unless explicitly stated otherwise, the description below is true -both for AIFF-C files and AIFF files. - -An AIFF-C file has the following structure. - - +-----------------+ - | FORM | - +-----------------+ - | | - +----+------------+ - | | AIFC | - | +------------+ - | | | - | | . | - | | . | - | | . | - +----+------------+ - -An AIFF file has the string "AIFF" instead of "AIFC". - -A chunk consists of an identifier (4 bytes) followed by a size (4 bytes, -big endian order), followed by the data. The size field does not include -the size of the 8 byte header. - -The following chunk types are recognized. - - FVER - (AIFF-C only). - MARK - <# of markers> (2 bytes) - list of markers: - (2 bytes, must be > 0) - (4 bytes) - ("pstring") - COMM - <# of channels> (2 bytes) - <# of sound frames> (4 bytes) - (2 bytes) - (10 bytes, IEEE 80-bit extended - floating point) - in AIFF-C files only: - (4 bytes) - ("pstring") - SSND - (4 bytes, not used by this program) - (4 bytes, not used by this program) - - -A pstring consists of 1 byte length, a string of characters, and 0 or 1 -byte pad to make the total length even. - -Usage. - -Reading AIFF files: - f = aifc.open(file, 'r') -where file is either the name of a file or an open file pointer. -The open file pointer must have methods read(), seek(), and close(). -In some types of audio files, if the setpos() method is not used, -the seek() method is not necessary. - -This returns an instance of a class with the following public methods: - getnchannels() -- returns number of audio channels (1 for - mono, 2 for stereo) - getsampwidth() -- returns sample width in bytes - getframerate() -- returns sampling frequency - getnframes() -- returns number of audio frames - getcomptype() -- returns compression type ('NONE' for AIFF files) - getcompname() -- returns human-readable version of - compression type ('not compressed' for AIFF files) - getparams() -- returns a namedtuple consisting of all of the - above in the above order - getmarkers() -- get the list of marks in the audio file or None - if there are no marks - getmark(id) -- get mark with the specified id (raises an error - if the mark does not exist) - readframes(n) -- returns at most n frames of audio - rewind() -- rewind to the beginning of the audio stream - setpos(pos) -- seek to the specified position - tell() -- return the current position - close() -- close the instance (make it unusable) -The position returned by tell(), the position given to setpos() and -the position of marks are all compatible and have nothing to do with -the actual position in the file. -The close() method is called automatically when the class instance -is destroyed. - -Writing AIFF files: - f = aifc.open(file, 'w') -where file is either the name of a file or an open file pointer. -The open file pointer must have methods write(), tell(), seek(), and -close(). - -This returns an instance of a class with the following public methods: - aiff() -- create an AIFF file (AIFF-C default) - aifc() -- create an AIFF-C file - setnchannels(n) -- set the number of channels - setsampwidth(n) -- set the sample width - setframerate(n) -- set the frame rate - setnframes(n) -- set the number of frames - setcomptype(type, name) - -- set the compression type and the - human-readable compression type - setparams(tuple) - -- set all parameters at once - setmark(id, pos, name) - -- add specified mark to the list of marks - tell() -- return current position in output file (useful - in combination with setmark()) - writeframesraw(data) - -- write audio frames without pathing up the - file header - writeframes(data) - -- write audio frames and patch up the file header - close() -- patch up the file header and close the - output file -You should set the parameters before the first writeframesraw or -writeframes. The total number of frames does not need to be set, -but when it is set to the correct value, the header does not have to -be patched up. -It is best to first set all parameters, perhaps possibly the -compression type, and then write audio frames using writeframesraw. -When all frames have been written, either call writeframes(b'') or -close() to patch up the sizes in the header. -Marks can be added anytime. If there are any marks, you must call -close() after all frames have been written. -The close() method is called automatically when the class instance -is destroyed. - -When a file is opened with the extension '.aiff', an AIFF file is -written, otherwise an AIFF-C file is written. This default can be -changed by calling aiff() or aifc() before the first writeframes or -writeframesraw. -""" - -import struct -import builtins -import warnings - -__all__ = ["Error", "open"] - - -warnings._deprecated(__name__, remove=(3, 13)) - - -class Error(Exception): - pass - -_AIFC_version = 0xA2805140 # Version 1 of AIFF-C - -def _read_long(file): - try: - return struct.unpack('>l', file.read(4))[0] - except struct.error: - raise EOFError from None - -def _read_ulong(file): - try: - return struct.unpack('>L', file.read(4))[0] - except struct.error: - raise EOFError from None - -def _read_short(file): - try: - return struct.unpack('>h', file.read(2))[0] - except struct.error: - raise EOFError from None - -def _read_ushort(file): - try: - return struct.unpack('>H', file.read(2))[0] - except struct.error: - raise EOFError from None - -def _read_string(file): - length = ord(file.read(1)) - if length == 0: - data = b'' - else: - data = file.read(length) - if length & 1 == 0: - dummy = file.read(1) - return data - -_HUGE_VAL = 1.79769313486231e+308 # See - -def _read_float(f): # 10 bytes - expon = _read_short(f) # 2 bytes - sign = 1 - if expon < 0: - sign = -1 - expon = expon + 0x8000 - himant = _read_ulong(f) # 4 bytes - lomant = _read_ulong(f) # 4 bytes - if expon == himant == lomant == 0: - f = 0.0 - elif expon == 0x7FFF: - f = _HUGE_VAL - else: - expon = expon - 16383 - f = (himant * 0x100000000 + lomant) * pow(2.0, expon - 63) - return sign * f - -def _write_short(f, x): - f.write(struct.pack('>h', x)) - -def _write_ushort(f, x): - f.write(struct.pack('>H', x)) - -def _write_long(f, x): - f.write(struct.pack('>l', x)) - -def _write_ulong(f, x): - f.write(struct.pack('>L', x)) - -def _write_string(f, s): - if len(s) > 255: - raise ValueError("string exceeds maximum pstring length") - f.write(struct.pack('B', len(s))) - f.write(s) - if len(s) & 1 == 0: - f.write(b'\x00') - -def _write_float(f, x): - import math - if x < 0: - sign = 0x8000 - x = x * -1 - else: - sign = 0 - if x == 0: - expon = 0 - himant = 0 - lomant = 0 - else: - fmant, expon = math.frexp(x) - if expon > 16384 or fmant >= 1 or fmant != fmant: # Infinity or NaN - expon = sign|0x7FFF - himant = 0 - lomant = 0 - else: # Finite - expon = expon + 16382 - if expon < 0: # denormalized - fmant = math.ldexp(fmant, expon) - expon = 0 - expon = expon | sign - fmant = math.ldexp(fmant, 32) - fsmant = math.floor(fmant) - himant = int(fsmant) - fmant = math.ldexp(fmant - fsmant, 32) - fsmant = math.floor(fmant) - lomant = int(fsmant) - _write_ushort(f, expon) - _write_ulong(f, himant) - _write_ulong(f, lomant) - -with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - from chunk import Chunk -from collections import namedtuple - -_aifc_params = namedtuple('_aifc_params', - 'nchannels sampwidth framerate nframes comptype compname') - -_aifc_params.nchannels.__doc__ = 'Number of audio channels (1 for mono, 2 for stereo)' -_aifc_params.sampwidth.__doc__ = 'Sample width in bytes' -_aifc_params.framerate.__doc__ = 'Sampling frequency' -_aifc_params.nframes.__doc__ = 'Number of audio frames' -_aifc_params.comptype.__doc__ = 'Compression type ("NONE" for AIFF files)' -_aifc_params.compname.__doc__ = ("""\ -A human-readable version of the compression type -('not compressed' for AIFF files)""") - - -class Aifc_read: - # Variables used in this class: - # - # These variables are available to the user though appropriate - # methods of this class: - # _file -- the open file with methods read(), close(), and seek() - # set through the __init__() method - # _nchannels -- the number of audio channels - # available through the getnchannels() method - # _nframes -- the number of audio frames - # available through the getnframes() method - # _sampwidth -- the number of bytes per audio sample - # available through the getsampwidth() method - # _framerate -- the sampling frequency - # available through the getframerate() method - # _comptype -- the AIFF-C compression type ('NONE' if AIFF) - # available through the getcomptype() method - # _compname -- the human-readable AIFF-C compression type - # available through the getcomptype() method - # _markers -- the marks in the audio file - # available through the getmarkers() and getmark() - # methods - # _soundpos -- the position in the audio stream - # available through the tell() method, set through the - # setpos() method - # - # These variables are used internally only: - # _version -- the AIFF-C version number - # _decomp -- the decompressor from builtin module cl - # _comm_chunk_read -- 1 iff the COMM chunk has been read - # _aifc -- 1 iff reading an AIFF-C file - # _ssnd_seek_needed -- 1 iff positioned correctly in audio - # file for readframes() - # _ssnd_chunk -- instantiation of a chunk class for the SSND chunk - # _framesize -- size of one frame in the file - - _file = None # Set here since __del__ checks it - - def initfp(self, file): - self._version = 0 - self._convert = None - self._markers = [] - self._soundpos = 0 - self._file = file - chunk = Chunk(file) - if chunk.getname() != b'FORM': - raise Error('file does not start with FORM id') - formdata = chunk.read(4) - if formdata == b'AIFF': - self._aifc = 0 - elif formdata == b'AIFC': - self._aifc = 1 - else: - raise Error('not an AIFF or AIFF-C file') - self._comm_chunk_read = 0 - self._ssnd_chunk = None - while 1: - self._ssnd_seek_needed = 1 - try: - chunk = Chunk(self._file) - except EOFError: - break - chunkname = chunk.getname() - if chunkname == b'COMM': - self._read_comm_chunk(chunk) - self._comm_chunk_read = 1 - elif chunkname == b'SSND': - self._ssnd_chunk = chunk - dummy = chunk.read(8) - self._ssnd_seek_needed = 0 - elif chunkname == b'FVER': - self._version = _read_ulong(chunk) - elif chunkname == b'MARK': - self._readmark(chunk) - chunk.skip() - if not self._comm_chunk_read or not self._ssnd_chunk: - raise Error('COMM chunk and/or SSND chunk missing') - - def __init__(self, f): - if isinstance(f, str): - file_object = builtins.open(f, 'rb') - try: - self.initfp(file_object) - except: - file_object.close() - raise - else: - # assume it is an open file object already - self.initfp(f) - - def __enter__(self): - return self - - def __exit__(self, *args): - self.close() - - # - # User visible methods. - # - def getfp(self): - return self._file - - def rewind(self): - self._ssnd_seek_needed = 1 - self._soundpos = 0 - - def close(self): - file = self._file - if file is not None: - self._file = None - file.close() - - def tell(self): - return self._soundpos - - def getnchannels(self): - return self._nchannels - - def getnframes(self): - return self._nframes - - def getsampwidth(self): - return self._sampwidth - - def getframerate(self): - return self._framerate - - def getcomptype(self): - return self._comptype - - def getcompname(self): - return self._compname - -## def getversion(self): -## return self._version - - def getparams(self): - return _aifc_params(self.getnchannels(), self.getsampwidth(), - self.getframerate(), self.getnframes(), - self.getcomptype(), self.getcompname()) - - def getmarkers(self): - if len(self._markers) == 0: - return None - return self._markers - - def getmark(self, id): - for marker in self._markers: - if id == marker[0]: - return marker - raise Error('marker {0!r} does not exist'.format(id)) - - def setpos(self, pos): - if pos < 0 or pos > self._nframes: - raise Error('position not in range') - self._soundpos = pos - self._ssnd_seek_needed = 1 - - def readframes(self, nframes): - if self._ssnd_seek_needed: - self._ssnd_chunk.seek(0) - dummy = self._ssnd_chunk.read(8) - pos = self._soundpos * self._framesize - if pos: - self._ssnd_chunk.seek(pos + 8) - self._ssnd_seek_needed = 0 - if nframes == 0: - return b'' - data = self._ssnd_chunk.read(nframes * self._framesize) - if self._convert and data: - data = self._convert(data) - self._soundpos = self._soundpos + len(data) // (self._nchannels - * self._sampwidth) - return data - - # - # Internal methods. - # - - def _alaw2lin(self, data): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', category=DeprecationWarning) - import audioop - return audioop.alaw2lin(data, 2) - - def _ulaw2lin(self, data): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', category=DeprecationWarning) - import audioop - return audioop.ulaw2lin(data, 2) - - def _adpcm2lin(self, data): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', category=DeprecationWarning) - import audioop - if not hasattr(self, '_adpcmstate'): - # first time - self._adpcmstate = None - data, self._adpcmstate = audioop.adpcm2lin(data, 2, self._adpcmstate) - return data - - def _sowt2lin(self, data): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', category=DeprecationWarning) - import audioop - return audioop.byteswap(data, 2) - - def _read_comm_chunk(self, chunk): - self._nchannels = _read_short(chunk) - self._nframes = _read_long(chunk) - self._sampwidth = (_read_short(chunk) + 7) // 8 - self._framerate = int(_read_float(chunk)) - if self._sampwidth <= 0: - raise Error('bad sample width') - if self._nchannels <= 0: - raise Error('bad # of channels') - self._framesize = self._nchannels * self._sampwidth - if self._aifc: - #DEBUG: SGI's soundeditor produces a bad size :-( - kludge = 0 - if chunk.chunksize == 18: - kludge = 1 - warnings.warn('Warning: bad COMM chunk size') - chunk.chunksize = 23 - #DEBUG end - self._comptype = chunk.read(4) - #DEBUG start - if kludge: - length = ord(chunk.file.read(1)) - if length & 1 == 0: - length = length + 1 - chunk.chunksize = chunk.chunksize + length - chunk.file.seek(-1, 1) - #DEBUG end - self._compname = _read_string(chunk) - if self._comptype != b'NONE': - if self._comptype == b'G722': - self._convert = self._adpcm2lin - elif self._comptype in (b'ulaw', b'ULAW'): - self._convert = self._ulaw2lin - elif self._comptype in (b'alaw', b'ALAW'): - self._convert = self._alaw2lin - elif self._comptype in (b'sowt', b'SOWT'): - self._convert = self._sowt2lin - else: - raise Error('unsupported compression type') - self._sampwidth = 2 - else: - self._comptype = b'NONE' - self._compname = b'not compressed' - - def _readmark(self, chunk): - nmarkers = _read_short(chunk) - # Some files appear to contain invalid counts. - # Cope with this by testing for EOF. - try: - for i in range(nmarkers): - id = _read_short(chunk) - pos = _read_long(chunk) - name = _read_string(chunk) - if pos or name: - # some files appear to have - # dummy markers consisting of - # a position 0 and name '' - self._markers.append((id, pos, name)) - except EOFError: - w = ('Warning: MARK chunk contains only %s marker%s instead of %s' % - (len(self._markers), '' if len(self._markers) == 1 else 's', - nmarkers)) - warnings.warn(w) - -class Aifc_write: - # Variables used in this class: - # - # These variables are user settable through appropriate methods - # of this class: - # _file -- the open file with methods write(), close(), tell(), seek() - # set through the __init__() method - # _comptype -- the AIFF-C compression type ('NONE' in AIFF) - # set through the setcomptype() or setparams() method - # _compname -- the human-readable AIFF-C compression type - # set through the setcomptype() or setparams() method - # _nchannels -- the number of audio channels - # set through the setnchannels() or setparams() method - # _sampwidth -- the number of bytes per audio sample - # set through the setsampwidth() or setparams() method - # _framerate -- the sampling frequency - # set through the setframerate() or setparams() method - # _nframes -- the number of audio frames written to the header - # set through the setnframes() or setparams() method - # _aifc -- whether we're writing an AIFF-C file or an AIFF file - # set through the aifc() method, reset through the - # aiff() method - # - # These variables are used internally only: - # _version -- the AIFF-C version number - # _comp -- the compressor from builtin module cl - # _nframeswritten -- the number of audio frames actually written - # _datalength -- the size of the audio samples written to the header - # _datawritten -- the size of the audio samples actually written - - _file = None # Set here since __del__ checks it - - def __init__(self, f): - if isinstance(f, str): - file_object = builtins.open(f, 'wb') - try: - self.initfp(file_object) - except: - file_object.close() - raise - - # treat .aiff file extensions as non-compressed audio - if f.endswith('.aiff'): - self._aifc = 0 - else: - # assume it is an open file object already - self.initfp(f) - - def initfp(self, file): - self._file = file - self._version = _AIFC_version - self._comptype = b'NONE' - self._compname = b'not compressed' - self._convert = None - self._nchannels = 0 - self._sampwidth = 0 - self._framerate = 0 - self._nframes = 0 - self._nframeswritten = 0 - self._datawritten = 0 - self._datalength = 0 - self._markers = [] - self._marklength = 0 - self._aifc = 1 # AIFF-C is default - - def __del__(self): - self.close() - - def __enter__(self): - return self - - def __exit__(self, *args): - self.close() - - # - # User visible methods. - # - def aiff(self): - if self._nframeswritten: - raise Error('cannot change parameters after starting to write') - self._aifc = 0 - - def aifc(self): - if self._nframeswritten: - raise Error('cannot change parameters after starting to write') - self._aifc = 1 - - def setnchannels(self, nchannels): - if self._nframeswritten: - raise Error('cannot change parameters after starting to write') - if nchannels < 1: - raise Error('bad # of channels') - self._nchannels = nchannels - - def getnchannels(self): - if not self._nchannels: - raise Error('number of channels not set') - return self._nchannels - - def setsampwidth(self, sampwidth): - if self._nframeswritten: - raise Error('cannot change parameters after starting to write') - if sampwidth < 1 or sampwidth > 4: - raise Error('bad sample width') - self._sampwidth = sampwidth - - def getsampwidth(self): - if not self._sampwidth: - raise Error('sample width not set') - return self._sampwidth - - def setframerate(self, framerate): - if self._nframeswritten: - raise Error('cannot change parameters after starting to write') - if framerate <= 0: - raise Error('bad frame rate') - self._framerate = framerate - - def getframerate(self): - if not self._framerate: - raise Error('frame rate not set') - return self._framerate - - def setnframes(self, nframes): - if self._nframeswritten: - raise Error('cannot change parameters after starting to write') - self._nframes = nframes - - def getnframes(self): - return self._nframeswritten - - def setcomptype(self, comptype, compname): - if self._nframeswritten: - raise Error('cannot change parameters after starting to write') - if comptype not in (b'NONE', b'ulaw', b'ULAW', - b'alaw', b'ALAW', b'G722', b'sowt', b'SOWT'): - raise Error('unsupported compression type') - self._comptype = comptype - self._compname = compname - - def getcomptype(self): - return self._comptype - - def getcompname(self): - return self._compname - -## def setversion(self, version): -## if self._nframeswritten: -## raise Error, 'cannot change parameters after starting to write' -## self._version = version - - def setparams(self, params): - nchannels, sampwidth, framerate, nframes, comptype, compname = params - if self._nframeswritten: - raise Error('cannot change parameters after starting to write') - if comptype not in (b'NONE', b'ulaw', b'ULAW', - b'alaw', b'ALAW', b'G722', b'sowt', b'SOWT'): - raise Error('unsupported compression type') - self.setnchannels(nchannels) - self.setsampwidth(sampwidth) - self.setframerate(framerate) - self.setnframes(nframes) - self.setcomptype(comptype, compname) - - def getparams(self): - if not self._nchannels or not self._sampwidth or not self._framerate: - raise Error('not all parameters set') - return _aifc_params(self._nchannels, self._sampwidth, self._framerate, - self._nframes, self._comptype, self._compname) - - def setmark(self, id, pos, name): - if id <= 0: - raise Error('marker ID must be > 0') - if pos < 0: - raise Error('marker position must be >= 0') - if not isinstance(name, bytes): - raise Error('marker name must be bytes') - for i in range(len(self._markers)): - if id == self._markers[i][0]: - self._markers[i] = id, pos, name - return - self._markers.append((id, pos, name)) - - def getmark(self, id): - for marker in self._markers: - if id == marker[0]: - return marker - raise Error('marker {0!r} does not exist'.format(id)) - - def getmarkers(self): - if len(self._markers) == 0: - return None - return self._markers - - def tell(self): - return self._nframeswritten - - def writeframesraw(self, data): - if not isinstance(data, (bytes, bytearray)): - data = memoryview(data).cast('B') - self._ensure_header_written(len(data)) - nframes = len(data) // (self._sampwidth * self._nchannels) - if self._convert: - data = self._convert(data) - self._file.write(data) - self._nframeswritten = self._nframeswritten + nframes - self._datawritten = self._datawritten + len(data) - - def writeframes(self, data): - self.writeframesraw(data) - if self._nframeswritten != self._nframes or \ - self._datalength != self._datawritten: - self._patchheader() - - def close(self): - if self._file is None: - return - try: - self._ensure_header_written(0) - if self._datawritten & 1: - # quick pad to even size - self._file.write(b'\x00') - self._datawritten = self._datawritten + 1 - self._writemarkers() - if self._nframeswritten != self._nframes or \ - self._datalength != self._datawritten or \ - self._marklength: - self._patchheader() - finally: - # Prevent ref cycles - self._convert = None - f = self._file - self._file = None - f.close() - - # - # Internal methods. - # - - def _lin2alaw(self, data): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', category=DeprecationWarning) - import audioop - return audioop.lin2alaw(data, 2) - - def _lin2ulaw(self, data): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', category=DeprecationWarning) - import audioop - return audioop.lin2ulaw(data, 2) - - def _lin2adpcm(self, data): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', category=DeprecationWarning) - import audioop - if not hasattr(self, '_adpcmstate'): - self._adpcmstate = None - data, self._adpcmstate = audioop.lin2adpcm(data, 2, self._adpcmstate) - return data - - def _lin2sowt(self, data): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', category=DeprecationWarning) - import audioop - return audioop.byteswap(data, 2) - - def _ensure_header_written(self, datasize): - if not self._nframeswritten: - if self._comptype in (b'ULAW', b'ulaw', - b'ALAW', b'alaw', b'G722', - b'sowt', b'SOWT'): - if not self._sampwidth: - self._sampwidth = 2 - if self._sampwidth != 2: - raise Error('sample width must be 2 when compressing ' - 'with ulaw/ULAW, alaw/ALAW, sowt/SOWT ' - 'or G7.22 (ADPCM)') - if not self._nchannels: - raise Error('# channels not specified') - if not self._sampwidth: - raise Error('sample width not specified') - if not self._framerate: - raise Error('sampling rate not specified') - self._write_header(datasize) - - def _init_compression(self): - if self._comptype == b'G722': - self._convert = self._lin2adpcm - elif self._comptype in (b'ulaw', b'ULAW'): - self._convert = self._lin2ulaw - elif self._comptype in (b'alaw', b'ALAW'): - self._convert = self._lin2alaw - elif self._comptype in (b'sowt', b'SOWT'): - self._convert = self._lin2sowt - - def _write_header(self, initlength): - if self._aifc and self._comptype != b'NONE': - self._init_compression() - self._file.write(b'FORM') - if not self._nframes: - self._nframes = initlength // (self._nchannels * self._sampwidth) - self._datalength = self._nframes * self._nchannels * self._sampwidth - if self._datalength & 1: - self._datalength = self._datalength + 1 - if self._aifc: - if self._comptype in (b'ulaw', b'ULAW', b'alaw', b'ALAW'): - self._datalength = self._datalength // 2 - if self._datalength & 1: - self._datalength = self._datalength + 1 - elif self._comptype == b'G722': - self._datalength = (self._datalength + 3) // 4 - if self._datalength & 1: - self._datalength = self._datalength + 1 - try: - self._form_length_pos = self._file.tell() - except (AttributeError, OSError): - self._form_length_pos = None - commlength = self._write_form_length(self._datalength) - if self._aifc: - self._file.write(b'AIFC') - self._file.write(b'FVER') - _write_ulong(self._file, 4) - _write_ulong(self._file, self._version) - else: - self._file.write(b'AIFF') - self._file.write(b'COMM') - _write_ulong(self._file, commlength) - _write_short(self._file, self._nchannels) - if self._form_length_pos is not None: - self._nframes_pos = self._file.tell() - _write_ulong(self._file, self._nframes) - if self._comptype in (b'ULAW', b'ulaw', b'ALAW', b'alaw', b'G722'): - _write_short(self._file, 8) - else: - _write_short(self._file, self._sampwidth * 8) - _write_float(self._file, self._framerate) - if self._aifc: - self._file.write(self._comptype) - _write_string(self._file, self._compname) - self._file.write(b'SSND') - if self._form_length_pos is not None: - self._ssnd_length_pos = self._file.tell() - _write_ulong(self._file, self._datalength + 8) - _write_ulong(self._file, 0) - _write_ulong(self._file, 0) - - def _write_form_length(self, datalength): - if self._aifc: - commlength = 18 + 5 + len(self._compname) - if commlength & 1: - commlength = commlength + 1 - verslength = 12 - else: - commlength = 18 - verslength = 0 - _write_ulong(self._file, 4 + verslength + self._marklength + \ - 8 + commlength + 16 + datalength) - return commlength - - def _patchheader(self): - curpos = self._file.tell() - if self._datawritten & 1: - datalength = self._datawritten + 1 - self._file.write(b'\x00') - else: - datalength = self._datawritten - if datalength == self._datalength and \ - self._nframes == self._nframeswritten and \ - self._marklength == 0: - self._file.seek(curpos, 0) - return - self._file.seek(self._form_length_pos, 0) - dummy = self._write_form_length(datalength) - self._file.seek(self._nframes_pos, 0) - _write_ulong(self._file, self._nframeswritten) - self._file.seek(self._ssnd_length_pos, 0) - _write_ulong(self._file, datalength + 8) - self._file.seek(curpos, 0) - self._nframes = self._nframeswritten - self._datalength = datalength - - def _writemarkers(self): - if len(self._markers) == 0: - return - self._file.write(b'MARK') - length = 2 - for marker in self._markers: - id, pos, name = marker - length = length + len(name) + 1 + 6 - if len(name) & 1 == 0: - length = length + 1 - _write_ulong(self._file, length) - self._marklength = length + 8 - _write_short(self._file, len(self._markers)) - for marker in self._markers: - id, pos, name = marker - _write_short(self._file, id) - _write_ulong(self._file, pos) - _write_string(self._file, name) - -def open(f, mode=None): - if mode is None: - if hasattr(f, 'mode'): - mode = f.mode - else: - mode = 'rb' - if mode in ('r', 'rb'): - return Aifc_read(f) - elif mode in ('w', 'wb'): - return Aifc_write(f) - else: - raise Error("mode must be 'r', 'rb', 'w', or 'wb'") - - -if __name__ == '__main__': - import sys - if not sys.argv[1:]: - sys.argv.append('/usr/demos/data/audio/bach.aiff') - fn = sys.argv[1] - with open(fn, 'r') as f: - print("Reading", fn) - print("nchannels =", f.getnchannels()) - print("nframes =", f.getnframes()) - print("sampwidth =", f.getsampwidth()) - print("framerate =", f.getframerate()) - print("comptype =", f.getcomptype()) - print("compname =", f.getcompname()) - if sys.argv[2:]: - gn = sys.argv[2] - print("Writing", gn) - with open(gn, 'w') as g: - g.setparams(f.getparams()) - while 1: - data = f.readframes(1024) - if not data: - break - g.writeframes(data) - print("Done.") diff --git a/Lib/imghdr.py b/Lib/imghdr.py deleted file mode 100644 index 6a372e66c7..0000000000 --- a/Lib/imghdr.py +++ /dev/null @@ -1,175 +0,0 @@ -"""Recognize image file formats based on their first few bytes.""" - -from os import PathLike -import warnings - -__all__ = ["what"] - - -warnings._deprecated(__name__, remove=(3, 13)) - - -#-------------------------# -# Recognize image headers # -#-------------------------# - -def what(file, h=None): - f = None - try: - if h is None: - if isinstance(file, (str, PathLike)): - f = open(file, 'rb') - h = f.read(32) - else: - location = file.tell() - h = file.read(32) - file.seek(location) - for tf in tests: - res = tf(h, f) - if res: - return res - finally: - if f: f.close() - return None - - -#---------------------------------# -# Subroutines per image file type # -#---------------------------------# - -tests = [] - -def test_jpeg(h, f): - """JPEG data with JFIF or Exif markers; and raw JPEG""" - if h[6:10] in (b'JFIF', b'Exif'): - return 'jpeg' - elif h[:4] == b'\xff\xd8\xff\xdb': - return 'jpeg' - -tests.append(test_jpeg) - -def test_png(h, f): - if h.startswith(b'\211PNG\r\n\032\n'): - return 'png' - -tests.append(test_png) - -def test_gif(h, f): - """GIF ('87 and '89 variants)""" - if h[:6] in (b'GIF87a', b'GIF89a'): - return 'gif' - -tests.append(test_gif) - -def test_tiff(h, f): - """TIFF (can be in Motorola or Intel byte order)""" - if h[:2] in (b'MM', b'II'): - return 'tiff' - -tests.append(test_tiff) - -def test_rgb(h, f): - """SGI image library""" - if h.startswith(b'\001\332'): - return 'rgb' - -tests.append(test_rgb) - -def test_pbm(h, f): - """PBM (portable bitmap)""" - if len(h) >= 3 and \ - h[0] == ord(b'P') and h[1] in b'14' and h[2] in b' \t\n\r': - return 'pbm' - -tests.append(test_pbm) - -def test_pgm(h, f): - """PGM (portable graymap)""" - if len(h) >= 3 and \ - h[0] == ord(b'P') and h[1] in b'25' and h[2] in b' \t\n\r': - return 'pgm' - -tests.append(test_pgm) - -def test_ppm(h, f): - """PPM (portable pixmap)""" - if len(h) >= 3 and \ - h[0] == ord(b'P') and h[1] in b'36' and h[2] in b' \t\n\r': - return 'ppm' - -tests.append(test_ppm) - -def test_rast(h, f): - """Sun raster file""" - if h.startswith(b'\x59\xA6\x6A\x95'): - return 'rast' - -tests.append(test_rast) - -def test_xbm(h, f): - """X bitmap (X10 or X11)""" - if h.startswith(b'#define '): - return 'xbm' - -tests.append(test_xbm) - -def test_bmp(h, f): - if h.startswith(b'BM'): - return 'bmp' - -tests.append(test_bmp) - -def test_webp(h, f): - if h.startswith(b'RIFF') and h[8:12] == b'WEBP': - return 'webp' - -tests.append(test_webp) - -def test_exr(h, f): - if h.startswith(b'\x76\x2f\x31\x01'): - return 'exr' - -tests.append(test_exr) - -#--------------------# -# Small test program # -#--------------------# - -def test(): - import sys - recursive = 0 - if sys.argv[1:] and sys.argv[1] == '-r': - del sys.argv[1:2] - recursive = 1 - try: - if sys.argv[1:]: - testall(sys.argv[1:], recursive, 1) - else: - testall(['.'], recursive, 1) - except KeyboardInterrupt: - sys.stderr.write('\n[Interrupted]\n') - sys.exit(1) - -def testall(list, recursive, toplevel): - import sys - import os - for filename in list: - if os.path.isdir(filename): - print(filename + '/:', end=' ') - if recursive or toplevel: - print('recursing down:') - import glob - names = glob.glob(os.path.join(glob.escape(filename), '*')) - testall(names, recursive, 0) - else: - print('*** directory (use -r) ***') - else: - print(filename + ':', end=' ') - sys.stdout.flush() - try: - print(what(filename)) - except OSError: - print('*** not found ***') - -if __name__ == '__main__': - test() diff --git a/Lib/sndhdr.py b/Lib/sndhdr.py deleted file mode 100644 index 594353136f..0000000000 --- a/Lib/sndhdr.py +++ /dev/null @@ -1,257 +0,0 @@ -"""Routines to help recognizing sound files. - -Function whathdr() recognizes various types of sound file headers. -It understands almost all headers that SOX can decode. - -The return tuple contains the following items, in this order: -- file type (as SOX understands it) -- sampling rate (0 if unknown or hard to decode) -- number of channels (0 if unknown or hard to decode) -- number of frames in the file (-1 if unknown or hard to decode) -- number of bits/sample, or 'U' for U-LAW, or 'A' for A-LAW - -If the file doesn't have a recognizable type, it returns None. -If the file can't be opened, OSError is raised. - -To compute the total time, divide the number of frames by the -sampling rate (a frame contains a sample for each channel). - -Function what() calls whathdr(). (It used to also use some -heuristics for raw data, but this doesn't work very well.) - -Finally, the function test() is a simple main program that calls -what() for all files mentioned on the argument list. For directory -arguments it calls what() for all files in that directory. Default -argument is "." (testing all files in the current directory). The -option -r tells it to recurse down directories found inside -explicitly given directories. -""" - -# The file structure is top-down except that the test program and its -# subroutine come last. - -__all__ = ['what', 'whathdr'] - -from collections import namedtuple - -SndHeaders = namedtuple('SndHeaders', - 'filetype framerate nchannels nframes sampwidth') - -SndHeaders.filetype.__doc__ = ("""The value for type indicates the data type -and will be one of the strings 'aifc', 'aiff', 'au','hcom', -'sndr', 'sndt', 'voc', 'wav', '8svx', 'sb', 'ub', or 'ul'.""") -SndHeaders.framerate.__doc__ = ("""The sampling_rate will be either the actual -value or 0 if unknown or difficult to decode.""") -SndHeaders.nchannels.__doc__ = ("""The number of channels or 0 if it cannot be -determined or if the value is difficult to decode.""") -SndHeaders.nframes.__doc__ = ("""The value for frames will be either the number -of frames or -1.""") -SndHeaders.sampwidth.__doc__ = ("""Either the sample size in bits or -'A' for A-LAW or 'U' for u-LAW.""") - -def what(filename): - """Guess the type of a sound file.""" - res = whathdr(filename) - return res - - -def whathdr(filename): - """Recognize sound headers.""" - with open(filename, 'rb') as f: - h = f.read(512) - for tf in tests: - res = tf(h, f) - if res: - return SndHeaders(*res) - return None - - -#-----------------------------------# -# Subroutines per sound header type # -#-----------------------------------# - -tests = [] - -def test_aifc(h, f): - import aifc - if not h.startswith(b'FORM'): - return None - if h[8:12] == b'AIFC': - fmt = 'aifc' - elif h[8:12] == b'AIFF': - fmt = 'aiff' - else: - return None - f.seek(0) - try: - a = aifc.open(f, 'r') - except (EOFError, aifc.Error): - return None - return (fmt, a.getframerate(), a.getnchannels(), - a.getnframes(), 8 * a.getsampwidth()) - -tests.append(test_aifc) - - -def test_au(h, f): - if h.startswith(b'.snd'): - func = get_long_be - elif h[:4] in (b'\0ds.', b'dns.'): - func = get_long_le - else: - return None - filetype = 'au' - hdr_size = func(h[4:8]) - data_size = func(h[8:12]) - encoding = func(h[12:16]) - rate = func(h[16:20]) - nchannels = func(h[20:24]) - sample_size = 1 # default - if encoding == 1: - sample_bits = 'U' - elif encoding == 2: - sample_bits = 8 - elif encoding == 3: - sample_bits = 16 - sample_size = 2 - else: - sample_bits = '?' - frame_size = sample_size * nchannels - if frame_size: - nframe = data_size / frame_size - else: - nframe = -1 - return filetype, rate, nchannels, nframe, sample_bits - -tests.append(test_au) - - -def test_hcom(h, f): - if h[65:69] != b'FSSD' or h[128:132] != b'HCOM': - return None - divisor = get_long_be(h[144:148]) - if divisor: - rate = 22050 / divisor - else: - rate = 0 - return 'hcom', rate, 1, -1, 8 - -tests.append(test_hcom) - - -def test_voc(h, f): - if not h.startswith(b'Creative Voice File\032'): - return None - sbseek = get_short_le(h[20:22]) - rate = 0 - if 0 <= sbseek < 500 and h[sbseek] == 1: - ratecode = 256 - h[sbseek+4] - if ratecode: - rate = int(1000000.0 / ratecode) - return 'voc', rate, 1, -1, 8 - -tests.append(test_voc) - - -def test_wav(h, f): - import wave - # 'RIFF' 'WAVE' 'fmt ' - if not h.startswith(b'RIFF') or h[8:12] != b'WAVE' or h[12:16] != b'fmt ': - return None - f.seek(0) - try: - w = wave.open(f, 'r') - except (EOFError, wave.Error): - return None - return ('wav', w.getframerate(), w.getnchannels(), - w.getnframes(), 8*w.getsampwidth()) - -tests.append(test_wav) - - -def test_8svx(h, f): - if not h.startswith(b'FORM') or h[8:12] != b'8SVX': - return None - # Should decode it to get #channels -- assume always 1 - return '8svx', 0, 1, 0, 8 - -tests.append(test_8svx) - - -def test_sndt(h, f): - if h.startswith(b'SOUND'): - nsamples = get_long_le(h[8:12]) - rate = get_short_le(h[20:22]) - return 'sndt', rate, 1, nsamples, 8 - -tests.append(test_sndt) - - -def test_sndr(h, f): - if h.startswith(b'\0\0'): - rate = get_short_le(h[2:4]) - if 4000 <= rate <= 25000: - return 'sndr', rate, 1, -1, 8 - -tests.append(test_sndr) - - -#-------------------------------------------# -# Subroutines to extract numbers from bytes # -#-------------------------------------------# - -def get_long_be(b): - return (b[0] << 24) | (b[1] << 16) | (b[2] << 8) | b[3] - -def get_long_le(b): - return (b[3] << 24) | (b[2] << 16) | (b[1] << 8) | b[0] - -def get_short_be(b): - return (b[0] << 8) | b[1] - -def get_short_le(b): - return (b[1] << 8) | b[0] - - -#--------------------# -# Small test program # -#--------------------# - -def test(): - import sys - recursive = 0 - if sys.argv[1:] and sys.argv[1] == '-r': - del sys.argv[1:2] - recursive = 1 - try: - if sys.argv[1:]: - testall(sys.argv[1:], recursive, 1) - else: - testall(['.'], recursive, 1) - except KeyboardInterrupt: - sys.stderr.write('\n[Interrupted]\n') - sys.exit(1) - -def testall(list, recursive, toplevel): - import sys - import os - for filename in list: - if os.path.isdir(filename): - print(filename + '/:', end=' ') - if recursive or toplevel: - print('recursing down:') - import glob - names = glob.glob(os.path.join(filename, '*')) - testall(names, recursive, 0) - else: - print('*** directory (use -r) ***') - else: - print(filename + ':', end=' ') - sys.stdout.flush() - try: - print(what(filename)) - except OSError: - print('*** not found ***') - -if __name__ == '__main__': - test() diff --git a/Lib/sunau.py b/Lib/sunau.py deleted file mode 100644 index 129502b0b4..0000000000 --- a/Lib/sunau.py +++ /dev/null @@ -1,531 +0,0 @@ -"""Stuff to parse Sun and NeXT audio files. - -An audio file consists of a header followed by the data. The structure -of the header is as follows. - - +---------------+ - | magic word | - +---------------+ - | header size | - +---------------+ - | data size | - +---------------+ - | encoding | - +---------------+ - | sample rate | - +---------------+ - | # of channels | - +---------------+ - | info | - | | - +---------------+ - -The magic word consists of the 4 characters '.snd'. Apart from the -info field, all header fields are 4 bytes in size. They are all -32-bit unsigned integers encoded in big-endian byte order. - -The header size really gives the start of the data. -The data size is the physical size of the data. From the other -parameters the number of frames can be calculated. -The encoding gives the way in which audio samples are encoded. -Possible values are listed below. -The info field currently consists of an ASCII string giving a -human-readable description of the audio file. The info field is -padded with NUL bytes to the header size. - -Usage. - -Reading audio files: - f = sunau.open(file, 'r') -where file is either the name of a file or an open file pointer. -The open file pointer must have methods read(), seek(), and close(). -When the setpos() and rewind() methods are not used, the seek() -method is not necessary. - -This returns an instance of a class with the following public methods: - getnchannels() -- returns number of audio channels (1 for - mono, 2 for stereo) - getsampwidth() -- returns sample width in bytes - getframerate() -- returns sampling frequency - getnframes() -- returns number of audio frames - getcomptype() -- returns compression type ('NONE' or 'ULAW') - getcompname() -- returns human-readable version of - compression type ('not compressed' matches 'NONE') - getparams() -- returns a namedtuple consisting of all of the - above in the above order - getmarkers() -- returns None (for compatibility with the - aifc module) - getmark(id) -- raises an error since the mark does not - exist (for compatibility with the aifc module) - readframes(n) -- returns at most n frames of audio - rewind() -- rewind to the beginning of the audio stream - setpos(pos) -- seek to the specified position - tell() -- return the current position - close() -- close the instance (make it unusable) -The position returned by tell() and the position given to setpos() -are compatible and have nothing to do with the actual position in the -file. -The close() method is called automatically when the class instance -is destroyed. - -Writing audio files: - f = sunau.open(file, 'w') -where file is either the name of a file or an open file pointer. -The open file pointer must have methods write(), tell(), seek(), and -close(). - -This returns an instance of a class with the following public methods: - setnchannels(n) -- set the number of channels - setsampwidth(n) -- set the sample width - setframerate(n) -- set the frame rate - setnframes(n) -- set the number of frames - setcomptype(type, name) - -- set the compression type and the - human-readable compression type - setparams(tuple)-- set all parameters at once - tell() -- return current position in output file - writeframesraw(data) - -- write audio frames without pathing up the - file header - writeframes(data) - -- write audio frames and patch up the file header - close() -- patch up the file header and close the - output file -You should set the parameters before the first writeframesraw or -writeframes. The total number of frames does not need to be set, -but when it is set to the correct value, the header does not have to -be patched up. -It is best to first set all parameters, perhaps possibly the -compression type, and then write audio frames using writeframesraw. -When all frames have been written, either call writeframes(b'') or -close() to patch up the sizes in the header. -The close() method is called automatically when the class instance -is destroyed. -""" - -from collections import namedtuple -import warnings - -_sunau_params = namedtuple('_sunau_params', - 'nchannels sampwidth framerate nframes comptype compname') - -# from -AUDIO_FILE_MAGIC = 0x2e736e64 -AUDIO_FILE_ENCODING_MULAW_8 = 1 -AUDIO_FILE_ENCODING_LINEAR_8 = 2 -AUDIO_FILE_ENCODING_LINEAR_16 = 3 -AUDIO_FILE_ENCODING_LINEAR_24 = 4 -AUDIO_FILE_ENCODING_LINEAR_32 = 5 -AUDIO_FILE_ENCODING_FLOAT = 6 -AUDIO_FILE_ENCODING_DOUBLE = 7 -AUDIO_FILE_ENCODING_ADPCM_G721 = 23 -AUDIO_FILE_ENCODING_ADPCM_G722 = 24 -AUDIO_FILE_ENCODING_ADPCM_G723_3 = 25 -AUDIO_FILE_ENCODING_ADPCM_G723_5 = 26 -AUDIO_FILE_ENCODING_ALAW_8 = 27 - -# from -AUDIO_UNKNOWN_SIZE = 0xFFFFFFFF # ((unsigned)(~0)) - -_simple_encodings = [AUDIO_FILE_ENCODING_MULAW_8, - AUDIO_FILE_ENCODING_LINEAR_8, - AUDIO_FILE_ENCODING_LINEAR_16, - AUDIO_FILE_ENCODING_LINEAR_24, - AUDIO_FILE_ENCODING_LINEAR_32, - AUDIO_FILE_ENCODING_ALAW_8] - -class Error(Exception): - pass - -def _read_u32(file): - x = 0 - for i in range(4): - byte = file.read(1) - if not byte: - raise EOFError - x = x*256 + ord(byte) - return x - -def _write_u32(file, x): - data = [] - for i in range(4): - d, m = divmod(x, 256) - data.insert(0, int(m)) - x = d - file.write(bytes(data)) - -class Au_read: - - def __init__(self, f): - if type(f) == type(''): - import builtins - f = builtins.open(f, 'rb') - self._opened = True - else: - self._opened = False - self.initfp(f) - - def __del__(self): - if self._file: - self.close() - - def __enter__(self): - return self - - def __exit__(self, *args): - self.close() - - def initfp(self, file): - self._file = file - self._soundpos = 0 - magic = int(_read_u32(file)) - if magic != AUDIO_FILE_MAGIC: - raise Error('bad magic number') - self._hdr_size = int(_read_u32(file)) - if self._hdr_size < 24: - raise Error('header size too small') - if self._hdr_size > 100: - raise Error('header size ridiculously large') - self._data_size = _read_u32(file) - if self._data_size != AUDIO_UNKNOWN_SIZE: - self._data_size = int(self._data_size) - self._encoding = int(_read_u32(file)) - if self._encoding not in _simple_encodings: - raise Error('encoding not (yet) supported') - if self._encoding in (AUDIO_FILE_ENCODING_MULAW_8, - AUDIO_FILE_ENCODING_ALAW_8): - self._sampwidth = 2 - self._framesize = 1 - elif self._encoding == AUDIO_FILE_ENCODING_LINEAR_8: - self._framesize = self._sampwidth = 1 - elif self._encoding == AUDIO_FILE_ENCODING_LINEAR_16: - self._framesize = self._sampwidth = 2 - elif self._encoding == AUDIO_FILE_ENCODING_LINEAR_24: - self._framesize = self._sampwidth = 3 - elif self._encoding == AUDIO_FILE_ENCODING_LINEAR_32: - self._framesize = self._sampwidth = 4 - else: - raise Error('unknown encoding') - self._framerate = int(_read_u32(file)) - self._nchannels = int(_read_u32(file)) - if not self._nchannels: - raise Error('bad # of channels') - self._framesize = self._framesize * self._nchannels - if self._hdr_size > 24: - self._info = file.read(self._hdr_size - 24) - self._info, _, _ = self._info.partition(b'\0') - else: - self._info = b'' - try: - self._data_pos = file.tell() - except (AttributeError, OSError): - self._data_pos = None - - def getfp(self): - return self._file - - def getnchannels(self): - return self._nchannels - - def getsampwidth(self): - return self._sampwidth - - def getframerate(self): - return self._framerate - - def getnframes(self): - if self._data_size == AUDIO_UNKNOWN_SIZE: - return AUDIO_UNKNOWN_SIZE - if self._encoding in _simple_encodings: - return self._data_size // self._framesize - return 0 # XXX--must do some arithmetic here - - def getcomptype(self): - if self._encoding == AUDIO_FILE_ENCODING_MULAW_8: - return 'ULAW' - elif self._encoding == AUDIO_FILE_ENCODING_ALAW_8: - return 'ALAW' - else: - return 'NONE' - - def getcompname(self): - if self._encoding == AUDIO_FILE_ENCODING_MULAW_8: - return 'CCITT G.711 u-law' - elif self._encoding == AUDIO_FILE_ENCODING_ALAW_8: - return 'CCITT G.711 A-law' - else: - return 'not compressed' - - def getparams(self): - return _sunau_params(self.getnchannels(), self.getsampwidth(), - self.getframerate(), self.getnframes(), - self.getcomptype(), self.getcompname()) - - def getmarkers(self): - return None - - def getmark(self, id): - raise Error('no marks') - - def readframes(self, nframes): - if self._encoding in _simple_encodings: - if nframes == AUDIO_UNKNOWN_SIZE: - data = self._file.read() - else: - data = self._file.read(nframes * self._framesize) - self._soundpos += len(data) // self._framesize - if self._encoding == AUDIO_FILE_ENCODING_MULAW_8: - import audioop - data = audioop.ulaw2lin(data, self._sampwidth) - return data - return None # XXX--not implemented yet - - def rewind(self): - if self._data_pos is None: - raise OSError('cannot seek') - self._file.seek(self._data_pos) - self._soundpos = 0 - - def tell(self): - return self._soundpos - - def setpos(self, pos): - if pos < 0 or pos > self.getnframes(): - raise Error('position not in range') - if self._data_pos is None: - raise OSError('cannot seek') - self._file.seek(self._data_pos + pos * self._framesize) - self._soundpos = pos - - def close(self): - file = self._file - if file: - self._file = None - if self._opened: - file.close() - -class Au_write: - - def __init__(self, f): - if type(f) == type(''): - import builtins - f = builtins.open(f, 'wb') - self._opened = True - else: - self._opened = False - self.initfp(f) - - def __del__(self): - if self._file: - self.close() - self._file = None - - def __enter__(self): - return self - - def __exit__(self, *args): - self.close() - - def initfp(self, file): - self._file = file - self._framerate = 0 - self._nchannels = 0 - self._sampwidth = 0 - self._framesize = 0 - self._nframes = AUDIO_UNKNOWN_SIZE - self._nframeswritten = 0 - self._datawritten = 0 - self._datalength = 0 - self._info = b'' - self._comptype = 'ULAW' # default is U-law - - def setnchannels(self, nchannels): - if self._nframeswritten: - raise Error('cannot change parameters after starting to write') - if nchannels not in (1, 2, 4): - raise Error('only 1, 2, or 4 channels supported') - self._nchannels = nchannels - - def getnchannels(self): - if not self._nchannels: - raise Error('number of channels not set') - return self._nchannels - - def setsampwidth(self, sampwidth): - if self._nframeswritten: - raise Error('cannot change parameters after starting to write') - if sampwidth not in (1, 2, 3, 4): - raise Error('bad sample width') - self._sampwidth = sampwidth - - def getsampwidth(self): - if not self._framerate: - raise Error('sample width not specified') - return self._sampwidth - - def setframerate(self, framerate): - if self._nframeswritten: - raise Error('cannot change parameters after starting to write') - self._framerate = framerate - - def getframerate(self): - if not self._framerate: - raise Error('frame rate not set') - return self._framerate - - def setnframes(self, nframes): - if self._nframeswritten: - raise Error('cannot change parameters after starting to write') - if nframes < 0: - raise Error('# of frames cannot be negative') - self._nframes = nframes - - def getnframes(self): - return self._nframeswritten - - def setcomptype(self, type, name): - if type in ('NONE', 'ULAW'): - self._comptype = type - else: - raise Error('unknown compression type') - - def getcomptype(self): - return self._comptype - - def getcompname(self): - if self._comptype == 'ULAW': - return 'CCITT G.711 u-law' - elif self._comptype == 'ALAW': - return 'CCITT G.711 A-law' - else: - return 'not compressed' - - def setparams(self, params): - nchannels, sampwidth, framerate, nframes, comptype, compname = params - self.setnchannels(nchannels) - self.setsampwidth(sampwidth) - self.setframerate(framerate) - self.setnframes(nframes) - self.setcomptype(comptype, compname) - - def getparams(self): - return _sunau_params(self.getnchannels(), self.getsampwidth(), - self.getframerate(), self.getnframes(), - self.getcomptype(), self.getcompname()) - - def tell(self): - return self._nframeswritten - - def writeframesraw(self, data): - if not isinstance(data, (bytes, bytearray)): - data = memoryview(data).cast('B') - self._ensure_header_written() - if self._comptype == 'ULAW': - import audioop - data = audioop.lin2ulaw(data, self._sampwidth) - nframes = len(data) // self._framesize - self._file.write(data) - self._nframeswritten = self._nframeswritten + nframes - self._datawritten = self._datawritten + len(data) - - def writeframes(self, data): - self.writeframesraw(data) - if self._nframeswritten != self._nframes or \ - self._datalength != self._datawritten: - self._patchheader() - - def close(self): - if self._file: - try: - self._ensure_header_written() - if self._nframeswritten != self._nframes or \ - self._datalength != self._datawritten: - self._patchheader() - self._file.flush() - finally: - file = self._file - self._file = None - if self._opened: - file.close() - - # - # private methods - # - - def _ensure_header_written(self): - if not self._nframeswritten: - if not self._nchannels: - raise Error('# of channels not specified') - if not self._sampwidth: - raise Error('sample width not specified') - if not self._framerate: - raise Error('frame rate not specified') - self._write_header() - - def _write_header(self): - if self._comptype == 'NONE': - if self._sampwidth == 1: - encoding = AUDIO_FILE_ENCODING_LINEAR_8 - self._framesize = 1 - elif self._sampwidth == 2: - encoding = AUDIO_FILE_ENCODING_LINEAR_16 - self._framesize = 2 - elif self._sampwidth == 3: - encoding = AUDIO_FILE_ENCODING_LINEAR_24 - self._framesize = 3 - elif self._sampwidth == 4: - encoding = AUDIO_FILE_ENCODING_LINEAR_32 - self._framesize = 4 - else: - raise Error('internal error') - elif self._comptype == 'ULAW': - encoding = AUDIO_FILE_ENCODING_MULAW_8 - self._framesize = 1 - else: - raise Error('internal error') - self._framesize = self._framesize * self._nchannels - _write_u32(self._file, AUDIO_FILE_MAGIC) - header_size = 25 + len(self._info) - header_size = (header_size + 7) & ~7 - _write_u32(self._file, header_size) - if self._nframes == AUDIO_UNKNOWN_SIZE: - length = AUDIO_UNKNOWN_SIZE - else: - length = self._nframes * self._framesize - try: - self._form_length_pos = self._file.tell() - except (AttributeError, OSError): - self._form_length_pos = None - _write_u32(self._file, length) - self._datalength = length - _write_u32(self._file, encoding) - _write_u32(self._file, self._framerate) - _write_u32(self._file, self._nchannels) - self._file.write(self._info) - self._file.write(b'\0'*(header_size - len(self._info) - 24)) - - def _patchheader(self): - if self._form_length_pos is None: - raise OSError('cannot seek') - self._file.seek(self._form_length_pos) - _write_u32(self._file, self._datawritten) - self._datalength = self._datawritten - self._file.seek(0, 2) - -def open(f, mode=None): - if mode is None: - if hasattr(f, 'mode'): - mode = f.mode - else: - mode = 'rb' - if mode in ('r', 'rb'): - return Au_read(f) - elif mode in ('w', 'wb'): - return Au_write(f) - else: - raise Error("mode must be 'r', 'rb', 'w', or 'wb'") - -def openfp(f, mode=None): - warnings.warn("sunau.openfp is deprecated since Python 3.7. " - "Use sunau.open instead.", DeprecationWarning, stacklevel=2) - return open(f, mode=mode) From fd4ad3e4d183b8d8faf73a34f5b13de1680ab58d Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Thu, 3 Apr 2025 11:25:41 -0700 Subject: [PATCH 167/295] Remove smtpd --- Lib/smtpd.py | 979 -------------------------------------- Lib/test/test_smtpd.py | 1018 ---------------------------------------- 2 files changed, 1997 deletions(-) delete mode 100755 Lib/smtpd.py delete mode 100644 Lib/test/test_smtpd.py diff --git a/Lib/smtpd.py b/Lib/smtpd.py deleted file mode 100755 index 963e0a7689..0000000000 --- a/Lib/smtpd.py +++ /dev/null @@ -1,979 +0,0 @@ -#! /usr/bin/env python3 -"""An RFC 5321 smtp proxy with optional RFC 1870 and RFC 6531 extensions. - -Usage: %(program)s [options] [localhost:localport [remotehost:remoteport]] - -Options: - - --nosetuid - -n - This program generally tries to setuid `nobody', unless this flag is - set. The setuid call will fail if this program is not run as root (in - which case, use this flag). - - --version - -V - Print the version number and exit. - - --class classname - -c classname - Use `classname' as the concrete SMTP proxy class. Uses `PureProxy' by - default. - - --size limit - -s limit - Restrict the total size of the incoming message to "limit" number of - bytes via the RFC 1870 SIZE extension. Defaults to 33554432 bytes. - - --smtputf8 - -u - Enable the SMTPUTF8 extension and behave as an RFC 6531 smtp proxy. - - --debug - -d - Turn on debugging prints. - - --help - -h - Print this message and exit. - -Version: %(__version__)s - -If localhost is not given then `localhost' is used, and if localport is not -given then 8025 is used. If remotehost is not given then `localhost' is used, -and if remoteport is not given, then 25 is used. -""" - -# Overview: -# -# This file implements the minimal SMTP protocol as defined in RFC 5321. It -# has a hierarchy of classes which implement the backend functionality for the -# smtpd. A number of classes are provided: -# -# SMTPServer - the base class for the backend. Raises NotImplementedError -# if you try to use it. -# -# DebuggingServer - simply prints each message it receives on stdout. -# -# PureProxy - Proxies all messages to a real smtpd which does final -# delivery. One known problem with this class is that it doesn't handle -# SMTP errors from the backend server at all. This should be fixed -# (contributions are welcome!). -# -# MailmanProxy - An experimental hack to work with GNU Mailman -# . Using this server as your real incoming smtpd, your -# mailhost will automatically recognize and accept mail destined to Mailman -# lists when those lists are created. Every message not destined for a list -# gets forwarded to a real backend smtpd, as with PureProxy. Again, errors -# are not handled correctly yet. -# -# -# Author: Barry Warsaw -# -# TODO: -# -# - support mailbox delivery -# - alias files -# - Handle more ESMTP extensions -# - handle error codes from the backend smtpd - -import sys -import os -import errno -import getopt -import time -import socket -import collections -from warnings import warn -from email._header_value_parser import get_addr_spec, get_angle_addr - -__all__ = [ - "SMTPChannel", "SMTPServer", "DebuggingServer", "PureProxy", - "MailmanProxy", -] - -warn( - 'The smtpd module is deprecated and unmaintained and will be removed ' - 'in Python 3.12. Please see aiosmtpd ' - '(https://aiosmtpd.readthedocs.io/) for the recommended replacement.', - DeprecationWarning, - stacklevel=2) - - -# These are imported after the above warning so that users get the correct -# deprecation warning. -import asyncore -import asynchat - - -program = sys.argv[0] -__version__ = 'Python SMTP proxy version 0.3' - - -class Devnull: - def write(self, msg): pass - def flush(self): pass - - -DEBUGSTREAM = Devnull() -NEWLINE = '\n' -COMMASPACE = ', ' -DATA_SIZE_DEFAULT = 33554432 - - -def usage(code, msg=''): - print(__doc__ % globals(), file=sys.stderr) - if msg: - print(msg, file=sys.stderr) - sys.exit(code) - - -class SMTPChannel(asynchat.async_chat): - COMMAND = 0 - DATA = 1 - - command_size_limit = 512 - command_size_limits = collections.defaultdict(lambda x=command_size_limit: x) - - @property - def max_command_size_limit(self): - try: - return max(self.command_size_limits.values()) - except ValueError: - return self.command_size_limit - - def __init__(self, server, conn, addr, data_size_limit=DATA_SIZE_DEFAULT, - map=None, enable_SMTPUTF8=False, decode_data=False): - asynchat.async_chat.__init__(self, conn, map=map) - self.smtp_server = server - self.conn = conn - self.addr = addr - self.data_size_limit = data_size_limit - self.enable_SMTPUTF8 = enable_SMTPUTF8 - self._decode_data = decode_data - if enable_SMTPUTF8 and decode_data: - raise ValueError("decode_data and enable_SMTPUTF8 cannot" - " be set to True at the same time") - if decode_data: - self._emptystring = '' - self._linesep = '\r\n' - self._dotsep = '.' - self._newline = NEWLINE - else: - self._emptystring = b'' - self._linesep = b'\r\n' - self._dotsep = ord(b'.') - self._newline = b'\n' - self._set_rset_state() - self.seen_greeting = '' - self.extended_smtp = False - self.command_size_limits.clear() - self.fqdn = socket.getfqdn() - try: - self.peer = conn.getpeername() - except OSError as err: - # a race condition may occur if the other end is closing - # before we can get the peername - self.close() - if err.errno != errno.ENOTCONN: - raise - return - print('Peer:', repr(self.peer), file=DEBUGSTREAM) - self.push('220 %s %s' % (self.fqdn, __version__)) - - def _set_post_data_state(self): - """Reset state variables to their post-DATA state.""" - self.smtp_state = self.COMMAND - self.mailfrom = None - self.rcpttos = [] - self.require_SMTPUTF8 = False - self.num_bytes = 0 - self.set_terminator(b'\r\n') - - def _set_rset_state(self): - """Reset all state variables except the greeting.""" - self._set_post_data_state() - self.received_data = '' - self.received_lines = [] - - - # properties for backwards-compatibility - @property - def __server(self): - warn("Access to __server attribute on SMTPChannel is deprecated, " - "use 'smtp_server' instead", DeprecationWarning, 2) - return self.smtp_server - @__server.setter - def __server(self, value): - warn("Setting __server attribute on SMTPChannel is deprecated, " - "set 'smtp_server' instead", DeprecationWarning, 2) - self.smtp_server = value - - @property - def __line(self): - warn("Access to __line attribute on SMTPChannel is deprecated, " - "use 'received_lines' instead", DeprecationWarning, 2) - return self.received_lines - @__line.setter - def __line(self, value): - warn("Setting __line attribute on SMTPChannel is deprecated, " - "set 'received_lines' instead", DeprecationWarning, 2) - self.received_lines = value - - @property - def __state(self): - warn("Access to __state attribute on SMTPChannel is deprecated, " - "use 'smtp_state' instead", DeprecationWarning, 2) - return self.smtp_state - @__state.setter - def __state(self, value): - warn("Setting __state attribute on SMTPChannel is deprecated, " - "set 'smtp_state' instead", DeprecationWarning, 2) - self.smtp_state = value - - @property - def __greeting(self): - warn("Access to __greeting attribute on SMTPChannel is deprecated, " - "use 'seen_greeting' instead", DeprecationWarning, 2) - return self.seen_greeting - @__greeting.setter - def __greeting(self, value): - warn("Setting __greeting attribute on SMTPChannel is deprecated, " - "set 'seen_greeting' instead", DeprecationWarning, 2) - self.seen_greeting = value - - @property - def __mailfrom(self): - warn("Access to __mailfrom attribute on SMTPChannel is deprecated, " - "use 'mailfrom' instead", DeprecationWarning, 2) - return self.mailfrom - @__mailfrom.setter - def __mailfrom(self, value): - warn("Setting __mailfrom attribute on SMTPChannel is deprecated, " - "set 'mailfrom' instead", DeprecationWarning, 2) - self.mailfrom = value - - @property - def __rcpttos(self): - warn("Access to __rcpttos attribute on SMTPChannel is deprecated, " - "use 'rcpttos' instead", DeprecationWarning, 2) - return self.rcpttos - @__rcpttos.setter - def __rcpttos(self, value): - warn("Setting __rcpttos attribute on SMTPChannel is deprecated, " - "set 'rcpttos' instead", DeprecationWarning, 2) - self.rcpttos = value - - @property - def __data(self): - warn("Access to __data attribute on SMTPChannel is deprecated, " - "use 'received_data' instead", DeprecationWarning, 2) - return self.received_data - @__data.setter - def __data(self, value): - warn("Setting __data attribute on SMTPChannel is deprecated, " - "set 'received_data' instead", DeprecationWarning, 2) - self.received_data = value - - @property - def __fqdn(self): - warn("Access to __fqdn attribute on SMTPChannel is deprecated, " - "use 'fqdn' instead", DeprecationWarning, 2) - return self.fqdn - @__fqdn.setter - def __fqdn(self, value): - warn("Setting __fqdn attribute on SMTPChannel is deprecated, " - "set 'fqdn' instead", DeprecationWarning, 2) - self.fqdn = value - - @property - def __peer(self): - warn("Access to __peer attribute on SMTPChannel is deprecated, " - "use 'peer' instead", DeprecationWarning, 2) - return self.peer - @__peer.setter - def __peer(self, value): - warn("Setting __peer attribute on SMTPChannel is deprecated, " - "set 'peer' instead", DeprecationWarning, 2) - self.peer = value - - @property - def __conn(self): - warn("Access to __conn attribute on SMTPChannel is deprecated, " - "use 'conn' instead", DeprecationWarning, 2) - return self.conn - @__conn.setter - def __conn(self, value): - warn("Setting __conn attribute on SMTPChannel is deprecated, " - "set 'conn' instead", DeprecationWarning, 2) - self.conn = value - - @property - def __addr(self): - warn("Access to __addr attribute on SMTPChannel is deprecated, " - "use 'addr' instead", DeprecationWarning, 2) - return self.addr - @__addr.setter - def __addr(self, value): - warn("Setting __addr attribute on SMTPChannel is deprecated, " - "set 'addr' instead", DeprecationWarning, 2) - self.addr = value - - # Overrides base class for convenience. - def push(self, msg): - asynchat.async_chat.push(self, bytes( - msg + '\r\n', 'utf-8' if self.require_SMTPUTF8 else 'ascii')) - - # Implementation of base class abstract method - def collect_incoming_data(self, data): - limit = None - if self.smtp_state == self.COMMAND: - limit = self.max_command_size_limit - elif self.smtp_state == self.DATA: - limit = self.data_size_limit - if limit and self.num_bytes > limit: - return - elif limit: - self.num_bytes += len(data) - if self._decode_data: - self.received_lines.append(str(data, 'utf-8')) - else: - self.received_lines.append(data) - - # Implementation of base class abstract method - def found_terminator(self): - line = self._emptystring.join(self.received_lines) - print('Data:', repr(line), file=DEBUGSTREAM) - self.received_lines = [] - if self.smtp_state == self.COMMAND: - sz, self.num_bytes = self.num_bytes, 0 - if not line: - self.push('500 Error: bad syntax') - return - if not self._decode_data: - line = str(line, 'utf-8') - i = line.find(' ') - if i < 0: - command = line.upper() - arg = None - else: - command = line[:i].upper() - arg = line[i+1:].strip() - max_sz = (self.command_size_limits[command] - if self.extended_smtp else self.command_size_limit) - if sz > max_sz: - self.push('500 Error: line too long') - return - method = getattr(self, 'smtp_' + command, None) - if not method: - self.push('500 Error: command "%s" not recognized' % command) - return - method(arg) - return - else: - if self.smtp_state != self.DATA: - self.push('451 Internal confusion') - self.num_bytes = 0 - return - if self.data_size_limit and self.num_bytes > self.data_size_limit: - self.push('552 Error: Too much mail data') - self.num_bytes = 0 - return - # Remove extraneous carriage returns and de-transparency according - # to RFC 5321, Section 4.5.2. - data = [] - for text in line.split(self._linesep): - if text and text[0] == self._dotsep: - data.append(text[1:]) - else: - data.append(text) - self.received_data = self._newline.join(data) - args = (self.peer, self.mailfrom, self.rcpttos, self.received_data) - kwargs = {} - if not self._decode_data: - kwargs = { - 'mail_options': self.mail_options, - 'rcpt_options': self.rcpt_options, - } - status = self.smtp_server.process_message(*args, **kwargs) - self._set_post_data_state() - if not status: - self.push('250 OK') - else: - self.push(status) - - # SMTP and ESMTP commands - def smtp_HELO(self, arg): - if not arg: - self.push('501 Syntax: HELO hostname') - return - # See issue #21783 for a discussion of this behavior. - if self.seen_greeting: - self.push('503 Duplicate HELO/EHLO') - return - self._set_rset_state() - self.seen_greeting = arg - self.push('250 %s' % self.fqdn) - - def smtp_EHLO(self, arg): - if not arg: - self.push('501 Syntax: EHLO hostname') - return - # See issue #21783 for a discussion of this behavior. - if self.seen_greeting: - self.push('503 Duplicate HELO/EHLO') - return - self._set_rset_state() - self.seen_greeting = arg - self.extended_smtp = True - self.push('250-%s' % self.fqdn) - if self.data_size_limit: - self.push('250-SIZE %s' % self.data_size_limit) - self.command_size_limits['MAIL'] += 26 - if not self._decode_data: - self.push('250-8BITMIME') - if self.enable_SMTPUTF8: - self.push('250-SMTPUTF8') - self.command_size_limits['MAIL'] += 10 - self.push('250 HELP') - - def smtp_NOOP(self, arg): - if arg: - self.push('501 Syntax: NOOP') - else: - self.push('250 OK') - - def smtp_QUIT(self, arg): - # args is ignored - self.push('221 Bye') - self.close_when_done() - - def _strip_command_keyword(self, keyword, arg): - keylen = len(keyword) - if arg[:keylen].upper() == keyword: - return arg[keylen:].strip() - return '' - - def _getaddr(self, arg): - if not arg: - return '', '' - if arg.lstrip().startswith('<'): - address, rest = get_angle_addr(arg) - else: - address, rest = get_addr_spec(arg) - if not address: - return address, rest - return address.addr_spec, rest - - def _getparams(self, params): - # Return params as dictionary. Return None if not all parameters - # appear to be syntactically valid according to RFC 1869. - result = {} - for param in params: - param, eq, value = param.partition('=') - if not param.isalnum() or eq and not value: - return None - result[param] = value if eq else True - return result - - def smtp_HELP(self, arg): - if arg: - extended = ' [SP ]' - lc_arg = arg.upper() - if lc_arg == 'EHLO': - self.push('250 Syntax: EHLO hostname') - elif lc_arg == 'HELO': - self.push('250 Syntax: HELO hostname') - elif lc_arg == 'MAIL': - msg = '250 Syntax: MAIL FROM:
' - if self.extended_smtp: - msg += extended - self.push(msg) - elif lc_arg == 'RCPT': - msg = '250 Syntax: RCPT TO:
' - if self.extended_smtp: - msg += extended - self.push(msg) - elif lc_arg == 'DATA': - self.push('250 Syntax: DATA') - elif lc_arg == 'RSET': - self.push('250 Syntax: RSET') - elif lc_arg == 'NOOP': - self.push('250 Syntax: NOOP') - elif lc_arg == 'QUIT': - self.push('250 Syntax: QUIT') - elif lc_arg == 'VRFY': - self.push('250 Syntax: VRFY
') - else: - self.push('501 Supported commands: EHLO HELO MAIL RCPT ' - 'DATA RSET NOOP QUIT VRFY') - else: - self.push('250 Supported commands: EHLO HELO MAIL RCPT DATA ' - 'RSET NOOP QUIT VRFY') - - def smtp_VRFY(self, arg): - if arg: - address, params = self._getaddr(arg) - if address: - self.push('252 Cannot VRFY user, but will accept message ' - 'and attempt delivery') - else: - self.push('502 Could not VRFY %s' % arg) - else: - self.push('501 Syntax: VRFY
') - - def smtp_MAIL(self, arg): - if not self.seen_greeting: - self.push('503 Error: send HELO first') - return - print('===> MAIL', arg, file=DEBUGSTREAM) - syntaxerr = '501 Syntax: MAIL FROM:
' - if self.extended_smtp: - syntaxerr += ' [SP ]' - if arg is None: - self.push(syntaxerr) - return - arg = self._strip_command_keyword('FROM:', arg) - address, params = self._getaddr(arg) - if not address: - self.push(syntaxerr) - return - if not self.extended_smtp and params: - self.push(syntaxerr) - return - if self.mailfrom: - self.push('503 Error: nested MAIL command') - return - self.mail_options = params.upper().split() - params = self._getparams(self.mail_options) - if params is None: - self.push(syntaxerr) - return - if not self._decode_data: - body = params.pop('BODY', '7BIT') - if body not in ['7BIT', '8BITMIME']: - self.push('501 Error: BODY can only be one of 7BIT, 8BITMIME') - return - if self.enable_SMTPUTF8: - smtputf8 = params.pop('SMTPUTF8', False) - if smtputf8 is True: - self.require_SMTPUTF8 = True - elif smtputf8 is not False: - self.push('501 Error: SMTPUTF8 takes no arguments') - return - size = params.pop('SIZE', None) - if size: - if not size.isdigit(): - self.push(syntaxerr) - return - elif self.data_size_limit and int(size) > self.data_size_limit: - self.push('552 Error: message size exceeds fixed maximum message size') - return - if len(params.keys()) > 0: - self.push('555 MAIL FROM parameters not recognized or not implemented') - return - self.mailfrom = address - print('sender:', self.mailfrom, file=DEBUGSTREAM) - self.push('250 OK') - - def smtp_RCPT(self, arg): - if not self.seen_greeting: - self.push('503 Error: send HELO first'); - return - print('===> RCPT', arg, file=DEBUGSTREAM) - if not self.mailfrom: - self.push('503 Error: need MAIL command') - return - syntaxerr = '501 Syntax: RCPT TO:
' - if self.extended_smtp: - syntaxerr += ' [SP ]' - if arg is None: - self.push(syntaxerr) - return - arg = self._strip_command_keyword('TO:', arg) - address, params = self._getaddr(arg) - if not address: - self.push(syntaxerr) - return - if not self.extended_smtp and params: - self.push(syntaxerr) - return - self.rcpt_options = params.upper().split() - params = self._getparams(self.rcpt_options) - if params is None: - self.push(syntaxerr) - return - # XXX currently there are no options we recognize. - if len(params.keys()) > 0: - self.push('555 RCPT TO parameters not recognized or not implemented') - return - self.rcpttos.append(address) - print('recips:', self.rcpttos, file=DEBUGSTREAM) - self.push('250 OK') - - def smtp_RSET(self, arg): - if arg: - self.push('501 Syntax: RSET') - return - self._set_rset_state() - self.push('250 OK') - - def smtp_DATA(self, arg): - if not self.seen_greeting: - self.push('503 Error: send HELO first'); - return - if not self.rcpttos: - self.push('503 Error: need RCPT command') - return - if arg: - self.push('501 Syntax: DATA') - return - self.smtp_state = self.DATA - self.set_terminator(b'\r\n.\r\n') - self.push('354 End data with .') - - # Commands that have not been implemented - def smtp_EXPN(self, arg): - self.push('502 EXPN not implemented') - - -class SMTPServer(asyncore.dispatcher): - # SMTPChannel class to use for managing client connections - channel_class = SMTPChannel - - def __init__(self, localaddr, remoteaddr, - data_size_limit=DATA_SIZE_DEFAULT, map=None, - enable_SMTPUTF8=False, decode_data=False): - self._localaddr = localaddr - self._remoteaddr = remoteaddr - self.data_size_limit = data_size_limit - self.enable_SMTPUTF8 = enable_SMTPUTF8 - self._decode_data = decode_data - if enable_SMTPUTF8 and decode_data: - raise ValueError("decode_data and enable_SMTPUTF8 cannot" - " be set to True at the same time") - asyncore.dispatcher.__init__(self, map=map) - try: - gai_results = socket.getaddrinfo(*localaddr, - type=socket.SOCK_STREAM) - self.create_socket(gai_results[0][0], gai_results[0][1]) - # try to re-use a server port if possible - self.set_reuse_addr() - self.bind(localaddr) - self.listen(5) - except: - self.close() - raise - else: - print('%s started at %s\n\tLocal addr: %s\n\tRemote addr:%s' % ( - self.__class__.__name__, time.ctime(time.time()), - localaddr, remoteaddr), file=DEBUGSTREAM) - - def handle_accepted(self, conn, addr): - print('Incoming connection from %s' % repr(addr), file=DEBUGSTREAM) - channel = self.channel_class(self, - conn, - addr, - self.data_size_limit, - self._map, - self.enable_SMTPUTF8, - self._decode_data) - - # API for "doing something useful with the message" - def process_message(self, peer, mailfrom, rcpttos, data, **kwargs): - """Override this abstract method to handle messages from the client. - - peer is a tuple containing (ipaddr, port) of the client that made the - socket connection to our smtp port. - - mailfrom is the raw address the client claims the message is coming - from. - - rcpttos is a list of raw addresses the client wishes to deliver the - message to. - - data is a string containing the entire full text of the message, - headers (if supplied) and all. It has been `de-transparencied' - according to RFC 821, Section 4.5.2. In other words, a line - containing a `.' followed by other text has had the leading dot - removed. - - kwargs is a dictionary containing additional information. It is - empty if decode_data=True was given as init parameter, otherwise - it will contain the following keys: - 'mail_options': list of parameters to the mail command. All - elements are uppercase strings. Example: - ['BODY=8BITMIME', 'SMTPUTF8']. - 'rcpt_options': same, for the rcpt command. - - This function should return None for a normal `250 Ok' response; - otherwise, it should return the desired response string in RFC 821 - format. - - """ - raise NotImplementedError - - -class DebuggingServer(SMTPServer): - - def _print_message_content(self, peer, data): - inheaders = 1 - lines = data.splitlines() - for line in lines: - # headers first - if inheaders and not line: - peerheader = 'X-Peer: ' + peer[0] - if not isinstance(data, str): - # decoded_data=false; make header match other binary output - peerheader = repr(peerheader.encode('utf-8')) - print(peerheader) - inheaders = 0 - if not isinstance(data, str): - # Avoid spurious 'str on bytes instance' warning. - line = repr(line) - print(line) - - def process_message(self, peer, mailfrom, rcpttos, data, **kwargs): - print('---------- MESSAGE FOLLOWS ----------') - if kwargs: - if kwargs.get('mail_options'): - print('mail options: %s' % kwargs['mail_options']) - if kwargs.get('rcpt_options'): - print('rcpt options: %s\n' % kwargs['rcpt_options']) - self._print_message_content(peer, data) - print('------------ END MESSAGE ------------') - - -class PureProxy(SMTPServer): - def __init__(self, *args, **kwargs): - if 'enable_SMTPUTF8' in kwargs and kwargs['enable_SMTPUTF8']: - raise ValueError("PureProxy does not support SMTPUTF8.") - super(PureProxy, self).__init__(*args, **kwargs) - - def process_message(self, peer, mailfrom, rcpttos, data): - lines = data.split('\n') - # Look for the last header - i = 0 - for line in lines: - if not line: - break - i += 1 - lines.insert(i, 'X-Peer: %s' % peer[0]) - data = NEWLINE.join(lines) - refused = self._deliver(mailfrom, rcpttos, data) - # TBD: what to do with refused addresses? - print('we got some refusals:', refused, file=DEBUGSTREAM) - - def _deliver(self, mailfrom, rcpttos, data): - import smtplib - refused = {} - try: - s = smtplib.SMTP() - s.connect(self._remoteaddr[0], self._remoteaddr[1]) - try: - refused = s.sendmail(mailfrom, rcpttos, data) - finally: - s.quit() - except smtplib.SMTPRecipientsRefused as e: - print('got SMTPRecipientsRefused', file=DEBUGSTREAM) - refused = e.recipients - except (OSError, smtplib.SMTPException) as e: - print('got', e.__class__, file=DEBUGSTREAM) - # All recipients were refused. If the exception had an associated - # error code, use it. Otherwise,fake it with a non-triggering - # exception code. - errcode = getattr(e, 'smtp_code', -1) - errmsg = getattr(e, 'smtp_error', 'ignore') - for r in rcpttos: - refused[r] = (errcode, errmsg) - return refused - - -class MailmanProxy(PureProxy): - def __init__(self, *args, **kwargs): - warn('MailmanProxy is deprecated and will be removed ' - 'in future', DeprecationWarning, 2) - if 'enable_SMTPUTF8' in kwargs and kwargs['enable_SMTPUTF8']: - raise ValueError("MailmanProxy does not support SMTPUTF8.") - super(PureProxy, self).__init__(*args, **kwargs) - - def process_message(self, peer, mailfrom, rcpttos, data): - from io import StringIO - from Mailman import Utils - from Mailman import Message - from Mailman import MailList - # If the message is to a Mailman mailing list, then we'll invoke the - # Mailman script directly, without going through the real smtpd. - # Otherwise we'll forward it to the local proxy for disposition. - listnames = [] - for rcpt in rcpttos: - local = rcpt.lower().split('@')[0] - # We allow the following variations on the theme - # listname - # listname-admin - # listname-owner - # listname-request - # listname-join - # listname-leave - parts = local.split('-') - if len(parts) > 2: - continue - listname = parts[0] - if len(parts) == 2: - command = parts[1] - else: - command = '' - if not Utils.list_exists(listname) or command not in ( - '', 'admin', 'owner', 'request', 'join', 'leave'): - continue - listnames.append((rcpt, listname, command)) - # Remove all list recipients from rcpttos and forward what we're not - # going to take care of ourselves. Linear removal should be fine - # since we don't expect a large number of recipients. - for rcpt, listname, command in listnames: - rcpttos.remove(rcpt) - # If there's any non-list destined recipients left, - print('forwarding recips:', ' '.join(rcpttos), file=DEBUGSTREAM) - if rcpttos: - refused = self._deliver(mailfrom, rcpttos, data) - # TBD: what to do with refused addresses? - print('we got refusals:', refused, file=DEBUGSTREAM) - # Now deliver directly to the list commands - mlists = {} - s = StringIO(data) - msg = Message.Message(s) - # These headers are required for the proper execution of Mailman. All - # MTAs in existence seem to add these if the original message doesn't - # have them. - if not msg.get('from'): - msg['From'] = mailfrom - if not msg.get('date'): - msg['Date'] = time.ctime(time.time()) - for rcpt, listname, command in listnames: - print('sending message to', rcpt, file=DEBUGSTREAM) - mlist = mlists.get(listname) - if not mlist: - mlist = MailList.MailList(listname, lock=0) - mlists[listname] = mlist - # dispatch on the type of command - if command == '': - # post - msg.Enqueue(mlist, tolist=1) - elif command == 'admin': - msg.Enqueue(mlist, toadmin=1) - elif command == 'owner': - msg.Enqueue(mlist, toowner=1) - elif command == 'request': - msg.Enqueue(mlist, torequest=1) - elif command in ('join', 'leave'): - # TBD: this is a hack! - if command == 'join': - msg['Subject'] = 'subscribe' - else: - msg['Subject'] = 'unsubscribe' - msg.Enqueue(mlist, torequest=1) - - -class Options: - setuid = True - classname = 'PureProxy' - size_limit = None - enable_SMTPUTF8 = False - - -def parseargs(): - global DEBUGSTREAM - try: - opts, args = getopt.getopt( - sys.argv[1:], 'nVhc:s:du', - ['class=', 'nosetuid', 'version', 'help', 'size=', 'debug', - 'smtputf8']) - except getopt.error as e: - usage(1, e) - - options = Options() - for opt, arg in opts: - if opt in ('-h', '--help'): - usage(0) - elif opt in ('-V', '--version'): - print(__version__) - sys.exit(0) - elif opt in ('-n', '--nosetuid'): - options.setuid = False - elif opt in ('-c', '--class'): - options.classname = arg - elif opt in ('-d', '--debug'): - DEBUGSTREAM = sys.stderr - elif opt in ('-u', '--smtputf8'): - options.enable_SMTPUTF8 = True - elif opt in ('-s', '--size'): - try: - int_size = int(arg) - options.size_limit = int_size - except: - print('Invalid size: ' + arg, file=sys.stderr) - sys.exit(1) - - # parse the rest of the arguments - if len(args) < 1: - localspec = 'localhost:8025' - remotespec = 'localhost:25' - elif len(args) < 2: - localspec = args[0] - remotespec = 'localhost:25' - elif len(args) < 3: - localspec = args[0] - remotespec = args[1] - else: - usage(1, 'Invalid arguments: %s' % COMMASPACE.join(args)) - - # split into host/port pairs - i = localspec.find(':') - if i < 0: - usage(1, 'Bad local spec: %s' % localspec) - options.localhost = localspec[:i] - try: - options.localport = int(localspec[i+1:]) - except ValueError: - usage(1, 'Bad local port: %s' % localspec) - i = remotespec.find(':') - if i < 0: - usage(1, 'Bad remote spec: %s' % remotespec) - options.remotehost = remotespec[:i] - try: - options.remoteport = int(remotespec[i+1:]) - except ValueError: - usage(1, 'Bad remote port: %s' % remotespec) - return options - - -if __name__ == '__main__': - options = parseargs() - # Become nobody - classname = options.classname - if "." in classname: - lastdot = classname.rfind(".") - mod = __import__(classname[:lastdot], globals(), locals(), [""]) - classname = classname[lastdot+1:] - else: - import __main__ as mod - class_ = getattr(mod, classname) - proxy = class_((options.localhost, options.localport), - (options.remotehost, options.remoteport), - options.size_limit, enable_SMTPUTF8=options.enable_SMTPUTF8) - if options.setuid: - try: - import pwd - except ImportError: - print('Cannot import module "pwd"; try running with -n option.', file=sys.stderr) - sys.exit(1) - nobody = pwd.getpwnam('nobody')[2] - try: - os.setuid(nobody) - except PermissionError: - print('Cannot setuid "nobody"; try running with -n option.', file=sys.stderr) - sys.exit(1) - try: - asyncore.loop() - except KeyboardInterrupt: - pass diff --git a/Lib/test/test_smtpd.py b/Lib/test/test_smtpd.py deleted file mode 100644 index d2e150d535..0000000000 --- a/Lib/test/test_smtpd.py +++ /dev/null @@ -1,1018 +0,0 @@ -import unittest -import textwrap -from test import support, mock_socket -from test.support import socket_helper -from test.support import warnings_helper -import socket -import io - -import warnings -with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - import smtpd - import asyncore - - -class DummyServer(smtpd.SMTPServer): - def __init__(self, *args, **kwargs): - smtpd.SMTPServer.__init__(self, *args, **kwargs) - self.messages = [] - if self._decode_data: - self.return_status = 'return status' - else: - self.return_status = b'return status' - - def process_message(self, peer, mailfrom, rcpttos, data, **kw): - self.messages.append((peer, mailfrom, rcpttos, data)) - if data == self.return_status: - return '250 Okish' - if 'mail_options' in kw and 'SMTPUTF8' in kw['mail_options']: - return '250 SMTPUTF8 message okish' - - -class DummyDispatcherBroken(Exception): - pass - - -class BrokenDummyServer(DummyServer): - def listen(self, num): - raise DummyDispatcherBroken() - - -class SMTPDServerTest(unittest.TestCase): - def setUp(self): - smtpd.socket = asyncore.socket = mock_socket - - def test_process_message_unimplemented(self): - server = smtpd.SMTPServer((socket_helper.HOST, 0), ('b', 0), - decode_data=True) - conn, addr = server.accept() - channel = smtpd.SMTPChannel(server, conn, addr, decode_data=True) - - def write_line(line): - channel.socket.queue_recv(line) - channel.handle_read() - - write_line(b'HELO example') - write_line(b'MAIL From:eggs@example') - write_line(b'RCPT To:spam@example') - write_line(b'DATA') - self.assertRaises(NotImplementedError, write_line, b'spam\r\n.\r\n') - - def test_decode_data_and_enable_SMTPUTF8_raises(self): - self.assertRaises( - ValueError, - smtpd.SMTPServer, - (socket_helper.HOST, 0), - ('b', 0), - enable_SMTPUTF8=True, - decode_data=True) - - def tearDown(self): - asyncore.close_all() - asyncore.socket = smtpd.socket = socket - - -class DebuggingServerTest(unittest.TestCase): - - def setUp(self): - smtpd.socket = asyncore.socket = mock_socket - - def send_data(self, channel, data, enable_SMTPUTF8=False): - def write_line(line): - channel.socket.queue_recv(line) - channel.handle_read() - write_line(b'EHLO example') - if enable_SMTPUTF8: - write_line(b'MAIL From:eggs@example BODY=8BITMIME SMTPUTF8') - else: - write_line(b'MAIL From:eggs@example') - write_line(b'RCPT To:spam@example') - write_line(b'DATA') - write_line(data) - write_line(b'.') - - def test_process_message_with_decode_data_true(self): - server = smtpd.DebuggingServer((socket_helper.HOST, 0), ('b', 0), - decode_data=True) - conn, addr = server.accept() - channel = smtpd.SMTPChannel(server, conn, addr, decode_data=True) - with support.captured_stdout() as s: - self.send_data(channel, b'From: test\n\nhello\n') - stdout = s.getvalue() - self.assertEqual(stdout, textwrap.dedent("""\ - ---------- MESSAGE FOLLOWS ---------- - From: test - X-Peer: peer-address - - hello - ------------ END MESSAGE ------------ - """)) - - def test_process_message_with_decode_data_false(self): - server = smtpd.DebuggingServer((socket_helper.HOST, 0), ('b', 0)) - conn, addr = server.accept() - channel = smtpd.SMTPChannel(server, conn, addr) - with support.captured_stdout() as s: - self.send_data(channel, b'From: test\n\nh\xc3\xa9llo\xff\n') - stdout = s.getvalue() - self.assertEqual(stdout, textwrap.dedent("""\ - ---------- MESSAGE FOLLOWS ---------- - b'From: test' - b'X-Peer: peer-address' - b'' - b'h\\xc3\\xa9llo\\xff' - ------------ END MESSAGE ------------ - """)) - - def test_process_message_with_enable_SMTPUTF8_true(self): - server = smtpd.DebuggingServer((socket_helper.HOST, 0), ('b', 0), - enable_SMTPUTF8=True) - conn, addr = server.accept() - channel = smtpd.SMTPChannel(server, conn, addr, enable_SMTPUTF8=True) - with support.captured_stdout() as s: - self.send_data(channel, b'From: test\n\nh\xc3\xa9llo\xff\n') - stdout = s.getvalue() - self.assertEqual(stdout, textwrap.dedent("""\ - ---------- MESSAGE FOLLOWS ---------- - b'From: test' - b'X-Peer: peer-address' - b'' - b'h\\xc3\\xa9llo\\xff' - ------------ END MESSAGE ------------ - """)) - - def test_process_SMTPUTF8_message_with_enable_SMTPUTF8_true(self): - server = smtpd.DebuggingServer((socket_helper.HOST, 0), ('b', 0), - enable_SMTPUTF8=True) - conn, addr = server.accept() - channel = smtpd.SMTPChannel(server, conn, addr, enable_SMTPUTF8=True) - with support.captured_stdout() as s: - self.send_data(channel, b'From: test\n\nh\xc3\xa9llo\xff\n', - enable_SMTPUTF8=True) - stdout = s.getvalue() - self.assertEqual(stdout, textwrap.dedent("""\ - ---------- MESSAGE FOLLOWS ---------- - mail options: ['BODY=8BITMIME', 'SMTPUTF8'] - b'From: test' - b'X-Peer: peer-address' - b'' - b'h\\xc3\\xa9llo\\xff' - ------------ END MESSAGE ------------ - """)) - - def tearDown(self): - asyncore.close_all() - asyncore.socket = smtpd.socket = socket - - -class TestFamilyDetection(unittest.TestCase): - def setUp(self): - smtpd.socket = asyncore.socket = mock_socket - - def tearDown(self): - asyncore.close_all() - asyncore.socket = smtpd.socket = socket - - @unittest.skipUnless(socket_helper.IPV6_ENABLED, "IPv6 not enabled") - def test_socket_uses_IPv6(self): - server = smtpd.SMTPServer((socket_helper.HOSTv6, 0), (socket_helper.HOSTv4, 0)) - self.assertEqual(server.socket.family, socket.AF_INET6) - - def test_socket_uses_IPv4(self): - server = smtpd.SMTPServer((socket_helper.HOSTv4, 0), (socket_helper.HOSTv6, 0)) - self.assertEqual(server.socket.family, socket.AF_INET) - - -class TestRcptOptionParsing(unittest.TestCase): - error_response = (b'555 RCPT TO parameters not recognized or not ' - b'implemented\r\n') - - def setUp(self): - smtpd.socket = asyncore.socket = mock_socket - self.old_debugstream = smtpd.DEBUGSTREAM - self.debug = smtpd.DEBUGSTREAM = io.StringIO() - - def tearDown(self): - asyncore.close_all() - asyncore.socket = smtpd.socket = socket - smtpd.DEBUGSTREAM = self.old_debugstream - - def write_line(self, channel, line): - channel.socket.queue_recv(line) - channel.handle_read() - - def test_params_rejected(self): - server = DummyServer((socket_helper.HOST, 0), ('b', 0)) - conn, addr = server.accept() - channel = smtpd.SMTPChannel(server, conn, addr) - self.write_line(channel, b'EHLO example') - self.write_line(channel, b'MAIL from: size=20') - self.write_line(channel, b'RCPT to: foo=bar') - self.assertEqual(channel.socket.last, self.error_response) - - def test_nothing_accepted(self): - server = DummyServer((socket_helper.HOST, 0), ('b', 0)) - conn, addr = server.accept() - channel = smtpd.SMTPChannel(server, conn, addr) - self.write_line(channel, b'EHLO example') - self.write_line(channel, b'MAIL from: size=20') - self.write_line(channel, b'RCPT to: ') - self.assertEqual(channel.socket.last, b'250 OK\r\n') - - -class TestMailOptionParsing(unittest.TestCase): - error_response = (b'555 MAIL FROM parameters not recognized or not ' - b'implemented\r\n') - - def setUp(self): - smtpd.socket = asyncore.socket = mock_socket - self.old_debugstream = smtpd.DEBUGSTREAM - self.debug = smtpd.DEBUGSTREAM = io.StringIO() - - def tearDown(self): - asyncore.close_all() - asyncore.socket = smtpd.socket = socket - smtpd.DEBUGSTREAM = self.old_debugstream - - def write_line(self, channel, line): - channel.socket.queue_recv(line) - channel.handle_read() - - def test_with_decode_data_true(self): - server = DummyServer((socket_helper.HOST, 0), ('b', 0), decode_data=True) - conn, addr = server.accept() - channel = smtpd.SMTPChannel(server, conn, addr, decode_data=True) - self.write_line(channel, b'EHLO example') - for line in [ - b'MAIL from: size=20 SMTPUTF8', - b'MAIL from: size=20 SMTPUTF8 BODY=8BITMIME', - b'MAIL from: size=20 BODY=UNKNOWN', - b'MAIL from: size=20 body=8bitmime', - ]: - self.write_line(channel, line) - self.assertEqual(channel.socket.last, self.error_response) - self.write_line(channel, b'MAIL from: size=20') - self.assertEqual(channel.socket.last, b'250 OK\r\n') - - def test_with_decode_data_false(self): - server = DummyServer((socket_helper.HOST, 0), ('b', 0)) - conn, addr = server.accept() - channel = smtpd.SMTPChannel(server, conn, addr) - self.write_line(channel, b'EHLO example') - for line in [ - b'MAIL from: size=20 SMTPUTF8', - b'MAIL from: size=20 SMTPUTF8 BODY=8BITMIME', - ]: - self.write_line(channel, line) - self.assertEqual(channel.socket.last, self.error_response) - self.write_line( - channel, - b'MAIL from: size=20 SMTPUTF8 BODY=UNKNOWN') - self.assertEqual( - channel.socket.last, - b'501 Error: BODY can only be one of 7BIT, 8BITMIME\r\n') - self.write_line( - channel, b'MAIL from: size=20 body=8bitmime') - self.assertEqual(channel.socket.last, b'250 OK\r\n') - - def test_with_enable_smtputf8_true(self): - server = DummyServer((socket_helper.HOST, 0), ('b', 0), enable_SMTPUTF8=True) - conn, addr = server.accept() - channel = smtpd.SMTPChannel(server, conn, addr, enable_SMTPUTF8=True) - self.write_line(channel, b'EHLO example') - self.write_line( - channel, - b'MAIL from: size=20 body=8bitmime smtputf8') - self.assertEqual(channel.socket.last, b'250 OK\r\n') - - -class SMTPDChannelTest(unittest.TestCase): - def setUp(self): - smtpd.socket = asyncore.socket = mock_socket - self.old_debugstream = smtpd.DEBUGSTREAM - self.debug = smtpd.DEBUGSTREAM = io.StringIO() - self.server = DummyServer((socket_helper.HOST, 0), ('b', 0), - decode_data=True) - conn, addr = self.server.accept() - self.channel = smtpd.SMTPChannel(self.server, conn, addr, - decode_data=True) - - def tearDown(self): - asyncore.close_all() - asyncore.socket = smtpd.socket = socket - smtpd.DEBUGSTREAM = self.old_debugstream - - def write_line(self, line): - self.channel.socket.queue_recv(line) - self.channel.handle_read() - - def test_broken_connect(self): - self.assertRaises( - DummyDispatcherBroken, BrokenDummyServer, - (socket_helper.HOST, 0), ('b', 0), decode_data=True) - - def test_decode_data_and_enable_SMTPUTF8_raises(self): - self.assertRaises( - ValueError, smtpd.SMTPChannel, - self.server, self.channel.conn, self.channel.addr, - enable_SMTPUTF8=True, decode_data=True) - - def test_server_accept(self): - self.server.handle_accept() - - def test_missing_data(self): - self.write_line(b'') - self.assertEqual(self.channel.socket.last, - b'500 Error: bad syntax\r\n') - - def test_EHLO(self): - self.write_line(b'EHLO example') - self.assertEqual(self.channel.socket.last, b'250 HELP\r\n') - - def test_EHLO_bad_syntax(self): - self.write_line(b'EHLO') - self.assertEqual(self.channel.socket.last, - b'501 Syntax: EHLO hostname\r\n') - - def test_EHLO_duplicate(self): - self.write_line(b'EHLO example') - self.write_line(b'EHLO example') - self.assertEqual(self.channel.socket.last, - b'503 Duplicate HELO/EHLO\r\n') - - def test_EHLO_HELO_duplicate(self): - self.write_line(b'EHLO example') - self.write_line(b'HELO example') - self.assertEqual(self.channel.socket.last, - b'503 Duplicate HELO/EHLO\r\n') - - def test_HELO(self): - name = smtpd.socket.getfqdn() - self.write_line(b'HELO example') - self.assertEqual(self.channel.socket.last, - '250 {}\r\n'.format(name).encode('ascii')) - - def test_HELO_EHLO_duplicate(self): - self.write_line(b'HELO example') - self.write_line(b'EHLO example') - self.assertEqual(self.channel.socket.last, - b'503 Duplicate HELO/EHLO\r\n') - - def test_HELP(self): - self.write_line(b'HELP') - self.assertEqual(self.channel.socket.last, - b'250 Supported commands: EHLO HELO MAIL RCPT ' + \ - b'DATA RSET NOOP QUIT VRFY\r\n') - - def test_HELP_command(self): - self.write_line(b'HELP MAIL') - self.assertEqual(self.channel.socket.last, - b'250 Syntax: MAIL FROM:
\r\n') - - def test_HELP_command_unknown(self): - self.write_line(b'HELP SPAM') - self.assertEqual(self.channel.socket.last, - b'501 Supported commands: EHLO HELO MAIL RCPT ' + \ - b'DATA RSET NOOP QUIT VRFY\r\n') - - def test_HELO_bad_syntax(self): - self.write_line(b'HELO') - self.assertEqual(self.channel.socket.last, - b'501 Syntax: HELO hostname\r\n') - - def test_HELO_duplicate(self): - self.write_line(b'HELO example') - self.write_line(b'HELO example') - self.assertEqual(self.channel.socket.last, - b'503 Duplicate HELO/EHLO\r\n') - - def test_HELO_parameter_rejected_when_extensions_not_enabled(self): - self.extended_smtp = False - self.write_line(b'HELO example') - self.write_line(b'MAIL from: SIZE=1234') - self.assertEqual(self.channel.socket.last, - b'501 Syntax: MAIL FROM:
\r\n') - - def test_MAIL_allows_space_after_colon(self): - self.write_line(b'HELO example') - self.write_line(b'MAIL from: ') - self.assertEqual(self.channel.socket.last, - b'250 OK\r\n') - - def test_extended_MAIL_allows_space_after_colon(self): - self.write_line(b'EHLO example') - self.write_line(b'MAIL from: size=20') - self.assertEqual(self.channel.socket.last, - b'250 OK\r\n') - - def test_NOOP(self): - self.write_line(b'NOOP') - self.assertEqual(self.channel.socket.last, b'250 OK\r\n') - - def test_HELO_NOOP(self): - self.write_line(b'HELO example') - self.write_line(b'NOOP') - self.assertEqual(self.channel.socket.last, b'250 OK\r\n') - - def test_NOOP_bad_syntax(self): - self.write_line(b'NOOP hi') - self.assertEqual(self.channel.socket.last, - b'501 Syntax: NOOP\r\n') - - def test_QUIT(self): - self.write_line(b'QUIT') - self.assertEqual(self.channel.socket.last, b'221 Bye\r\n') - - def test_HELO_QUIT(self): - self.write_line(b'HELO example') - self.write_line(b'QUIT') - self.assertEqual(self.channel.socket.last, b'221 Bye\r\n') - - def test_QUIT_arg_ignored(self): - self.write_line(b'QUIT bye bye') - self.assertEqual(self.channel.socket.last, b'221 Bye\r\n') - - def test_bad_state(self): - self.channel.smtp_state = 'BAD STATE' - self.write_line(b'HELO example') - self.assertEqual(self.channel.socket.last, - b'451 Internal confusion\r\n') - - def test_command_too_long(self): - self.write_line(b'HELO example') - self.write_line(b'MAIL from: ' + - b'a' * self.channel.command_size_limit + - b'@example') - self.assertEqual(self.channel.socket.last, - b'500 Error: line too long\r\n') - - def test_MAIL_command_limit_extended_with_SIZE(self): - self.write_line(b'EHLO example') - fill_len = self.channel.command_size_limit - len('MAIL from:<@example>') - self.write_line(b'MAIL from:<' + - b'a' * fill_len + - b'@example> SIZE=1234') - self.assertEqual(self.channel.socket.last, b'250 OK\r\n') - - self.write_line(b'MAIL from:<' + - b'a' * (fill_len + 26) + - b'@example> SIZE=1234') - self.assertEqual(self.channel.socket.last, - b'500 Error: line too long\r\n') - - def test_MAIL_command_rejects_SMTPUTF8_by_default(self): - self.write_line(b'EHLO example') - self.write_line( - b'MAIL from: BODY=8BITMIME SMTPUTF8') - self.assertEqual(self.channel.socket.last[0:1], b'5') - - def test_data_longer_than_default_data_size_limit(self): - # Hack the default so we don't have to generate so much data. - self.channel.data_size_limit = 1048 - self.write_line(b'HELO example') - self.write_line(b'MAIL From:eggs@example') - self.write_line(b'RCPT To:spam@example') - self.write_line(b'DATA') - self.write_line(b'A' * self.channel.data_size_limit + - b'A\r\n.') - self.assertEqual(self.channel.socket.last, - b'552 Error: Too much mail data\r\n') - - def test_MAIL_size_parameter(self): - self.write_line(b'EHLO example') - self.write_line(b'MAIL FROM: SIZE=512') - self.assertEqual(self.channel.socket.last, - b'250 OK\r\n') - - def test_MAIL_invalid_size_parameter(self): - self.write_line(b'EHLO example') - self.write_line(b'MAIL FROM: SIZE=invalid') - self.assertEqual(self.channel.socket.last, - b'501 Syntax: MAIL FROM:
[SP ]\r\n') - - def test_MAIL_RCPT_unknown_parameters(self): - self.write_line(b'EHLO example') - self.write_line(b'MAIL FROM: ham=green') - self.assertEqual(self.channel.socket.last, - b'555 MAIL FROM parameters not recognized or not implemented\r\n') - - self.write_line(b'MAIL FROM:') - self.write_line(b'RCPT TO: ham=green') - self.assertEqual(self.channel.socket.last, - b'555 RCPT TO parameters not recognized or not implemented\r\n') - - def test_MAIL_size_parameter_larger_than_default_data_size_limit(self): - self.channel.data_size_limit = 1048 - self.write_line(b'EHLO example') - self.write_line(b'MAIL FROM: SIZE=2096') - self.assertEqual(self.channel.socket.last, - b'552 Error: message size exceeds fixed maximum message size\r\n') - - def test_need_MAIL(self): - self.write_line(b'HELO example') - self.write_line(b'RCPT to:spam@example') - self.assertEqual(self.channel.socket.last, - b'503 Error: need MAIL command\r\n') - - def test_MAIL_syntax_HELO(self): - self.write_line(b'HELO example') - self.write_line(b'MAIL from eggs@example') - self.assertEqual(self.channel.socket.last, - b'501 Syntax: MAIL FROM:
\r\n') - - def test_MAIL_syntax_EHLO(self): - self.write_line(b'EHLO example') - self.write_line(b'MAIL from eggs@example') - self.assertEqual(self.channel.socket.last, - b'501 Syntax: MAIL FROM:
[SP ]\r\n') - - def test_MAIL_missing_address(self): - self.write_line(b'HELO example') - self.write_line(b'MAIL from:') - self.assertEqual(self.channel.socket.last, - b'501 Syntax: MAIL FROM:
\r\n') - - def test_MAIL_chevrons(self): - self.write_line(b'HELO example') - self.write_line(b'MAIL from:') - self.assertEqual(self.channel.socket.last, b'250 OK\r\n') - - def test_MAIL_empty_chevrons(self): - self.write_line(b'EHLO example') - self.write_line(b'MAIL from:<>') - self.assertEqual(self.channel.socket.last, b'250 OK\r\n') - - def test_MAIL_quoted_localpart(self): - self.write_line(b'EHLO example') - self.write_line(b'MAIL from: <"Fred Blogs"@example.com>') - self.assertEqual(self.channel.socket.last, b'250 OK\r\n') - self.assertEqual(self.channel.mailfrom, '"Fred Blogs"@example.com') - - def test_MAIL_quoted_localpart_no_angles(self): - self.write_line(b'EHLO example') - self.write_line(b'MAIL from: "Fred Blogs"@example.com') - self.assertEqual(self.channel.socket.last, b'250 OK\r\n') - self.assertEqual(self.channel.mailfrom, '"Fred Blogs"@example.com') - - def test_MAIL_quoted_localpart_with_size(self): - self.write_line(b'EHLO example') - self.write_line(b'MAIL from: <"Fred Blogs"@example.com> SIZE=1000') - self.assertEqual(self.channel.socket.last, b'250 OK\r\n') - self.assertEqual(self.channel.mailfrom, '"Fred Blogs"@example.com') - - def test_MAIL_quoted_localpart_with_size_no_angles(self): - self.write_line(b'EHLO example') - self.write_line(b'MAIL from: "Fred Blogs"@example.com SIZE=1000') - self.assertEqual(self.channel.socket.last, b'250 OK\r\n') - self.assertEqual(self.channel.mailfrom, '"Fred Blogs"@example.com') - - def test_nested_MAIL(self): - self.write_line(b'HELO example') - self.write_line(b'MAIL from:eggs@example') - self.write_line(b'MAIL from:spam@example') - self.assertEqual(self.channel.socket.last, - b'503 Error: nested MAIL command\r\n') - - def test_VRFY(self): - self.write_line(b'VRFY eggs@example') - self.assertEqual(self.channel.socket.last, - b'252 Cannot VRFY user, but will accept message and attempt ' + \ - b'delivery\r\n') - - def test_VRFY_syntax(self): - self.write_line(b'VRFY') - self.assertEqual(self.channel.socket.last, - b'501 Syntax: VRFY
\r\n') - - def test_EXPN_not_implemented(self): - self.write_line(b'EXPN') - self.assertEqual(self.channel.socket.last, - b'502 EXPN not implemented\r\n') - - def test_no_HELO_MAIL(self): - self.write_line(b'MAIL from:') - self.assertEqual(self.channel.socket.last, - b'503 Error: send HELO first\r\n') - - def test_need_RCPT(self): - self.write_line(b'HELO example') - self.write_line(b'MAIL From:eggs@example') - self.write_line(b'DATA') - self.assertEqual(self.channel.socket.last, - b'503 Error: need RCPT command\r\n') - - def test_RCPT_syntax_HELO(self): - self.write_line(b'HELO example') - self.write_line(b'MAIL From: eggs@example') - self.write_line(b'RCPT to eggs@example') - self.assertEqual(self.channel.socket.last, - b'501 Syntax: RCPT TO:
\r\n') - - def test_RCPT_syntax_EHLO(self): - self.write_line(b'EHLO example') - self.write_line(b'MAIL From: eggs@example') - self.write_line(b'RCPT to eggs@example') - self.assertEqual(self.channel.socket.last, - b'501 Syntax: RCPT TO:
[SP ]\r\n') - - def test_RCPT_lowercase_to_OK(self): - self.write_line(b'HELO example') - self.write_line(b'MAIL From: eggs@example') - self.write_line(b'RCPT to: ') - self.assertEqual(self.channel.socket.last, b'250 OK\r\n') - - def test_no_HELO_RCPT(self): - self.write_line(b'RCPT to eggs@example') - self.assertEqual(self.channel.socket.last, - b'503 Error: send HELO first\r\n') - - def test_data_dialog(self): - self.write_line(b'HELO example') - self.write_line(b'MAIL From:eggs@example') - self.assertEqual(self.channel.socket.last, b'250 OK\r\n') - self.write_line(b'RCPT To:spam@example') - self.assertEqual(self.channel.socket.last, b'250 OK\r\n') - - self.write_line(b'DATA') - self.assertEqual(self.channel.socket.last, - b'354 End data with .\r\n') - self.write_line(b'data\r\nmore\r\n.') - self.assertEqual(self.channel.socket.last, b'250 OK\r\n') - self.assertEqual(self.server.messages, - [(('peer-address', 'peer-port'), - 'eggs@example', - ['spam@example'], - 'data\nmore')]) - - def test_DATA_syntax(self): - self.write_line(b'HELO example') - self.write_line(b'MAIL From:eggs@example') - self.write_line(b'RCPT To:spam@example') - self.write_line(b'DATA spam') - self.assertEqual(self.channel.socket.last, b'501 Syntax: DATA\r\n') - - def test_no_HELO_DATA(self): - self.write_line(b'DATA spam') - self.assertEqual(self.channel.socket.last, - b'503 Error: send HELO first\r\n') - - def test_data_transparency_section_4_5_2(self): - self.write_line(b'HELO example') - self.write_line(b'MAIL From:eggs@example') - self.write_line(b'RCPT To:spam@example') - self.write_line(b'DATA') - self.write_line(b'..\r\n.\r\n') - self.assertEqual(self.channel.received_data, '.') - - def test_multiple_RCPT(self): - self.write_line(b'HELO example') - self.write_line(b'MAIL From:eggs@example') - self.write_line(b'RCPT To:spam@example') - self.write_line(b'RCPT To:ham@example') - self.write_line(b'DATA') - self.write_line(b'data\r\n.') - self.assertEqual(self.server.messages, - [(('peer-address', 'peer-port'), - 'eggs@example', - ['spam@example','ham@example'], - 'data')]) - - def test_manual_status(self): - # checks that the Channel is able to return a custom status message - self.write_line(b'HELO example') - self.write_line(b'MAIL From:eggs@example') - self.write_line(b'RCPT To:spam@example') - self.write_line(b'DATA') - self.write_line(b'return status\r\n.') - self.assertEqual(self.channel.socket.last, b'250 Okish\r\n') - - def test_RSET(self): - self.write_line(b'HELO example') - self.write_line(b'MAIL From:eggs@example') - self.write_line(b'RCPT To:spam@example') - self.write_line(b'RSET') - self.assertEqual(self.channel.socket.last, b'250 OK\r\n') - self.write_line(b'MAIL From:foo@example') - self.write_line(b'RCPT To:eggs@example') - self.write_line(b'DATA') - self.write_line(b'data\r\n.') - self.assertEqual(self.server.messages, - [(('peer-address', 'peer-port'), - 'foo@example', - ['eggs@example'], - 'data')]) - - def test_HELO_RSET(self): - self.write_line(b'HELO example') - self.write_line(b'RSET') - self.assertEqual(self.channel.socket.last, b'250 OK\r\n') - - def test_RSET_syntax(self): - self.write_line(b'RSET hi') - self.assertEqual(self.channel.socket.last, b'501 Syntax: RSET\r\n') - - def test_unknown_command(self): - self.write_line(b'UNKNOWN_CMD') - self.assertEqual(self.channel.socket.last, - b'500 Error: command "UNKNOWN_CMD" not ' + \ - b'recognized\r\n') - - def test_attribute_deprecations(self): - with warnings_helper.check_warnings(('', DeprecationWarning)): - spam = self.channel._SMTPChannel__server - with warnings_helper.check_warnings(('', DeprecationWarning)): - self.channel._SMTPChannel__server = 'spam' - with warnings_helper.check_warnings(('', DeprecationWarning)): - spam = self.channel._SMTPChannel__line - with warnings_helper.check_warnings(('', DeprecationWarning)): - self.channel._SMTPChannel__line = 'spam' - with warnings_helper.check_warnings(('', DeprecationWarning)): - spam = self.channel._SMTPChannel__state - with warnings_helper.check_warnings(('', DeprecationWarning)): - self.channel._SMTPChannel__state = 'spam' - with warnings_helper.check_warnings(('', DeprecationWarning)): - spam = self.channel._SMTPChannel__greeting - with warnings_helper.check_warnings(('', DeprecationWarning)): - self.channel._SMTPChannel__greeting = 'spam' - with warnings_helper.check_warnings(('', DeprecationWarning)): - spam = self.channel._SMTPChannel__mailfrom - with warnings_helper.check_warnings(('', DeprecationWarning)): - self.channel._SMTPChannel__mailfrom = 'spam' - with warnings_helper.check_warnings(('', DeprecationWarning)): - spam = self.channel._SMTPChannel__rcpttos - with warnings_helper.check_warnings(('', DeprecationWarning)): - self.channel._SMTPChannel__rcpttos = 'spam' - with warnings_helper.check_warnings(('', DeprecationWarning)): - spam = self.channel._SMTPChannel__data - with warnings_helper.check_warnings(('', DeprecationWarning)): - self.channel._SMTPChannel__data = 'spam' - with warnings_helper.check_warnings(('', DeprecationWarning)): - spam = self.channel._SMTPChannel__fqdn - with warnings_helper.check_warnings(('', DeprecationWarning)): - self.channel._SMTPChannel__fqdn = 'spam' - with warnings_helper.check_warnings(('', DeprecationWarning)): - spam = self.channel._SMTPChannel__peer - with warnings_helper.check_warnings(('', DeprecationWarning)): - self.channel._SMTPChannel__peer = 'spam' - with warnings_helper.check_warnings(('', DeprecationWarning)): - spam = self.channel._SMTPChannel__conn - with warnings_helper.check_warnings(('', DeprecationWarning)): - self.channel._SMTPChannel__conn = 'spam' - with warnings_helper.check_warnings(('', DeprecationWarning)): - spam = self.channel._SMTPChannel__addr - with warnings_helper.check_warnings(('', DeprecationWarning)): - self.channel._SMTPChannel__addr = 'spam' - -@unittest.skipUnless(socket_helper.IPV6_ENABLED, "IPv6 not enabled") -class SMTPDChannelIPv6Test(SMTPDChannelTest): - def setUp(self): - smtpd.socket = asyncore.socket = mock_socket - self.old_debugstream = smtpd.DEBUGSTREAM - self.debug = smtpd.DEBUGSTREAM = io.StringIO() - self.server = DummyServer((socket_helper.HOSTv6, 0), ('b', 0), - decode_data=True) - conn, addr = self.server.accept() - self.channel = smtpd.SMTPChannel(self.server, conn, addr, - decode_data=True) - -class SMTPDChannelWithDataSizeLimitTest(unittest.TestCase): - - def setUp(self): - smtpd.socket = asyncore.socket = mock_socket - self.old_debugstream = smtpd.DEBUGSTREAM - self.debug = smtpd.DEBUGSTREAM = io.StringIO() - self.server = DummyServer((socket_helper.HOST, 0), ('b', 0), - decode_data=True) - conn, addr = self.server.accept() - # Set DATA size limit to 32 bytes for easy testing - self.channel = smtpd.SMTPChannel(self.server, conn, addr, 32, - decode_data=True) - - def tearDown(self): - asyncore.close_all() - asyncore.socket = smtpd.socket = socket - smtpd.DEBUGSTREAM = self.old_debugstream - - def write_line(self, line): - self.channel.socket.queue_recv(line) - self.channel.handle_read() - - def test_data_limit_dialog(self): - self.write_line(b'HELO example') - self.write_line(b'MAIL From:eggs@example') - self.assertEqual(self.channel.socket.last, b'250 OK\r\n') - self.write_line(b'RCPT To:spam@example') - self.assertEqual(self.channel.socket.last, b'250 OK\r\n') - - self.write_line(b'DATA') - self.assertEqual(self.channel.socket.last, - b'354 End data with .\r\n') - self.write_line(b'data\r\nmore\r\n.') - self.assertEqual(self.channel.socket.last, b'250 OK\r\n') - self.assertEqual(self.server.messages, - [(('peer-address', 'peer-port'), - 'eggs@example', - ['spam@example'], - 'data\nmore')]) - - def test_data_limit_dialog_too_much_data(self): - self.write_line(b'HELO example') - self.write_line(b'MAIL From:eggs@example') - self.assertEqual(self.channel.socket.last, b'250 OK\r\n') - self.write_line(b'RCPT To:spam@example') - self.assertEqual(self.channel.socket.last, b'250 OK\r\n') - - self.write_line(b'DATA') - self.assertEqual(self.channel.socket.last, - b'354 End data with .\r\n') - self.write_line(b'This message is longer than 32 bytes\r\n.') - self.assertEqual(self.channel.socket.last, - b'552 Error: Too much mail data\r\n') - - -class SMTPDChannelWithDecodeDataFalse(unittest.TestCase): - - def setUp(self): - smtpd.socket = asyncore.socket = mock_socket - self.old_debugstream = smtpd.DEBUGSTREAM - self.debug = smtpd.DEBUGSTREAM = io.StringIO() - self.server = DummyServer((socket_helper.HOST, 0), ('b', 0)) - conn, addr = self.server.accept() - self.channel = smtpd.SMTPChannel(self.server, conn, addr) - - def tearDown(self): - asyncore.close_all() - asyncore.socket = smtpd.socket = socket - smtpd.DEBUGSTREAM = self.old_debugstream - - def write_line(self, line): - self.channel.socket.queue_recv(line) - self.channel.handle_read() - - def test_ascii_data(self): - self.write_line(b'HELO example') - self.write_line(b'MAIL From:eggs@example') - self.write_line(b'RCPT To:spam@example') - self.write_line(b'DATA') - self.write_line(b'plain ascii text') - self.write_line(b'.') - self.assertEqual(self.channel.received_data, b'plain ascii text') - - def test_utf8_data(self): - self.write_line(b'HELO example') - self.write_line(b'MAIL From:eggs@example') - self.write_line(b'RCPT To:spam@example') - self.write_line(b'DATA') - self.write_line(b'utf8 enriched text: \xc5\xbc\xc5\xba\xc4\x87') - self.write_line(b'and some plain ascii') - self.write_line(b'.') - self.assertEqual( - self.channel.received_data, - b'utf8 enriched text: \xc5\xbc\xc5\xba\xc4\x87\n' - b'and some plain ascii') - - -class SMTPDChannelWithDecodeDataTrue(unittest.TestCase): - - def setUp(self): - smtpd.socket = asyncore.socket = mock_socket - self.old_debugstream = smtpd.DEBUGSTREAM - self.debug = smtpd.DEBUGSTREAM = io.StringIO() - self.server = DummyServer((socket_helper.HOST, 0), ('b', 0), - decode_data=True) - conn, addr = self.server.accept() - # Set decode_data to True - self.channel = smtpd.SMTPChannel(self.server, conn, addr, - decode_data=True) - - def tearDown(self): - asyncore.close_all() - asyncore.socket = smtpd.socket = socket - smtpd.DEBUGSTREAM = self.old_debugstream - - def write_line(self, line): - self.channel.socket.queue_recv(line) - self.channel.handle_read() - - def test_ascii_data(self): - self.write_line(b'HELO example') - self.write_line(b'MAIL From:eggs@example') - self.write_line(b'RCPT To:spam@example') - self.write_line(b'DATA') - self.write_line(b'plain ascii text') - self.write_line(b'.') - self.assertEqual(self.channel.received_data, 'plain ascii text') - - def test_utf8_data(self): - self.write_line(b'HELO example') - self.write_line(b'MAIL From:eggs@example') - self.write_line(b'RCPT To:spam@example') - self.write_line(b'DATA') - self.write_line(b'utf8 enriched text: \xc5\xbc\xc5\xba\xc4\x87') - self.write_line(b'and some plain ascii') - self.write_line(b'.') - self.assertEqual( - self.channel.received_data, - 'utf8 enriched text: żźć\nand some plain ascii') - - -class SMTPDChannelTestWithEnableSMTPUTF8True(unittest.TestCase): - def setUp(self): - smtpd.socket = asyncore.socket = mock_socket - self.old_debugstream = smtpd.DEBUGSTREAM - self.debug = smtpd.DEBUGSTREAM = io.StringIO() - self.server = DummyServer((socket_helper.HOST, 0), ('b', 0), - enable_SMTPUTF8=True) - conn, addr = self.server.accept() - self.channel = smtpd.SMTPChannel(self.server, conn, addr, - enable_SMTPUTF8=True) - - def tearDown(self): - asyncore.close_all() - asyncore.socket = smtpd.socket = socket - smtpd.DEBUGSTREAM = self.old_debugstream - - def write_line(self, line): - self.channel.socket.queue_recv(line) - self.channel.handle_read() - - def test_MAIL_command_accepts_SMTPUTF8_when_announced(self): - self.write_line(b'EHLO example') - self.write_line( - 'MAIL from: BODY=8BITMIME SMTPUTF8'.encode( - 'utf-8') - ) - self.assertEqual(self.channel.socket.last, b'250 OK\r\n') - - def test_process_smtputf8_message(self): - self.write_line(b'EHLO example') - for mail_parameters in [b'', b'BODY=8BITMIME SMTPUTF8']: - self.write_line(b'MAIL from: ' + mail_parameters) - self.assertEqual(self.channel.socket.last[0:3], b'250') - self.write_line(b'rcpt to:') - self.assertEqual(self.channel.socket.last[0:3], b'250') - self.write_line(b'data') - self.assertEqual(self.channel.socket.last[0:3], b'354') - self.write_line(b'c\r\n.') - if mail_parameters == b'': - self.assertEqual(self.channel.socket.last, b'250 OK\r\n') - else: - self.assertEqual(self.channel.socket.last, - b'250 SMTPUTF8 message okish\r\n') - - def test_utf8_data(self): - self.write_line(b'EHLO example') - self.write_line( - 'MAIL From: naïve@examplé BODY=8BITMIME SMTPUTF8'.encode('utf-8')) - self.assertEqual(self.channel.socket.last[0:3], b'250') - self.write_line('RCPT To:späm@examplé'.encode('utf-8')) - self.assertEqual(self.channel.socket.last[0:3], b'250') - self.write_line(b'DATA') - self.assertEqual(self.channel.socket.last[0:3], b'354') - self.write_line(b'utf8 enriched text: \xc5\xbc\xc5\xba\xc4\x87') - self.write_line(b'.') - self.assertEqual( - self.channel.received_data, - b'utf8 enriched text: \xc5\xbc\xc5\xba\xc4\x87') - - def test_MAIL_command_limit_extended_with_SIZE_and_SMTPUTF8(self): - self.write_line(b'ehlo example') - fill_len = (512 + 26 + 10) - len('mail from:<@example>') - self.write_line(b'MAIL from:<' + - b'a' * (fill_len + 1) + - b'@example>') - self.assertEqual(self.channel.socket.last, - b'500 Error: line too long\r\n') - self.write_line(b'MAIL from:<' + - b'a' * fill_len + - b'@example>') - self.assertEqual(self.channel.socket.last, b'250 OK\r\n') - - def test_multiple_emails_with_extended_command_length(self): - self.write_line(b'ehlo example') - fill_len = (512 + 26 + 10) - len('mail from:<@example>') - for char in [b'a', b'b', b'c']: - self.write_line(b'MAIL from:<' + char * fill_len + b'a@example>') - self.assertEqual(self.channel.socket.last[0:3], b'500') - self.write_line(b'MAIL from:<' + char * fill_len + b'@example>') - self.assertEqual(self.channel.socket.last[0:3], b'250') - self.write_line(b'rcpt to:') - self.assertEqual(self.channel.socket.last[0:3], b'250') - self.write_line(b'data') - self.assertEqual(self.channel.socket.last[0:3], b'354') - self.write_line(b'test\r\n.') - self.assertEqual(self.channel.socket.last[0:3], b'250') - - -class MiscTestCase(unittest.TestCase): - def test__all__(self): - not_exported = { - "program", "Devnull", "DEBUGSTREAM", "NEWLINE", "COMMASPACE", - "DATA_SIZE_DEFAULT", "usage", "Options", "parseargs", - } - support.check__all__(self, smtpd, not_exported=not_exported) - - -if __name__ == "__main__": - unittest.main() From e7f04612f61242bd65bfb7f01a5f13d36fa54af1 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Thu, 3 Apr 2025 10:10:26 -0700 Subject: [PATCH 168/295] remove chunk.py --- Lib/chunk.py | 173 --------------------------------------------------- 1 file changed, 173 deletions(-) delete mode 100644 Lib/chunk.py diff --git a/Lib/chunk.py b/Lib/chunk.py deleted file mode 100644 index 618781efd1..0000000000 --- a/Lib/chunk.py +++ /dev/null @@ -1,173 +0,0 @@ -"""Simple class to read IFF chunks. - -An IFF chunk (used in formats such as AIFF, TIFF, RMFF (RealMedia File -Format)) has the following structure: - -+----------------+ -| ID (4 bytes) | -+----------------+ -| size (4 bytes) | -+----------------+ -| data | -| ... | -+----------------+ - -The ID is a 4-byte string which identifies the type of chunk. - -The size field (a 32-bit value, encoded using big-endian byte order) -gives the size of the whole chunk, including the 8-byte header. - -Usually an IFF-type file consists of one or more chunks. The proposed -usage of the Chunk class defined here is to instantiate an instance at -the start of each chunk and read from the instance until it reaches -the end, after which a new instance can be instantiated. At the end -of the file, creating a new instance will fail with an EOFError -exception. - -Usage: -while True: - try: - chunk = Chunk(file) - except EOFError: - break - chunktype = chunk.getname() - while True: - data = chunk.read(nbytes) - if not data: - pass - # do something with data - -The interface is file-like. The implemented methods are: -read, close, seek, tell, isatty. -Extra methods are: skip() (called by close, skips to the end of the chunk), -getname() (returns the name (ID) of the chunk) - -The __init__ method has one required argument, a file-like object -(including a chunk instance), and one optional argument, a flag which -specifies whether or not chunks are aligned on 2-byte boundaries. The -default is 1, i.e. aligned. -""" - -import warnings - -warnings._deprecated(__name__, remove=(3, 13)) - -class Chunk: - def __init__(self, file, align=True, bigendian=True, inclheader=False): - import struct - self.closed = False - self.align = align # whether to align to word (2-byte) boundaries - if bigendian: - strflag = '>' - else: - strflag = '<' - self.file = file - self.chunkname = file.read(4) - if len(self.chunkname) < 4: - raise EOFError - try: - self.chunksize = struct.unpack_from(strflag+'L', file.read(4))[0] - except struct.error: - raise EOFError from None - if inclheader: - self.chunksize = self.chunksize - 8 # subtract header - self.size_read = 0 - try: - self.offset = self.file.tell() - except (AttributeError, OSError): - self.seekable = False - else: - self.seekable = True - - def getname(self): - """Return the name (ID) of the current chunk.""" - return self.chunkname - - def getsize(self): - """Return the size of the current chunk.""" - return self.chunksize - - def close(self): - if not self.closed: - try: - self.skip() - finally: - self.closed = True - - def isatty(self): - if self.closed: - raise ValueError("I/O operation on closed file") - return False - - def seek(self, pos, whence=0): - """Seek to specified position into the chunk. - Default position is 0 (start of chunk). - If the file is not seekable, this will result in an error. - """ - - if self.closed: - raise ValueError("I/O operation on closed file") - if not self.seekable: - raise OSError("cannot seek") - if whence == 1: - pos = pos + self.size_read - elif whence == 2: - pos = pos + self.chunksize - if pos < 0 or pos > self.chunksize: - raise RuntimeError - self.file.seek(self.offset + pos, 0) - self.size_read = pos - - def tell(self): - if self.closed: - raise ValueError("I/O operation on closed file") - return self.size_read - - def read(self, size=-1): - """Read at most size bytes from the chunk. - If size is omitted or negative, read until the end - of the chunk. - """ - - if self.closed: - raise ValueError("I/O operation on closed file") - if self.size_read >= self.chunksize: - return b'' - if size < 0: - size = self.chunksize - self.size_read - if size > self.chunksize - self.size_read: - size = self.chunksize - self.size_read - data = self.file.read(size) - self.size_read = self.size_read + len(data) - if self.size_read == self.chunksize and \ - self.align and \ - (self.chunksize & 1): - dummy = self.file.read(1) - self.size_read = self.size_read + len(dummy) - return data - - def skip(self): - """Skip the rest of the chunk. - If you are not interested in the contents of the chunk, - this method should be called so that the file points to - the start of the next chunk. - """ - - if self.closed: - raise ValueError("I/O operation on closed file") - if self.seekable: - try: - n = self.chunksize - self.size_read - # maybe fix alignment - if self.align and (self.chunksize & 1): - n = n + 1 - self.file.seek(n, 1) - self.size_read = self.size_read + n - return - except OSError: - pass - while self.size_read < self.chunksize: - n = min(8192, self.chunksize - self.size_read) - dummy = self.read(n) - if not dummy: - raise EOFError From 2c94b809aeddc047e93cb0bd0f47720f36f01d67 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Fri, 4 Apr 2025 16:02:29 +0900 Subject: [PATCH 169/295] move cspell to last step --- .github/workflows/ci.yaml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 586e00be26..05516c9270 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -313,13 +313,6 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: install extra dictionaries - run: npm install @cspell/dict-en_us @cspell/dict-cpp @cspell/dict-python @cspell/dict-rust @cspell/dict-win32 @cspell/dict-shell - - name: spell checker - uses: streetsidesoftware/cspell-action@v6 - with: - files: '**/*.rs' - incremental_files_only: true - uses: dtolnay/rust-toolchain@stable with: components: rustfmt, clippy @@ -339,6 +332,14 @@ jobs: - name: check wasm code with prettier # prettier doesn't handle ignore files very well: https://github.com/prettier/prettier/issues/8506 run: cd wasm && git ls-files -z | xargs -0 prettier --check -u + # Keep cspell check as the last step. This is optional test. + - name: install extra dictionaries + run: npm install @cspell/dict-en_us @cspell/dict-cpp @cspell/dict-python @cspell/dict-rust @cspell/dict-win32 @cspell/dict-shell + - name: spell checker + uses: streetsidesoftware/cspell-action@v6 + with: + files: '**/*.rs' + incremental_files_only: true miri: if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }} From 7ac61f384013b3e94828053122cb869b74fed079 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Fri, 4 Apr 2025 15:38:36 +0900 Subject: [PATCH 170/295] fix cspell warnings --- .cspell.dict/cpython.txt | 5 ++ .cspell.dict/python-more.txt | 107 ++++++++++++++++++++++++++++++++++- .cspell.dict/rust-more.txt | 25 +++++++- .cspell.json | 17 ++++++ common/src/boxvec.rs | 1 + common/src/hash.rs | 4 +- common/src/rc.rs | 2 +- common/src/static_cell.rs | 4 +- derive/src/lib.rs | 10 ++-- src/lib.rs | 6 +- stdlib/src/array.rs | 6 +- stdlib/src/binascii.rs | 4 +- stdlib/src/csv.rs | 8 +-- stdlib/src/fcntl.rs | 2 + stdlib/src/grp.rs | 1 + stdlib/src/locale.rs | 2 + stdlib/src/math.rs | 26 ++++----- stdlib/src/random.rs | 4 +- stdlib/src/syslog.rs | 2 +- stdlib/src/tkinter.rs | 4 +- stdlib/src/zlib.rs | 2 +- vm/src/anystr.rs | 2 +- vm/src/builtins/bytearray.rs | 4 +- vm/src/builtins/bytes.rs | 6 +- vm/src/builtins/complex.rs | 2 +- vm/src/builtins/dict.rs | 8 +-- vm/src/builtins/function.rs | 16 +++--- vm/src/builtins/int.rs | 15 ++--- vm/src/builtins/memory.rs | 8 +-- vm/src/builtins/str.rs | 4 +- vm/src/builtins/super.rs | 4 +- vm/src/bytesinner.rs | 52 ++++++++++------- vm/src/cformat.rs | 2 + vm/src/exceptions.rs | 10 ++-- vm/src/frame.rs | 20 ++++--- vm/src/function/argument.rs | 2 +- vm/src/function/fspath.rs | 2 +- vm/src/function/protocol.rs | 14 ++--- vm/src/import.rs | 8 +-- vm/src/object/core.rs | 3 +- vm/src/ospath.rs | 10 ++-- vm/src/protocol/buffer.rs | 22 +++---- vm/src/protocol/iter.rs | 6 +- vm/src/protocol/object.rs | 22 +++---- vm/src/stdlib/builtins.rs | 14 +++-- vm/src/stdlib/io.rs | 96 +++++++++++++++---------------- vm/src/stdlib/itertools.rs | 14 ++--- vm/src/stdlib/nt.rs | 10 ++-- vm/src/stdlib/operator.rs | 6 +- vm/src/stdlib/os.rs | 2 +- vm/src/stdlib/sre.rs | 34 +++++------ vm/src/stdlib/time.rs | 5 +- vm/src/vm/thread.rs | 8 +-- vm/src/vm/vm_new.rs | 4 +- vm/src/vm/vm_ops.rs | 12 ++-- vm/sre_engine/src/engine.rs | 32 +++++------ wtf8/src/lib.rs | 6 +- 57 files changed, 450 insertions(+), 277 deletions(-) diff --git a/.cspell.dict/cpython.txt b/.cspell.dict/cpython.txt index 0ac387634d..f7e282e4bc 100644 --- a/.cspell.dict/cpython.txt +++ b/.cspell.dict/cpython.txt @@ -10,14 +10,19 @@ cellarg cellvar cellvars cmpop +weakreflist +XXPRIME dictoffset elts +xstat excepthandler +fileutils finalbody freevar freevars fromlist heaptype +HIGHRES IMMUTABLETYPE kwonlyarg kwonlyargs diff --git a/.cspell.dict/python-more.txt b/.cspell.dict/python-more.txt index a482c880cc..526f5ba166 100644 --- a/.cspell.dict/python-more.txt +++ b/.cspell.dict/python-more.txt @@ -1,17 +1,34 @@ +abiflags abstractmethods +aenter +aexit aiter anext +appendleft +argcount arrayiterator arraytype asend +asyncgen athrow +backslashreplace basicsize +bdfl +bigcharset +breakpointhook cformat +chunksize classcell +closefd closesocket codepoint codepoints +codesize +contextvar cpython +cratio +dealloc +debugbuild decompressor defaultaction descr @@ -19,15 +36,29 @@ dictcomp dictitems dictkeys dictview +digestmod +dllhandle docstring docstrings dunder +endianness +endpos eventmask +excepthook +exceptiongroup +exitfuncs +extendleft +fastlocals fdel +fedcba fget fileencoding fillchar +fillvalue finallyhandler +firstiter +firstlineno +fnctl frombytes fromhex fromunicode @@ -35,45 +66,87 @@ fset fspath fstring fstrings +ftruncate genexpr getattro +getcodesize +getdefaultencoding +getfilesystemencodeerrors +getfilesystemencoding getformat +getframe getnewargs +getpip +getrandom +getrecursionlimit +getrefcount getweakrefcount getweakrefs +getwindowsversion +gmtoff +groupdict +groupindex +hamt hostnames +idfunc idiv impls +indexgroup infj instancecheck instanceof +irepeat isabstractmethod +isbytes +iscased +istext itemiterator itemsize iternext +keepends +keyfunc keyiterator kwarg kwargs +kwdefaults +kwonlyargcount +lastgroup linearization linearize listcomp +lvalue mappingproxy +maskpri +maxdigits +MAXGROUPS +MAXREPEAT maxsplit +maxunicode memoryview memoryviewiterator metaclass metaclasses metatype +mformat mro mros +multiarch +namereplace nanj +nbytes +ncallbacks ndigits ndim +nlocals nonbytes origname posixsubprocess +posonly +posonlyargcount +profilefunc +pycodecs +pycs pyexpat -pytraverse PYTHONDEBUG PYTHONHOME PYTHONINSPECT @@ -82,11 +155,19 @@ PYTHONPATH PYTHONPATH PYTHONVERBOSE PYTHONWARNINGS +pytraverse qualname +quotetabs radd rdiv rdivmod +readall +readbuffer reconstructor +releaselevel +reverseitemiterator +reverseiterator +reversekeyiterator reversevalueiterator rfloordiv rlshift @@ -95,22 +176,42 @@ rpow rrshift rsub rtruediv +rvalue scproxy setattro setcomp showwarnmsg -warnmsg +signum +slotnames stacklevel +stacksize +startpos subclasscheck subclasshook +suboffset +sumprod +surrogateescape +surrogatepass +sysconfigdata +sysvars +titlecased +unimportable unionable unraisablehook +unsliceable +urandom valueiterator vararg varargs varnames warningregistry +warnmsg +warnoptions warnopts weakproxy winver -xopts \ No newline at end of file +withdata +xmlcharrefreplace +xoptions +xopts +yieldfrom diff --git a/.cspell.dict/rust-more.txt b/.cspell.dict/rust-more.txt index f2177dd4c7..d75529789f 100644 --- a/.cspell.dict/rust-more.txt +++ b/.cspell.dict/rust-more.txt @@ -3,27 +3,42 @@ bidi biguint bindgen bitflags +bitor bstr byteorder +byteset +caseless chrono consts +cranelift cstring +datelike +deserializer flate2 fract +getres hasher +hexf +hexversion idents +illumos indexmap insta keccak lalrpop +lexopt libc +libloading libz longlong Manually maplit memmap +memmem metas modpow +msvc +muldiv nanos objclass peekable @@ -31,17 +46,25 @@ powc powf prepended punct +puruspe replacen rsplitn rustc rustfmt +rustyline seekfrom +siphash splitn subsec +thiserror +timelike timsort trai ulonglong unic unistd +unsync +wasmbind +widestring winapi -winsock \ No newline at end of file +winsock diff --git a/.cspell.json b/.cspell.json index 562b300ffa..99718a6515 100644 --- a/.cspell.json +++ b/.cspell.json @@ -47,16 +47,24 @@ // words - list of words to be always considered correct "words": [ // RustPython + "aiterable", + "alnum", "baseclass", + "boxvec", "Bytecode", "cfgs", "codegen", + "coro", "dedentations", "dedents", "deduped", "downcasted", "dumpable", + "emscripten", + "excs", + "finalizer", "GetSet", + "groupref", "internable", "makeunicodedata", "miri", @@ -100,6 +108,15 @@ "unraisable", "wasi", "zelf", + // unix + "CLOEXEC", + "codeset", + "endgrent", + "getrusage", + "nanosleep", + "WRLCK", + // win32 + "birthtime", ], // flagWords - list of words to be always considered incorrect "flagWords": [ diff --git a/common/src/boxvec.rs b/common/src/boxvec.rs index 1a1d57c169..4501835477 100644 --- a/common/src/boxvec.rs +++ b/common/src/boxvec.rs @@ -1,3 +1,4 @@ +// cspell:ignore //! An unresizable vector backed by a `Box<[T]>` #![allow(clippy::needless_lifetimes)] diff --git a/common/src/hash.rs b/common/src/hash.rs index 8fef70c8b9..1ae561650c 100644 --- a/common/src/hash.rs +++ b/common/src/hash.rs @@ -53,14 +53,14 @@ impl HashSecret { fix_sentinel(mod_int(self.hash_one(data) as _)) } - pub fn hash_iter<'a, T: 'a, I, F, E>(&self, iter: I, hashf: F) -> Result + pub fn hash_iter<'a, T: 'a, I, F, E>(&self, iter: I, hash_func: F) -> Result where I: IntoIterator, F: Fn(&'a T) -> Result, { let mut hasher = self.build_hasher(); for element in iter { - let item_hash = hashf(element)?; + let item_hash = hash_func(element)?; item_hash.hash(&mut hasher); } Ok(fix_sentinel(mod_int(hasher.finish() as PyHash))) diff --git a/common/src/rc.rs b/common/src/rc.rs index 81207e840c..40c7cf97a8 100644 --- a/common/src/rc.rs +++ b/common/src/rc.rs @@ -3,7 +3,7 @@ use std::rc::Rc; #[cfg(feature = "threading")] use std::sync::Arc; -// type aliases instead of newtypes because you can't do `fn method(self: PyRc)` with a +// type aliases instead of new-types because you can't do `fn method(self: PyRc)` with a // newtype; requires the arbitrary_self_types unstable feature #[cfg(feature = "threading")] diff --git a/common/src/static_cell.rs b/common/src/static_cell.rs index 7f16dad399..407b83ae0a 100644 --- a/common/src/static_cell.rs +++ b/common/src/static_cell.rs @@ -13,7 +13,7 @@ mod non_threading { impl StaticCell { #[doc(hidden)] - pub const fn _from_localkey(inner: &'static LocalKey>) -> Self { + pub const fn _from_local_key(inner: &'static LocalKey>) -> Self { Self { inner } } @@ -58,7 +58,7 @@ mod non_threading { ::std::thread_local! { $vis static $name: $crate::lock::OnceCell<&'static $t> = $crate::lock::OnceCell::new(); } - $crate::static_cell::StaticCell::_from_localkey(&$name) + $crate::static_cell::StaticCell::_from_local_key(&$name) };)+ }; } diff --git a/derive/src/lib.rs b/derive/src/lib.rs index a96c2aef6e..2a7b3d68fc 100644 --- a/derive/src/lib.rs +++ b/derive/src/lib.rs @@ -34,7 +34,7 @@ pub fn derive_from_args(input: TokenStream) -> TokenStream { /// - `IMMUTABLETYPE`: class attributes are immutable. /// - `with`: which trait implementations are to be included in the python class. /// ```rust, ignore -/// #[pyclass(module = "mymodule", name = "MyClass", base = "BaseClass")] +/// #[pyclass(module = "my_module", name = "MyClass", base = "BaseClass")] /// struct MyStruct { /// x: i32, /// } @@ -161,8 +161,8 @@ pub fn pyexception(attr: TokenStream, item: TokenStream) -> TokenStream { /// - `name`: the name of the python module, /// by default, it is the name of the module, but this can be configured. /// ```rust, ignore -/// // This will create a module named `mymodule` -/// #[pymodule(name = "mymodule")] +/// // This will create a module named `my_module` +/// #[pymodule(name = "my_module")] /// mod module { /// } /// ``` @@ -173,7 +173,7 @@ pub fn pyexception(attr: TokenStream, item: TokenStream) -> TokenStream { /// } /// /// #[pymodule(with(submodule))] -/// mod mymodule { +/// mod my_module { /// } /// ``` /// - `with`: declare the list of submodules that this module contains (see `sub` for example). @@ -190,7 +190,7 @@ pub fn pyexception(attr: TokenStream, item: TokenStream) -> TokenStream { /// #### Examples /// ```rust, ignore /// #[pymodule] -/// mod mymodule { +/// mod my_module { /// #[pyattr] /// const MY_CONSTANT: i32 = 42; /// #[pyattr] diff --git a/src/lib.rs b/src/lib.rs index 67a2a16eef..3fa5292e94 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,13 +9,13 @@ //! use rustpython_vm::{pymodule, py_freeze}; //! fn main() { //! rustpython::run(|vm| { -//! vm.add_native_module("mymod".to_owned(), Box::new(mymod::make_module)); -//! vm.add_frozen(py_freeze!(source = "def foo(): pass", module_name = "otherthing")); +//! vm.add_native_module("my_mod".to_owned(), Box::new(my_mod::make_module)); +//! vm.add_frozen(py_freeze!(source = "def foo(): pass", module_name = "other_thing")); //! }); //! } //! //! #[pymodule] -//! mod mymod { +//! mod my_mod { //! use rustpython_vm::builtins::PyStrRef; //TODO: use rustpython_vm::prelude::*; //! diff --git a/stdlib/src/array.rs b/stdlib/src/array.rs index fd83f0a5ad..db4394e44f 100644 --- a/stdlib/src/array.rs +++ b/stdlib/src/array.rs @@ -880,14 +880,14 @@ mod array { return Err(vm.new_value_error("negative count".to_owned())); } let n = vm.check_repeat_or_overflow_error(itemsize, n)?; - let nbytes = n * itemsize; + let n_bytes = n * itemsize; - let b = vm.call_method(&f, "read", (nbytes,))?; + let b = vm.call_method(&f, "read", (n_bytes,))?; let b = b .downcast::() .map_err(|_| vm.new_type_error("read() didn't return bytes".to_owned()))?; - let not_enough_bytes = b.len() != nbytes; + let not_enough_bytes = b.len() != n_bytes; self._from_bytes(b.as_bytes(), itemsize, vm)?; diff --git a/stdlib/src/binascii.rs b/stdlib/src/binascii.rs index f154a2251b..a8df1fb60b 100644 --- a/stdlib/src/binascii.rs +++ b/stdlib/src/binascii.rs @@ -1,4 +1,4 @@ -// spell-checker:ignore hexlify unhexlify uuencodes +// spell-checker:ignore hexlify unhexlify uuencodes CRCTAB pub(super) use decl::crc32; pub(crate) use decl::make_module; @@ -339,7 +339,7 @@ mod decl { || (buffer[idx + 1] >= b'a' && buffer[idx + 1] <= b'f') || (buffer[idx + 1] >= b'0' && buffer[idx + 1] <= b'9')) { - // hexval + // hex val if let (Some(ch1), Some(ch2)) = (unhex_nibble(buffer[idx]), unhex_nibble(buffer[idx + 1])) { diff --git a/stdlib/src/csv.rs b/stdlib/src/csv.rs index 214209ab9e..39c15fd952 100644 --- a/stdlib/src/csv.rs +++ b/stdlib/src/csv.rs @@ -981,14 +981,14 @@ mod _csv { String::from_utf8(input.to_vec()).unwrap() }; loop { - let (res, nread, nwritten, nends) = reader.read_record( + let (res, n_read, n_written, n_ends) = reader.read_record( &input.as_bytes()[input_offset..], &mut buffer[output_offset..], &mut output_ends[output_ends_offset..], ); - input_offset += nread; - output_offset += nwritten; - output_ends_offset += nends; + input_offset += n_read; + output_offset += n_written; + output_ends_offset += n_ends; match res { csv_core::ReadRecordResult::InputEmpty => {} csv_core::ReadRecordResult::OutputFull => resize_buf(buffer), diff --git a/stdlib/src/fcntl.rs b/stdlib/src/fcntl.rs index 307d6e4351..7dff14ccd8 100644 --- a/stdlib/src/fcntl.rs +++ b/stdlib/src/fcntl.rs @@ -1,3 +1,5 @@ +// cspell:disable + pub(crate) use fcntl::make_module; #[pymodule] diff --git a/stdlib/src/grp.rs b/stdlib/src/grp.rs index 2cdad56588..9c946dd582 100644 --- a/stdlib/src/grp.rs +++ b/stdlib/src/grp.rs @@ -1,3 +1,4 @@ +// cspell:disable pub(crate) use grp::make_module; #[pymodule] diff --git a/stdlib/src/locale.rs b/stdlib/src/locale.rs index 9ca71a0957..dfc6c93497 100644 --- a/stdlib/src/locale.rs +++ b/stdlib/src/locale.rs @@ -1,3 +1,5 @@ +// cspell:ignore abday abmon yesexpr + pub(crate) use _locale::make_module; #[cfg(windows)] diff --git a/stdlib/src/math.rs b/stdlib/src/math.rs index f86ebb591e..93929e3566 100644 --- a/stdlib/src/math.rs +++ b/stdlib/src/math.rs @@ -136,7 +136,7 @@ mod math { if base.is_sign_negative() { return Err(vm.new_value_error("math domain error".to_owned())); } - log2(x, vm).map(|logx| logx / base.log2()) + log2(x, vm).map(|log_x| log_x / base.log2()) } #[pyfunction] @@ -188,7 +188,7 @@ mod math { #[pyfunction] fn log10(x: PyObjectRef, vm: &VirtualMachine) -> PyResult { - log2(x, vm).map(|logx| logx / 10f64.log2()) + log2(x, vm).map(|log_x| log_x / 10f64.log2()) } #[pyfunction] @@ -588,16 +588,16 @@ mod math { where F: Fn(&BigInt, &PyInt) -> BigInt, { - let argvec = args.into_vec(); + let arg_vec = args.into_vec(); - if argvec.is_empty() { + if arg_vec.is_empty() { return default; - } else if argvec.len() == 1 { - return op(argvec[0].as_bigint(), &argvec[0]); + } else if arg_vec.len() == 1 { + return op(arg_vec[0].as_bigint(), &arg_vec[0]); } - let mut res = argvec[0].as_bigint().clone(); - for num in &argvec[1..] { + let mut res = arg_vec[0].as_bigint().clone(); + for num in &arg_vec[1..] { res = op(&res, num) } res @@ -895,15 +895,15 @@ mod math { return Err(vm.new_value_error("math domain error".to_owned())); } - let absx = x.abs(); - let absy = y.abs(); - let modulus = absx % absy; + let abs_x = x.abs(); + let abs_y = y.abs(); + let modulus = abs_x % abs_y; - let c = absy - modulus; + let c = abs_y - modulus; let r = match modulus.partial_cmp(&c) { Some(Ordering::Less) => modulus, Some(Ordering::Greater) => -c, - _ => modulus - 2.0 * fmod(0.5 * (absx - modulus), absy), + _ => modulus - 2.0 * fmod(0.5 * (abs_x - modulus), abs_y), }; return Ok(1.0_f64.copysign(x) * r); diff --git a/stdlib/src/random.rs b/stdlib/src/random.rs index 31e523b68b..a2aaff2612 100644 --- a/stdlib/src/random.rs +++ b/stdlib/src/random.rs @@ -79,7 +79,7 @@ mod _random { }; let words = (k - 1) / 32 + 1; - let wordarray = (0..words) + let word_array = (0..words) .map(|_| { let word = gen_u32(k); k = k.wrapping_sub(32); @@ -87,7 +87,7 @@ mod _random { }) .collect::>(); - let uint = BigUint::new(wordarray); + let uint = BigUint::new(word_array); // very unlikely but might as well check let sign = if uint.is_zero() { Sign::NoSign diff --git a/stdlib/src/syslog.rs b/stdlib/src/syslog.rs index 3b36f9ea74..69e9d1cb9e 100644 --- a/stdlib/src/syslog.rs +++ b/stdlib/src/syslog.rs @@ -1,4 +1,4 @@ -// spell-checker:ignore logoption openlog setlogmask upto +// spell-checker:ignore logoption openlog setlogmask upto NDELAY pub(crate) use syslog::make_module; diff --git a/stdlib/src/tkinter.rs b/stdlib/src/tkinter.rs index 1d14c9f38c..907dc55002 100644 --- a/stdlib/src/tkinter.rs +++ b/stdlib/src/tkinter.rs @@ -1,3 +1,5 @@ +// cspell:ignore createcommand + pub(crate) use self::_tkinter::make_module; #[pymodule] @@ -45,7 +47,7 @@ mod _tkinter { #[pyfunction] fn create(args: FuncArgs, _vm: &VirtualMachine) -> PyResult { - // TODO: handle arguements + // TODO: handle arguments // TODO: this means creating 2 tk instances is not possible. let tk = Tk::new(()).unwrap(); Ok(TkApp { diff --git a/stdlib/src/zlib.rs b/stdlib/src/zlib.rs index 40e364f8d4..9c19b74066 100644 --- a/stdlib/src/zlib.rs +++ b/stdlib/src/zlib.rs @@ -1,4 +1,4 @@ -// spell-checker:ignore compressobj decompressobj zdict chunksize zlibmodule miniz +// spell-checker:ignore compressobj decompressobj zdict chunksize zlibmodule miniz chunker pub(crate) use zlib::make_module; diff --git a/vm/src/anystr.rs b/vm/src/anystr.rs index 6bc8a4dd13..79b15b6a3f 100644 --- a/vm/src/anystr.rs +++ b/vm/src/anystr.rs @@ -200,7 +200,7 @@ pub trait AnyStr { F: Fn(&Self) -> PyObjectRef; #[inline] - fn py_startsendswith<'a, T, F>( + fn py_starts_ends_with<'a, T, F>( &self, affix: &'a PyObject, func_name: &str, diff --git a/vm/src/builtins/bytearray.rs b/vm/src/builtins/bytearray.rs index 36cf8cadcd..3d4822cf48 100644 --- a/vm/src/builtins/bytearray.rs +++ b/vm/src/builtins/bytearray.rs @@ -375,7 +375,7 @@ impl PyByteArray { Some(x) => x, None => return Ok(false), }; - substr.py_startsendswith( + substr.py_starts_ends_with( &affix, "endswith", "bytes", @@ -396,7 +396,7 @@ impl PyByteArray { Some(x) => x, None => return Ok(false), }; - substr.py_startsendswith( + substr.py_starts_ends_with( &affix, "startswith", "bytes", diff --git a/vm/src/builtins/bytes.rs b/vm/src/builtins/bytes.rs index eff4190eda..434de6a76c 100644 --- a/vm/src/builtins/bytes.rs +++ b/vm/src/builtins/bytes.rs @@ -299,7 +299,7 @@ impl PyBytes { Some(x) => x, None => return Ok(false), }; - substr.py_startsendswith( + substr.py_starts_ends_with( &affix, "endswith", "bytes", @@ -319,7 +319,7 @@ impl PyBytes { Some(x) => x, None => return Ok(false), }; - substr.py_startsendswith( + substr.py_starts_ends_with( &affix, "startswith", "bytes", @@ -541,7 +541,7 @@ impl PyRef { /// Other possible values are 'ignore', 'replace' /// For a list of possible encodings, /// see https://docs.python.org/3/library/codecs.html#standard-encodings - /// currently, only 'utf-8' and 'ascii' emplemented + /// currently, only 'utf-8' and 'ascii' implemented #[pymethod] fn decode(self, args: DecodeArgs, vm: &VirtualMachine) -> PyResult { bytes_decode(self.into(), args, vm) diff --git a/vm/src/builtins/complex.rs b/vm/src/builtins/complex.rs index d48707261c..02324704b3 100644 --- a/vm/src/builtins/complex.rs +++ b/vm/src/builtins/complex.rs @@ -53,7 +53,7 @@ impl From for PyComplex { impl PyObjectRef { /// Tries converting a python object into a complex, returns an option of whether the complex - /// and whether the object was a complex originally or coereced into one + /// and whether the object was a complex originally or coerced into one pub fn try_complex(&self, vm: &VirtualMachine) -> PyResult> { if let Some(complex) = self.payload_if_exact::(vm) { return Ok(Some((complex.value, true))); diff --git a/vm/src/builtins/dict.rs b/vm/src/builtins/dict.rs index a19b11fcfb..fc2f206dd0 100644 --- a/vm/src/builtins/dict.rs +++ b/vm/src/builtins/dict.rs @@ -281,8 +281,8 @@ impl PyDict { #[pymethod(magic)] fn or(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let dicted: Result = other.downcast(); - if let Ok(other) = dicted { + let other_dict: Result = other.downcast(); + if let Ok(other) = other_dict { let self_cp = self.copy(); self_cp.merge_dict(other, vm)?; return Ok(self_cp.into_pyobject(vm)); @@ -397,8 +397,8 @@ impl PyRef { #[pymethod(magic)] fn ror(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let dicted: Result = other.downcast(); - if let Ok(other) = dicted { + let other_dict: Result = other.downcast(); + if let Ok(other) = other_dict { let other_cp = other.copy(); other_cp.merge_dict(self, vm)?; return Ok(other_cp.into_pyobject(vm)); diff --git a/vm/src/builtins/function.rs b/vm/src/builtins/function.rs index f7b5d39993..3181f1068f 100644 --- a/vm/src/builtins/function.rs +++ b/vm/src/builtins/function.rs @@ -198,9 +198,9 @@ impl PyFunction { // function definition calls for if nargs < nexpected_args { let defaults = get_defaults!().0.as_ref().map(|tup| tup.as_slice()); - let ndefs = defaults.map_or(0, |d| d.len()); + let n_defs = defaults.map_or(0, |d| d.len()); - let nrequired = code.arg_count as usize - ndefs; + let nrequired = code.arg_count as usize - n_defs; // Given the number of defaults available, check all the arguments for which we // _don't_ have defaults; if any are missing, raise an exception @@ -642,9 +642,9 @@ impl PyBoundMethod { vm: &VirtualMachine, ) -> (Option, (PyObjectRef, Option)) { let builtins_getattr = vm.builtins.get_attr("getattr", vm).ok(); - let funcself = self.object.clone(); - let funcname = self.function.get_attr("__name__", vm).ok(); - (builtins_getattr, (funcself, funcname)) + let func_self = self.object.clone(); + let func_name = self.function.get_attr("__name__", vm).ok(); + (builtins_getattr, (func_self, func_name)) } #[pygetset(magic)] @@ -700,16 +700,16 @@ impl Representable for PyBoundMethod { #[inline] fn repr_str(zelf: &Py, vm: &VirtualMachine) -> PyResult { #[allow(clippy::needless_match)] // False positive on nightly - let funcname = + let func_name = if let Some(qname) = vm.get_attribute_opt(zelf.function.clone(), "__qualname__")? { Some(qname) } else { vm.get_attribute_opt(zelf.function.clone(), "__name__")? }; - let funcname: Option = funcname.and_then(|o| o.downcast().ok()); + let func_name: Option = func_name.and_then(|o| o.downcast().ok()); Ok(format!( "", - funcname.as_ref().map_or("?", |s| s.as_str()), + func_name.as_ref().map_or("?", |s| s.as_str()), &zelf.object.repr(vm)?.as_str(), )) } diff --git a/vm/src/builtins/int.rs b/vm/src/builtins/int.rs index d644343f1c..5f12f2490e 100644 --- a/vm/src/builtins/int.rs +++ b/vm/src/builtins/int.rs @@ -524,13 +524,14 @@ impl PyInt { // Malachite division uses floor rounding, Python uses half-even let remainder = &value - &rounded; - let halfpow10 = &pow10 / BigInt::from(2); - let correction = - if remainder > halfpow10 || (remainder == halfpow10 && quotient.is_odd()) { - pow10 - } else { - BigInt::from(0) - }; + let half_pow10 = &pow10 / BigInt::from(2); + let correction = if remainder > half_pow10 + || (remainder == half_pow10 && quotient.is_odd()) + { + pow10 + } else { + BigInt::from(0) + }; let rounded = (rounded + correction) * sign; return Ok(vm.ctx.new_int(rounded)); } diff --git a/vm/src/builtins/memory.rs b/vm/src/builtins/memory.rs index c5af12dc1f..09239e3e49 100644 --- a/vm/src/builtins/memory.rs +++ b/vm/src/builtins/memory.rs @@ -43,7 +43,7 @@ pub struct PyMemoryView { // avoid double release when memoryview had released the buffer before drop buffer: ManuallyDrop, // the released memoryview does not mean the buffer is destroyed - // because the possible another memeoryview is viewing from it + // because the possible another memoryview is viewing from it released: AtomicCell, // start does NOT mean the bytes before start will not be visited, // it means the point we starting to get the absolute position via @@ -103,7 +103,7 @@ impl PyMemoryView { }) } - /// don't use this function to create the memeoryview if the buffer is exporting + /// don't use this function to create the memoryview if the buffer is exporting /// via another memoryview, use PyMemoryView::new_view() or PyMemoryView::from_object /// to reduce the chain pub fn from_buffer_range( @@ -262,8 +262,8 @@ impl PyMemoryView { // no suboffset set, stride must be positive self.start += stride as usize * range.start; } - let newlen = range.len(); - self.desc.dim_desc[dim].0 = newlen; + let new_len = range.len(); + self.desc.dim_desc[dim].0 = new_len; } fn init_slice(&mut self, slice: &PySlice, dim: usize, vm: &VirtualMachine) -> PyResult<()> { diff --git a/vm/src/builtins/str.rs b/vm/src/builtins/str.rs index 8aafc63c3b..90c702a14d 100644 --- a/vm/src/builtins/str.rs +++ b/vm/src/builtins/str.rs @@ -836,7 +836,7 @@ impl PyStr { Some(x) => x, None => return Ok(false), }; - substr.py_startsendswith( + substr.py_starts_ends_with( &affix, "endswith", "str", @@ -856,7 +856,7 @@ impl PyStr { Some(x) => x, None => return Ok(false), }; - substr.py_startsendswith( + substr.py_starts_ends_with( &affix, "startswith", "str", diff --git a/vm/src/builtins/super.rs b/vm/src/builtins/super.rs index 5f363ebea5..442d162c78 100644 --- a/vm/src/builtins/super.rs +++ b/vm/src/builtins/super.rs @@ -29,7 +29,7 @@ impl PySuperInner { let obj = if vm.is_none(&obj) { None } else { - let obj_type = supercheck(typ.clone(), obj.clone(), vm)?; + let obj_type = super_check(typ.clone(), obj.clone(), vm)?; Some((obj, obj_type)) }; Ok(Self { typ, obj }) @@ -236,7 +236,7 @@ impl Representable for PySuper { } } -fn supercheck(ty: PyTypeRef, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { +fn super_check(ty: PyTypeRef, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { if let Ok(cls) = obj.clone().downcast::() { if cls.fast_issubclass(&ty) { return Ok(cls); diff --git a/vm/src/bytesinner.rs b/vm/src/bytesinner.rs index 63d5148e04..88d2b9744f 100644 --- a/vm/src/bytesinner.rs +++ b/vm/src/bytesinner.rs @@ -748,10 +748,10 @@ impl PyBytesInner { self.elements.py_zfill(width) } - // len(self)>=1, from="", len(to)>=1, maxcount>=1 - fn replace_interleave(&self, to: PyBytesInner, maxcount: Option) -> Vec { + // len(self)>=1, from="", len(to)>=1, max_count>=1 + fn replace_interleave(&self, to: PyBytesInner, max_count: Option) -> Vec { let place_count = self.elements.len() + 1; - let count = maxcount.map_or(place_count, |v| std::cmp::min(v, place_count)) - 1; + let count = max_count.map_or(place_count, |v| std::cmp::min(v, place_count)) - 1; let capacity = self.elements.len() + count * to.len(); let mut result = Vec::with_capacity(capacity); let to_slice = to.elements.as_slice(); @@ -764,8 +764,12 @@ impl PyBytesInner { result } - fn replace_delete(&self, from: PyBytesInner, maxcount: Option) -> Vec { - let count = count_substring(self.elements.as_slice(), from.elements.as_slice(), maxcount); + fn replace_delete(&self, from: PyBytesInner, max_count: Option) -> Vec { + let count = count_substring( + self.elements.as_slice(), + from.elements.as_slice(), + max_count, + ); if count == 0 { // no matches return self.elements.clone(); @@ -793,7 +797,7 @@ impl PyBytesInner { &self, from: PyBytesInner, to: PyBytesInner, - maxcount: Option, + max_count: Option, ) -> Vec { let len = from.len(); let mut iter = self.elements.find_iter(&from.elements); @@ -801,7 +805,7 @@ impl PyBytesInner { let mut new = if let Some(offset) = iter.next() { let mut new = self.elements.clone(); new[offset..offset + len].clone_from_slice(to.elements.as_slice()); - if maxcount == Some(1) { + if max_count == Some(1) { return new; } else { new @@ -810,7 +814,7 @@ impl PyBytesInner { return self.elements.clone(); }; - let mut count = maxcount.unwrap_or(usize::MAX) - 1; + let mut count = max_count.unwrap_or(usize::MAX) - 1; for offset in iter { new[offset..offset + len].clone_from_slice(to.elements.as_slice()); count -= 1; @@ -825,10 +829,14 @@ impl PyBytesInner { &self, from: PyBytesInner, to: PyBytesInner, - maxcount: Option, + max_count: Option, vm: &VirtualMachine, ) -> PyResult> { - let count = count_substring(self.elements.as_slice(), from.elements.as_slice(), maxcount); + let count = count_substring( + self.elements.as_slice(), + from.elements.as_slice(), + max_count, + ); if count == 0 { // no matches, return unchanged return Ok(self.elements.clone()); @@ -866,19 +874,19 @@ impl PyBytesInner { &self, from: PyBytesInner, to: PyBytesInner, - maxcount: OptionalArg, + max_count: OptionalArg, vm: &VirtualMachine, ) -> PyResult> { // stringlib_replace in CPython - let maxcount = match maxcount { - OptionalArg::Present(maxcount) if maxcount >= 0 => { - if maxcount == 0 || (self.elements.is_empty() && !from.is_empty()) { + let max_count = match max_count { + OptionalArg::Present(max_count) if max_count >= 0 => { + if max_count == 0 || (self.elements.is_empty() && !from.is_empty()) { // nothing to do; return the original bytes return Ok(self.elements.clone()); } else if self.elements.is_empty() && from.is_empty() { return Ok(to.elements); } - Some(maxcount as usize) + Some(max_count as usize) } _ => None, }; @@ -892,7 +900,7 @@ impl PyBytesInner { // insert the 'to' bytes everywhere. // >>> b"Python".replace(b"", b".") // b'.P.y.t.h.o.n.' - return Ok(self.replace_interleave(to, maxcount)); + return Ok(self.replace_interleave(to, max_count)); } // Except for b"".replace(b"", b"A") == b"A" there is no way beyond this @@ -904,13 +912,13 @@ impl PyBytesInner { if to.elements.is_empty() { // delete all occurrences of 'from' bytes - Ok(self.replace_delete(from, maxcount)) + Ok(self.replace_delete(from, max_count)) } else if from.len() == to.len() { // Handle special case where both bytes have the same length - Ok(self.replace_in_place(from, to, maxcount)) + Ok(self.replace_in_place(from, to, max_count)) } else { // Otherwise use the more generic algorithms - self.replace_general(from, to, maxcount, vm) + self.replace_general(from, to, max_count, vm) } } @@ -978,10 +986,10 @@ where } #[inline] -fn count_substring(haystack: &[u8], needle: &[u8], maxcount: Option) -> usize { +fn count_substring(haystack: &[u8], needle: &[u8], max_count: Option) -> usize { let substrings = haystack.find_iter(needle); - if let Some(maxcount) = maxcount { - std::cmp::min(substrings.take(maxcount).count(), maxcount) + if let Some(max_count) = max_count { + std::cmp::min(substrings.take(max_count).count(), max_count) } else { substrings.count() } diff --git a/vm/src/cformat.rs b/vm/src/cformat.rs index 93c409172c..2904b9432e 100644 --- a/vm/src/cformat.rs +++ b/vm/src/cformat.rs @@ -1,3 +1,5 @@ +//cspell:ignore bytesobject + //! Implementation of Printf-Style string formatting //! as per the [Python Docs](https://docs.python.org/3/library/stdtypes.html#printf-style-string-formatting). diff --git a/vm/src/exceptions.rs b/vm/src/exceptions.rs index 58f2a51b68..708a93fe61 100644 --- a/vm/src/exceptions.rs +++ b/vm/src/exceptions.rs @@ -212,10 +212,10 @@ impl VirtualMachine { if let Some(text) = maybe_text { // if text ends with \n, remove it let rtext = text.as_str().trim_end_matches('\n'); - let ltext = rtext.trim_start_matches([' ', '\n', '\x0c']); // \x0c is \f - let spaces = (rtext.len() - ltext.len()) as isize; + let l_text = rtext.trim_start_matches([' ', '\n', '\x0c']); // \x0c is \f + let spaces = (rtext.len() - l_text.len()) as isize; - writeln!(output, " {}", ltext)?; + writeln!(output, " {}", l_text)?; let maybe_offset: Option = getattr("offset").and_then(|obj| obj.try_to_value::(vm).ok()); @@ -237,7 +237,7 @@ impl VirtualMachine { let colno = offset - 1 - spaces; let end_colno = end_offset - 1 - spaces; if colno >= 0 { - let caretspace = ltext.chars().collect::>()[..colno as usize] + let caret_space = l_text.chars().collect::>()[..colno as usize] .iter() .map(|c| if c.is_whitespace() { *c } else { ' ' }) .collect::(); @@ -250,7 +250,7 @@ impl VirtualMachine { writeln!( output, " {}{}", - caretspace, + caret_space, "^".repeat(error_width as usize) )?; } diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 78f03a04d8..7976a5254f 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -350,7 +350,7 @@ impl ExecutingFrame<'_> { fn run(&mut self, vm: &VirtualMachine) -> PyResult { flame_guard!(format!("Frame::run({})", self.code.obj_name)); // Execute until return or exception: - let instrs = &self.code.instructions; + let instructions = &self.code.instructions; let mut arg_state = bytecode::OpArgState::default(); loop { let idx = self.lasti() as usize; @@ -359,7 +359,7 @@ impl ExecutingFrame<'_> { // self.code.locations[idx], self.code.source_path // ); self.update_lasti(|i| *i += 1); - let bytecode::CodeUnit { op, arg } = instrs[idx]; + let bytecode::CodeUnit { op, arg } = instructions[idx]; let arg = arg_state.extend(arg); let mut do_extend_arg = false; let result = self.execute_instruction(op, arg, &mut do_extend_arg, vm); @@ -805,14 +805,14 @@ impl ExecutingFrame<'_> { dict.set_item(&*key, value, vm)?; Ok(None) } - bytecode::Instruction::BinaryOperation { op } => self.execute_binop(vm, op.get(arg)), + bytecode::Instruction::BinaryOperation { op } => self.execute_bin_op(vm, op.get(arg)), bytecode::Instruction::BinaryOperationInplace { op } => { - self.execute_binop_inplace(vm, op.get(arg)) + self.execute_bin_op_inplace(vm, op.get(arg)) } bytecode::Instruction::LoadAttr { idx } => self.load_attr(vm, idx.get(arg)), bytecode::Instruction::StoreAttr { idx } => self.store_attr(vm, idx.get(arg)), bytecode::Instruction::DeleteAttr { idx } => self.delete_attr(vm, idx.get(arg)), - bytecode::Instruction::UnaryOperation { op } => self.execute_unop(vm, op.get(arg)), + bytecode::Instruction::UnaryOperation { op } => self.execute_unary_op(vm, op.get(arg)), bytecode::Instruction::TestOperation { op } => self.execute_test(vm, op.get(arg)), bytecode::Instruction::CompareOperation { op } => self.execute_compare(vm, op.get(arg)), bytecode::Instruction::ReturnValue => { @@ -1792,7 +1792,7 @@ impl ExecutingFrame<'_> { } #[cfg_attr(feature = "flame-it", flame("Frame"))] - fn execute_binop(&mut self, vm: &VirtualMachine, op: bytecode::BinaryOperator) -> FrameResult { + fn execute_bin_op(&mut self, vm: &VirtualMachine, op: bytecode::BinaryOperator) -> FrameResult { let b_ref = &self.pop_value(); let a_ref = &self.pop_value(); let value = match op { @@ -1814,7 +1814,7 @@ impl ExecutingFrame<'_> { self.push_value(value); Ok(None) } - fn execute_binop_inplace( + fn execute_bin_op_inplace( &mut self, vm: &VirtualMachine, op: bytecode::BinaryOperator, @@ -1842,7 +1842,11 @@ impl ExecutingFrame<'_> { } #[cfg_attr(feature = "flame-it", flame("Frame"))] - fn execute_unop(&mut self, vm: &VirtualMachine, op: bytecode::UnaryOperator) -> FrameResult { + fn execute_unary_op( + &mut self, + vm: &VirtualMachine, + op: bytecode::UnaryOperator, + ) -> FrameResult { let a = self.pop_value(); let value = match op { bytecode::UnaryOperator::Minus => vm._neg(&a)?, diff --git a/vm/src/function/argument.rs b/vm/src/function/argument.rs index 197cfe7b96..5033ee7627 100644 --- a/vm/src/function/argument.rs +++ b/vm/src/function/argument.rs @@ -276,7 +276,7 @@ impl ArgumentError { vm.new_type_error(format!("{name} is an invalid keyword argument")) } ArgumentError::RequiredKeywordArgument(name) => { - vm.new_type_error(format!("Required keyqord only argument {name}")) + vm.new_type_error(format!("Required keyword only argument {name}")) } ArgumentError::Exception(ex) => ex, } diff --git a/vm/src/function/fspath.rs b/vm/src/function/fspath.rs index 83bd452151..74051644e0 100644 --- a/vm/src/function/fspath.rs +++ b/vm/src/function/fspath.rs @@ -94,7 +94,7 @@ impl FsPath { } #[cfg(windows)] - pub fn to_widecstring(&self, vm: &VirtualMachine) -> PyResult { + pub fn to_wide_cstring(&self, vm: &VirtualMachine) -> PyResult { widestring::WideCString::from_os_str(self.as_os_str(vm)?) .map_err(|err| err.into_pyexception(vm)) } diff --git a/vm/src/function/protocol.rs b/vm/src/function/protocol.rs index 2f4b4d160a..0f146fed95 100644 --- a/vm/src/function/protocol.rs +++ b/vm/src/function/protocol.rs @@ -76,7 +76,7 @@ impl TryFromObject for ArgCallable { /// objects using a generic type parameter that implements `TryFromObject`. pub struct ArgIterable { iterable: PyObjectRef, - iterfn: Option, + iter_fn: Option, _item: PhantomData, } @@ -92,7 +92,7 @@ impl ArgIterable { /// This operation may fail if an exception is raised while invoking the /// `__iter__` method of the iterable object. pub fn iter<'a>(&self, vm: &'a VirtualMachine) -> PyResult> { - let iter = PyIter::new(match self.iterfn { + let iter = PyIter::new(match self.iter_fn { Some(f) => f(self.iterable.clone(), vm)?, None => PySequenceIterator::new(self.iterable.clone(), vm)?.into_pyobject(vm), }); @@ -105,17 +105,17 @@ where T: TryFromObject, { fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { - let iterfn = { + let iter_fn = { let cls = obj.class(); - let iterfn = cls.mro_find_map(|x| x.slots.iter.load()); - if iterfn.is_none() && !cls.has_attr(identifier!(vm, __getitem__)) { + let iter_fn = cls.mro_find_map(|x| x.slots.iter.load()); + if iter_fn.is_none() && !cls.has_attr(identifier!(vm, __getitem__)) { return Err(vm.new_type_error(format!("'{}' object is not iterable", cls.name()))); } - iterfn + iter_fn }; Ok(Self { iterable: obj, - iterfn, + iter_fn, _item: PhantomData, }) } diff --git a/vm/src/import.rs b/vm/src/import.rs index 0ce116d014..416c40a844 100644 --- a/vm/src/import.rs +++ b/vm/src/import.rs @@ -81,7 +81,7 @@ pub fn make_frozen(vm: &VirtualMachine, name: &str) -> PyResult> { pub fn import_frozen(vm: &VirtualMachine, module_name: &str) -> PyResult { let frozen = make_frozen(vm, module_name)?; - let module = import_codeobj(vm, module_name, frozen, false)?; + let module = import_code_obj(vm, module_name, frozen, false)?; debug_assert!(module.get_attr(identifier!(vm, __name__), vm).is_ok()); // TODO: give a correct origname here module.set_attr("__origname__", vm.ctx.new_str(module_name.to_owned()), vm)?; @@ -116,7 +116,7 @@ pub fn import_file( vm.compile_opts(), ) .map_err(|err| vm.new_syntax_error(&err, Some(content)))?; - import_codeobj(vm, module_name, code, true) + import_code_obj(vm, module_name, code, true) } #[cfg(feature = "rustpython-compiler")] @@ -129,10 +129,10 @@ pub fn import_source(vm: &VirtualMachine, module_name: &str, content: &str) -> P vm.compile_opts(), ) .map_err(|err| vm.new_syntax_error(&err, Some(content)))?; - import_codeobj(vm, module_name, code, false) + import_code_obj(vm, module_name, code, false) } -pub fn import_codeobj( +pub fn import_code_obj( vm: &VirtualMachine, module_name: &str, code_obj: PyRef, diff --git a/vm/src/object/core.rs b/vm/src/object/core.rs index 56ab419c01..bbe900f7cd 100644 --- a/vm/src/object/core.rs +++ b/vm/src/object/core.rs @@ -207,7 +207,7 @@ impl WeakRefList { hash: Radium::new(crate::common::hash::SENTINEL), }; let weak = PyRef::new_ref(obj, cls, dict); - // SAFETY: we don't actually own the PyObjectWeaks inside `list`, and every time we take + // SAFETY: we don't actually own the PyObjectWeak's inside `list`, and every time we take // one out of the list we immediately wrap it in ManuallyDrop or forget it inner.list.push_front(unsafe { ptr::read(&weak) }); inner.ref_count += 1; @@ -1301,6 +1301,7 @@ mod tests { #[test] fn miri_test_drop() { + //cspell:ignore dfghjkl let ctx = crate::Context::genesis(); let obj = ctx.new_bytes(b"dfghjkl".to_vec()); drop(obj); diff --git a/vm/src/ospath.rs b/vm/src/ospath.rs index c1b1859164..26d1582825 100644 --- a/vm/src/ospath.rs +++ b/vm/src/ospath.rs @@ -70,7 +70,7 @@ impl OsPath { } #[cfg(windows)] - pub fn to_widecstring(&self, vm: &VirtualMachine) -> PyResult { + pub fn to_wide_cstring(&self, vm: &VirtualMachine) -> PyResult { widestring::WideCString::from_os_str(&self.path).map_err(|err| err.to_pyexception(vm)) } @@ -167,18 +167,18 @@ impl<'a> IOErrorBuilder<'a> { impl ToPyException for IOErrorBuilder<'_> { fn to_pyexception(&self, vm: &VirtualMachine) -> PyBaseExceptionRef { - let excp = self.error.to_pyexception(vm); + let exc = self.error.to_pyexception(vm); if let Some(filename) = &self.filename { - excp.as_object() + exc.as_object() .set_attr("filename", filename.filename(vm), vm) .unwrap(); } if let Some(filename2) = &self.filename2 { - excp.as_object() + exc.as_object() .set_attr("filename2", filename2.filename(vm), vm) .unwrap(); } - excp + exc } } diff --git a/vm/src/protocol/buffer.rs b/vm/src/protocol/buffer.rs index e3b03b4f80..a3b7f125f5 100644 --- a/vm/src/protocol/buffer.rs +++ b/vm/src/protocol/buffer.rs @@ -131,7 +131,7 @@ impl PyBuffer { // drop PyBuffer without calling release // after this function, the owner should use forget() - // or wrap PyBuffer in the ManaullyDrop to prevent drop() + // or wrap PyBuffer in the ManuallyDrop to prevent drop() pub(crate) unsafe fn drop_without_release(&mut self) { // SAFETY: requirements forwarded from caller unsafe { @@ -267,7 +267,7 @@ impl BufferDescriptor { Ok(pos) } - pub fn for_each_segment(&self, try_conti: bool, mut f: F) + pub fn for_each_segment(&self, try_contiguous: bool, mut f: F) where F: FnMut(Range), { @@ -275,20 +275,20 @@ impl BufferDescriptor { f(0..self.itemsize as isize); return; } - if try_conti && self.is_last_dim_contiguous() { + if try_contiguous && self.is_last_dim_contiguous() { self._for_each_segment::<_, true>(0, 0, &mut f); } else { self._for_each_segment::<_, false>(0, 0, &mut f); } } - fn _for_each_segment(&self, mut index: isize, dim: usize, f: &mut F) + fn _for_each_segment(&self, mut index: isize, dim: usize, f: &mut F) where F: FnMut(Range), { let (shape, stride, suboffset) = self.dim_desc[dim]; if dim + 1 == self.ndim() { - if CONTI { + if CONTIGUOUS { f(index..index + (shape * self.itemsize) as isize); } else { for _ in 0..shape { @@ -300,13 +300,13 @@ impl BufferDescriptor { return; } for _ in 0..shape { - self._for_each_segment::(index + suboffset, dim + 1, f); + self._for_each_segment::(index + suboffset, dim + 1, f); index += stride; } } /// zip two BufferDescriptor with the same shape - pub fn zip_eq(&self, other: &Self, try_conti: bool, mut f: F) + pub fn zip_eq(&self, other: &Self, try_contiguous: bool, mut f: F) where F: FnMut(Range, Range) -> bool, { @@ -314,14 +314,14 @@ impl BufferDescriptor { f(0..self.itemsize as isize, 0..other.itemsize as isize); return; } - if try_conti && self.is_last_dim_contiguous() { + if try_contiguous && self.is_last_dim_contiguous() { self._zip_eq::<_, true>(other, 0, 0, 0, &mut f); } else { self._zip_eq::<_, false>(other, 0, 0, 0, &mut f); } } - fn _zip_eq( + fn _zip_eq( &self, other: &Self, mut a_index: isize, @@ -335,7 +335,7 @@ impl BufferDescriptor { let (_b_shape, b_stride, b_suboffset) = other.dim_desc[dim]; debug_assert_eq!(shape, _b_shape); if dim + 1 == self.ndim() { - if CONTI { + if CONTIGUOUS { if f( a_index..a_index + (shape * self.itemsize) as isize, b_index..b_index + (shape * other.itemsize) as isize, @@ -360,7 +360,7 @@ impl BufferDescriptor { } for _ in 0..shape { - self._zip_eq::( + self._zip_eq::( other, a_index + a_suboffset, b_index + b_suboffset, diff --git a/vm/src/protocol/iter.rs b/vm/src/protocol/iter.rs index a7491a3897..254134991c 100644 --- a/vm/src/protocol/iter.rs +++ b/vm/src/protocol/iter.rs @@ -125,12 +125,12 @@ impl TryFromObject for PyIter { // in the vm when a for loop is entered. Next, it is used when the builtin // function 'iter' is called. fn try_from_object(vm: &VirtualMachine, iter_target: PyObjectRef) -> PyResult { - let getiter = { + let get_iter = { let cls = iter_target.class(); cls.mro_find_map(|x| x.slots.iter.load()) }; - if let Some(getiter) = getiter { - let iter = getiter(iter_target, vm)?; + if let Some(get_iter) = get_iter { + let iter = get_iter(iter_target, vm)?; if PyIter::check(&iter) { Ok(Self(iter)) } else { diff --git a/vm/src/protocol/object.rs b/vm/src/protocol/object.rs index 4cdcb68257..256baa0fdf 100644 --- a/vm/src/protocol/object.rs +++ b/vm/src/protocol/object.rs @@ -28,8 +28,8 @@ impl PyObjectRef { // int PyObject_GenericSetDict(PyObject *o, PyObject *value, void *context) #[inline(always)] - pub fn rich_compare(self, other: Self, opid: PyComparisonOp, vm: &VirtualMachine) -> PyResult { - self._cmp(&other, opid, vm).map(|res| res.to_pyobject(vm)) + pub fn rich_compare(self, other: Self, op_id: PyComparisonOp, vm: &VirtualMachine) -> PyResult { + self._cmp(&other, op_id, vm).map(|res| res.to_pyobject(vm)) } pub fn bytes(self, vm: &VirtualMachine) -> PyResult { @@ -323,17 +323,17 @@ impl PyObject { match op { PyComparisonOp::Eq => Ok(Either::B(self.is(&other))), PyComparisonOp::Ne => Ok(Either::B(!self.is(&other))), - _ => Err(vm.new_unsupported_binop_error(self, other, op.operator_token())), + _ => Err(vm.new_unsupported_bin_op_error(self, other, op.operator_token())), } } #[inline(always)] pub fn rich_compare_bool( &self, other: &Self, - opid: PyComparisonOp, + op_id: PyComparisonOp, vm: &VirtualMachine, ) -> PyResult { - match self._cmp(other, opid, vm)? { + match self._cmp(other, op_id, vm)? { Either::A(obj) => obj.try_to_bool(vm), Either::B(other) => Ok(other), } @@ -479,13 +479,13 @@ impl PyObject { let r = if let Ok(typ) = cls.try_to_ref::(vm) { if self.class().fast_issubclass(typ) { true - } else if let Ok(icls) = + } else if let Ok(i_cls) = PyTypeRef::try_from_object(vm, self.get_attr(identifier!(vm, __class__), vm)?) { - if icls.is(self.class()) { + if i_cls.is(self.class()) { false } else { - icls.fast_issubclass(typ) + i_cls.fast_issubclass(typ) } } else { false @@ -497,11 +497,11 @@ impl PyObject { cls.class() ) })?; - let icls: PyObjectRef = self.get_attr(identifier!(vm, __class__), vm)?; - if vm.is_none(&icls) { + let i_cls: PyObjectRef = self.get_attr(identifier!(vm, __class__), vm)?; + if vm.is_none(&i_cls) { false } else { - icls.abstract_issubclass(cls, vm)? + i_cls.abstract_issubclass(cls, vm)? } }; Ok(r) diff --git a/vm/src/stdlib/builtins.rs b/vm/src/stdlib/builtins.rs index 9dcb35aae9..9a21dd34dd 100644 --- a/vm/src/stdlib/builtins.rs +++ b/vm/src/stdlib/builtins.rs @@ -588,12 +588,14 @@ mod builtins { iterator.class().name() ))); } - PyIter::new(iterator).next(vm).map(|iret| match iret { - PyIterReturn::Return(obj) => PyIterReturn::Return(obj), - PyIterReturn::StopIteration(v) => { - default_value.map_or(PyIterReturn::StopIteration(v), PyIterReturn::Return) - } - }) + PyIter::new(iterator) + .next(vm) + .map(|iter_ret| match iter_ret { + PyIterReturn::Return(obj) => PyIterReturn::Return(obj), + PyIterReturn::StopIteration(v) => { + default_value.map_or(PyIterReturn::StopIteration(v), PyIterReturn::Return) + } + }) } #[pyfunction] diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index 33ef118acd..6f13e0878d 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -673,14 +673,14 @@ mod _io { } fn _readinto( zelf: PyObjectRef, - bufobj: PyObjectRef, + buf_obj: PyObjectRef, method: &str, vm: &VirtualMachine, ) -> PyResult { - let b = ArgMemoryBuffer::try_from_borrowed_object(vm, &bufobj)?; + let b = ArgMemoryBuffer::try_from_borrowed_object(vm, &buf_obj)?; let l = b.len(); let data = vm.call_method(&zelf, method, (l,))?; - if data.is(&bufobj) { + if data.is(&buf_obj) { return Ok(l); } let mut buf = b.borrow_buf_mut(); @@ -929,25 +929,25 @@ mod _io { ) -> PyResult> { let len = buf_range.len(); let res = if let Some(buf) = buf { - let memobj = PyMemoryView::from_buffer_range(buf, buf_range, vm)?.to_pyobject(vm); + let mem_obj = PyMemoryView::from_buffer_range(buf, buf_range, vm)?.to_pyobject(vm); // TODO: loop if write() raises an interrupt - vm.call_method(self.raw.as_ref().unwrap(), "write", (memobj,))? + vm.call_method(self.raw.as_ref().unwrap(), "write", (mem_obj,))? } else { let v = std::mem::take(&mut self.buffer); - let writebuf = VecBuffer::from(v).into_ref(&vm.ctx); - let memobj = PyMemoryView::from_buffer_range( - writebuf.clone().into_pybuffer(true), + let write_buf = VecBuffer::from(v).into_ref(&vm.ctx); + let mem_obj = PyMemoryView::from_buffer_range( + write_buf.clone().into_pybuffer(true), buf_range, vm, )? .into_ref(&vm.ctx); // TODO: loop if write() raises an interrupt - let res = vm.call_method(self.raw.as_ref().unwrap(), "write", (memobj.clone(),)); + let res = vm.call_method(self.raw.as_ref().unwrap(), "write", (mem_obj.clone(),)); - memobj.release(); - self.buffer = writebuf.take(); + mem_obj.release(); + self.buffer = write_buf.take(); res? }; @@ -1159,9 +1159,9 @@ mod _io { let res = match v { Either::A(v) => { let v = v.unwrap_or(&mut self.buffer); - let readbuf = VecBuffer::from(std::mem::take(v)).into_ref(&vm.ctx); - let memobj = PyMemoryView::from_buffer_range( - readbuf.clone().into_pybuffer(false), + let read_buf = VecBuffer::from(std::mem::take(v)).into_ref(&vm.ctx); + let mem_obj = PyMemoryView::from_buffer_range( + read_buf.clone().into_pybuffer(false), buf_range, vm, )? @@ -1169,17 +1169,17 @@ mod _io { // TODO: loop if readinto() raises an interrupt let res = - vm.call_method(self.raw.as_ref().unwrap(), "readinto", (memobj.clone(),)); + vm.call_method(self.raw.as_ref().unwrap(), "readinto", (mem_obj.clone(),)); - memobj.release(); - std::mem::swap(v, &mut readbuf.take()); + mem_obj.release(); + std::mem::swap(v, &mut read_buf.take()); res? } Either::B(buf) => { - let memobj = PyMemoryView::from_buffer_range(buf, buf_range, vm)?; + let mem_obj = PyMemoryView::from_buffer_range(buf, buf_range, vm)?; // TODO: loop if readinto() raises an interrupt - vm.call_method(self.raw.as_ref().unwrap(), "readinto", (memobj,))? + vm.call_method(self.raw.as_ref().unwrap(), "readinto", (mem_obj,))? } }; @@ -2305,14 +2305,14 @@ mod _io { let incremental_encoder = codec.get_incremental_encoder(Some(errors.to_owned()), vm)?; let encoding_name = vm.get_attribute_opt(incremental_encoder.clone(), "name")?; - let encodefunc = encoding_name.and_then(|name| { + let encode_func = encoding_name.and_then(|name| { let name = name.payload::()?; match name.as_str() { "utf-8" => Some(textio_encode_utf8 as EncodeFunc), _ => None, } }); - Some((incremental_encoder, encodefunc)) + Some((incremental_encoder, encode_func)) } else { None }; @@ -2600,12 +2600,12 @@ mod _io { while skip_bytes > 0 { cookie.set_decoder_state(decoder, vm)?; let input = &next_input.as_bytes()[..skip_bytes as usize]; - let ndecoded = decoder_decode(input)?; - if ndecoded.chars <= num_to_skip.chars { + let n_decoded = decoder_decode(input)?; + if n_decoded.chars <= num_to_skip.chars { let (dec_buffer, dec_flags) = decoder_getstate()?; if dec_buffer.is_empty() { cookie.dec_flags = dec_flags; - num_to_skip -= ndecoded; + num_to_skip -= n_decoded; break; } skip_bytes -= dec_buffer.len() as isize; @@ -2625,23 +2625,23 @@ mod _io { cookie.set_num_to_skip(num_to_skip); if num_to_skip.chars != 0 { - let mut ndecoded = Utf8size::default(); + let mut n_decoded = Utf8size::default(); let mut input = next_input.as_bytes(); input = &input[skip_bytes..]; while !input.is_empty() { let (byte1, rest) = input.split_at(1); let n = decoder_decode(byte1)?; - ndecoded += n; + n_decoded += n; cookie.bytes_to_feed += 1; let (dec_buffer, dec_flags) = decoder_getstate()?; - if dec_buffer.is_empty() && ndecoded.chars < num_to_skip.chars { + if dec_buffer.is_empty() && n_decoded.chars < num_to_skip.chars { cookie.start_pos += cookie.bytes_to_feed as Offset; - num_to_skip -= ndecoded; + num_to_skip -= n_decoded; cookie.dec_flags = dec_flags; cookie.bytes_to_feed = 0; - ndecoded = Utf8size::default(); + n_decoded = Utf8size::default(); } - if ndecoded.chars >= num_to_skip.chars { + if n_decoded.chars >= num_to_skip.chars { break; } input = rest; @@ -2650,7 +2650,7 @@ mod _io { let decoded = vm.call_method(decoder, "decode", (vm.ctx.new_bytes(vec![]), true))?; let decoded = check_decoded(decoded, vm)?; - let final_decoded_chars = ndecoded.chars + decoded.char_len(); + let final_decoded_chars = n_decoded.chars + decoded.char_len(); cookie.need_eof = true; if final_decoded_chars < num_to_skip.chars { return Err( @@ -2739,7 +2739,7 @@ mod _io { let mut textio = self.lock(vm)?; textio.check_closed(vm)?; - let (encoder, encodefunc) = textio + let (encoder, encode_func) = textio .encoder .as_ref() .ok_or_else(|| new_unsupported_operation(vm, "not writable".to_owned()))?; @@ -2767,8 +2767,8 @@ mod _io { } else { obj }; - let chunk = if let Some(encodefunc) = *encodefunc { - encodefunc(chunk) + let chunk = if let Some(encode_func) = *encode_func { + encode_func(chunk) } else { let b = vm.call_method(encoder, "encode", (chunk.clone(),))?; b.downcast::() @@ -2866,7 +2866,7 @@ mod _io { } let mut start; - let mut endpos; + let mut end_pos; let mut offset_to_buffer; let mut chunked = Utf8size::default(); let mut remaining: Option = None; @@ -2883,7 +2883,7 @@ mod _io { textio.set_decoded_chars(None); textio.snapshot = None; start = Utf8size::default(); - endpos = Utf8size::default(); + end_pos = Utf8size::default(); offset_to_buffer = Utf8size::default(); break 'outer None; } @@ -2918,11 +2918,11 @@ mod _io { let nl_res = textio.newline.find_newline(line_from_start); match nl_res { Ok(p) | Err(p) => { - endpos = start + Utf8size::len_str(&line_from_start[..p]); + end_pos = start + Utf8size::len_str(&line_from_start[..p]); if let Some(limit) = limit { - // original CPython logic: endpos = start + limit - chunked - if chunked.chars + endpos.chars >= limit { - endpos = start + // original CPython logic: end_pos = start + limit - chunked + if chunked.chars + end_pos.chars >= limit { + end_pos = start + Utf8size { chars: limit - chunked.chars, bytes: crate::common::str::codepoint_range_end( @@ -2939,21 +2939,21 @@ mod _io { if nl_res.is_ok() { break Some(line); } - if endpos.bytes > start.bytes { - let chunk = SlicedStr(line.clone(), start.bytes..endpos.bytes); + if end_pos.bytes > start.bytes { + let chunk = SlicedStr(line.clone(), start.bytes..end_pos.bytes); chunked += chunk.utf8_len(); chunks.push(chunk); } let line_len = line.byte_len(); - if endpos.bytes < line_len { - remaining = Some(SlicedStr(line, endpos.bytes..line_len)); + if end_pos.bytes < line_len { + remaining = Some(SlicedStr(line, end_pos.bytes..line_len)); } textio.set_decoded_chars(None); }; let cur_line = cur_line.map(|line| { - textio.decoded_chars_used = endpos - offset_to_buffer; - SlicedStr(line, start.bytes..endpos.bytes) + textio.decoded_chars_used = end_pos - offset_to_buffer; + SlicedStr(line, start.bytes..end_pos.bytes) }); // don't need to care about chunked.chars anymore let mut chunked = chunked.bytes; @@ -3166,7 +3166,7 @@ mod _io { #[derive(Debug)] struct IncrementalNewlineDecoderData { decoder: PyObjectRef, - // afaict, this is used for nothing + // currently this is used for nothing // errors: PyObjectRef, pendingcr: bool, translate: bool, @@ -4237,7 +4237,7 @@ mod fileio { #[cfg(any(unix, target_os = "wasi"))] let fd = Fd::open(&path.clone().into_cstring(vm)?, flags, 0o666); #[cfg(windows)] - let fd = Fd::wopen(&path.to_widecstring(vm)?, flags, 0o666); + let fd = Fd::wopen(&path.to_wide_cstring(vm)?, flags, 0o666); let filename = OsPathOrFd::Path(path); match fd { Ok(fd) => (fd.0, filename), diff --git a/vm/src/stdlib/itertools.rs b/vm/src/stdlib/itertools.rs index dab62987d6..18641ac3b6 100644 --- a/vm/src/stdlib/itertools.rs +++ b/vm/src/stdlib/itertools.rs @@ -1087,7 +1087,7 @@ mod decl { #[derive(Debug, PyPayload)] struct PyItertoolsAccumulate { iterable: PyIter, - binop: Option, + bin_op: Option, initial: Option, acc_value: PyRwLock>, } @@ -1107,7 +1107,7 @@ mod decl { fn py_new(cls: PyTypeRef, args: AccumulateArgs, vm: &VirtualMachine) -> PyResult { PyItertoolsAccumulate { iterable: args.iterable, - binop: args.func.flatten(), + bin_op: args.func.flatten(), initial: args.initial.flatten(), acc_value: PyRwLock::new(None), } @@ -1127,7 +1127,7 @@ mod decl { #[pymethod(magic)] fn reduce(zelf: PyRef, vm: &VirtualMachine) -> PyTupleRef { let class = zelf.class().to_owned(); - let binop = zelf.binop.clone(); + let bin_op = zelf.bin_op.clone(); let it = zelf.iterable.clone(); let acc_value = zelf.acc_value.read().clone(); if let Some(initial) = &zelf.initial { @@ -1136,7 +1136,7 @@ mod decl { source: PyRwLock::new(Some(chain_args.to_pyobject(vm).get_iter(vm).unwrap())), active: PyRwLock::new(None), }; - let tup = vm.new_tuple((chain, binop)); + let tup = vm.new_tuple((chain, bin_op)); return vm.new_tuple((class, tup, acc_value)); } match acc_value { @@ -1151,7 +1151,7 @@ mod decl { .into_pyobject(vm); let acc = Self { iterable: PyIter::new(chain), - binop, + bin_op, initial: None, acc_value: PyRwLock::new(None), }; @@ -1161,7 +1161,7 @@ mod decl { } _ => {} } - let tup = vm.new_tuple((it, binop)); + let tup = vm.new_tuple((it, bin_op)); vm.new_tuple((class, tup, acc_value)) } } @@ -1191,7 +1191,7 @@ mod decl { return Ok(PyIterReturn::StopIteration(v)); } }; - match &zelf.binop { + match &zelf.bin_op { None => vm._add(&value, &obj)?, Some(op) => op.call((value, obj), vm)?, } diff --git a/vm/src/stdlib/nt.rs b/vm/src/stdlib/nt.rs index b4899bb225..428d3421fd 100644 --- a/vm/src/stdlib/nt.rs +++ b/vm/src/stdlib/nt.rs @@ -41,7 +41,7 @@ pub(crate) mod module { #[pyfunction] pub(super) fn access(path: OsPath, mode: u8, vm: &VirtualMachine) -> PyResult { - let attr = unsafe { FileSystem::GetFileAttributesW(path.to_widecstring(vm)?.as_ptr()) }; + let attr = unsafe { FileSystem::GetFileAttributesW(path.to_wide_cstring(vm)?.as_ptr()) }; Ok(attr != FileSystem::INVALID_FILE_ATTRIBUTES && (mode & 2 == 0 || attr & FileSystem::FILE_ATTRIBUTE_READONLY == 0 @@ -256,7 +256,7 @@ pub(crate) mod module { #[pyfunction] fn _getfullpathname(path: OsPath, vm: &VirtualMachine) -> PyResult { - let wpath = path.to_widecstring(vm)?; + let wpath = path.to_wide_cstring(vm)?; let mut buffer = vec![0u16; Foundation::MAX_PATH as usize]; let ret = unsafe { FileSystem::GetFullPathNameW( @@ -289,7 +289,7 @@ pub(crate) mod module { #[pyfunction] fn _getvolumepathname(path: OsPath, vm: &VirtualMachine) -> PyResult { - let wide = path.to_widecstring(vm)?; + let wide = path.to_wide_cstring(vm)?; let buflen = std::cmp::max(wide.len(), Foundation::MAX_PATH as usize); let mut buffer = vec![0u16; buflen]; let ret = unsafe { @@ -344,7 +344,7 @@ pub(crate) mod module { fn _getdiskusage(path: OsPath, vm: &VirtualMachine) -> PyResult<(u64, u64)> { use FileSystem::GetDiskFreeSpaceExW; - let wpath = path.to_widecstring(vm)?; + let wpath = path.to_wide_cstring(vm)?; let mut _free_to_me: u64 = 0; let mut total: u64 = 0; let mut free: u64 = 0; @@ -437,7 +437,7 @@ pub(crate) mod module { let mode = mode.unwrap_or(0o777); let [] = dir_fd.0; let _ = mode; - let wide = path.to_widecstring(vm)?; + let wide = path.to_wide_cstring(vm)?; let res = unsafe { FileSystem::CreateDirectoryW(wide.as_ptr(), std::ptr::null_mut()) }; if res == 0 { return Err(errno_err(vm)); diff --git a/vm/src/stdlib/operator.rs b/vm/src/stdlib/operator.rs index d8ff1715fa..38f931b0e7 100644 --- a/vm/src/stdlib/operator.rs +++ b/vm/src/stdlib/operator.rs @@ -389,15 +389,15 @@ mod _operator { type Args = FuncArgs; fn py_new(cls: PyTypeRef, args: Self::Args, vm: &VirtualMachine) -> PyResult { - let nattr = args.args.len(); + let n_attr = args.args.len(); // Check we get no keyword and at least one positional. if !args.kwargs.is_empty() { return Err(vm.new_type_error("attrgetter() takes no keyword arguments".to_owned())); } - if nattr == 0 { + if n_attr == 0 { return Err(vm.new_type_error("attrgetter expected 1 argument, got 0.".to_owned())); } - let mut attrs = Vec::with_capacity(nattr); + let mut attrs = Vec::with_capacity(n_attr); for o in args.args { if let Ok(r) = o.try_into_value(vm) { attrs.push(r); diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 641ba54dea..48e16ad41f 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -219,7 +219,7 @@ pub(super) mod _os { #[cfg(windows)] let fd = { let [] = dir_fd.0; - let name = name.to_widecstring(vm)?; + let name = name.to_wide_cstring(vm)?; let flags = flags | libc::O_NOINHERIT; Fd::wopen(&name, flags, mode) }; diff --git a/vm/src/stdlib/sre.rs b/vm/src/stdlib/sre.rs index 7b67c038f4..fd41aa2b7d 100644 --- a/vm/src/stdlib/sre.rs +++ b/vm/src/stdlib/sre.rs @@ -287,7 +287,7 @@ mod _sre { with_sre_str!(zelf, &string_args.string, vm, |s| { let req = s.create_request(&zelf, string_args.pos, string_args.endpos); let state = State::default(); - let mut matchlist: Vec = Vec::new(); + let mut match_list: Vec = Vec::new(); let mut iter = SearchIter { req, state }; while iter.next().is_some() { @@ -301,10 +301,10 @@ mod _sre { .into() }; - matchlist.push(item); + match_list.push(item); } - Ok(matchlist) + Ok(match_list) }) } @@ -362,7 +362,7 @@ mod _sre { with_sre_str!(zelf, &split_args.string, vm, |s| { let req = s.create_request(&zelf, 0, usize::MAX); let state = State::default(); - let mut splitlist: Vec = Vec::new(); + let mut split_list: Vec = Vec::new(); let mut iter = SearchIter { req, state }; let mut n = 0; let mut last = 0; @@ -370,13 +370,13 @@ mod _sre { while (split_args.maxsplit == 0 || n < split_args.maxsplit) && iter.next().is_some() { /* get segment before this match */ - splitlist.push(s.slice(last, iter.state.start, vm)); + split_list.push(s.slice(last, iter.state.start, vm)); let m = Match::new(&mut iter.state, zelf.clone(), split_args.string.clone()); // add groups (if any) for i in 1..=zelf.groups { - splitlist.push(m.get_slice(i, s, vm).unwrap_or_else(|| vm.ctx.none())); + split_list.push(m.get_slice(i, s, vm).unwrap_or_else(|| vm.ctx.none())); } n += 1; @@ -384,9 +384,9 @@ mod _sre { } // get segment following last match (even if empty) - splitlist.push(req.string.slice(last, s.count(), vm)); + split_list.push(req.string.slice(last, s.count(), vm)); - Ok(splitlist) + Ok(split_list) }) } @@ -444,7 +444,7 @@ mod _sre { with_sre_str!(zelf, &string, vm, |s| { let req = s.create_request(&zelf, 0, usize::MAX); let state = State::default(); - let mut sublist: Vec = Vec::new(); + let mut sub_list: Vec = Vec::new(); let mut iter = SearchIter { req, state }; let mut n = 0; let mut last_pos = 0; @@ -452,26 +452,26 @@ mod _sre { while (count == 0 || n < count) && iter.next().is_some() { if last_pos < iter.state.start { /* get segment before this match */ - sublist.push(s.slice(last_pos, iter.state.start, vm)); + sub_list.push(s.slice(last_pos, iter.state.start, vm)); } match &filter { - FilterType::Literal(literal) => sublist.push(literal.clone()), + FilterType::Literal(literal) => sub_list.push(literal.clone()), FilterType::Callable(callable) => { let m = Match::new(&mut iter.state, zelf.clone(), string.clone()) .into_ref(&vm.ctx); - sublist.push(callable.invoke((m,), vm)?); + sub_list.push(callable.invoke((m,), vm)?); } FilterType::Template(template) => { let m = Match::new(&mut iter.state, zelf.clone(), string.clone()); // template.expand(m)? // let mut list = vec![template.literal.clone()]; - sublist.push(template.literal.clone()); + sub_list.push(template.literal.clone()); for (index, literal) in template.items.iter().cloned() { if let Some(item) = m.get_slice(index, s, vm) { - sublist.push(item); + sub_list.push(item); } - sublist.push(literal); + sub_list.push(literal); } } }; @@ -481,9 +481,9 @@ mod _sre { } /* get segment following last match */ - sublist.push(s.slice(last_pos, iter.req.end, vm)); + sub_list.push(s.slice(last_pos, iter.req.end, vm)); - let list = PyList::from(sublist).into_pyobject(vm); + let list = PyList::from(sub_list).into_pyobject(vm); let join_type: PyObjectRef = if zelf.isbytes { vm.ctx.new_bytes(vec![]).into() diff --git a/vm/src/stdlib/time.rs b/vm/src/stdlib/time.rs index 10d51bd39a..5f41304c19 100644 --- a/vm/src/stdlib/time.rs +++ b/vm/src/stdlib/time.rs @@ -1,3 +1,4 @@ +//cspell:ignore cfmt //! The python `time` module. // See also: @@ -377,10 +378,10 @@ mod decl { #[cfg(any(windows, all(target_arch = "wasm32", target_os = "emscripten")))] pub(super) fn time_muldiv(ticks: i64, mul: i64, div: i64) -> u64 { - let intpart = ticks / div; + let int_part = ticks / div; let ticks = ticks % div; let remaining = (ticks * mul) / div; - (intpart * mul + remaining) as u64 + (int_part * mul + remaining) as u64 } #[cfg(all(target_arch = "wasm32", target_os = "emscripten"))] diff --git a/vm/src/vm/thread.rs b/vm/src/vm/thread.rs index 9d29bfae54..ea5a2d995a 100644 --- a/vm/src/vm/thread.rs +++ b/vm/src/vm/thread.rs @@ -39,13 +39,13 @@ pub fn with_vm(obj: &PyObject, f: F) -> Option where F: Fn(&VirtualMachine) -> R, { - let vm_owns_obj = |intp: NonNull| { + let vm_owns_obj = |interp: NonNull| { // SAFETY: all references in VM_STACK should be valid - let vm = unsafe { intp.as_ref() }; + let vm = unsafe { interp.as_ref() }; obj.fast_isinstance(vm.ctx.types.object_type) }; VM_STACK.with(|vms| { - let intp = match vms.borrow().iter().copied().exactly_one() { + let interp = match vms.borrow().iter().copied().exactly_one() { Ok(x) => { debug_assert!(vm_owns_obj(x)); x @@ -54,7 +54,7 @@ where }; // SAFETY: all references in VM_STACK should be valid, and should not be changed or moved // at least until this function returns and the stack unwinds to an enter_vm() call - let vm = unsafe { intp.as_ref() }; + let vm = unsafe { interp.as_ref() }; let prev = VM_CURRENT.with(|current| current.replace(vm)); let ret = f(vm); VM_CURRENT.with(|current| current.replace(prev)); diff --git a/vm/src/vm/vm_new.rs b/vm/src/vm/vm_new.rs index 3ceb783a48..9a7a7fe748 100644 --- a/vm/src/vm/vm_new.rs +++ b/vm/src/vm/vm_new.rs @@ -155,7 +155,7 @@ impl VirtualMachine { )) } - pub fn new_unsupported_binop_error( + pub fn new_unsupported_bin_op_error( &self, a: &PyObject, b: &PyObject, @@ -169,7 +169,7 @@ impl VirtualMachine { )) } - pub fn new_unsupported_ternop_error( + pub fn new_unsupported_ternary_op_error( &self, a: &PyObject, b: &PyObject, diff --git a/vm/src/vm/vm_ops.rs b/vm/src/vm/vm_ops.rs index 5235393a69..c6be959a60 100644 --- a/vm/src/vm/vm_ops.rs +++ b/vm/src/vm/vm_ops.rs @@ -206,7 +206,7 @@ impl VirtualMachine { if !result.is(&self.ctx.not_implemented) { return Ok(result); } - Err(self.new_unsupported_binop_error(a, b, op)) + Err(self.new_unsupported_bin_op_error(a, b, op)) } /// Binary in-place operators @@ -250,7 +250,7 @@ impl VirtualMachine { if !result.is(&self.ctx.not_implemented) { return Ok(result); } - Err(self.new_unsupported_binop_error(a, b, op)) + Err(self.new_unsupported_bin_op_error(a, b, op)) } fn ternary_op( @@ -384,7 +384,7 @@ impl VirtualMachine { return Ok(result); } } - Err(self.new_unsupported_binop_error(a, b, "+")) + Err(self.new_unsupported_bin_op_error(a, b, "+")) } pub fn _iadd(&self, a: &PyObject, b: &PyObject) -> PyResult { @@ -398,7 +398,7 @@ impl VirtualMachine { return Ok(result); } } - Err(self.new_unsupported_binop_error(a, b, "+=")) + Err(self.new_unsupported_bin_op_error(a, b, "+=")) } pub fn _mul(&self, a: &PyObject, b: &PyObject) -> PyResult { @@ -419,7 +419,7 @@ impl VirtualMachine { })?; return seq_b.repeat(n, self); } - Err(self.new_unsupported_binop_error(a, b, "*")) + Err(self.new_unsupported_bin_op_error(a, b, "*")) } pub fn _imul(&self, a: &PyObject, b: &PyObject) -> PyResult { @@ -448,7 +448,7 @@ impl VirtualMachine { * used. */ return seq_b.repeat(n, self); } - Err(self.new_unsupported_binop_error(a, b, "*=")) + Err(self.new_unsupported_bin_op_error(a, b, "*=")) } pub fn _abs(&self, a: &PyObject) -> PyResult { diff --git a/vm/sre_engine/src/engine.rs b/vm/sre_engine/src/engine.rs index bf0a6046fa..9b27f55031 100644 --- a/vm/sre_engine/src/engine.rs +++ b/vm/sre_engine/src/engine.rs @@ -549,7 +549,7 @@ fn _match(req: &Request<'_, S>, state: &mut State, mut ctx: MatchCo break 'result false; }; - let mut gctx = MatchContext { + let mut g_ctx = MatchContext { cursor: req.string.create_cursor(group_start), ..ctx }; @@ -557,12 +557,12 @@ fn _match(req: &Request<'_, S>, state: &mut State, mut ctx: MatchCo for _ in group_start..group_end { #[allow(clippy::redundant_closure_call)] if ctx.at_end(req) - || $f(ctx.peek_char::()) != $f(gctx.peek_char::()) + || $f(ctx.peek_char::()) != $f(g_ctx.peek_char::()) { break 'result false; } ctx.advance_char::(); - gctx.advance_char::(); + g_ctx.advance_char::(); } ctx.skip_code(2); @@ -627,8 +627,8 @@ fn _match(req: &Request<'_, S>, state: &mut State, mut ctx: MatchCo break 'context next_ctx; } SreOpcode::AT => { - let atcode = SreAtCode::try_from(ctx.peek_code(req, 1)).unwrap(); - if at(req, &ctx, atcode) { + let at_code = SreAtCode::try_from(ctx.peek_code(req, 1)).unwrap(); + if at(req, &ctx, at_code) { ctx.skip_code(2); } else { break 'result false; @@ -642,8 +642,8 @@ fn _match(req: &Request<'_, S>, state: &mut State, mut ctx: MatchCo continue 'context; } SreOpcode::CATEGORY => { - let catcode = SreCatCode::try_from(ctx.peek_code(req, 1)).unwrap(); - if ctx.at_end(req) || !category(catcode, ctx.peek_char::()) { + let cat_code = SreCatCode::try_from(ctx.peek_code(req, 1)).unwrap(); + if ctx.at_end(req) || !category(cat_code, ctx.peek_char::()) { break 'result false; } ctx.skip_code(2); @@ -1179,8 +1179,8 @@ impl MatchContext { } } -fn at(req: &Request<'_, S>, ctx: &MatchContext, atcode: SreAtCode) -> bool { - match atcode { +fn at(req: &Request<'_, S>, ctx: &MatchContext, at_code: SreAtCode) -> bool { + match at_code { SreAtCode::BEGINNING | SreAtCode::BEGINNING_STRING => ctx.at_beginning(), SreAtCode::BEGINNING_LINE => ctx.at_beginning() || is_linebreak(ctx.back_peek_char::()), SreAtCode::BOUNDARY => ctx.at_boundary(req, is_word), @@ -1210,8 +1210,8 @@ fn charset_loc_ignore(set: &[u32], c: u32) -> bool { up != lo && charset(set, up) } -fn category(catcode: SreCatCode, c: u32) -> bool { - match catcode { +fn category(cat_code: SreCatCode, c: u32) -> bool { + match cat_code { SreCatCode::DIGIT => is_digit(c), SreCatCode::NOT_DIGIT => !is_digit(c), SreCatCode::SPACE => is_space(c), @@ -1250,13 +1250,13 @@ fn charset(set: &[u32], ch: u32) -> bool { } SreOpcode::CATEGORY => { /* */ - let catcode = match SreCatCode::try_from(set[i + 1]) { + let cat_code = match SreCatCode::try_from(set[i + 1]) { Ok(code) => code, Err(_) => { break; } }; - if category(catcode, ch) { + if category(cat_code, ch) { return ok; } i += 2; @@ -1270,14 +1270,14 @@ fn charset(set: &[u32], ch: u32) -> bool { i += 1 + 8; } SreOpcode::BIGCHARSET => { - /* <256 blockindices> */ + /* <256 block_indices> */ let count = set[i + 1] as usize; if ch < 0x10000 { let set = &set[i + 2..]; let block_index = ch >> 8; - let (_, blockindices, _) = unsafe { set.align_to::() }; + let (_, block_indices, _) = unsafe { set.align_to::() }; let blocks = &set[64..]; - let block = blockindices[block_index as usize]; + let block = block_indices[block_index as usize]; if blocks[((block as u32 * 256 + (ch & 255)) / 32) as usize] & (1u32 << (ch & 31)) != 0 diff --git a/wtf8/src/lib.rs b/wtf8/src/lib.rs index ff4dcf8900..64ea42d06e 100644 --- a/wtf8/src/lib.rs +++ b/wtf8/src/lib.rs @@ -22,7 +22,7 @@ //! string has no surrogates, it can be viewed as a UTF-8 Rust [`str`] without //! needing any copies or re-encoding. //! -//! This implementation is mostly copied from the WTF-8 implentation in the +//! This implementation is mostly copied from the WTF-8 implementation in the //! Rust 1.85 standard library, which is used as the backing for [`OsStr`] on //! Windows targets. As previously mentioned, however, it is modified to not //! join two surrogates into one codepoint when concatenating strings, in order @@ -463,8 +463,8 @@ impl Wtf8Buf { pub fn pop(&mut self) -> Option { let ch = self.code_points().next_back()?; - let newlen = self.len() - ch.len_wtf8(); - self.bytes.truncate(newlen); + let new_len = self.len() - ch.len_wtf8(); + self.bytes.truncate(new_len); Some(ch) } From d7113e11db71419478b65c92a64389442b2e4f1b Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Fri, 4 Apr 2025 16:42:14 +0900 Subject: [PATCH 171/295] Fix more cspell warnings --- .cspell.dict/cpython.txt | 15 +++- .cspell.dict/python-more.txt | 36 +++++++++ .cspell.dict/rust-more.txt | 10 +++ .cspell.json | 8 ++ common/src/format.rs | 1 + common/src/os.rs | 4 +- common/src/static_cell.rs | 4 +- common/src/str.rs | 8 +- src/lib.rs | 8 +- stdlib/src/csv.rs | 22 +++--- stdlib/src/json/machinery.rs | 1 + stdlib/src/locale.rs | 2 +- stdlib/src/math.rs | 2 +- stdlib/src/multiprocessing.rs | 6 +- stdlib/src/overlapped.rs | 4 +- vm/src/anystr.rs | 4 +- vm/src/builtins/bytearray.rs | 2 +- vm/src/builtins/bytes.rs | 2 +- vm/src/builtins/dict.rs | 10 +-- vm/src/builtins/float.rs | 2 - vm/src/builtins/function.rs | 28 +++---- vm/src/builtins/genericalias.rs | 4 +- vm/src/builtins/int.rs | 2 +- vm/src/builtins/memory.rs | 2 +- vm/src/builtins/module.rs | 2 +- vm/src/builtins/set.rs | 6 +- vm/src/builtins/tuple.rs | 2 +- vm/src/builtins/type.rs | 4 +- vm/src/{bytesinner.rs => bytes_inner.rs} | 1 + vm/src/{dictdatatype.rs => dict_inner.rs} | 4 +- vm/src/function/fspath.rs | 8 +- vm/src/intern.rs | 2 +- vm/src/lib.rs | 4 +- vm/src/object/core.rs | 4 +- vm/src/object/traverse.rs | 2 +- vm/src/protocol/object.rs | 4 +- vm/src/protocol/sequence.rs | 4 +- vm/src/py_io.rs | 8 +- vm/src/scope.rs | 2 +- vm/src/stdlib/ast/python.rs | 6 +- vm/src/stdlib/io.rs | 6 +- vm/src/stdlib/itertools.rs | 8 +- vm/src/stdlib/marshal.rs | 1 + vm/src/stdlib/operator.rs | 6 +- vm/src/stdlib/os.rs | 10 +-- vm/src/stdlib/sre.rs | 16 ++-- vm/src/stdlib/sys.rs | 16 ++-- vm/src/stdlib/thread.rs | 16 ++-- vm/src/stdlib/time.rs | 4 +- vm/src/types/slot.rs | 90 +++++++++++------------ vm/src/vm/mod.rs | 6 +- vm/sre_engine/benches/benches.rs | 10 +-- vm/sre_engine/src/engine.rs | 2 +- vm/sre_engine/tests/tests.rs | 15 ++-- wasm/lib/src/convert.rs | 8 +- wasm/lib/src/js_module.rs | 6 +- wasm/lib/src/vm_class.rs | 2 +- 57 files changed, 268 insertions(+), 204 deletions(-) rename vm/src/{bytesinner.rs => bytes_inner.rs} (99%) rename vm/src/{dictdatatype.rs => dict_inner.rs} (99%) diff --git a/.cspell.dict/cpython.txt b/.cspell.dict/cpython.txt index f7e282e4bc..1840918a4d 100644 --- a/.cspell.dict/cpython.txt +++ b/.cspell.dict/cpython.txt @@ -6,15 +6,14 @@ badsyntax basetype boolop bxor +cached_tsver cellarg cellvar cellvars cmpop -weakreflist -XXPRIME +denom dictoffset elts -xstat excepthandler fileutils finalbody @@ -30,17 +29,25 @@ linearise maxdepth mult nkwargs +numer orelse +pathconfig patma posonlyarg posonlyargs prec +PYTHREAD_NAME +SA_ONSTACK stackdepth +tok_oldval unaryop unparse unparser VARKEYWORDS varkwarg wbits +weakreflist withitem -withs \ No newline at end of file +withs +xstat +XXPRIME \ No newline at end of file diff --git a/.cspell.dict/python-more.txt b/.cspell.dict/python-more.txt index 526f5ba166..2edfe95bdf 100644 --- a/.cspell.dict/python-more.txt +++ b/.cspell.dict/python-more.txt @@ -80,6 +80,7 @@ getpip getrandom getrecursionlimit getrefcount +getsizeof getweakrefcount getweakrefs getwindowsversion @@ -90,6 +91,7 @@ hamt hostnames idfunc idiv +idxs impls indexgroup infj @@ -99,6 +101,7 @@ irepeat isabstractmethod isbytes iscased +isfinal istext itemiterator itemsize @@ -111,9 +114,11 @@ kwargs kwdefaults kwonlyargcount lastgroup +lastindex linearization linearize listcomp +longrange lvalue mappingproxy maskpri @@ -137,25 +142,39 @@ nbytes ncallbacks ndigits ndim +nldecoder nlocals +NOARGS nonbytes +Nonprintable origname +ospath +pendingcr +phello +platlibdir posixsubprocess posonly posonlyargcount +prepending profilefunc +pycache pycodecs pycs pyexpat +PYTHONBREAKPOINT PYTHONDEBUG +PYTHONHASHSEED PYTHONHOME PYTHONINSPECT PYTHONOPTIMIZE PYTHONPATH PYTHONPATH +PYTHONSAFEPATH PYTHONVERBOSE +PYTHONWARNDEFAULTENCODING PYTHONWARNINGS pytraverse +PYVENV qualname quotetabs radd @@ -164,6 +183,7 @@ rdivmod readall readbuffer reconstructor +refcnt releaselevel reverseitemiterator reverseiterator @@ -178,23 +198,37 @@ rsub rtruediv rvalue scproxy +seennl setattro setcomp +setrecursionlimit showwarnmsg signum slotnames +STACKLESS stacklevel stacksize startpos +subclassable subclasscheck subclasshook suboffset +suboffsets +SUBPATTERN sumprod surrogateescape surrogatepass +sysconf sysconfigdata sysvars +teedata +thisclass titlecased +tobytes +tolist +toreadonly +TPFLAGS +tracefunc unimportable unionable unraisablehook @@ -208,7 +242,9 @@ warningregistry warnmsg warnoptions warnopts +weaklist weakproxy +weakrefs winver withdata xmlcharrefreplace diff --git a/.cspell.dict/rust-more.txt b/.cspell.dict/rust-more.txt index d75529789f..99e87e532c 100644 --- a/.cspell.dict/rust-more.txt +++ b/.cspell.dict/rust-more.txt @@ -1,4 +1,5 @@ ahash +arrayvec bidi biguint bindgen @@ -14,6 +15,8 @@ cranelift cstring datelike deserializer +fdiv +flamescope flate2 fract getres @@ -44,16 +47,21 @@ objclass peekable powc powf +powi prepended punct puruspe replacen +rmatch +rposition rsplitn rustc rustfmt rustyline +seedable seekfrom siphash +siphasher splitn subsec thiserror @@ -63,8 +71,10 @@ trai ulonglong unic unistd +unraw unsync wasmbind +wasmtime widestring winapi winsock diff --git a/.cspell.json b/.cspell.json index 99718a6515..caa53a2879 100644 --- a/.cspell.json +++ b/.cspell.json @@ -46,6 +46,7 @@ ], // words - list of words to be always considered correct "words": [ + "RUSTPYTHONPATH", // RustPython "aiterable", "alnum", @@ -69,9 +70,11 @@ "makeunicodedata", "miri", "notrace", + "openat", "pyarg", "pyarg", "pyargs", + "pyast", "PyAttr", "pyc", "PyClass", @@ -80,6 +83,7 @@ "PyFunction", "pygetset", "pyimpl", + "pylib", "pymember", "PyMethod", "PyModule", @@ -100,6 +104,7 @@ "richcompare", "RustPython", "struc", + "sysmodule", "tracebacks", "typealiases", "Unconstructible", @@ -112,11 +117,14 @@ "CLOEXEC", "codeset", "endgrent", + "gethrvtime", "getrusage", "nanosleep", + "sigaction", "WRLCK", // win32 "birthtime", + "IFEXEC", ], // flagWords - list of words to be always considered incorrect "flagWords": [ diff --git a/common/src/format.rs b/common/src/format.rs index 75d0996796..4c1ce6c5c2 100644 --- a/common/src/format.rs +++ b/common/src/format.rs @@ -1,3 +1,4 @@ +// cspell:ignore ddfe use itertools::{Itertools, PeekingNext}; use malachite_bigint::{BigInt, Sign}; use num_traits::FromPrimitive; diff --git a/common/src/os.rs b/common/src/os.rs index 06ea1432e9..d37f28d28a 100644 --- a/common/src/os.rs +++ b/common/src/os.rs @@ -62,13 +62,13 @@ pub fn last_posix_errno() -> i32 { } #[cfg(unix)] -pub fn bytes_as_osstr(b: &[u8]) -> Result<&std::ffi::OsStr, Utf8Error> { +pub fn bytes_as_os_str(b: &[u8]) -> Result<&std::ffi::OsStr, Utf8Error> { use std::os::unix::ffi::OsStrExt; Ok(std::ffi::OsStr::from_bytes(b)) } #[cfg(not(unix))] -pub fn bytes_as_osstr(b: &[u8]) -> Result<&std::ffi::OsStr, Utf8Error> { +pub fn bytes_as_os_str(b: &[u8]) -> Result<&std::ffi::OsStr, Utf8Error> { Ok(std::str::from_utf8(b)?.as_ref()) } diff --git a/common/src/static_cell.rs b/common/src/static_cell.rs index 407b83ae0a..30e34f187f 100644 --- a/common/src/static_cell.rs +++ b/common/src/static_cell.rs @@ -76,7 +76,7 @@ mod threading { impl StaticCell { #[doc(hidden)] - pub const fn _from_oncecell(inner: OnceCell) -> Self { + pub const fn _from_once_cell(inner: OnceCell) -> Self { Self { inner } } @@ -108,7 +108,7 @@ mod threading { ($($(#[$attr:meta])* $vis:vis static $name:ident: $t:ty;)+) => { $($(#[$attr])* $vis static $name: $crate::static_cell::StaticCell<$t> = - $crate::static_cell::StaticCell::_from_oncecell($crate::lock::OnceCell::new());)+ + $crate::static_cell::StaticCell::_from_once_cell($crate::lock::OnceCell::new());)+ }; } } diff --git a/common/src/str.rs b/common/src/str.rs index ca1723e7ef..fa26959e0b 100644 --- a/common/src/str.rs +++ b/common/src/str.rs @@ -360,8 +360,8 @@ pub fn get_chars(s: &str, range: impl RangeBounds) -> &str { } #[inline] -pub fn char_range_end(s: &str, nchars: usize) -> Option { - let i = match nchars.checked_sub(1) { +pub fn char_range_end(s: &str, n_chars: usize) -> Option { + let i = match n_chars.checked_sub(1) { Some(last_char_index) => { let (index, c) = s.char_indices().nth(last_char_index)?; index + c.len_utf8() @@ -395,8 +395,8 @@ pub fn get_codepoints(w: &Wtf8, range: impl RangeBounds) -> &Wtf8 { } #[inline] -pub fn codepoint_range_end(s: &Wtf8, nchars: usize) -> Option { - let i = match nchars.checked_sub(1) { +pub fn codepoint_range_end(s: &Wtf8, n_chars: usize) -> Option { + let i = match n_chars.checked_sub(1) { Some(last_char_index) => { let (index, c) = s.code_point_indices().nth(last_char_index)?; index + c.len_wtf8() diff --git a/src/lib.rs b/src/lib.rs index 3fa5292e94..e415d847cf 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -233,14 +233,14 @@ fn write_profile(settings: &Settings) -> Result<(), Box> enum ProfileFormat { Html, Text, - Speedscope, + SpeedScore, } let profile_output = settings.profile_output.as_deref(); let profile_format = match settings.profile_format.as_deref() { Some("html") => ProfileFormat::Html, Some("text") => ProfileFormat::Text, None if profile_output == Some("-".as_ref()) => ProfileFormat::Text, - Some("speedscope") | None => ProfileFormat::Speedscope, + Some("speedscope") | None => ProfileFormat::SpeedScore, Some(other) => { error!("Unknown profile format {}", other); // TODO: Need to change to ExitCode or Termination @@ -251,7 +251,7 @@ fn write_profile(settings: &Settings) -> Result<(), Box> let profile_output = profile_output.unwrap_or_else(|| match profile_format { ProfileFormat::Html => "flame-graph.html".as_ref(), ProfileFormat::Text => "flame.txt".as_ref(), - ProfileFormat::Speedscope => "flamescope.json".as_ref(), + ProfileFormat::SpeedScore => "flamescope.json".as_ref(), }); let profile_output: Box = if profile_output == "-" { @@ -265,7 +265,7 @@ fn write_profile(settings: &Settings) -> Result<(), Box> match profile_format { ProfileFormat::Html => flame::dump_html(profile_output)?, ProfileFormat::Text => flame::dump_text_to_writer(profile_output)?, - ProfileFormat::Speedscope => flamescope::dump(profile_output)?, + ProfileFormat::SpeedScore => flamescope::dump(profile_output)?, } Ok(()) diff --git a/stdlib/src/csv.rs b/stdlib/src/csv.rs index 39c15fd952..730d3b2feb 100644 --- a/stdlib/src/csv.rs +++ b/stdlib/src/csv.rs @@ -277,7 +277,7 @@ mod _csv { .map_err(|_| vm.new_type_error("argument 1 must be a dialect object".to_owned()))?, OptionalArg::Missing => opts.result(vm)?, }; - let dialect = opts.update_pydialect(dialect); + let dialect = opts.update_py_dialect(dialect); GLOBAL_HASHMAP .lock() .insert(name.as_str().to_owned(), dialect); @@ -665,7 +665,7 @@ mod _csv { } impl FormatOptions { - fn update_pydialect(&self, mut res: PyDialect) -> PyDialect { + fn update_py_dialect(&self, mut res: PyDialect) -> PyDialect { macro_rules! check_and_fill { ($res:ident, $e:ident) => {{ if let Some(t) = self.$e { @@ -699,18 +699,18 @@ mod _csv { DialectItem::Str(name) => { let g = GLOBAL_HASHMAP.lock(); if let Some(dialect) = g.get(name) { - Ok(self.update_pydialect(*dialect)) + Ok(self.update_py_dialect(*dialect)) } else { - Err(new_csv_error(vm, format!("{} is not registed.", name))) + Err(new_csv_error(vm, format!("{} is not registered.", name))) } // TODO // Maybe need to update the obj from HashMap } - DialectItem::Obj(o) => Ok(self.update_pydialect(*o)), + DialectItem::Obj(o) => Ok(self.update_py_dialect(*o)), DialectItem::None => { let g = GLOBAL_HASHMAP.lock(); let res = *g.get("excel").unwrap(); - Ok(self.update_pydialect(res)) + Ok(self.update_py_dialect(res)) } } } @@ -1084,8 +1084,8 @@ mod _csv { macro_rules! handle_res { ($x:expr) => {{ - let (res, nwritten) = $x; - buffer_offset += nwritten; + let (res, n_written) = $x; + buffer_offset += n_written; match res { csv_core::WriteResult::InputEmpty => break, csv_core::WriteResult::OutputFull => resize_buf(buffer), @@ -1118,10 +1118,10 @@ mod _csv { } loop { - let (res, nread, nwritten) = + let (res, n_read, n_written) = writer.field(&data[input_offset..], &mut buffer[buffer_offset..]); - input_offset += nread; - handle_res!((res, nwritten)); + input_offset += n_read; + handle_res!((res, n_written)); } } diff --git a/stdlib/src/json/machinery.rs b/stdlib/src/json/machinery.rs index 4612b5263d..a4344e363c 100644 --- a/stdlib/src/json/machinery.rs +++ b/stdlib/src/json/machinery.rs @@ -1,3 +1,4 @@ +// cspell:ignore LOJKINE // derived from https://github.com/lovasoa/json_in_type // BSD 2-Clause License diff --git a/stdlib/src/locale.rs b/stdlib/src/locale.rs index dfc6c93497..6cde173fb1 100644 --- a/stdlib/src/locale.rs +++ b/stdlib/src/locale.rs @@ -1,4 +1,4 @@ -// cspell:ignore abday abmon yesexpr +// cspell:ignore abday abmon yesexpr noexpr CRNCYSTR RADIXCHAR AMPM THOUSEP pub(crate) use _locale::make_module; diff --git a/stdlib/src/math.rs b/stdlib/src/math.rs index 93929e3566..6665ee8b49 100644 --- a/stdlib/src/math.rs +++ b/stdlib/src/math.rs @@ -652,7 +652,7 @@ mod math { partials.truncate(i); if x != 0.0 { if !x.is_finite() { - // a nonfinite x could arise either as + // a non-finite x could arise either as // a result of intermediate overflow, or // as a result of a nan or inf in the // summands diff --git a/stdlib/src/multiprocessing.rs b/stdlib/src/multiprocessing.rs index 2db922e16b..4a98c1afad 100644 --- a/stdlib/src/multiprocessing.rs +++ b/stdlib/src/multiprocessing.rs @@ -19,12 +19,12 @@ mod _multiprocessing { #[pyfunction] fn recv(socket: usize, size: usize, vm: &VirtualMachine) -> PyResult { let mut buf = vec![0; size]; - let nread = + let n_read = unsafe { WinSock::recv(socket as SOCKET, buf.as_mut_ptr() as *mut _, size as i32, 0) }; - if nread < 0 { + if n_read < 0 { Err(os::errno_err(vm)) } else { - Ok(nread) + Ok(n_read) } } diff --git a/stdlib/src/overlapped.rs b/stdlib/src/overlapped.rs index 007fa67423..6fd8a1516d 100644 --- a/stdlib/src/overlapped.rs +++ b/stdlib/src/overlapped.rs @@ -184,14 +184,14 @@ mod _overlapped { buf: buf.as_ptr() as *mut _, len: buf.len() as _, }; - let mut nread: u32 = 0; + let mut n_read: u32 = 0; // TODO: optimization with MaybeUninit let ret = unsafe { windows_sys::Win32::Networking::WinSock::WSARecv( handle as _, &wsabuf, 1, - &mut nread, + &mut n_read, &mut flags, &mut inner.overlapped, None, diff --git a/vm/src/anystr.rs b/vm/src/anystr.rs index 79b15b6a3f..03582215ba 100644 --- a/vm/src/anystr.rs +++ b/vm/src/anystr.rs @@ -167,7 +167,7 @@ pub trait AnyStr { full_obj: impl FnOnce() -> PyObjectRef, split: SP, splitn: SN, - splitw: SW, + split_whitespace: SW, ) -> PyResult> where T: TryFromObject + AnyStrWrapper, @@ -188,7 +188,7 @@ pub trait AnyStr { splitn(self, pattern, (args.maxsplit + 1) as usize, vm) } } else { - splitw(self, args.maxsplit, vm) + split_whitespace(self, args.maxsplit, vm) }; Ok(splits) } diff --git a/vm/src/builtins/bytearray.rs b/vm/src/builtins/bytearray.rs index 3d4822cf48..ce2232d8eb 100644 --- a/vm/src/builtins/bytearray.rs +++ b/vm/src/builtins/bytearray.rs @@ -9,7 +9,7 @@ use crate::{ anystr::{self, AnyStr}, atomic_func, byte::{bytes_from_object, value_from_object}, - bytesinner::{ + bytes_inner::{ ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions, ByteInnerSplitOptions, ByteInnerTranslateOptions, DecodeArgs, PyBytesInner, bytes_decode, }, diff --git a/vm/src/builtins/bytes.rs b/vm/src/builtins/bytes.rs index 434de6a76c..77b9f9d526 100644 --- a/vm/src/builtins/bytes.rs +++ b/vm/src/builtins/bytes.rs @@ -6,7 +6,7 @@ use crate::{ TryFromBorrowedObject, TryFromObject, VirtualMachine, anystr::{self, AnyStr}, atomic_func, - bytesinner::{ + bytes_inner::{ ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions, ByteInnerSplitOptions, ByteInnerTranslateOptions, DecodeArgs, PyBytesInner, bytes_decode, }, diff --git a/vm/src/builtins/dict.rs b/vm/src/builtins/dict.rs index fc2f206dd0..f78543a5f5 100644 --- a/vm/src/builtins/dict.rs +++ b/vm/src/builtins/dict.rs @@ -12,7 +12,7 @@ use crate::{ }, class::{PyClassDef, PyClassImpl}, common::ascii, - dictdatatype::{self, DictKey}, + dict_inner::{self, DictKey}, function::{ArgIterable, KwArgs, OptionalArg, PyArithmeticValue::*, PyComparisonValue}, iter::PyExactSizeIterator, protocol::{PyIterIter, PyIterReturn, PyMappingMethods, PyNumberMethods, PySequenceMethods}, @@ -27,7 +27,7 @@ use rustpython_common::lock::PyMutex; use std::fmt; use std::sync::LazyLock; -pub type DictContentType = dictdatatype::Dict; +pub type DictContentType = dict_inner::Dict; #[pyclass(module = false, name = "dict", unhashable = true, traverse)] #[derive(Default)] @@ -154,7 +154,7 @@ impl PyDict { self.entries.contains(vm, key).unwrap() } - pub fn size(&self) -> dictdatatype::DictSize { + pub fn size(&self) -> dict_inner::DictSize { self.entries.size() } } @@ -811,7 +811,7 @@ macro_rules! dict_view { #[pyclass(module = false, name = $iter_class_name)] #[derive(Debug)] pub(crate) struct $iter_name { - pub size: dictdatatype::DictSize, + pub size: dict_inner::DictSize, pub internal: PyMutex>, } @@ -884,7 +884,7 @@ macro_rules! dict_view { #[pyclass(module = false, name = $reverse_iter_class_name)] #[derive(Debug)] pub(crate) struct $reverse_iter_name { - pub size: dictdatatype::DictSize, + pub size: dict_inner::DictSize, internal: PyMutex>, } diff --git a/vm/src/builtins/float.rs b/vm/src/builtins/float.rs index 27f1f3273f..85f2a07bb9 100644 --- a/vm/src/builtins/float.rs +++ b/vm/src/builtins/float.rs @@ -1,5 +1,3 @@ -// spell-checker:ignore numer denom - use super::{ PyByteArray, PyBytes, PyInt, PyIntRef, PyStr, PyStrRef, PyType, PyTypeRef, try_bigint_to_f64, }; diff --git a/vm/src/builtins/function.rs b/vm/src/builtins/function.rs index 3181f1068f..e054ac4348 100644 --- a/vm/src/builtins/function.rs +++ b/vm/src/builtins/function.rs @@ -90,13 +90,13 @@ impl PyFunction { ) -> PyResult<()> { let code = &*self.code; let nargs = func_args.args.len(); - let nexpected_args = code.arg_count as usize; + let n_expected_args = code.arg_count as usize; let total_args = code.arg_count as usize + code.kwonlyarg_count as usize; // let arg_names = self.code.arg_names(); // This parses the arguments from args and kwargs into // the proper variables keeping into account default values - // and starargs and kwargs. + // and star-args and kwargs. // See also: PyEval_EvalCodeWithName in cpython: // https://github.com/python/cpython/blob/main/Python/ceval.c#L3681 @@ -108,7 +108,7 @@ impl PyFunction { // zip short-circuits if either iterator returns None, which is the behavior we want -- // only fill as much as there is to fill with as much as we have for (local, arg) in Iterator::zip( - fastlocals.iter_mut().take(nexpected_args), + fastlocals.iter_mut().take(n_expected_args), args_iter.by_ref().take(nargs), ) { *local = Some(arg); @@ -122,11 +122,11 @@ impl PyFunction { vararg_offset += 1; } else { // Check the number of positional arguments - if nargs > nexpected_args { + if nargs > n_expected_args { return Err(vm.new_type_error(format!( "{}() takes {} positional arguments but {} were given", self.qualname(), - nexpected_args, + n_expected_args, nargs ))); } @@ -141,7 +141,7 @@ impl PyFunction { None }; - let argpos = |range: std::ops::Range<_>, name: &str| { + let arg_pos = |range: std::ops::Range<_>, name: &str| { code.varnames .iter() .enumerate() @@ -155,7 +155,7 @@ impl PyFunction { // Handle keyword arguments for (name, value) in func_args.kwargs { // Check if we have a parameter with this name: - if let Some(pos) = argpos(code.posonlyarg_count as usize..total_args, &name) { + if let Some(pos) = arg_pos(code.posonlyarg_count as usize..total_args, &name) { let slot = &mut fastlocals[pos]; if slot.is_some() { return Err(vm.new_type_error(format!( @@ -167,7 +167,7 @@ impl PyFunction { *slot = Some(value); } else if let Some(kwargs) = kwargs.as_ref() { kwargs.set_item(&name, value, vm)?; - } else if argpos(0..code.posonlyarg_count as usize, &name).is_some() { + } else if arg_pos(0..code.posonlyarg_count as usize, &name).is_some() { posonly_passed_as_kwarg.push(name); } else { return Err(vm.new_type_error(format!( @@ -196,15 +196,15 @@ impl PyFunction { // Add missing positional arguments, if we have fewer positional arguments than the // function definition calls for - if nargs < nexpected_args { + if nargs < n_expected_args { let defaults = get_defaults!().0.as_ref().map(|tup| tup.as_slice()); let n_defs = defaults.map_or(0, |d| d.len()); - let nrequired = code.arg_count as usize - n_defs; + let n_required = code.arg_count as usize - n_defs; // Given the number of defaults available, check all the arguments for which we // _don't_ have defaults; if any are missing, raise an exception - let mut missing: Vec<_> = (nargs..nrequired) + let mut missing: Vec<_> = (nargs..n_required) .filter_map(|i| { if fastlocals[i].is_none() { Some(&code.varnames[i]) @@ -247,13 +247,13 @@ impl PyFunction { } if let Some(defaults) = defaults { - let n = std::cmp::min(nargs, nexpected_args); - let i = n.saturating_sub(nrequired); + let n = std::cmp::min(nargs, n_expected_args); + let i = n.saturating_sub(n_required); // We have sufficient defaults, so iterate over the corresponding names and use // the default if we don't already have a value for i in i..defaults.len() { - let slot = &mut fastlocals[nrequired + i]; + let slot = &mut fastlocals[n_required + i]; if slot.is_none() { *slot = Some(defaults[i].clone()); } diff --git a/vm/src/builtins/genericalias.rs b/vm/src/builtins/genericalias.rs index 549985bcfb..18649718dd 100644 --- a/vm/src/builtins/genericalias.rs +++ b/vm/src/builtins/genericalias.rs @@ -253,7 +253,7 @@ fn tuple_index(tuple: &PyTupleRef, item: &PyObjectRef) -> Option { fn subs_tvars( obj: PyObjectRef, params: &PyTupleRef, - argitems: &[PyObjectRef], + arg_items: &[PyObjectRef], vm: &VirtualMachine, ) -> PyResult { obj.get_attr(identifier!(vm, __parameters__), vm) @@ -267,7 +267,7 @@ fn subs_tvars( .iter() .map(|arg| { if let Some(idx) = tuple_index(params, arg) { - argitems[idx].clone() + arg_items[idx].clone() } else { arg.clone() } diff --git a/vm/src/builtins/int.rs b/vm/src/builtins/int.rs index 5f12f2490e..80aaae03eb 100644 --- a/vm/src/builtins/int.rs +++ b/vm/src/builtins/int.rs @@ -3,7 +3,7 @@ use crate::{ AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyRefExact, PyResult, TryFromBorrowedObject, VirtualMachine, builtins::PyStrRef, - bytesinner::PyBytesInner, + bytes_inner::PyBytesInner, class::PyClassImpl, common::{ format::FormatSpec, diff --git a/vm/src/builtins/memory.rs b/vm/src/builtins/memory.rs index 09239e3e49..801d94fb36 100644 --- a/vm/src/builtins/memory.rs +++ b/vm/src/builtins/memory.rs @@ -6,7 +6,7 @@ use crate::{ AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromBorrowedObject, TryFromObject, VirtualMachine, atomic_func, buffer::FormatSpec, - bytesinner::bytes_to_hex, + bytes_inner::bytes_to_hex, class::PyClassImpl, common::{ borrow::{BorrowedValue, BorrowedValueMut}, diff --git a/vm/src/builtins/module.rs b/vm/src/builtins/module.rs index 8c8f22cf58..2cdc13a59c 100644 --- a/vm/src/builtins/module.rs +++ b/vm/src/builtins/module.rs @@ -122,7 +122,7 @@ impl Py { name.downcast::().ok() } - // TODO: to be replaced by the commented-out dict method above once dictoffsets land + // TODO: to be replaced by the commented-out dict method above once dictoffset land pub fn dict(&self) -> PyDictRef { self.as_object().dict().unwrap() } diff --git a/vm/src/builtins/set.rs b/vm/src/builtins/set.rs index 3e10e5c6b7..43e6ee1f7d 100644 --- a/vm/src/builtins/set.rs +++ b/vm/src/builtins/set.rs @@ -11,7 +11,7 @@ use crate::{ class::PyClassImpl, common::{ascii, hash::PyHash, lock::PyMutex, rc::PyRc}, convert::ToPyResult, - dictdatatype::{self, DictSize}, + dict_inner::{self, DictSize}, function::{ArgIterable, OptionalArg, PosArgs, PyArithmeticValue, PyComparisonValue}, protocol::{PyIterReturn, PyNumberMethods, PySequenceMethods}, recursion::ReprGuard, @@ -30,7 +30,7 @@ use rustpython_common::{ use std::sync::LazyLock; use std::{fmt, ops::Deref}; -pub type SetContentType = dictdatatype::Dict<()>; +pub type SetContentType = dict_inner::Dict<()>; #[pyclass(module = false, name = "set", unhashable = true, traverse)] #[derive(Default)] @@ -460,7 +460,7 @@ impl PySetInner { hash = self.content.try_fold_keys(hash, |h, element| { Ok(h ^ _shuffle_bits(element.hash(vm)? as u64)) })?; - // Disperse patterns arising in nested frozensets + // Disperse patterns arising in nested frozen-sets hash ^= (hash >> 11) ^ (hash >> 25); hash = hash.wrapping_mul(69069).wrapping_add(907133923); // -1 is reserved as an error code diff --git a/vm/src/builtins/tuple.rs b/vm/src/builtins/tuple.rs index 1b6e281657..1dc7861071 100644 --- a/vm/src/builtins/tuple.rs +++ b/vm/src/builtins/tuple.rs @@ -332,7 +332,7 @@ impl PyTuple { #[pymethod(magic)] fn getnewargs(zelf: PyRef, vm: &VirtualMachine) -> (PyTupleRef,) { // the arguments to pass to tuple() is just one tuple - so we'll be doing tuple(tup), which - // should just return tup, or tuplesubclass(tup), which'll copy/validate (e.g. for a + // should just return tup, or tuple_subclass(tup), which'll copy/validate (e.g. for a // structseq) let tup_arg = if zelf.class().is(vm.ctx.types.tuple_type) { zelf diff --git a/vm/src/builtins/type.rs b/vm/src/builtins/type.rs index 776c777cb3..a0c7ee58ff 100644 --- a/vm/src/builtins/type.rs +++ b/vm/src/builtins/type.rs @@ -320,7 +320,7 @@ impl PyType { } } - // This is used for class initialisation where the vm is not yet available. + // This is used for class initialization where the vm is not yet available. pub fn set_str_attr>( &self, attr_name: &str, @@ -451,7 +451,7 @@ impl Py { F: Fn(&Self) -> Option, { // the hot path will be primitive types which usually hit the result from itself. - // try std::intrinsics::likely once it is stablized + // try std::intrinsics::likely once it is stabilized if let Some(r) = f(self) { Some(r) } else { diff --git a/vm/src/bytesinner.rs b/vm/src/bytes_inner.rs similarity index 99% rename from vm/src/bytesinner.rs rename to vm/src/bytes_inner.rs index 88d2b9744f..10394721e7 100644 --- a/vm/src/bytesinner.rs +++ b/vm/src/bytes_inner.rs @@ -1,3 +1,4 @@ +// cspell:ignore unchunked use crate::{ AsObject, PyObject, PyObjectRef, PyPayload, PyResult, TryFromBorrowedObject, VirtualMachine, anystr::{self, AnyStr, AnyStrContainer, AnyStrWrapper}, diff --git a/vm/src/dictdatatype.rs b/vm/src/dict_inner.rs similarity index 99% rename from vm/src/dictdatatype.rs rename to vm/src/dict_inner.rs index ab37b7dc85..c49ab752de 100644 --- a/vm/src/dictdatatype.rs +++ b/vm/src/dict_inner.rs @@ -20,7 +20,7 @@ use num_traits::ToPrimitive; use std::{fmt, mem::size_of, ops::ControlFlow}; // HashIndex is intended to be same size with hash::PyHash -// but it doesn't mean the values are compatible with actual pyhash value +// but it doesn't mean the values are compatible with actual PyHash value /// hash value of an object returned by __hash__ type HashValue = hash::PyHash; @@ -691,7 +691,7 @@ impl Dict { type LookupResult = (IndexEntry, IndexIndex); /// Types implementing this trait can be used to index -/// the dictionary. Typical usecases are: +/// the dictionary. Typical use-cases are: /// - PyObjectRef -> arbitrary python type used as key /// - str -> string reference used as key, this is often used internally pub trait DictKey { diff --git a/vm/src/function/fspath.rs b/vm/src/function/fspath.rs index 74051644e0..28145e490a 100644 --- a/vm/src/function/fspath.rs +++ b/vm/src/function/fspath.rs @@ -62,7 +62,7 @@ impl FsPath { // TODO: FS encodings match self { FsPath::Str(s) => vm.fsencode(s), - FsPath::Bytes(b) => Self::bytes_as_osstr(b.as_bytes(), vm).map(Cow::Borrowed), + FsPath::Bytes(b) => Self::bytes_as_os_str(b.as_bytes(), vm).map(Cow::Borrowed), } } @@ -84,7 +84,7 @@ impl FsPath { pub fn to_path_buf(&self, vm: &VirtualMachine) -> PyResult { let path = match self { FsPath::Str(s) => PathBuf::from(s.as_str()), - FsPath::Bytes(b) => PathBuf::from(Self::bytes_as_osstr(b, vm)?), + FsPath::Bytes(b) => PathBuf::from(Self::bytes_as_os_str(b, vm)?), }; Ok(path) } @@ -99,8 +99,8 @@ impl FsPath { .map_err(|err| err.into_pyexception(vm)) } - pub fn bytes_as_osstr<'a>(b: &'a [u8], vm: &VirtualMachine) -> PyResult<&'a std::ffi::OsStr> { - rustpython_common::os::bytes_as_osstr(b) + pub fn bytes_as_os_str<'a>(b: &'a [u8], vm: &VirtualMachine) -> PyResult<&'a std::ffi::OsStr> { + rustpython_common::os::bytes_as_os_str(b) .map_err(|_| vm.new_unicode_decode_error("can't decode path for utf-8".to_owned())) } } diff --git a/vm/src/intern.rs b/vm/src/intern.rs index bb9220d069..08e41bb5b5 100644 --- a/vm/src/intern.rs +++ b/vm/src/intern.rs @@ -281,7 +281,7 @@ impl InternableString for PyRefExact { } pub trait MaybeInternedString: - AsRef + crate::dictdatatype::DictKey + sealed::SealedMaybeInterned + AsRef + crate::dict_inner::DictKey + sealed::SealedMaybeInterned { fn as_interned(&self) -> Option<&'static PyStrInterned>; } diff --git a/vm/src/lib.rs b/vm/src/lib.rs index 2e4afa3ea1..9561a9cc23 100644 --- a/vm/src/lib.rs +++ b/vm/src/lib.rs @@ -43,14 +43,14 @@ mod anystr; pub mod buffer; pub mod builtins; pub mod byte; -mod bytesinner; +mod bytes_inner; pub mod cformat; pub mod class; mod codecs; pub mod compiler; pub mod convert; mod coroutine; -mod dictdatatype; +mod dict_inner; #[cfg(feature = "rustpython-compiler")] pub mod eval; pub mod exceptions; diff --git a/vm/src/object/core.rs b/vm/src/object/core.rs index bbe900f7cd..a6049884d8 100644 --- a/vm/src/object/core.rs +++ b/vm/src/object/core.rs @@ -370,11 +370,11 @@ impl PyWeak { let dealloc = { let mut guard = unsafe { self.parent.as_ref().lock() }; let offset = std::mem::offset_of!(PyInner, payload); - let pyinner = (self as *const Self) + let py_inner = (self as *const Self) .cast::() .wrapping_sub(offset) .cast::>(); - let node_ptr = unsafe { NonNull::new_unchecked(pyinner as *mut Py) }; + let node_ptr = unsafe { NonNull::new_unchecked(py_inner as *mut Py) }; // the list doesn't have ownership over its PyRef! we're being dropped // right now so that should be obvious!! std::mem::forget(unsafe { guard.list.remove(node_ptr) }); diff --git a/vm/src/object/traverse.rs b/vm/src/object/traverse.rs index 9ff0f88343..c105d23feb 100644 --- a/vm/src/object/traverse.rs +++ b/vm/src/object/traverse.rs @@ -26,7 +26,7 @@ pub unsafe trait Traverse { /// If some field is not called, the worst results is just memory leak, /// but if some field is called repeatedly, panic and deadlock can happen. /// - /// - _**DO NOT**_ clone a `PyObjectRef` or `Pyef` in `traverse()` + /// - _**DO NOT**_ clone a `PyObjectRef` or `PyRef` in `traverse()` fn traverse(&self, traverse_fn: &mut TraverseFn<'_>); } diff --git a/vm/src/protocol/object.rs b/vm/src/protocol/object.rs index 256baa0fdf..1e972eb540 100644 --- a/vm/src/protocol/object.rs +++ b/vm/src/protocol/object.rs @@ -7,10 +7,10 @@ use crate::{ PyAsyncGen, PyBytes, PyDict, PyDictRef, PyGenericAlias, PyInt, PyList, PyStr, PyStrRef, PyTuple, PyTupleRef, PyType, PyTypeRef, pystr::AsPyStr, }, - bytesinner::ByteInnerNewOptions, + bytes_inner::ByteInnerNewOptions, common::{hash::PyHash, str::to_ascii}, convert::{ToPyObject, ToPyResult}, - dictdatatype::DictKey, + dict_inner::DictKey, function::{Either, OptionalArg, PyArithmeticValue, PySetterValue}, object::PyPayload, protocol::{PyIter, PyMapping, PySequence}, diff --git a/vm/src/protocol/sequence.rs b/vm/src/protocol/sequence.rs index 5d5622c156..0681c3e664 100644 --- a/vm/src/protocol/sequence.rs +++ b/vm/src/protocol/sequence.rs @@ -118,7 +118,7 @@ impl PySequence<'_> { return f(self, other, vm); } - // if both arguments apear to be sequences, try fallback to __add__ + // if both arguments appear to be sequences, try fallback to __add__ if self.check() && other.to_sequence().check() { let ret = vm.binary_op1(self.obj, other, PyNumberBinaryOp::Add)?; if let PyArithmeticValue::Implemented(ret) = PyArithmeticValue::from_object(vm, ret) { @@ -156,7 +156,7 @@ impl PySequence<'_> { return f(self, other, vm); } - // if both arguments apear to be sequences, try fallback to __iadd__ + // if both arguments appear to be sequences, try fallback to __iadd__ if self.check() && other.to_sequence().check() { let ret = vm._iadd(self.obj, other)?; if let PyArithmeticValue::Implemented(ret) = PyArithmeticValue::from_object(vm, ret) { diff --git a/vm/src/py_io.rs b/vm/src/py_io.rs index c50f09e2bf..87df9a73d8 100644 --- a/vm/src/py_io.rs +++ b/vm/src/py_io.rs @@ -70,12 +70,12 @@ pub fn file_readline(obj: &PyObject, size: Option, vm: &VirtualMachine) - }; let ret = match_class!(match ret { s @ PyStr => { - let sval = s.as_str(); - if sval.is_empty() { + let s_val = s.as_str(); + if s_val.is_empty() { return Err(eof_err()); } - if let Some(nonl) = sval.strip_suffix('\n') { - vm.ctx.new_str(nonl).into() + if let Some(no_nl) = s_val.strip_suffix('\n') { + vm.ctx.new_str(no_nl).into() } else { s.into() } diff --git a/vm/src/scope.rs b/vm/src/scope.rs index e01209857c..7515468d78 100644 --- a/vm/src/scope.rs +++ b/vm/src/scope.rs @@ -141,7 +141,7 @@ impl Scope { // impl Sealed for super::PyStrRef {} // } // pub trait PyName: -// sealed::Sealed + crate::dictdatatype::DictKey + Clone + ToPyObject +// sealed::Sealed + crate::dict_inner::DictKey + Clone + ToPyObject // { // } // impl PyName for str {} diff --git a/vm/src/stdlib/ast/python.rs b/vm/src/stdlib/ast/python.rs index 50f8294c76..74c4db888a 100644 --- a/vm/src/stdlib/ast/python.rs +++ b/vm/src/stdlib/ast/python.rs @@ -19,8 +19,8 @@ pub(crate) mod _ast { fn init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { let fields = zelf.get_attr("_fields", vm)?; let fields: Vec = fields.try_to_value(vm)?; - let numargs = args.args.len(); - if numargs > fields.len() { + let n_args = args.args.len(); + if n_args > fields.len() { return Err(vm.new_type_error(format!( "{} constructor takes at most {} positional argument{}", zelf.class().name(), @@ -33,7 +33,7 @@ pub(crate) mod _ast { } for (key, value) in args.kwargs { if let Some(pos) = fields.iter().position(|f| f.as_str() == key) { - if pos < numargs { + if pos < n_args { return Err(vm.new_type_error(format!( "{} got multiple values for argument '{}'", zelf.class().name(), diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index 6f13e0878d..3e1979e3d0 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -1360,7 +1360,7 @@ mod _io { }) } - pub fn repr_fileobj_name(obj: &PyObject, vm: &VirtualMachine) -> PyResult> { + pub fn repr_file_obj_name(obj: &PyObject, vm: &VirtualMachine) -> PyResult> { let name = match obj.get_attr("name", vm) { Ok(name) => Some(name), Err(e) @@ -1549,7 +1549,7 @@ mod _io { #[pyslot] fn slot_repr(zelf: &PyObject, vm: &VirtualMachine) -> PyResult { - let name_repr = repr_fileobj_name(zelf, vm)?; + let name_repr = repr_file_obj_name(zelf, vm)?; let cls = zelf.class(); let slot_name = cls.slot_name(); let repr = if let Some(name_repr) = name_repr { @@ -4293,7 +4293,7 @@ mod fileio { if fd < 0 { return Ok("<_io.FileIO [closed]>".to_owned()); } - let name_repr = repr_fileobj_name(zelf.as_object(), vm)?; + let name_repr = repr_file_obj_name(zelf.as_object(), vm)?; let mode = zelf.mode(); let closefd = if zelf.closefd.load() { "True" } else { "False" }; let repr = if let Some(name_repr) = name_repr { diff --git a/vm/src/stdlib/itertools.rs b/vm/src/stdlib/itertools.rs index 18641ac3b6..addfc991ff 100644 --- a/vm/src/stdlib/itertools.rs +++ b/vm/src/stdlib/itertools.rs @@ -87,7 +87,7 @@ mod decl { fn setstate(zelf: PyRef, state: PyTupleRef, vm: &VirtualMachine) -> PyResult<()> { let args = state.as_slice(); if args.is_empty() { - let msg = String::from("function takes at leat 1 arguments (0 given)"); + let msg = String::from("function takes at least 1 arguments (0 given)"); return Err(vm.new_type_error(msg)); } if args.len() > 2 { @@ -1892,14 +1892,14 @@ mod decl { return Ok(PyIterReturn::StopIteration(None)); } let mut result: Vec = Vec::new(); - let mut numactive = zelf.iterators.len(); + let mut num_active = zelf.iterators.len(); for idx in 0..zelf.iterators.len() { let next_obj = match zelf.iterators[idx].next(vm)? { PyIterReturn::Return(obj) => obj, PyIterReturn::StopIteration(v) => { - numactive -= 1; - if numactive == 0 { + num_active -= 1; + if num_active == 0 { return Ok(PyIterReturn::StopIteration(v)); } zelf.fillvalue.read().clone() diff --git a/vm/src/stdlib/marshal.rs b/vm/src/stdlib/marshal.rs index 564ee5bf6c..17d8ccd3e1 100644 --- a/vm/src/stdlib/marshal.rs +++ b/vm/src/stdlib/marshal.rs @@ -1,3 +1,4 @@ +// cspell:ignore pyfrozen pycomplex pub(crate) use decl::make_module; #[pymodule(name = "marshal")] diff --git a/vm/src/stdlib/operator.rs b/vm/src/stdlib/operator.rs index 38f931b0e7..fbb8147e9f 100644 --- a/vm/src/stdlib/operator.rs +++ b/vm/src/stdlib/operator.rs @@ -532,9 +532,9 @@ mod _operator { fn reduce(zelf: PyRef, vm: &VirtualMachine) -> PyResult { // With no kwargs, return (type(obj), (name, *args)) tuple. if zelf.args.kwargs.is_empty() { - let mut pargs = vec![zelf.name.as_object().to_owned()]; - pargs.append(&mut zelf.args.args.clone()); - Ok(vm.new_tuple((zelf.class().to_owned(), vm.ctx.new_tuple(pargs)))) + let mut py_args = vec![zelf.name.as_object().to_owned()]; + py_args.append(&mut zelf.args.args.clone()); + Ok(vm.new_tuple((zelf.class().to_owned(), vm.ctx.new_tuple(py_args)))) } else { // If we have kwargs, create a partial function that contains them and pass back that // along with the args. diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 48e16ad41f..08a5051fe7 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -120,8 +120,8 @@ pub(super) struct FollowSymlinks( #[pyarg(named, name = "follow_symlinks", default = true)] pub bool, ); -fn bytes_as_osstr<'a>(b: &'a [u8], vm: &VirtualMachine) -> PyResult<&'a ffi::OsStr> { - rustpython_common::os::bytes_as_osstr(b) +fn bytes_as_os_str<'a>(b: &'a [u8], vm: &VirtualMachine) -> PyResult<&'a ffi::OsStr> { + rustpython_common::os::bytes_as_os_str(b) .map_err(|_| vm.new_unicode_decode_error("can't decode path for utf-8".to_owned())) } @@ -393,8 +393,8 @@ pub(super) mod _os { if key.is_empty() || key.contains(&b'=') { return Err(vm.new_value_error("illegal environment variable name".to_string())); } - let key = super::bytes_as_osstr(key, vm)?; - let value = super::bytes_as_osstr(value, vm)?; + let key = super::bytes_as_os_str(key, vm)?; + let value = super::bytes_as_os_str(value, vm)?; // SAFETY: requirements forwarded from the caller unsafe { env::set_var(key, value) }; Ok(()) @@ -415,7 +415,7 @@ pub(super) mod _os { ), )); } - let key = super::bytes_as_osstr(key, vm)?; + let key = super::bytes_as_os_str(key, vm)?; // SAFETY: requirements forwarded from the caller unsafe { env::remove_var(key) }; Ok(()) diff --git a/vm/src/stdlib/sre.rs b/vm/src/stdlib/sre.rs index fd41aa2b7d..fdb48c7524 100644 --- a/vm/src/stdlib/sre.rs +++ b/vm/src/stdlib/sre.rs @@ -228,7 +228,7 @@ mod _sre { } #[pymethod(name = "match")] - fn pymatch( + fn py_match( zelf: PyRef, string_args: StringArgs, vm: &VirtualMachine, @@ -242,7 +242,7 @@ mod _sre { let req = x.create_request(&zelf, pos, endpos); let mut state = State::default(); Ok(state - .pymatch(&req) + .py_match(&req) .then(|| Match::new(&mut state, zelf.clone(), string).into_ref(&vm.ctx))) }) } @@ -257,7 +257,7 @@ mod _sre { let mut req = x.create_request(&zelf, string_args.pos, string_args.endpos); req.match_all = true; let mut state = State::default(); - Ok(state.pymatch(&req).then(|| { + Ok(state.py_match(&req).then(|| { Match::new(&mut state, zelf.clone(), string_args.string).into_ref(&vm.ctx) })) }) @@ -346,11 +346,11 @@ mod _sre { #[pymethod] fn sub(zelf: PyRef, sub_args: SubArgs, vm: &VirtualMachine) -> PyResult { - Self::subx(zelf, sub_args, false, vm) + Self::sub_impl(zelf, sub_args, false, vm) } #[pymethod] fn subn(zelf: PyRef, sub_args: SubArgs, vm: &VirtualMachine) -> PyResult { - Self::subx(zelf, sub_args, true, vm) + Self::sub_impl(zelf, sub_args, true, vm) } #[pymethod] @@ -407,7 +407,7 @@ mod _sre { self.pattern.clone() } - fn subx( + fn sub_impl( zelf: PyRef, sub_args: SubArgs, subn: bool, @@ -860,12 +860,12 @@ mod _sre { } #[pymethod(name = "match")] - fn pymatch(&self, vm: &VirtualMachine) -> PyResult>> { + fn py_match(&self, vm: &VirtualMachine) -> PyResult>> { with_sre_str!(self.pattern, &self.string.clone(), vm, |s| { let mut req = s.create_request(&self.pattern, self.start.load(), self.end); let mut state = State::default(); req.must_advance = self.must_advance.load(); - let has_matched = state.pymatch(&req); + let has_matched = state.py_match(&req); self.must_advance .store(state.cursor.position == state.start); diff --git a/vm/src/stdlib/sys.rs b/vm/src/stdlib/sys.rs index fdfe2faf69..befa0f8dff 100644 --- a/vm/src/stdlib/sys.rs +++ b/vm/src/stdlib/sys.rs @@ -513,19 +513,19 @@ mod sys { } // Get the size of the version information block - let verblock_size = + let ver_block_size = GetFileVersionInfoSizeW(kernel32_path.as_ptr(), std::ptr::null_mut()); - if verblock_size == 0 { + if ver_block_size == 0 { return Err(std::io::Error::last_os_error()); } // Allocate a buffer to hold the version information - let mut verblock = vec![0u8; verblock_size as usize]; + let mut ver_block = vec![0u8; ver_block_size as usize]; if GetFileVersionInfoW( kernel32_path.as_ptr(), 0, - verblock_size, - verblock.as_mut_ptr() as *mut _, + ver_block_size, + ver_block.as_mut_ptr() as *mut _, ) == 0 { return Err(std::io::Error::last_os_error()); @@ -540,7 +540,7 @@ mod sys { let mut ffi_ptr: *mut VS_FIXEDFILEINFO = std::ptr::null_mut(); let mut ffi_len: u32 = 0; if VerQueryValueW( - verblock.as_ptr() as *const _, + ver_block.as_ptr() as *const _, sub_block.as_ptr(), &mut ffi_ptr as *mut *mut VS_FIXEDFILEINFO as *mut *mut _, &mut ffi_len as *mut u32, @@ -572,10 +572,10 @@ mod sys { let mut version: OSVERSIONINFOEXW = unsafe { std::mem::zeroed() }; version.dwOSVersionInfoSize = std::mem::size_of::() as u32; let result = unsafe { - let osvi = &mut version as *mut OSVERSIONINFOEXW as *mut OSVERSIONINFOW; + let os_vi = &mut version as *mut OSVERSIONINFOEXW as *mut OSVERSIONINFOW; // SAFETY: GetVersionExW accepts a pointer of OSVERSIONINFOW, but windows-sys crate's type currently doesn't allow to do so. // https://docs.microsoft.com/en-us/windows/win32/api/sysinfoapi/nf-sysinfoapi-getversionexw#parameters - GetVersionExW(osvi) + GetVersionExW(os_vi) }; if result == 0 { diff --git a/vm/src/stdlib/thread.rs b/vm/src/stdlib/thread.rs index ad80f1f1e1..0ee087e5e4 100644 --- a/vm/src/stdlib/thread.rs +++ b/vm/src/stdlib/thread.rs @@ -334,8 +334,8 @@ pub(crate) mod _thread { ); } } - SENTINELS.with(|sents| { - for lock in sents.replace(Default::default()) { + SENTINELS.with(|sentinels| { + for lock in sentinels.replace(Default::default()) { if lock.mu.is_locked() { unsafe { lock.mu.unlock() }; } @@ -360,7 +360,7 @@ pub(crate) mod _thread { #[pyfunction] fn _set_sentinel(vm: &VirtualMachine) -> PyRef { let lock = Lock { mu: RawMutex::INIT }.into_ref(&vm.ctx); - SENTINELS.with(|sents| sents.borrow_mut().push(lock.clone())); + SENTINELS.with(|sentinels| sentinels.borrow_mut().push(lock.clone())); lock } @@ -385,7 +385,7 @@ pub(crate) mod _thread { #[pyclass(with(GetAttr, SetAttr), flags(BASETYPE))] impl Local { - fn ldict(&self, vm: &VirtualMachine) -> PyDictRef { + fn l_dict(&self, vm: &VirtualMachine) -> PyDictRef { self.data.get_or(|| vm.ctx.new_dict()).clone() } @@ -401,12 +401,12 @@ pub(crate) mod _thread { impl GetAttr for Local { fn getattro(zelf: &Py, attr: &Py, vm: &VirtualMachine) -> PyResult { - let ldict = zelf.ldict(vm); + let l_dict = zelf.l_dict(vm); if attr.as_str() == "__dict__" { - Ok(ldict.into()) + Ok(l_dict.into()) } else { zelf.as_object() - .generic_getattr_opt(attr, Some(ldict), vm)? + .generic_getattr_opt(attr, Some(l_dict), vm)? .ok_or_else(|| { vm.new_attribute_error(format!( "{} has no attribute '{}'", @@ -431,7 +431,7 @@ pub(crate) mod _thread { zelf.class().name() ))) } else { - let dict = zelf.ldict(vm); + let dict = zelf.l_dict(vm); if let PySetterValue::Assign(value) = value { dict.set_item(attr, value, vm)?; } else { diff --git a/vm/src/stdlib/time.rs b/vm/src/stdlib/time.rs index 5f41304c19..cc543e9249 100644 --- a/vm/src/stdlib/time.rs +++ b/vm/src/stdlib/time.rs @@ -243,8 +243,8 @@ mod decl { let timestamp = match value { Either::A(float) => { let secs = float.trunc() as i64; - let nsecs = (float.fract() * 1e9) as u32; - DateTime::::from_timestamp(secs, nsecs) + let nano_secs = (float.fract() * 1e9) as u32; + DateTime::::from_timestamp(secs, nano_secs) } Either::B(int) => DateTime::::from_timestamp(int, 0), }; diff --git a/vm/src/types/slot.rs b/vm/src/types/slot.rs index 2d8c825817..e2121973ec 100644 --- a/vm/src/types/slot.rs +++ b/vm/src/types/slot.rs @@ -390,7 +390,7 @@ impl PyType { }}; } - macro_rules! toggle_subslot { + macro_rules! toggle_sub_slot { ($group:ident, $name:ident, $func:expr) => { self.slots .$group @@ -520,90 +520,90 @@ impl PyType { toggle_slot!(del, del_wrapper); } _ if name == identifier!(ctx, __int__) => { - toggle_subslot!(as_number, int, number_unary_op_wrapper!(__int__)); + toggle_sub_slot!(as_number, int, number_unary_op_wrapper!(__int__)); } _ if name == identifier!(ctx, __index__) => { - toggle_subslot!(as_number, index, number_unary_op_wrapper!(__index__)); + toggle_sub_slot!(as_number, index, number_unary_op_wrapper!(__index__)); } _ if name == identifier!(ctx, __float__) => { - toggle_subslot!(as_number, float, number_unary_op_wrapper!(__float__)); + toggle_sub_slot!(as_number, float, number_unary_op_wrapper!(__float__)); } _ if name == identifier!(ctx, __add__) => { - toggle_subslot!(as_number, add, number_binary_op_wrapper!(__add__)); + toggle_sub_slot!(as_number, add, number_binary_op_wrapper!(__add__)); } _ if name == identifier!(ctx, __radd__) => { - toggle_subslot!( + toggle_sub_slot!( as_number, right_add, number_binary_right_op_wrapper!(__radd__) ); } _ if name == identifier!(ctx, __iadd__) => { - toggle_subslot!(as_number, inplace_add, number_binary_op_wrapper!(__iadd__)); + toggle_sub_slot!(as_number, inplace_add, number_binary_op_wrapper!(__iadd__)); } _ if name == identifier!(ctx, __sub__) => { - toggle_subslot!(as_number, subtract, number_binary_op_wrapper!(__sub__)); + toggle_sub_slot!(as_number, subtract, number_binary_op_wrapper!(__sub__)); } _ if name == identifier!(ctx, __rsub__) => { - toggle_subslot!( + toggle_sub_slot!( as_number, right_subtract, number_binary_right_op_wrapper!(__rsub__) ); } _ if name == identifier!(ctx, __isub__) => { - toggle_subslot!( + toggle_sub_slot!( as_number, inplace_subtract, number_binary_op_wrapper!(__isub__) ); } _ if name == identifier!(ctx, __mul__) => { - toggle_subslot!(as_number, multiply, number_binary_op_wrapper!(__mul__)); + toggle_sub_slot!(as_number, multiply, number_binary_op_wrapper!(__mul__)); } _ if name == identifier!(ctx, __rmul__) => { - toggle_subslot!( + toggle_sub_slot!( as_number, right_multiply, number_binary_right_op_wrapper!(__rmul__) ); } _ if name == identifier!(ctx, __imul__) => { - toggle_subslot!( + toggle_sub_slot!( as_number, inplace_multiply, number_binary_op_wrapper!(__imul__) ); } _ if name == identifier!(ctx, __mod__) => { - toggle_subslot!(as_number, remainder, number_binary_op_wrapper!(__mod__)); + toggle_sub_slot!(as_number, remainder, number_binary_op_wrapper!(__mod__)); } _ if name == identifier!(ctx, __rmod__) => { - toggle_subslot!( + toggle_sub_slot!( as_number, right_remainder, number_binary_right_op_wrapper!(__rmod__) ); } _ if name == identifier!(ctx, __imod__) => { - toggle_subslot!( + toggle_sub_slot!( as_number, inplace_remainder, number_binary_op_wrapper!(__imod__) ); } _ if name == identifier!(ctx, __divmod__) => { - toggle_subslot!(as_number, divmod, number_binary_op_wrapper!(__divmod__)); + toggle_sub_slot!(as_number, divmod, number_binary_op_wrapper!(__divmod__)); } _ if name == identifier!(ctx, __rdivmod__) => { - toggle_subslot!( + toggle_sub_slot!( as_number, right_divmod, number_binary_right_op_wrapper!(__rdivmod__) ); } _ if name == identifier!(ctx, __pow__) => { - toggle_subslot!(as_number, power, |a, b, c, vm| { + toggle_sub_slot!(as_number, power, |a, b, c, vm| { let args = if vm.is_none(c) { vec![b.to_owned()] } else { @@ -613,7 +613,7 @@ impl PyType { }); } _ if name == identifier!(ctx, __rpow__) => { - toggle_subslot!(as_number, right_power, |a, b, c, vm| { + toggle_sub_slot!(as_number, right_power, |a, b, c, vm| { let args = if vm.is_none(c) { vec![a.to_owned()] } else { @@ -623,141 +623,141 @@ impl PyType { }); } _ if name == identifier!(ctx, __ipow__) => { - toggle_subslot!(as_number, inplace_power, |a, b, _, vm| { + toggle_sub_slot!(as_number, inplace_power, |a, b, _, vm| { vm.call_special_method(a, identifier!(vm, __ipow__), (b.to_owned(),)) }); } _ if name == identifier!(ctx, __lshift__) => { - toggle_subslot!(as_number, lshift, number_binary_op_wrapper!(__lshift__)); + toggle_sub_slot!(as_number, lshift, number_binary_op_wrapper!(__lshift__)); } _ if name == identifier!(ctx, __rlshift__) => { - toggle_subslot!( + toggle_sub_slot!( as_number, right_lshift, number_binary_right_op_wrapper!(__rlshift__) ); } _ if name == identifier!(ctx, __ilshift__) => { - toggle_subslot!( + toggle_sub_slot!( as_number, inplace_lshift, number_binary_op_wrapper!(__ilshift__) ); } _ if name == identifier!(ctx, __rshift__) => { - toggle_subslot!(as_number, rshift, number_binary_op_wrapper!(__rshift__)); + toggle_sub_slot!(as_number, rshift, number_binary_op_wrapper!(__rshift__)); } _ if name == identifier!(ctx, __rrshift__) => { - toggle_subslot!( + toggle_sub_slot!( as_number, right_rshift, number_binary_right_op_wrapper!(__rrshift__) ); } _ if name == identifier!(ctx, __irshift__) => { - toggle_subslot!( + toggle_sub_slot!( as_number, inplace_rshift, number_binary_op_wrapper!(__irshift__) ); } _ if name == identifier!(ctx, __and__) => { - toggle_subslot!(as_number, and, number_binary_op_wrapper!(__and__)); + toggle_sub_slot!(as_number, and, number_binary_op_wrapper!(__and__)); } _ if name == identifier!(ctx, __rand__) => { - toggle_subslot!( + toggle_sub_slot!( as_number, right_and, number_binary_right_op_wrapper!(__rand__) ); } _ if name == identifier!(ctx, __iand__) => { - toggle_subslot!(as_number, inplace_and, number_binary_op_wrapper!(__iand__)); + toggle_sub_slot!(as_number, inplace_and, number_binary_op_wrapper!(__iand__)); } _ if name == identifier!(ctx, __xor__) => { - toggle_subslot!(as_number, xor, number_binary_op_wrapper!(__xor__)); + toggle_sub_slot!(as_number, xor, number_binary_op_wrapper!(__xor__)); } _ if name == identifier!(ctx, __rxor__) => { - toggle_subslot!( + toggle_sub_slot!( as_number, right_xor, number_binary_right_op_wrapper!(__rxor__) ); } _ if name == identifier!(ctx, __ixor__) => { - toggle_subslot!(as_number, inplace_xor, number_binary_op_wrapper!(__ixor__)); + toggle_sub_slot!(as_number, inplace_xor, number_binary_op_wrapper!(__ixor__)); } _ if name == identifier!(ctx, __or__) => { - toggle_subslot!(as_number, or, number_binary_op_wrapper!(__or__)); + toggle_sub_slot!(as_number, or, number_binary_op_wrapper!(__or__)); } _ if name == identifier!(ctx, __ror__) => { - toggle_subslot!( + toggle_sub_slot!( as_number, right_or, number_binary_right_op_wrapper!(__ror__) ); } _ if name == identifier!(ctx, __ior__) => { - toggle_subslot!(as_number, inplace_or, number_binary_op_wrapper!(__ior__)); + toggle_sub_slot!(as_number, inplace_or, number_binary_op_wrapper!(__ior__)); } _ if name == identifier!(ctx, __floordiv__) => { - toggle_subslot!( + toggle_sub_slot!( as_number, floor_divide, number_binary_op_wrapper!(__floordiv__) ); } _ if name == identifier!(ctx, __rfloordiv__) => { - toggle_subslot!( + toggle_sub_slot!( as_number, right_floor_divide, number_binary_right_op_wrapper!(__rfloordiv__) ); } _ if name == identifier!(ctx, __ifloordiv__) => { - toggle_subslot!( + toggle_sub_slot!( as_number, inplace_floor_divide, number_binary_op_wrapper!(__ifloordiv__) ); } _ if name == identifier!(ctx, __truediv__) => { - toggle_subslot!( + toggle_sub_slot!( as_number, true_divide, number_binary_op_wrapper!(__truediv__) ); } _ if name == identifier!(ctx, __rtruediv__) => { - toggle_subslot!( + toggle_sub_slot!( as_number, right_true_divide, number_binary_right_op_wrapper!(__rtruediv__) ); } _ if name == identifier!(ctx, __itruediv__) => { - toggle_subslot!( + toggle_sub_slot!( as_number, inplace_true_divide, number_binary_op_wrapper!(__itruediv__) ); } _ if name == identifier!(ctx, __matmul__) => { - toggle_subslot!( + toggle_sub_slot!( as_number, matrix_multiply, number_binary_op_wrapper!(__matmul__) ); } _ if name == identifier!(ctx, __rmatmul__) => { - toggle_subslot!( + toggle_sub_slot!( as_number, right_matrix_multiply, number_binary_right_op_wrapper!(__rmatmul__) ); } _ if name == identifier!(ctx, __imatmul__) => { - toggle_subslot!( + toggle_sub_slot!( as_number, inplace_matrix_multiply, number_binary_op_wrapper!(__imatmul__) diff --git a/vm/src/vm/mod.rs b/vm/src/vm/mod.rs index 752943319d..4b1cece8c5 100644 --- a/vm/src/vm/mod.rs +++ b/vm/src/vm/mod.rs @@ -648,7 +648,7 @@ impl VirtualMachine { list_borrow = value.payload::().unwrap().borrow_vec(); &list_borrow } else { - return self.map_pyiter(value, func); + return self.map_py_iter(value, func); }; slice.iter().map(|obj| func(obj.clone())).collect() } @@ -682,12 +682,12 @@ impl VirtualMachine { ref t @ PyTuple => Ok(t.iter().cloned().map(f).collect()), // TODO: put internal iterable type obj => { - Ok(self.map_pyiter(obj, f)) + Ok(self.map_py_iter(obj, f)) } }) } - fn map_pyiter(&self, value: &PyObject, mut f: F) -> PyResult> + fn map_py_iter(&self, value: &PyObject, mut f: F) -> PyResult> where F: FnMut(PyObjectRef) -> PyResult, { diff --git a/vm/sre_engine/benches/benches.rs b/vm/sre_engine/benches/benches.rs index ee49b036de..e2372d783e 100644 --- a/vm/sre_engine/benches/benches.rs +++ b/vm/sre_engine/benches/benches.rs @@ -92,20 +92,20 @@ fn basic(c: &mut Criterion) { let (req, mut state) = p.state(s); assert!(state.search(req)); let (req, mut state) = p.state(s); - assert!(state.pymatch(&req)); + assert!(state.py_match(&req)); let (mut req, mut state) = p.state(s); req.match_all = true; - assert!(state.pymatch(&req)); + assert!(state.py_match(&req)); let s2 = format!("{}{}{}", " ".repeat(10000), s, " ".repeat(10000)); let (req, mut state) = p.state_range(s2.as_str(), 0..usize::MAX); assert!(state.search(req)); let (req, mut state) = p.state_range(s2.as_str(), 10000..usize::MAX); - assert!(state.pymatch(&req)); + assert!(state.py_match(&req)); let (req, mut state) = p.state_range(s2.as_str(), 10000..10000 + s.len()); - assert!(state.pymatch(&req)); + assert!(state.py_match(&req)); let (mut req, mut state) = p.state_range(s2.as_str(), 10000..10000 + s.len()); req.match_all = true; - assert!(state.pymatch(&req)); + assert!(state.py_match(&req)); }); }); } diff --git a/vm/sre_engine/src/engine.rs b/vm/sre_engine/src/engine.rs index 9b27f55031..1e0b15fd01 100644 --- a/vm/sre_engine/src/engine.rs +++ b/vm/sre_engine/src/engine.rs @@ -129,7 +129,7 @@ impl State { req.string.adjust_cursor(&mut self.cursor, start); } - pub fn pymatch(&mut self, req: &Request<'_, S>) -> bool { + pub fn py_match(&mut self, req: &Request<'_, S>) -> bool { self.start = req.start; req.string.adjust_cursor(&mut self.cursor, self.start); diff --git a/vm/sre_engine/tests/tests.rs b/vm/sre_engine/tests/tests.rs index 5499afa281..0ada32e5db 100644 --- a/vm/sre_engine/tests/tests.rs +++ b/vm/sre_engine/tests/tests.rs @@ -1,3 +1,4 @@ +// cspell:disable use rustpython_sre_engine::{Request, State, StrDrive}; struct Pattern { @@ -21,7 +22,7 @@ fn test_2427() { #[rustfmt::skip] let lookbehind = Pattern { pattern: "(?x)++x", code: &[14, 4, 0, 2, 4294967295, 28, 8, 1, 4294967295, 27, 4, 16, 120, 1, 1, 16, 120, 1] }; // END GENERATED let (req, mut state) = p.state("xxx"); - assert!(!state.pymatch(&req)); + assert!(!state.py_match(&req)); } #[test] @@ -162,7 +163,7 @@ fn test_bug_20998() { // END GENERATED let (mut req, mut state) = p.state("ABC"); req.match_all = true; - assert!(state.pymatch(&req)); + assert!(state.py_match(&req)); assert_eq!(state.cursor.position, 3); } @@ -173,7 +174,7 @@ fn test_bigcharset() { #[rustfmt::skip] let p = Pattern { pattern: "[a-z]*", code: &[14, 4, 0, 0, 4294967295, 24, 97, 0, 4294967295, 39, 92, 10, 3, 33685760, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 0, 0, 0, 134217726, 0, 0, 0, 0, 0, 131072, 0, 2147483648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1] }; // END GENERATED let (req, mut state) = p.state("x "); - assert!(state.pymatch(&req)); + assert!(state.py_match(&req)); assert_eq!(state.cursor.position, 1); } diff --git a/wasm/lib/src/convert.rs b/wasm/lib/src/convert.rs index 4f6e4db35c..bccf5564fa 100644 --- a/wasm/lib/src/convert.rs +++ b/wasm/lib/src/convert.rs @@ -33,8 +33,8 @@ extern "C" { } pub fn py_err_to_js_err(vm: &VirtualMachine, py_err: &PyBaseExceptionRef) -> JsValue { - let jserr = vm.try_class("_js", "JSError").ok(); - let js_arg = if jserr.is_some_and(|jserr| py_err.fast_isinstance(&jserr)) { + let js_err = vm.try_class("_js", "JSError").ok(); + let js_arg = if js_err.is_some_and(|js_err| py_err.fast_isinstance(&js_err)) { py_err.get_arg(0) } else { None @@ -116,7 +116,7 @@ pub fn py_to_js(vm: &VirtualMachine, py_obj: PyObjectRef) -> JsValue { } } let result = py_obj.call(py_func_args, vm); - pyresult_to_jsresult(vm, result) + pyresult_to_js_result(vm, result) }) }; let closure = Closure::wrap(Box::new(closure) @@ -164,7 +164,7 @@ pub fn object_entries(obj: &Object) -> impl Iterator Result { +pub fn pyresult_to_js_result(vm: &VirtualMachine, result: PyResult) -> Result { result .map(|value| py_to_js(vm, value)) .map_err(|err| py_err_to_js_err(vm, &err)) diff --git a/wasm/lib/src/js_module.rs b/wasm/lib/src/js_module.rs index a5b7281481..f159c467d0 100644 --- a/wasm/lib/src/js_module.rs +++ b/wasm/lib/src/js_module.rs @@ -326,7 +326,7 @@ mod _js { .map(|arg| PyJsValue::new(arg).into_pyobject(vm)), ); let res = py_obj.call(pyargs, vm); - convert::pyresult_to_jsresult(vm, res) + convert::pyresult_to_js_result(vm, res) }) }; let closure: ClosureType = if once { @@ -500,7 +500,7 @@ mod _js { Some(on_fulfill) => stored_vm.interp.enter(move |vm| { let val = convert::js_to_py(vm, val); let res = on_fulfill.invoke((val,), vm); - convert::pyresult_to_jsresult(vm, res) + convert::pyresult_to_js_result(vm, res) }), None => Ok(val), }, @@ -508,7 +508,7 @@ mod _js { Some(on_reject) => stored_vm.interp.enter(move |vm| { let err = new_js_error(vm, err); let res = on_reject.invoke((err,), vm); - convert::pyresult_to_jsresult(vm, res) + convert::pyresult_to_js_result(vm, res) }), None => Err(err), }, diff --git a/wasm/lib/src/vm_class.rs b/wasm/lib/src/vm_class.rs index c04877f7e3..d84603e986 100644 --- a/wasm/lib/src/vm_class.rs +++ b/wasm/lib/src/vm_class.rs @@ -340,7 +340,7 @@ impl WASMVirtualMachine { let code = vm.compile(source, mode, source_path); let code = code.map_err(convert::syntax_err)?; let result = vm.run_code_obj(code, scope.clone()); - convert::pyresult_to_jsresult(vm, result) + convert::pyresult_to_js_result(vm, result) })? } From 883e0cab292383a6e70db7a3150f36c62e478988 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Thu, 3 Apr 2025 09:23:06 -0700 Subject: [PATCH 172/295] build docs on CI and deny warnings --- .github/workflows/ci.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 05516c9270..b728c81bc2 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -325,6 +325,8 @@ jobs: python-version: ${{ env.PYTHON_VERSION }} - name: install ruff run: python -m pip install ruff==0.0.291 # astral-sh/ruff#7778 + - name: Ensure docs generate no warnings + run: cargo doc -- -Dwarnings - name: run python lint run: ruff extra_tests wasm examples --exclude='./.*',./Lib,./vm/Lib,./benches/ --select=E9,F63,F7,F82 --show-source - name: install prettier From 5c854fc6902177f81dc8bb662b36b713d67615ea Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Thu, 3 Apr 2025 09:23:22 -0700 Subject: [PATCH 173/295] clear out warnings --- common/src/linked_list.rs | 2 +- src/lib.rs | 4 ++-- src/settings.rs | 4 ++++ src/shell.rs | 2 ++ stdlib/src/pystruct.rs | 4 ++-- vm/src/builtins/mod.rs | 4 ++-- vm/src/dict_inner.rs | 6 +++--- vm/src/function/builtin.rs | 2 +- vm/src/import.rs | 5 ++--- vm/src/lib.rs | 7 ++++--- vm/src/object/core.rs | 16 ++++++++-------- vm/src/object/traverse.rs | 6 +++--- vm/src/prelude.rs | 4 ++++ vm/src/protocol/buffer.rs | 2 +- vm/src/protocol/object.rs | 2 +- vm/src/readline.rs | 5 +++++ vm/src/suggestion.rs | 3 +++ vm/src/version.rs | 3 +-- vm/src/vm/interpreter.rs | 13 ++++++------- vm/src/vm/mod.rs | 4 ++-- vm/src/vm/vm_ops.rs | 4 ++-- wtf8/src/lib.rs | 2 +- 22 files changed, 60 insertions(+), 44 deletions(-) diff --git a/common/src/linked_list.rs b/common/src/linked_list.rs index 7f55d727fb..83577b71d1 100644 --- a/common/src/linked_list.rs +++ b/common/src/linked_list.rs @@ -1,4 +1,4 @@ -//! This module is modified from tokio::util::linked_list: https://github.com/tokio-rs/tokio/blob/master/tokio/src/util/linked_list.rs +//! This module is modified from tokio::util::linked_list: //! Tokio is licensed under the MIT license: //! //! Copyright (c) 2021 Tokio Contributors diff --git a/src/lib.rs b/src/lib.rs index e415d847cf..98de3de5c6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,9 +1,9 @@ //! This is the `rustpython` binary. If you're looking to embed RustPython into your application, -//! you're likely looking for the [`rustpython-vm`](https://docs.rs/rustpython-vm) crate. +//! you're likely looking for the [`rustpython_vm`] crate. //! //! You can install `rustpython` with `cargo install rustpython`, or if you'd like to inject your //! own native modules you can make a binary crate that depends on the `rustpython` crate (and -//! probably `rustpython-vm`, too), and make a `main.rs` that looks like: +//! probably [`rustpython_vm`], too), and make a `main.rs` that looks like: //! //! ```no_run //! use rustpython_vm::{pymodule, py_freeze}; diff --git a/src/settings.rs b/src/settings.rs index 76c46ac43a..00ee55bddd 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -13,7 +13,11 @@ pub enum RunMode { } pub enum InstallPipMode { + /// Install pip using the ensurepip pip module. This has a higher chance of + /// success, but may not install the latest version of pip. Ensurepip, + /// Install pip using the get-pip.py script, which retrieves the latest pip version. + /// This can be broken due to incompatibilities with cpython. GetPip, } diff --git a/src/shell.rs b/src/shell.rs index 98ee6eee21..04090a49fc 100644 --- a/src/shell.rs +++ b/src/shell.rs @@ -71,6 +71,8 @@ fn shell_exec( } } + +/// Enter a repl loop pub fn run_shell(vm: &VirtualMachine, scope: Scope) -> PyResult<()> { let mut repl = Readline::new(helper::ShellHelper::new(vm, scope.globals.clone())); let mut full_input = String::new(); diff --git a/stdlib/src/pystruct.rs b/stdlib/src/pystruct.rs index 220970dd20..9426470911 100644 --- a/stdlib/src/pystruct.rs +++ b/stdlib/src/pystruct.rs @@ -1,9 +1,9 @@ //! Python struct module. //! -//! Docs: https://docs.python.org/3/library/struct.html +//! Docs: //! //! Use this rust module to do byte packing: -//! https://docs.rs/byteorder/1.2.6/byteorder/ +//! pub(crate) use _struct::make_module; diff --git a/vm/src/builtins/mod.rs b/vm/src/builtins/mod.rs index ae3b7eea2a..8540e6887c 100644 --- a/vm/src/builtins/mod.rs +++ b/vm/src/builtins/mod.rs @@ -1,6 +1,6 @@ //! This package contains the python basic/builtin types -//! 7 common PyRef type aliases are exposed - PyBytesRef, PyDictRef, PyIntRef, PyListRef, PyStrRef, PyTypeRef, PyTupleRef -//! Do not add more PyRef type aliases. They will be rare enough to use directly PyRef. +//! 7 common PyRef type aliases are exposed - [`PyBytesRef`], [`PyDictRef`], [`PyIntRef`], [`PyListRef`], [`PyStrRef`], [`PyTypeRef`], [`PyTupleRef`] +//! Do not add more PyRef type aliases. They will be rare enough to use directly `PyRef`. pub(crate) mod asyncgenerator; pub use asyncgenerator::PyAsyncGen; diff --git a/vm/src/dict_inner.rs b/vm/src/dict_inner.rs index c49ab752de..56ebc5ebaf 100644 --- a/vm/src/dict_inner.rs +++ b/vm/src/dict_inner.rs @@ -1,7 +1,7 @@ //! Ordered dictionary implementation. -//! Inspired by: https://morepypy.blogspot.com/2015/01/faster-more-memory-efficient-and-more.html -//! And: https://www.youtube.com/watch?v=p33CVV29OG8 -//! And: http://code.activestate.com/recipes/578375/ +//! Inspired by: +//! And: +//! And: use crate::{ AsObject, Py, PyExact, PyObject, PyObjectRef, PyRefExact, PyResult, VirtualMachine, diff --git a/vm/src/function/builtin.rs b/vm/src/function/builtin.rs index b8a408453d..186dc7aeb8 100644 --- a/vm/src/function/builtin.rs +++ b/vm/src/function/builtin.rs @@ -65,7 +65,7 @@ const fn zst_ref_out_of_thin_air(x: T) -> &'static T { } } -/// Get the [`STATIC_FUNC`](IntoPyNativeFn::STATIC_FUNC) of the passed function. The same +/// Get the STATIC_FUNC of the passed function. The same /// requirements of zero-sizedness apply, see that documentation for details. /// /// Equivalent to [`IntoPyNativeFn::into_func()`], but usable in a const context. This is only diff --git a/vm/src/import.rs b/vm/src/import.rs index 416c40a844..90aadbdbf2 100644 --- a/vm/src/import.rs +++ b/vm/src/import.rs @@ -1,6 +1,5 @@ -/* - * Import mechanics - */ +//! Import mechanics + use crate::{ AsObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, builtins::{PyBaseExceptionRef, PyCode, list, traceback::PyTraceback}, diff --git a/vm/src/lib.rs b/vm/src/lib.rs index 9561a9cc23..8ae5ff15db 100644 --- a/vm/src/lib.rs +++ b/vm/src/lib.rs @@ -1,9 +1,10 @@ -//! This crate contains most python logic. +//! This crate contains most of the python logic. //! -//! - Compilation -//! - Bytecode +//! - Interpreter //! - Import mechanics //! - Base objects +//! +//! Some stdlib modules are implemented here, but most of them are in the `rustpython-stdlib` module. The // to allow `mod foo {}` in foo.rs; clippy thinks this is a mistake/misunderstanding of // how `mod` works, but we want this sometimes for pymodule declarations diff --git a/vm/src/object/core.rs b/vm/src/object/core.rs index a6049884d8..8edcb4dfd6 100644 --- a/vm/src/object/core.rs +++ b/vm/src/object/core.rs @@ -1,14 +1,14 @@ //! Essential types for object models //! -//! +-------------------------+--------------+---------------+ -//! | Management | Typed | Untyped | -//! +-------------------------+--------------+---------------+ -//! | Interpreter-independent | Py | PyObject | -//! | Reference-counted | PyRef | PyObjectRef | -//! | Weak | PyWeakRef | PyRef | -//! +-------------------------+--------------+---------------+ +//! +-------------------------+--------------+-----------------------+ +//! | Management | Typed | Untyped | +//! +-------------------------+------------------+-------------------+ +//! | Interpreter-independent | [`Py`] | [`PyObject`] | +//! | Reference-counted | [`PyRef`] | [`PyObjectRef`] | +//! | Weak | [`PyWeakRef`] | [`PyRef`] | +//! +-------------------------+--------------+-----------------------+ //! -//! PyRef may looking like to be called as PyObjectWeak by the rule, +//! [`PyRef`] may looking like to be called as PyObjectWeak by the rule, //! but not to do to remember it is a PyRef object. use super::{ PyAtomicRef, diff --git a/vm/src/object/traverse.rs b/vm/src/object/traverse.rs index c105d23feb..46e5daff05 100644 --- a/vm/src/object/traverse.rs +++ b/vm/src/object/traverse.rs @@ -17,16 +17,16 @@ pub trait MaybeTraverse { fn try_traverse(&self, traverse_fn: &mut TraverseFn<'_>); } -/// Type that need traverse it's children should impl `Traverse`(Not `MaybeTraverse`) +/// Type that need traverse it's children should impl [`Traverse`] (not [`MaybeTraverse`]) /// # Safety -/// Please carefully read [`traverse()`] and follow the guideline +/// Please carefully read [`Traverse::traverse()`] and follow the guideline pub unsafe trait Traverse { /// impl `traverse()` with caution! Following those guideline so traverse doesn't cause memory error!: /// - Make sure that every owned object(Every PyObjectRef/PyRef) is called with traverse_fn **at most once**. /// If some field is not called, the worst results is just memory leak, /// but if some field is called repeatedly, panic and deadlock can happen. /// - /// - _**DO NOT**_ clone a `PyObjectRef` or `PyRef` in `traverse()` + /// - _**DO NOT**_ clone a [`PyObjectRef`] or [`PyRef`] in [`Traverse::traverse()`] fn traverse(&self, traverse_fn: &mut TraverseFn<'_>); } diff --git a/vm/src/prelude.rs b/vm/src/prelude.rs index 0bd0fe88be..b277f1468a 100644 --- a/vm/src/prelude.rs +++ b/vm/src/prelude.rs @@ -1,3 +1,7 @@ +//! The prelude imports the various objects and traits. +//! +//! The intention is that one can include `use rustpython_vm::prelude::*`. + pub use crate::{ object::{ AsObject, Py, PyExact, PyObject, PyObjectRef, PyPayload, PyRef, PyRefExact, PyResult, diff --git a/vm/src/protocol/buffer.rs b/vm/src/protocol/buffer.rs index a3b7f125f5..fcd44c11d3 100644 --- a/vm/src/protocol/buffer.rs +++ b/vm/src/protocol/buffer.rs @@ -1,5 +1,5 @@ //! Buffer protocol -//! https://docs.python.org/3/c-api/buffer.html +//! use crate::{ Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromBorrowedObject, VirtualMachine, diff --git a/vm/src/protocol/object.rs b/vm/src/protocol/object.rs index 1e972eb540..eab24f82d0 100644 --- a/vm/src/protocol/object.rs +++ b/vm/src/protocol/object.rs @@ -1,5 +1,5 @@ //! Object Protocol -//! https://docs.python.org/3/c-api/object.html +//! use crate::{ AsObject, Py, PyObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine, diff --git a/vm/src/readline.rs b/vm/src/readline.rs index 53647270e1..54a77f1289 100644 --- a/vm/src/readline.rs +++ b/vm/src/readline.rs @@ -1,3 +1,8 @@ +//! Readline interface for REPLs +//! +//! This module provides a common interface for reading lines from the console, with support for history and completion. +//! It uses the [`rustyline`] crate on non-WASM platforms and a custom implementation on WASM platforms. + use std::{io, path::Path}; type OtherError = Box; diff --git a/vm/src/suggestion.rs b/vm/src/suggestion.rs index 2bc9992d43..01f53d70ca 100644 --- a/vm/src/suggestion.rs +++ b/vm/src/suggestion.rs @@ -1,3 +1,6 @@ +//! This module provides functionality to suggest similar names for attributes or variables. +//! This is used during tracebacks. + use crate::{ AsObject, Py, PyObjectRef, VirtualMachine, builtins::{PyStr, PyStrRef}, diff --git a/vm/src/version.rs b/vm/src/version.rs index 7413f8f139..9d472e8be0 100644 --- a/vm/src/version.rs +++ b/vm/src/version.rs @@ -1,5 +1,4 @@ -/* Several function to retrieve version information. - */ +//! Several function to retrieve version information. use chrono::{Local, prelude::DateTime}; use std::time::{Duration, UNIX_EPOCH}; diff --git a/vm/src/vm/interpreter.rs b/vm/src/vm/interpreter.rs index cc669e0661..02c71bf136 100644 --- a/vm/src/vm/interpreter.rs +++ b/vm/src/vm/interpreter.rs @@ -64,7 +64,7 @@ impl Interpreter { /// /// To finalize the vm once all desired `enter`s are called, calling `finalize` will be helpful. /// - /// See also [`run`] for managed way to run the interpreter. + /// See also [`Interpreter::run`] for managed way to run the interpreter. pub fn enter(&self, f: F) -> R where F: FnOnce(&VirtualMachine) -> R, @@ -72,13 +72,12 @@ impl Interpreter { thread::enter_vm(&self.vm, || f(&self.vm)) } - /// Run [`enter`] and call `expect_pyresult` for the result. + /// Run [`Interpreter::enter`] and call [`VirtualMachine::expect_pyresult`] for the result. /// /// This function is useful when you want to expect a result from the function, /// but also print useful panic information when exception raised. /// - /// See [`enter`] for more information. - /// See [`expect_pyresult`] for more information. + /// See also [`Interpreter::enter`] and [`VirtualMachine::expect_pyresult`] for more information. pub fn enter_and_expect(&self, f: F, msg: &str) -> R where F: FnOnce(&VirtualMachine) -> PyResult, @@ -92,11 +91,11 @@ impl Interpreter { /// Run a function with the main virtual machine and return exit code. /// /// To enter vm context only once and safely terminate the vm, this function is preferred. - /// Unlike [`enter`], `run` calls finalize and returns exit code. + /// Unlike [`Interpreter::enter`], `run` calls finalize and returns exit code. /// You will not be able to obtain Python exception in this way. /// - /// See [`finalize`] for the finalization steps. - /// See also [`enter`] for pure function call to obtain Python exception. + /// See [`Interpreter::finalize`] for the finalization steps. + /// See also [`Interpreter::enter`] for pure function call to obtain Python exception. pub fn run(self, f: F) -> u8 where F: FnOnce(&VirtualMachine) -> PyResult<()>, diff --git a/vm/src/vm/mod.rs b/vm/src/vm/mod.rs index 4b1cece8c5..7baaae7770 100644 --- a/vm/src/vm/mod.rs +++ b/vm/src/vm/mod.rs @@ -61,7 +61,7 @@ pub const MAX_MEMORY_SIZE: usize = isize::MAX as usize; /// Top level container of a python virtual machine. In theory you could /// create more instances of this struct and have them operate fully isolated. /// -/// To construct this, please refer to the [`Interpreter`](Interpreter) +/// To construct this, please refer to the [`Interpreter`] pub struct VirtualMachine { pub builtins: PyRef, pub sys_module: PyRef, @@ -564,7 +564,7 @@ impl VirtualMachine { /// Call Python __import__ function without from_list. /// Roughly equivalent to `import module_name` or `import top.submodule`. /// - /// See also [`import_from`] for more advanced import. + /// See also [`VirtualMachine::import_from`] for more advanced import. /// See also [`rustpython_vm::import::import_source`] and other primitive import functions. #[inline] pub fn import<'a>(&self, module_name: impl AsPyStr<'a>, level: usize) -> PyResult { diff --git a/vm/src/vm/vm_ops.rs b/vm/src/vm/vm_ops.rs index c6be959a60..df33e822aa 100644 --- a/vm/src/vm/vm_ops.rs +++ b/vm/src/vm/vm_ops.rs @@ -152,9 +152,9 @@ impl VirtualMachine { /// Calling scheme used for binary operations: /// /// Order operations are tried until either a valid result or error: - /// b.rop(b,a)[*], a.op(a,b), b.rop(b,a) + /// `b.rop(b,a)[*], a.op(a,b), b.rop(b,a)` /// - /// [*] only when Py_TYPE(a) != Py_TYPE(b) && Py_TYPE(b) is a subclass of Py_TYPE(a) + /// `[*]` - only when Py_TYPE(a) != Py_TYPE(b) && Py_TYPE(b) is a subclass of Py_TYPE(a) pub fn binary_op1(&self, a: &PyObject, b: &PyObject, op_slot: PyNumberBinaryOp) -> PyResult { let class_a = a.class(); let class_b = b.class(); diff --git a/wtf8/src/lib.rs b/wtf8/src/lib.rs index 64ea42d06e..3ba28a5146 100644 --- a/wtf8/src/lib.rs +++ b/wtf8/src/lib.rs @@ -19,7 +19,7 @@ //! //! We use WTF-8 over something more similar to CPython's string implementation //! because of its compatibility with UTF-8, meaning that in the case where a -//! string has no surrogates, it can be viewed as a UTF-8 Rust [`str`] without +//! string has no surrogates, it can be viewed as a UTF-8 Rust [`prim@str`] without //! needing any copies or re-encoding. //! //! This implementation is mostly copied from the WTF-8 implementation in the From 36cce6b17464bb7744daf5e0d44eb24b223174ba Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sat, 5 Apr 2025 00:14:03 +0900 Subject: [PATCH 174/295] run fmt --- src/shell.rs | 1 - vm/src/lib.rs | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/shell.rs b/src/shell.rs index 04090a49fc..f733e03ddc 100644 --- a/src/shell.rs +++ b/src/shell.rs @@ -71,7 +71,6 @@ fn shell_exec( } } - /// Enter a repl loop pub fn run_shell(vm: &VirtualMachine, scope: Scope) -> PyResult<()> { let mut repl = Readline::new(helper::ShellHelper::new(vm, scope.globals.clone())); diff --git a/vm/src/lib.rs b/vm/src/lib.rs index 8ae5ff15db..de0042c619 100644 --- a/vm/src/lib.rs +++ b/vm/src/lib.rs @@ -3,7 +3,7 @@ //! - Interpreter //! - Import mechanics //! - Base objects -//! +//! //! Some stdlib modules are implemented here, but most of them are in the `rustpython-stdlib` module. The // to allow `mod foo {}` in foo.rs; clippy thinks this is a mistake/misunderstanding of From ab4dffb53ce2bb80a37efc000328c6d596296b18 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Fri, 4 Apr 2025 11:37:40 -0700 Subject: [PATCH 175/295] this should just fail if warnings happen because of RUSTFLAGS --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b728c81bc2..afd3201e28 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -326,7 +326,7 @@ jobs: - name: install ruff run: python -m pip install ruff==0.0.291 # astral-sh/ruff#7778 - name: Ensure docs generate no warnings - run: cargo doc -- -Dwarnings + run: cargo doc - name: run python lint run: ruff extra_tests wasm examples --exclude='./.*',./Lib,./vm/Lib,./benches/ --select=E9,F63,F7,F82 --show-source - name: install prettier From c2665e38ba636919d8deff103eef2d5215f5c05d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 4 Apr 2025 20:54:25 +0000 Subject: [PATCH 176/295] Bump openssl from 0.10.71 to 0.10.72 Bumps [openssl](https://github.com/sfackler/rust-openssl) from 0.10.71 to 0.10.72. - [Release notes](https://github.com/sfackler/rust-openssl/releases) - [Commits](https://github.com/sfackler/rust-openssl/compare/openssl-v0.10.71...openssl-v0.10.72) --- updated-dependencies: - dependency-name: openssl dependency-version: 0.10.72 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- Cargo.lock | 10 +++++----- stdlib/Cargo.toml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 577ef516bc..706f70f2d9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1336,7 +1336,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" dependencies = [ "cfg-if", - "windows-targets 0.48.5", + "windows-targets 0.52.6", ] [[package]] @@ -1659,9 +1659,9 @@ checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9" [[package]] name = "openssl" -version = "0.10.71" +version = "0.10.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e14130c6a98cd258fdcb0fb6d744152343ff729cbfcb28c656a9d12b999fbcd" +checksum = "fedfea7d58a1f73118430a55da6a286e7b044961736ce96a16a17068ea25e5da" dependencies = [ "bitflags 2.8.0", "cfg-if", @@ -1700,9 +1700,9 @@ dependencies = [ [[package]] name = "openssl-sys" -version = "0.9.106" +version = "0.9.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bb61ea9811cc39e3c2069f40b8b8e2e70d8569b361f879786cc7ed48b777cdd" +checksum = "8288979acd84749c744a9014b4382d42b8f7b2592847b5afb2ed29e5d16ede07" dependencies = [ "cc", "libc", diff --git a/stdlib/Cargo.toml b/stdlib/Cargo.toml index 0ec23bf132..a8393206d6 100644 --- a/stdlib/Cargo.toml +++ b/stdlib/Cargo.toml @@ -106,7 +106,7 @@ rustix = { workspace = true } gethostname = "1.0.0" socket2 = { version = "0.5.8", features = ["all"] } dns-lookup = "2" -openssl = { version = "0.10.66", optional = true } +openssl = { version = "0.10.72", optional = true } openssl-sys = { version = "0.9.80", optional = true } openssl-probe = { version = "0.1.5", optional = true } foreign-types-shared = { version = "0.1.1", optional = true } From d800a6bb9852bdf5c8cadd291d7a8969a9a81301 Mon Sep 17 00:00:00 2001 From: Hanif Ariffin Date: Sat, 5 Apr 2025 13:53:40 +0800 Subject: [PATCH 177/295] Update test_math from CPython 3.13.2 (#5610) Implemnted fma in math module. --- Lib/test/{ => mathdata}/cmath_testcases.txt | 3 + Lib/test/{ => mathdata}/ieee754.txt | 0 Lib/test/{ => mathdata}/math_testcases.txt | 0 Lib/test/test_math.py | 244 +++++++++++++++++++- stdlib/src/math.rs | 24 ++ 5 files changed, 268 insertions(+), 3 deletions(-) rename Lib/test/{ => mathdata}/cmath_testcases.txt (99%) rename Lib/test/{ => mathdata}/ieee754.txt (100%) rename Lib/test/{ => mathdata}/math_testcases.txt (100%) diff --git a/Lib/test/cmath_testcases.txt b/Lib/test/mathdata/cmath_testcases.txt similarity index 99% rename from Lib/test/cmath_testcases.txt rename to Lib/test/mathdata/cmath_testcases.txt index dd7e458ddc..0165e17634 100644 --- a/Lib/test/cmath_testcases.txt +++ b/Lib/test/mathdata/cmath_testcases.txt @@ -1536,6 +1536,7 @@ sqrt0141 sqrt -1.797e+308 -9.9999999999999999e+306 -> 3.7284476432057307e+152 -1 sqrt0150 sqrt 1.7976931348623157e+308 0.0 -> 1.3407807929942596355e+154 0.0 sqrt0151 sqrt 2.2250738585072014e-308 0.0 -> 1.4916681462400413487e-154 0.0 sqrt0152 sqrt 5e-324 0.0 -> 2.2227587494850774834e-162 0.0 +sqrt0153 sqrt 5e-324 1.0 -> 0.7071067811865476 0.7071067811865476 -- special values sqrt1000 sqrt 0.0 0.0 -> 0.0 0.0 @@ -1744,6 +1745,7 @@ cosh0023 cosh 2.218885944363501 2.0015727395883687 -> -1.94294321081968 4.129026 -- large real part cosh0030 cosh 710.5 2.3519999999999999 -> -1.2967465239355998e+308 1.3076707908857333e+308 cosh0031 cosh -710.5 0.69999999999999996 -> 1.4085466381392499e+308 -1.1864024666450239e+308 +cosh0032 cosh 720.0 0.0 -> inf 0.0 overflow -- Additional real values (mpmath) cosh0050 cosh 1e-150 0.0 -> 1.0 0.0 @@ -1853,6 +1855,7 @@ sinh0023 sinh 0.043713693678420068 0.22512549887532657 -> 0.042624198673416713 0 -- large real part sinh0030 sinh 710.5 -2.3999999999999999 -> -1.3579970564885919e+308 -1.24394470907798e+308 sinh0031 sinh -710.5 0.80000000000000004 -> -1.2830671601735164e+308 1.3210954193997678e+308 +sinh0032 sinh 720.0 0.0 -> inf 0.0 overflow -- Additional real values (mpmath) sinh0050 sinh 1e-100 0.0 -> 1.00000000000000002e-100 0.0 diff --git a/Lib/test/ieee754.txt b/Lib/test/mathdata/ieee754.txt similarity index 100% rename from Lib/test/ieee754.txt rename to Lib/test/mathdata/ieee754.txt diff --git a/Lib/test/math_testcases.txt b/Lib/test/mathdata/math_testcases.txt similarity index 100% rename from Lib/test/math_testcases.txt rename to Lib/test/mathdata/math_testcases.txt diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py index fa79456ed4..bb02041644 100644 --- a/Lib/test/test_math.py +++ b/Lib/test/test_math.py @@ -33,8 +33,8 @@ else: file = __file__ test_dir = os.path.dirname(file) or os.curdir -math_testcases = os.path.join(test_dir, 'math_testcases.txt') -test_file = os.path.join(test_dir, 'cmath_testcases.txt') +math_testcases = os.path.join(test_dir, 'mathdata', 'math_testcases.txt') +test_file = os.path.join(test_dir, 'mathdata', 'cmath_testcases.txt') def to_ulps(x): @@ -2628,9 +2628,247 @@ def test_fractions(self): self.assertAllNotClose(fraction_examples, rel_tol=1e-9) +class FMATests(unittest.TestCase): + """ Tests for math.fma. """ + + def test_fma_nan_results(self): + # Selected representative values. + values = [ + -math.inf, -1e300, -2.3, -1e-300, -0.0, + 0.0, 1e-300, 2.3, 1e300, math.inf, math.nan + ] + + # If any input is a NaN, the result should be a NaN, too. + for a, b in itertools.product(values, repeat=2): + self.assertIsNaN(math.fma(math.nan, a, b)) + self.assertIsNaN(math.fma(a, math.nan, b)) + self.assertIsNaN(math.fma(a, b, math.nan)) + + def test_fma_infinities(self): + # Cases involving infinite inputs or results. + positives = [1e-300, 2.3, 1e300, math.inf] + finites = [-1e300, -2.3, -1e-300, -0.0, 0.0, 1e-300, 2.3, 1e300] + non_nans = [-math.inf, -2.3, -0.0, 0.0, 2.3, math.inf] + + # ValueError due to inf * 0 computation. + for c in non_nans: + for infinity in [math.inf, -math.inf]: + for zero in [0.0, -0.0]: + with self.assertRaises(ValueError): + math.fma(infinity, zero, c) + with self.assertRaises(ValueError): + math.fma(zero, infinity, c) + + # ValueError when a*b and c both infinite of opposite signs. + for b in positives: + with self.assertRaises(ValueError): + math.fma(math.inf, b, -math.inf) + with self.assertRaises(ValueError): + math.fma(math.inf, -b, math.inf) + with self.assertRaises(ValueError): + math.fma(-math.inf, -b, -math.inf) + with self.assertRaises(ValueError): + math.fma(-math.inf, b, math.inf) + with self.assertRaises(ValueError): + math.fma(b, math.inf, -math.inf) + with self.assertRaises(ValueError): + math.fma(-b, math.inf, math.inf) + with self.assertRaises(ValueError): + math.fma(-b, -math.inf, -math.inf) + with self.assertRaises(ValueError): + math.fma(b, -math.inf, math.inf) + + # Infinite result when a*b and c both infinite of the same sign. + for b in positives: + self.assertEqual(math.fma(math.inf, b, math.inf), math.inf) + self.assertEqual(math.fma(math.inf, -b, -math.inf), -math.inf) + self.assertEqual(math.fma(-math.inf, -b, math.inf), math.inf) + self.assertEqual(math.fma(-math.inf, b, -math.inf), -math.inf) + self.assertEqual(math.fma(b, math.inf, math.inf), math.inf) + self.assertEqual(math.fma(-b, math.inf, -math.inf), -math.inf) + self.assertEqual(math.fma(-b, -math.inf, math.inf), math.inf) + self.assertEqual(math.fma(b, -math.inf, -math.inf), -math.inf) + + # Infinite result when a*b finite, c infinite. + for a, b in itertools.product(finites, finites): + self.assertEqual(math.fma(a, b, math.inf), math.inf) + self.assertEqual(math.fma(a, b, -math.inf), -math.inf) + + # Infinite result when a*b infinite, c finite. + for b, c in itertools.product(positives, finites): + self.assertEqual(math.fma(math.inf, b, c), math.inf) + self.assertEqual(math.fma(-math.inf, b, c), -math.inf) + self.assertEqual(math.fma(-math.inf, -b, c), math.inf) + self.assertEqual(math.fma(math.inf, -b, c), -math.inf) + + self.assertEqual(math.fma(b, math.inf, c), math.inf) + self.assertEqual(math.fma(b, -math.inf, c), -math.inf) + self.assertEqual(math.fma(-b, -math.inf, c), math.inf) + self.assertEqual(math.fma(-b, math.inf, c), -math.inf) + + # gh-73468: On some platforms, libc fma() doesn't implement IEE 754-2008 + # properly: it doesn't use the right sign when the result is zero. + @unittest.skipIf( + sys.platform.startswith(("freebsd", "wasi", "netbsd")) + or (sys.platform == "android" and platform.machine() == "x86_64"), + f"this platform doesn't implement IEE 754-2008 properly") + def test_fma_zero_result(self): + nonnegative_finites = [0.0, 1e-300, 2.3, 1e300] + + # Zero results from exact zero inputs. + for b in nonnegative_finites: + self.assertIsPositiveZero(math.fma(0.0, b, 0.0)) + self.assertIsPositiveZero(math.fma(0.0, b, -0.0)) + self.assertIsNegativeZero(math.fma(0.0, -b, -0.0)) + self.assertIsPositiveZero(math.fma(0.0, -b, 0.0)) + self.assertIsPositiveZero(math.fma(-0.0, -b, 0.0)) + self.assertIsPositiveZero(math.fma(-0.0, -b, -0.0)) + self.assertIsNegativeZero(math.fma(-0.0, b, -0.0)) + self.assertIsPositiveZero(math.fma(-0.0, b, 0.0)) + + self.assertIsPositiveZero(math.fma(b, 0.0, 0.0)) + self.assertIsPositiveZero(math.fma(b, 0.0, -0.0)) + self.assertIsNegativeZero(math.fma(-b, 0.0, -0.0)) + self.assertIsPositiveZero(math.fma(-b, 0.0, 0.0)) + self.assertIsPositiveZero(math.fma(-b, -0.0, 0.0)) + self.assertIsPositiveZero(math.fma(-b, -0.0, -0.0)) + self.assertIsNegativeZero(math.fma(b, -0.0, -0.0)) + self.assertIsPositiveZero(math.fma(b, -0.0, 0.0)) + + # Exact zero result from nonzero inputs. + self.assertIsPositiveZero(math.fma(2.0, 2.0, -4.0)) + self.assertIsPositiveZero(math.fma(2.0, -2.0, 4.0)) + self.assertIsPositiveZero(math.fma(-2.0, -2.0, -4.0)) + self.assertIsPositiveZero(math.fma(-2.0, 2.0, 4.0)) + + # Underflow to zero. + tiny = 1e-300 + self.assertIsPositiveZero(math.fma(tiny, tiny, 0.0)) + self.assertIsNegativeZero(math.fma(tiny, -tiny, 0.0)) + self.assertIsPositiveZero(math.fma(-tiny, -tiny, 0.0)) + self.assertIsNegativeZero(math.fma(-tiny, tiny, 0.0)) + self.assertIsPositiveZero(math.fma(tiny, tiny, -0.0)) + self.assertIsNegativeZero(math.fma(tiny, -tiny, -0.0)) + self.assertIsPositiveZero(math.fma(-tiny, -tiny, -0.0)) + self.assertIsNegativeZero(math.fma(-tiny, tiny, -0.0)) + + # Corner case where rounding the multiplication would + # give the wrong result. + x = float.fromhex('0x1p-500') + y = float.fromhex('0x1p-550') + z = float.fromhex('0x1p-1000') + self.assertIsNegativeZero(math.fma(x-y, x+y, -z)) + self.assertIsPositiveZero(math.fma(y-x, x+y, z)) + self.assertIsNegativeZero(math.fma(y-x, -(x+y), -z)) + self.assertIsPositiveZero(math.fma(x-y, -(x+y), z)) + + def test_fma_overflow(self): + a = b = float.fromhex('0x1p512') + c = float.fromhex('0x1p1023') + # Overflow from multiplication. + with self.assertRaises(OverflowError): + math.fma(a, b, 0.0) + self.assertEqual(math.fma(a, b/2.0, 0.0), c) + # Overflow from the addition. + with self.assertRaises(OverflowError): + math.fma(a, b/2.0, c) + # No overflow, even though a*b overflows a float. + self.assertEqual(math.fma(a, b, -c), c) + + # Extreme case: a * b is exactly at the overflow boundary, so the + # tiniest offset makes a difference between overflow and a finite + # result. + a = float.fromhex('0x1.ffffffc000000p+511') + b = float.fromhex('0x1.0000002000000p+512') + c = float.fromhex('0x0.0000000000001p-1022') + with self.assertRaises(OverflowError): + math.fma(a, b, 0.0) + with self.assertRaises(OverflowError): + math.fma(a, b, c) + self.assertEqual(math.fma(a, b, -c), + float.fromhex('0x1.fffffffffffffp+1023')) + + # Another extreme case: here a*b is about as large as possible subject + # to math.fma(a, b, c) being finite. + a = float.fromhex('0x1.ae565943785f9p+512') + b = float.fromhex('0x1.3094665de9db8p+512') + c = float.fromhex('0x1.fffffffffffffp+1023') + self.assertEqual(math.fma(a, b, -c), c) + + def test_fma_single_round(self): + a = float.fromhex('0x1p-50') + self.assertEqual(math.fma(a - 1.0, a + 1.0, 1.0), a*a) + + def test_random(self): + # A collection of randomly generated inputs for which the naive FMA + # (with two rounds) gives a different result from a singly-rounded FMA. + + # tuples (a, b, c, expected) + test_values = [ + ('0x1.694adde428b44p-1', '0x1.371b0d64caed7p-1', + '0x1.f347e7b8deab8p-4', '0x1.19f10da56c8adp-1'), + ('0x1.605401ccc6ad6p-2', '0x1.ce3a40bf56640p-2', + '0x1.96e3bf7bf2e20p-2', '0x1.1af6d8aa83101p-1'), + ('0x1.e5abd653a67d4p-2', '0x1.a2e400209b3e6p-1', + '0x1.a90051422ce13p-1', '0x1.37d68cc8c0fbbp+0'), + ('0x1.f94e8efd54700p-2', '0x1.123065c812cebp-1', + '0x1.458f86fb6ccd0p-1', '0x1.ccdcee26a3ff3p-1'), + ('0x1.bd926f1eedc96p-1', '0x1.eee9ca68c5740p-1', + '0x1.960c703eb3298p-2', '0x1.3cdcfb4fdb007p+0'), + ('0x1.27348350fbccdp-1', '0x1.3b073914a53f1p-1', + '0x1.e300da5c2b4cbp-1', '0x1.4c51e9a3c4e29p+0'), + ('0x1.2774f00b3497bp-1', '0x1.7038ec336bff0p-2', + '0x1.2f6f2ccc3576bp-1', '0x1.99ad9f9c2688bp-1'), + ('0x1.51d5a99300e5cp-1', '0x1.5cd74abd445a1p-1', + '0x1.8880ab0bbe530p-1', '0x1.3756f96b91129p+0'), + ('0x1.73cb965b821b8p-2', '0x1.218fd3d8d5371p-1', + '0x1.d1ea966a1f758p-2', '0x1.5217b8fd90119p-1'), + ('0x1.4aa98e890b046p-1', '0x1.954d85dff1041p-1', + '0x1.122b59317ebdfp-1', '0x1.0bf644b340cc5p+0'), + ('0x1.e28f29e44750fp-1', '0x1.4bcc4fdcd18fep-1', + '0x1.fd47f81298259p-1', '0x1.9b000afbc9995p+0'), + ('0x1.d2e850717fe78p-3', '0x1.1dd7531c303afp-1', + '0x1.e0869746a2fc2p-2', '0x1.316df6eb26439p-1'), + ('0x1.cf89c75ee6fbap-2', '0x1.b23decdc66825p-1', + '0x1.3d1fe76ac6168p-1', '0x1.00d8ea4c12abbp+0'), + ('0x1.3265ae6f05572p-2', '0x1.16d7ec285f7a2p-1', + '0x1.0b8405b3827fbp-1', '0x1.5ef33c118a001p-1'), + ('0x1.c4d1bf55ec1a5p-1', '0x1.bc59618459e12p-2', + '0x1.ce5b73dc1773dp-1', '0x1.496cf6164f99bp+0'), + ('0x1.d350026ac3946p-1', '0x1.9a234e149a68cp-2', + '0x1.f5467b1911fd6p-2', '0x1.b5cee3225caa5p-1'), + ] + for a_hex, b_hex, c_hex, expected_hex in test_values: + a = float.fromhex(a_hex) + b = float.fromhex(b_hex) + c = float.fromhex(c_hex) + expected = float.fromhex(expected_hex) + self.assertEqual(math.fma(a, b, c), expected) + self.assertEqual(math.fma(b, a, c), expected) + + # Custom assertions. + def assertIsNaN(self, value): + self.assertTrue( + math.isnan(value), + msg="Expected a NaN, got {!r}".format(value) + ) + + def assertIsPositiveZero(self, value): + self.assertTrue( + value == 0 and math.copysign(1, value) > 0, + msg="Expected a positive zero, got {!r}".format(value) + ) + + def assertIsNegativeZero(self, value): + self.assertTrue( + value == 0 and math.copysign(1, value) < 0, + msg="Expected a negative zero, got {!r}".format(value) + ) + + def load_tests(loader, tests, pattern): from doctest import DocFileSuite - tests.addTest(DocFileSuite("ieee754.txt")) + tests.addTest(DocFileSuite(os.path.join("mathdata", "ieee754.txt"))) return tests if __name__ == '__main__': diff --git a/stdlib/src/math.rs b/stdlib/src/math.rs index 6665ee8b49..a7da60949c 100644 --- a/stdlib/src/math.rs +++ b/stdlib/src/math.rs @@ -975,4 +975,28 @@ mod math { Ok(result) } + + #[pyfunction] + fn fma( + x: ArgIntoFloat, + y: ArgIntoFloat, + z: ArgIntoFloat, + vm: &VirtualMachine, + ) -> PyResult { + let result = (*x).mul_add(*y, *z); + + if result.is_finite() { + return Ok(result); + } + + if result.is_nan() { + if !x.is_nan() && !y.is_nan() && !z.is_nan() { + return Err(vm.new_value_error("invalid operation in fma".to_string())); + } + } else if x.is_finite() && y.is_finite() && z.is_finite() { + return Err(vm.new_overflow_error("overflow in fma".to_string())); + } + + Ok(result) + } } From 2230d6c751eb8fdd2d7554df529574ea53b906b8 Mon Sep 17 00:00:00 2001 From: Hanif Ariffin Date: Sat, 5 Apr 2025 14:33:33 +0800 Subject: [PATCH 178/295] Fix not throwing the same error as CPython in test_pathlib.test_expanduser (#5578) * Fix not throwing the same error as CPython when trying to expanduser of a non-existent user Signed-off-by: Hanif Ariffin * add pwd test * Skip pwd test on windows --------- Signed-off-by: Hanif Ariffin Co-authored-by: Jeong YunWon Co-authored-by: Jeong, YunWon <69878+youknowone@users.noreply.github.com> --- extra_tests/snippets/stdlib_pwd.py | 12 ++++++++++++ vm/src/stdlib/pwd.rs | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 extra_tests/snippets/stdlib_pwd.py diff --git a/extra_tests/snippets/stdlib_pwd.py b/extra_tests/snippets/stdlib_pwd.py new file mode 100644 index 0000000000..6ef3a64d02 --- /dev/null +++ b/extra_tests/snippets/stdlib_pwd.py @@ -0,0 +1,12 @@ +import sys +# windows doesn't support pwd +if sys.platform.startswith("win"): + exit(0) + +from testutils import assert_raises +import pwd + +with assert_raises(KeyError): + fake_name = 'fake_user' + while pwd.getpwnam(fake_name): + fake_name += '1' diff --git a/vm/src/stdlib/pwd.rs b/vm/src/stdlib/pwd.rs index b95910c73f..20b4edb448 100644 --- a/vm/src/stdlib/pwd.rs +++ b/vm/src/stdlib/pwd.rs @@ -59,7 +59,7 @@ mod pwd { if pw_name.contains('\0') { return Err(exceptions::cstring_error(vm)); } - let user = User::from_name(name.as_str()).map_err(|err| err.into_pyexception(vm))?; + let user = User::from_name(name.as_str()).ok().flatten(); let user = user.ok_or_else(|| { vm.new_key_error( vm.ctx From 98137eb79c8813ad3b6de3e583d05572a0c89b35 Mon Sep 17 00:00:00 2001 From: Noa Date: Sun, 21 Aug 2022 15:10:50 -0500 Subject: [PATCH 179/295] Switch to const-initialized thread_local variables where appropriate --- common/src/static_cell.rs | 4 +++- common/src/str.rs | 5 ++++- vm/src/stdlib/thread.rs | 4 +++- wasm/lib/src/vm_class.rs | 4 +++- 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/common/src/static_cell.rs b/common/src/static_cell.rs index 30e34f187f..a8beee0820 100644 --- a/common/src/static_cell.rs +++ b/common/src/static_cell.rs @@ -56,7 +56,9 @@ mod non_threading { $($(#[$attr])* $vis static $name: $crate::static_cell::StaticCell<$t> = { ::std::thread_local! { - $vis static $name: $crate::lock::OnceCell<&'static $t> = $crate::lock::OnceCell::new(); + $vis static $name: $crate::lock::OnceCell<&'static $t> = const { + $crate::lock::OnceCell::new() + }; } $crate::static_cell::StaticCell::_from_local_key(&$name) };)+ diff --git a/common/src/str.rs b/common/src/str.rs index fa26959e0b..ca5e0d117f 100644 --- a/common/src/str.rs +++ b/common/src/str.rs @@ -486,7 +486,10 @@ pub mod levenshtein { pub fn levenshtein_distance(a: &str, b: &str, max_cost: usize) -> usize { thread_local! { - static BUFFER: RefCell<[usize; MAX_STRING_SIZE]> = const { RefCell::new([0usize; MAX_STRING_SIZE]) }; + #[allow(clippy::declare_interior_mutable_const)] + static BUFFER: RefCell<[usize; MAX_STRING_SIZE]> = const { + RefCell::new([0usize; MAX_STRING_SIZE]) + }; } if a == b { diff --git a/vm/src/stdlib/thread.rs b/vm/src/stdlib/thread.rs index 0ee087e5e4..b3e345b20a 100644 --- a/vm/src/stdlib/thread.rs +++ b/vm/src/stdlib/thread.rs @@ -355,7 +355,9 @@ pub(crate) mod _thread { Err(vm.new_exception_empty(vm.ctx.exceptions.system_exit.to_owned())) } - thread_local!(static SENTINELS: RefCell>> = RefCell::default()); + thread_local! { + static SENTINELS: RefCell>> = const { RefCell::new(Vec::new()) }; + } #[pyfunction] fn _set_sentinel(vm: &VirtualMachine) -> PyRef { diff --git a/wasm/lib/src/vm_class.rs b/wasm/lib/src/vm_class.rs index d84603e986..bbd895c989 100644 --- a/wasm/lib/src/vm_class.rs +++ b/wasm/lib/src/vm_class.rs @@ -86,7 +86,9 @@ pub fn add_init_func(f: fn(&mut VirtualMachine)) { // https://rustwasm.github.io/2018/10/24/multithreading-rust-and-wasm.html#atomic-instructions thread_local! { static STORED_VMS: RefCell>> = RefCell::default(); - static VM_INIT_FUNCS: RefCell> = RefCell::default(); + static VM_INIT_FUNCS: RefCell> = const { + RefCell::new(Vec::new()) + }; } pub fn get_vm_id(vm: &VirtualMachine) -> &str { From be56911598b2cbe0719b655a6861bf317ca2aa8a Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sun, 6 Apr 2025 01:21:28 -0700 Subject: [PATCH 180/295] _tkinter pt. 2 (#5640) --- .cspell.dict/python-more.txt | 1 + Cargo.lock | 791 +++++++++++++---------------------- Lib/tkinter/__init__.py | 4 +- flamegraph.svg | 1 + stdlib/Cargo.toml | 6 +- stdlib/src/tkinter.rs | 487 +++++++++++++++++++-- 6 files changed, 745 insertions(+), 545 deletions(-) create mode 100644 flamegraph.svg diff --git a/.cspell.dict/python-more.txt b/.cspell.dict/python-more.txt index 2edfe95bdf..32d13f59b9 100644 --- a/.cspell.dict/python-more.txt +++ b/.cspell.dict/python-more.txt @@ -224,6 +224,7 @@ sysvars teedata thisclass titlecased +tkapp tobytes tolist toreadonly diff --git a/Cargo.lock b/Cargo.lock index 706f70f2d9..f3a8f59af3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -59,9 +59,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.96" +version = "1.0.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b964d184e89d9b6b67dd2715bc8e74cf3107fb2b529990c90cf517326150bf4" +checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f" [[package]] name = "approx" @@ -116,37 +116,24 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" -[[package]] -name = "bind_syn" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d6608ba072b4bc847774fac76963956592b5cdfa3751afcefa252fb61cb85b9" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.98", -] - [[package]] name = "bindgen" -version = "0.64.0" +version = "0.71.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4243e6031260db77ede97ad86c27e501d646a27ab57b59a574f725d98ab1fb4" +checksum = "5f58bf3d7db68cfbac37cfc485a8d711e87e064c3d0fe0435b92f7a407f9d6b3" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.9.0", "cexpr", "clang-sys", - "lazy_static 1.5.0", - "lazycell", + "itertools 0.11.0", "log", - "peeking_take_while", + "prettyplease", "proc-macro2", "quote", "regex", - "rustc-hash 1.1.0", + "rustc-hash", "shlex", - "syn 1.0.109", - "which 4.4.2", + "syn 2.0.100", ] [[package]] @@ -157,9 +144,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.8.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36" +checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" [[package]] name = "blake2" @@ -201,15 +188,9 @@ dependencies = [ [[package]] name = "bytemuck" -version = "1.21.0" +version = "1.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef657dfab802224e671f5818e9a4935f9b1957ed18e58292690cc39e7a4092a3" - -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +checksum = "b6b1fc10dbac614ebc03540c9dbd60e83887fda27794998c6528f1782047d540" [[package]] name = "bzip2" @@ -223,12 +204,11 @@ dependencies = [ [[package]] name = "bzip2-sys" -version = "0.1.12+1.0.8" +version = "0.1.13+1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72ebc2f1a417f01e1da30ef264ee86ae31d2dcd2d603ea283d3c244a883ca2a9" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" dependencies = [ "cc", - "libc", "pkg-config", ] @@ -258,35 +238,13 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.14" +version = "1.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c3d1b2e905a3a7b00a6141adb0e4c0bb941d11caf55349d863942a1cc44e3c9" +checksum = "1fcb57c740ae1daf453ae85f16e37396f672b039e00d9d866e07ddb24e328e3a" dependencies = [ "shlex", ] -[[package]] -name = "cex" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b0114a3f232423fadbbdbb692688e3e68c3b58b4b063ac3a7d0190d561080da" -dependencies = [ - "cex_derive", - "enumx", -] - -[[package]] -name = "cex_derive" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b5048cd656d7d2e739960fa33d9f95693005792dc8aad0af8b8f0b7d76c938d" -dependencies = [ - "indexmap 1.9.3", - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "cexpr" version = "0.6.0" @@ -310,16 +268,16 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrono" -version = "0.4.39" +version = "0.4.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825" +checksum = "1a7964611d71df112cb1730f2ee67324fcf4d0fc6606acbbe9bfe06df124637c" dependencies = [ "android-tzdata", "iana-time-zone", "js-sys", "num-traits", "wasm-bindgen", - "windows-targets 0.52.6", + "windows-link", ] [[package]] @@ -344,19 +302,6 @@ dependencies = [ "unicode-width 0.1.14", ] -[[package]] -name = "clib" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fda1a698cd341f055d3ae1fdbfb1cc441e7f9bacce795356ecc685e69134e957" -dependencies = [ - "anyhow", - "bindgen", - "inwelling", - "pkg-config", - "toml", -] - [[package]] name = "clipboard-win" version = "5.4.0" @@ -382,9 +327,9 @@ dependencies = [ [[package]] name = "console" -version = "0.15.10" +version = "0.15.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea3c6ecd8059b57859df5c69830340ed3c41d30e3da0c1cbed90a96ac853041b" +checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8" dependencies = [ "encode_unicode", "libc", @@ -484,10 +429,10 @@ dependencies = [ "cranelift-entity", "cranelift-isle", "gimli", - "hashbrown 0.15.2", + "hashbrown", "log", "regalloc2", - "rustc-hash 2.1.1", + "rustc-hash", "serde", "smallvec", "target-lexicon 0.13.2", @@ -711,7 +656,7 @@ checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", "unicode-xid", ] @@ -761,15 +706,15 @@ dependencies = [ [[package]] name = "dyn-clone" -version = "1.0.18" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "feeef44e73baff3a26d371801df019877a9866a8c493d315ab00177843314f35" +checksum = "1c7a8fb8a9fbf66c1f703fe16184d10ca0ee9d23be5b4436400408ba54a95005" [[package]] name = "either" -version = "1.13.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" [[package]] name = "encode_unicode" @@ -783,27 +728,6 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" -[[package]] -name = "enumx" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32875abeb14f7fe2c2b8ad15e58f41701f455d124d0a03bc88132d5face2663f" -dependencies = [ - "enumx_derive", -] - -[[package]] -name = "enumx_derive" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa5d66efdd1eab6ea85ba31bdb58bed1e4ce218c1361061384ece88f40ebeb49" -dependencies = [ - "indexmap 1.9.3", - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "env_logger" version = "0.9.3" @@ -851,13 +775,13 @@ checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" [[package]] name = "fd-lock" -version = "4.0.2" +version = "4.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e5768da2206272c81ef0b5e951a41862938a6070da63bcea197899942d3b947" +checksum = "0ce92ff622d6dadf7349484f42c93271a0d49b7cc4d466a936405bacbe10aa78" dependencies = [ "cfg-if", - "rustix", - "windows-sys 0.52.0", + "rustix 1.0.3", + "windows-sys 0.59.0", ] [[package]] @@ -891,7 +815,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8168cbad48fdda10be94de9c6319f9e8ac5d3cf0a1abda1864269dfcca3d302a" dependencies = [ "flame", - "indexmap 2.7.1", + "indexmap", "serde", "serde_json", ] @@ -946,11 +870,11 @@ dependencies = [ [[package]] name = "gethostname" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fd4b8790c0792e3b11895efdf5f289ebe8b59107a6624f1cce68f24ff8c7035" +checksum = "ed7131e57abbde63513e0e6636f76668a1ca9798dcae2df4e283cae9ee83859e" dependencies = [ - "rustix", + "rustix 1.0.3", "windows-targets 0.52.6", ] @@ -978,16 +902,16 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8" +checksum = "73fea8450eea4bac3940448fb7ae50d91f034f941199fcd9d909a5a07aa455f0" dependencies = [ "cfg-if", "js-sys", "libc", - "wasi 0.13.3+wasi-0.2.2", + "r-efi", + "wasi 0.14.2+wasi-0.2.4", "wasm-bindgen", - "windows-targets 0.52.6", ] [[package]] @@ -997,7 +921,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" dependencies = [ "fallible-iterator", - "indexmap 2.7.1", + "indexmap", "stable_deref_trait", ] @@ -1015,20 +939,14 @@ checksum = "1b43ede17f21864e81be2fa654110bf1e793774238d86ef8555c37e6519c0403" [[package]] name = "half" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" +checksum = "7db2ff139bba50379da6aa0766b52fdcb62cb5b263009b09ed58ba604e14bbd1" dependencies = [ "cfg-if", "crunchy", ] -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" - [[package]] name = "hashbrown" version = "0.15.2" @@ -1038,30 +956,12 @@ dependencies = [ "foldhash", ] -[[package]] -name = "heck" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" -dependencies = [ - "unicode-segmentation", -] - [[package]] name = "heck" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" -[[package]] -name = "heredom" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a7a4d76fa670b51cfb56e908ad8bfd44a14fee853ea764790e46634d3fcdf4d" -dependencies = [ - "tuplex", -] - [[package]] name = "hermit-abi" version = "0.1.19" @@ -1100,16 +1000,17 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.61" +version = "0.1.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" +checksum = "b0c919e5debc312ad217002b8048a17b7d83f80703865bbfcfebb0458b0b27d8" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", + "log", "wasm-bindgen", - "windows-core", + "windows-core 0.61.0", ] [[package]] @@ -1123,35 +1024,25 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" -dependencies = [ - "autocfg", - "hashbrown 0.12.3", -] - -[[package]] -name = "indexmap" -version = "2.7.1" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c9c992b02b5b4c94ea26e32fe5bccb7aa7d9f390ab5c1221ff895bc7ea8b652" +checksum = "3954d50fe15b02142bf25d3b8bdadb634ec3948f103d04ffe3031bc8fe9d7058" dependencies = [ "equivalent", - "hashbrown 0.15.2", + "hashbrown", ] [[package]] name = "indoc" -version = "2.0.5" +version = "2.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" +checksum = "f4c7245a08504955605670dbf141fceab975f15ca21570696aebe9d2e71576bd" [[package]] name = "insta" -version = "1.42.1" +version = "1.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71c1b125e30d93896b365e156c33dadfffab45ee8400afcbba4752f59de08a86" +checksum = "50259abbaa67d11d2bcafc7ba1d094ed7a0c70e3ce893f0d0997f73558cb3084" dependencies = [ "console", "linked-hash-map", @@ -1160,26 +1051,16 @@ dependencies = [ "similar", ] -[[package]] -name = "inwelling" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f6292c68ffca1fa94ca8f95ca5ad2885d79d96377f1d37ced6a47cd26cfaf8c" -dependencies = [ - "toml", - "walkdir", -] - [[package]] name = "is-macro" version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d57a3e447e24c22647738e4607f1df1e0ec6f72e16182c4cd199f647cdfb0e4" dependencies = [ - "heck 0.5.0", + "heck", "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -1211,9 +1092,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.14" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" [[package]] name = "js-sys" @@ -1246,9 +1127,13 @@ dependencies = [ [[package]] name = "lambert_w" -version = "1.0.17" +version = "1.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45bf98425154bfe790a47b72ac452914f6df9ebfb202bc59e089e29db00258cf" +checksum = "0eeec1be8d026f51b1cf70ed28442b9f0ece61ff196cd3a99d8b4492a83a864b" +dependencies = [ + "num-complex", + "num-traits", +] [[package]] name = "lazy_static" @@ -1262,12 +1147,6 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" -[[package]] -name = "lazycell" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" - [[package]] name = "lexical-parse-float" version = "1.0.5" @@ -1300,15 +1179,15 @@ dependencies = [ [[package]] name = "lexopt" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baff4b617f7df3d896f97fe922b64817f6cd9a756bb81d40f8883f2f66dcb401" +checksum = "9fa0e2a1fcbe2f6be6c42e342259976206b383122fc152e872795338b5a3f3a7" [[package]] name = "libc" -version = "0.2.169" +version = "0.2.171" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" +checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" [[package]] name = "libffi" @@ -1351,7 +1230,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.0", "libc", ] @@ -1387,6 +1266,12 @@ version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" +[[package]] +name = "linux-raw-sys" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe7db12097d22ec582439daf8618b8fdd1a7bef6270e9af3b1ebcd30893cf413" + [[package]] name = "lock_api" version = "0.4.12" @@ -1399,9 +1284,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.25" +version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f" +checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" [[package]] name = "lz4_flex" @@ -1437,7 +1322,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5063891d2cec8fd20cabccbd3fc277bf8d5666f481fb3f79d999559b39a62713" dependencies = [ - "hashbrown 0.15.2", + "hashbrown", "itertools 0.11.0", "libm", "ryu", @@ -1533,9 +1418,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3b1c9bd4fe1f0f8b387f6eb9eb3b4a1aa26185e5750efb9140301703f62cd1b" +checksum = "8e3e04debbb59698c15bacbb6d93584a8c0ca9cc3213cb423d31f760d8843ce5" dependencies = [ "adler2", ] @@ -1549,12 +1434,6 @@ dependencies = [ "rand_core 0.9.3", ] -[[package]] -name = "mutf8" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97b444426a4c188e9ad33560853ebd52309ab72811f536a9e6f37907fd12cf45" - [[package]] name = "nibble_vec" version = "0.1.0" @@ -1570,7 +1449,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.0", "cfg-if", "cfg_aliases", "libc", @@ -1639,23 +1518,22 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af1844ef2428cc3e1cb900be36181049ef3d3193c63e43026cfe202983b27a56" dependencies = [ - "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] name = "once_cell" -version = "1.20.3" +version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "945462a4b81e43c4e3ba96bd7b49d834c6f61198356aa858733bc4acf3cbe62e" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" [[package]] name = "oorandom" -version = "11.1.4" +version = "11.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" [[package]] name = "openssl" @@ -1663,7 +1541,7 @@ version = "0.10.72" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fedfea7d58a1f73118430a55da6a286e7b044961736ce96a16a17068ea25e5da" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.0", "cfg-if", "foreign-types", "libc", @@ -1680,7 +1558,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -1745,7 +1623,7 @@ checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.5.8", + "redox_syscall 0.5.10", "smallvec", "windows-targets 0.52.6", ] @@ -1756,12 +1634,6 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" -[[package]] -name = "peeking_take_while" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" - [[package]] name = "phf" version = "0.11.3" @@ -1802,29 +1674,29 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.9" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfe2e71e1471fe07709406bf725f710b02927c9c54b2b5b2ec0e8087d97c327d" +checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.9" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6e859e6e5bd50440ab63c47e3ebabc90f26251f7c73c3d3e837b74a1cc3fa67" +checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] name = "pkg-config" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" [[package]] name = "plotters" @@ -1862,38 +1734,39 @@ checksum = "52a40bc70c2c58040d2d8b167ba9a5ff59fc9dab7ad44771cfde3dcfde7a09c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] name = "portable-atomic" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "280dc24453071f1b63954171985a0b0d30058d287960968b9b2aca264c8d4ee6" +checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" [[package]] name = "ppv-lite86" -version = "0.2.20" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" dependencies = [ - "zerocopy 0.7.35", + "zerocopy 0.8.24", ] [[package]] -name = "proc-macro-crate" -version = "3.3.0" +name = "prettyplease" +version = "0.2.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edce586971a4dfaa28950c6f18ed55e0406c1ab88bbce2c6f6293a7aaba73d35" +checksum = "5316f57387668042f561aae71480de936257848f9c43ce528e311d89a07cadeb" dependencies = [ - "toml_edit 0.22.24", + "proc-macro2", + "syn 2.0.100", ] [[package]] name = "proc-macro2" -version = "1.0.93" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99" +checksum = "a31971752e70b8b2686d7e46ec17fb38dad4051d94024c88df49b667caea9c84" dependencies = [ "unicode-ident", ] @@ -1955,7 +1828,7 @@ dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -1964,22 +1837,28 @@ version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "36c011a03ba1e50152b4b394b479826cad97e7a21eb52df179cd91ac411cbfbe" dependencies = [ - "heck 0.5.0", + "heck", "proc-macro2", "pyo3-build-config", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] name = "quote" -version = "1.0.38" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" + [[package]] name = "radium" version = "0.7.0" @@ -2015,7 +1894,7 @@ checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.3", - "zerocopy 0.8.20", + "zerocopy 0.8.24", ] [[package]] @@ -2053,7 +1932,7 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" dependencies = [ - "getrandom 0.3.1", + "getrandom 0.3.2", ] [[package]] @@ -2084,11 +1963,11 @@ checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" [[package]] name = "redox_syscall" -version = "0.5.8" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" +checksum = "0b8c0c260b63a8219631167be35e6a988e9554dbd323f8bd08439c8ed1302bd1" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.0", ] [[package]] @@ -2110,9 +1989,9 @@ checksum = "145c1c267e14f20fb0f88aa76a1c5ffec42d592c1d28b3cd9148ae35916158d3" dependencies = [ "allocator-api2", "bumpalo", - "hashbrown 0.15.2", + "hashbrown", "log", - "rustc-hash 2.1.1", + "rustc-hash", "smallvec", ] @@ -2175,7 +2054,7 @@ dependencies = [ "pmutil", "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -2184,7 +2063,7 @@ version = "0.0.0" source = "git+https://github.com/astral-sh/ruff.git?tag=0.11.0#2cd25ef6410fb5fca96af1578728a3d828d2d53a" dependencies = [ "aho-corasick", - "bitflags 2.8.0", + "bitflags 2.9.0", "compact_str", "is-macro", "itertools 0.14.0", @@ -2192,7 +2071,7 @@ dependencies = [ "ruff_python_trivia", "ruff_source_file", "ruff_text_size", - "rustc-hash 2.1.1", + "rustc-hash", ] [[package]] @@ -2200,14 +2079,14 @@ name = "ruff_python_parser" version = "0.0.0" source = "git+https://github.com/astral-sh/ruff.git?tag=0.11.0#2cd25ef6410fb5fca96af1578728a3d828d2d53a" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.0", "bstr", "compact_str", "memchr", "ruff_python_ast", "ruff_python_trivia", "ruff_text_size", - "rustc-hash 2.1.1", + "rustc-hash", "static_assertions", "unicode-ident", "unicode-normalization", @@ -2239,12 +2118,6 @@ name = "ruff_text_size" version = "0.0.0" source = "git+https://github.com/astral-sh/ruff.git?tag=0.11.0#2cd25ef6410fb5fca96af1578728a3d828d2d53a" -[[package]] -name = "rustc-hash" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" - [[package]] name = "rustc-hash" version = "2.1.1" @@ -2257,10 +2130,23 @@ version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.0", "errno", "libc", - "linux-raw-sys", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustix" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e56a18552996ac8d29ecc3b190b4fdbb2d91ca4ec396de7bbffaf43f3d637e96" +dependencies = [ + "bitflags 2.9.0", + "errno", + "libc", + "linux-raw-sys 0.9.3", "windows-sys 0.59.0", ] @@ -2291,8 +2177,8 @@ name = "rustpython-codegen" version = "0.4.0" dependencies = [ "ahash", - "bitflags 2.8.0", - "indexmap 2.7.1", + "bitflags 2.9.0", + "indexmap", "insta", "itertools 0.14.0", "log", @@ -2307,7 +2193,7 @@ dependencies = [ "rustpython-compiler-source", "rustpython-literal", "rustpython-wtf8", - "thiserror 2.0.11", + "thiserror 2.0.12", "unicode_names2", ] @@ -2316,10 +2202,10 @@ name = "rustpython-common" version = "0.4.0" dependencies = [ "ascii", - "bitflags 2.8.0", + "bitflags 2.9.0", "bstr", "cfg-if", - "getrandom 0.3.1", + "getrandom 0.3.2", "itertools 0.14.0", "libc", "lock_api", @@ -2352,14 +2238,14 @@ dependencies = [ "rustpython-codegen", "rustpython-compiler-core", "rustpython-compiler-source", - "thiserror 2.0.11", + "thiserror 2.0.12", ] [[package]] name = "rustpython-compiler-core" version = "0.4.0" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.0", "itertools 0.14.0", "lz4_flex", "malachite-bigint", @@ -2384,7 +2270,7 @@ dependencies = [ "proc-macro2", "rustpython-compiler", "rustpython-derive-impl", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -2397,9 +2283,9 @@ dependencies = [ "quote", "rustpython-compiler-core", "rustpython-doc", - "syn 2.0.98", + "syn 2.0.100", "syn-ext", - "textwrap 0.16.1", + "textwrap 0.16.2", ] [[package]] @@ -2422,7 +2308,7 @@ dependencies = [ "num-traits", "rustpython-compiler-core", "rustpython-derive", - "thiserror 2.0.11", + "thiserror 2.0.12", ] [[package]] @@ -2451,7 +2337,7 @@ dependencies = [ name = "rustpython-sre_engine" version = "0.4.0" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.0", "criterion", "num_enum", "optional", @@ -2479,7 +2365,7 @@ dependencies = [ "foreign-types-shared", "gethostname", "hex", - "indexmap 2.7.1", + "indexmap", "itertools 0.14.0", "junction", "libc", @@ -2504,7 +2390,7 @@ dependencies = [ "paste", "puruspe", "rand_core 0.9.3", - "rustix", + "rustix 0.38.44", "rustpython-common", "rustpython-derive", "rustpython-vm", @@ -2514,9 +2400,9 @@ dependencies = [ "sha3", "socket2", "system-configuration", - "tcl", + "tcl-sys", "termios", - "tk", + "tk-sys", "ucd", "unic-char-property", "unic-normal", @@ -2538,7 +2424,7 @@ version = "0.4.0" dependencies = [ "ahash", "ascii", - "bitflags 2.8.0", + "bitflags 2.9.0", "bstr", "caseless", "cfg-if", @@ -2548,11 +2434,11 @@ dependencies = [ "exitcode", "flame", "flamer", - "getrandom 0.3.1", + "getrandom 0.3.2", "glob", - "half 2.4.1", + "half 2.5.0", "hex", - "indexmap 2.7.1", + "indexmap", "is-macro", "itertools 0.14.0", "junction", @@ -2578,7 +2464,7 @@ dependencies = [ "ruff_python_parser", "ruff_source_file", "ruff_text_size", - "rustix", + "rustix 0.38.44", "rustpython-codegen", "rustpython-common", "rustpython-compiler", @@ -2592,9 +2478,9 @@ dependencies = [ "schannel", "serde", "static_assertions", - "strum 0.27.1", - "strum_macros 0.27.1", - "thiserror 2.0.11", + "strum", + "strum_macros", + "thiserror 2.0.12", "thread_local", "timsort", "uname", @@ -2604,7 +2490,7 @@ dependencies = [ "unicode-casing", "unicode_names2", "wasm-bindgen", - "which 6.0.3", + "which", "widestring", "windows", "windows-sys 0.59.0", @@ -2642,9 +2528,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.19" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7c45b9784283f1b2e7fb61b42047c2fd678ef0960d4f6f1eba131594cc369d4" +checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2" [[package]] name = "rustyline" @@ -2652,7 +2538,7 @@ version = "15.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2ee1e066dc922e513bda599c6ccb5f3bb2b0ea5870a579448f2622993f0a9a2f" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.0", "cfg-if", "clipboard-win", "fd-lock", @@ -2670,9 +2556,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.19" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ea1a2d0a644769cc99faa24c3ad26b379b786fe7c36fd3c546254801650e6dd" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" [[package]] name = "same-file" @@ -2700,9 +2586,9 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "serde" -version = "1.0.218" +version = "1.0.219" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8dfc9d19bdbf6d17e22319da49161d5d0108e4188e8b680aef6299eed22df60" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" dependencies = [ "serde_derive", ] @@ -2731,20 +2617,20 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.218" +version = "1.0.219" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f09503e191f4e797cb8aac08e9a4a4695c5edf6a2e70e376d961ddd5c969f82b" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] name = "serde_json" -version = "1.0.139" +version = "1.0.140" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44f86c3acccc9c65b153fe1b85a3be07fe5515274ec9f0653b4a0875731c72a6" +checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" dependencies = [ "itoa", "memchr", @@ -2752,15 +2638,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_spanned" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" -dependencies = [ - "serde", -] - [[package]] name = "sha-1" version = "0.10.1" @@ -2819,9 +2696,9 @@ checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd" [[package]] name = "socket2" -version = "0.5.8" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8" +checksum = "4f5fd57c80058a56cf5c777ab8a126398ece8e442983605d280a44ce79d0edef" dependencies = [ "libc", "windows-sys 0.52.0", @@ -2839,41 +2716,23 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" -[[package]] -name = "strum" -version = "0.19.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b89a286a7e3b5720b9a477b23253bc50debac207c8d21505f8e70b36792f11b5" - [[package]] name = "strum" version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f64def088c51c9510a8579e3c5d67c65349dcf755e5479ad3d010aa6454e2c32" -[[package]] -name = "strum_macros" -version = "0.19.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e61bb0be289045cb80bfce000512e32d09f8337e54c186725da381377ad1f8d5" -dependencies = [ - "heck 0.3.3", - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "strum_macros" version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c77a8c5abcaf0f9ce05d62342b7d298c346515365c36b673df4ebe3ced01fde8" dependencies = [ - "heck 0.5.0", + "heck", "proc-macro2", "quote", "rustversion", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -2895,9 +2754,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.98" +version = "2.0.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36147f1a48ae0ec2b5b3bc5b537d267457555a10dc06f3dbc8cb11ba3006d3b1" +checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0" dependencies = [ "proc-macro2", "quote", @@ -2912,7 +2771,7 @@ checksum = "b126de4ef6c2a628a68609dd00733766c3b015894698a438ebdf374933fc31d1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -2949,33 +2808,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e502f78cdbb8ba4718f566c418c52bc729126ffd16baee5baa718cf25dd5a69a" [[package]] -name = "tcl" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e0d0928e8b4dca8ebd485f687f725bb34e454c7a28c1d353bf7d1b8060581bf" -dependencies = [ - "cex", - "clib", - "enumx", - "inwelling", - "mutf8", - "serde", - "serde_derive", - "tcl_derive", - "tuplex", -] - -[[package]] -name = "tcl_derive" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "625d95e672231bbf31dead6861b0ad72bcb71a2891b26b0c4924cd1cc9687b93" +name = "tcl-sys" +version = "0.1.0" +source = "git+https://github.com/arihant2math/tkinter.git?tag=v0.1.0#09a4f62e894df64692b34e6c7f81af1e1ae376dd" dependencies = [ - "bind_syn", - "proc-macro2", - "quote", - "syn 2.0.98", - "uuid", + "bindgen", + "pkg-config", ] [[package]] @@ -3007,9 +2845,9 @@ dependencies = [ [[package]] name = "textwrap" -version = "0.16.1" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9" +checksum = "c13547615a44dc9c452a8a534638acdf07120d4b6847c8178705da06306a3057" [[package]] name = "thiserror" @@ -3022,11 +2860,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.11" +version = "2.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d452f284b73e6d76dd36758a0c8684b1d5be31f92b89d07fd5822175732206fc" +checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" dependencies = [ - "thiserror-impl 2.0.11", + "thiserror-impl 2.0.12", ] [[package]] @@ -3037,18 +2875,18 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] name = "thiserror-impl" -version = "2.0.11" +version = "2.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2" +checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -3090,9 +2928,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.8.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "022db8904dfa342efe721985167e9fcd16c29b226db4397ed752a761cfce81e8" +checksum = "09b3661f17e86524eccd4371ab0429194e0d7c008abb45f7a7495b1719463c71" dependencies = [ "tinyvec_macros", ] @@ -3104,78 +2942,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] -name = "tk" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6fbe29c813c9eee5e0d4d996a4a615f6538220f0ad181269a413f21c13eb077" -dependencies = [ - "bitflags 1.3.2", - "cex", - "clib", - "enumx", - "heredom", - "inwelling", - "num_enum", - "once_cell", - "serde", - "strum 0.19.5", - "strum_macros 0.19.4", - "tcl", - "tcl_derive", - "tuplex", -] - -[[package]] -name = "toml" -version = "0.7.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd79e69d3b627db300ff956027cc6c3798cef26d22526befdfcd12feeb6d2257" -dependencies = [ - "serde", - "serde_spanned", - "toml_datetime", - "toml_edit 0.19.15", -] - -[[package]] -name = "toml_datetime" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" -dependencies = [ - "serde", -] - -[[package]] -name = "toml_edit" -version = "0.19.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" -dependencies = [ - "indexmap 2.7.1", - "serde", - "serde_spanned", - "toml_datetime", - "winnow 0.5.40", -] - -[[package]] -name = "toml_edit" -version = "0.22.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474" +name = "tk-sys" +version = "0.1.0" +source = "git+https://github.com/arihant2math/tkinter.git?tag=v0.1.0#09a4f62e894df64692b34e6c7f81af1e1ae376dd" dependencies = [ - "indexmap 2.7.1", - "toml_datetime", - "winnow 0.7.4", + "bindgen", + "pkg-config", ] -[[package]] -name = "tuplex" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "676ac81d5454c4dcf37955d34fa8626ede3490f744b86ca14a7b90168d2a08aa" - [[package]] name = "twox-hash" version = "1.6.3" @@ -3320,9 +3094,9 @@ checksum = "623f59e6af2a98bdafeb93fa277ac8e1e40440973001ca15cf4ae1541cd16d56" [[package]] name = "unicode-ident" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00e2473a93778eb0bad35909dff6a10d28e63f792f16ed15e404fca9d5eeedbe" +checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" [[package]] name = "unicode-normalization" @@ -3381,9 +3155,9 @@ dependencies = [ [[package]] name = "unindent" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" +checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" [[package]] name = "utf8parse" @@ -3393,12 +3167,11 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.13.2" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c1f41ffb7cf259f1ecc2876861a17e7142e63ead296f671f81f6ae85903e0d6" +checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9" dependencies = [ "atomic", - "getrandom 0.3.1", ] [[package]] @@ -3437,9 +3210,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasi" -version = "0.13.3+wasi-0.2.2" +version = "0.14.2+wasi-0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26816d2e1a4a36a2940b96c5296ce403917633dff8f3440e9b236ed6f6bacad2" +checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" dependencies = [ "wit-bindgen-rt", ] @@ -3466,7 +3239,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", "wasm-bindgen-shared", ] @@ -3501,7 +3274,7 @@ checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3537,18 +3310,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "which" -version = "4.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" -dependencies = [ - "either", - "home", - "once_cell", - "rustix", -] - [[package]] name = "which" version = "6.0.3" @@ -3557,15 +3318,15 @@ checksum = "b4ee928febd44d98f2f459a4a79bd4d928591333a494a10a868418ac1b39cf1f" dependencies = [ "either", "home", - "rustix", + "rustix 0.38.44", "winsafe", ] [[package]] name = "widestring" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7219d36b6eac893fa81e84ebe06485e7dcbb616177469b142df14f1f4deb1311" +checksum = "dd7cf3379ca1aac9eea11fba24fd7e315d621f8dfe35c8d7d2be8b793726e07d" [[package]] name = "winapi" @@ -3604,7 +3365,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be" dependencies = [ - "windows-core", + "windows-core 0.52.0", "windows-targets 0.52.6", ] @@ -3617,6 +3378,65 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-core" +version = "0.61.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4763c1de310c86d75a878046489e2e5ba02c649d185f21c67d4cf8a56d098980" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "windows-interface" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "windows-link" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" + +[[package]] +name = "windows-result" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c64fd11a4fd95df68efcfee5f44a294fe71b8bc6a91993e2791938abcc712252" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2ba9642430ee452d5a7aa78d72907ebe8cfda358e8cb7918a2050581322f97" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-sys" version = "0.48.0" @@ -3765,24 +3585,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "winnow" -version = "0.5.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876" -dependencies = [ - "memchr", -] - -[[package]] -name = "winnow" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e97b544156e9bebe1a0ffbc03484fc1ffe3100cbce3ffb17eac35f7cdd7ab36" -dependencies = [ - "memchr", -] - [[package]] name = "winreg" version = "0.55.0" @@ -3801,11 +3603,11 @@ checksum = "d135d17ab770252ad95e9a872d365cf3090e3be864a34ab46f48555993efc904" [[package]] name = "wit-bindgen-rt" -version = "0.33.0" +version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c" +checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.0", ] [[package]] @@ -3820,17 +3622,16 @@ version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ - "byteorder", "zerocopy-derive 0.7.35", ] [[package]] name = "zerocopy" -version = "0.8.20" +version = "0.8.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dde3bb8c68a8f3f1ed4ac9221aad6b10cece3e60a8e2ea54a6a2dec806d0084c" +checksum = "2586fea28e186957ef732a5f8b3be2da217d65c5969d4b1e17f973ebbe876879" dependencies = [ - "zerocopy-derive 0.8.20", + "zerocopy-derive 0.8.24", ] [[package]] @@ -3841,18 +3642,18 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] name = "zerocopy-derive" -version = "0.8.20" +version = "0.8.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eea57037071898bf96a6da35fd626f4f27e9cee3ead2a6c703cf09d472b2e700" +checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index 8b2502b4c0..df3b936ccd 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -2451,7 +2451,9 @@ def __init__(self, screenName=None, baseName=None, className='Tk', self.tk = None if baseName is None: import os - baseName = os.path.basename(sys.argv[0]) + # TODO: RUSTPYTHON + # baseName = os.path.basename(sys.argv[0]) + baseName = "" # sys.argv[0] baseName, ext = os.path.splitext(baseName) if ext not in ('.py', '.pyc'): baseName = baseName + ext diff --git a/flamegraph.svg b/flamegraph.svg new file mode 100644 index 0000000000..a196068928 --- /dev/null +++ b/flamegraph.svg @@ -0,0 +1 @@ +ERROR: No valid input provided to flamegraph \ No newline at end of file diff --git a/stdlib/Cargo.toml b/stdlib/Cargo.toml index a8393206d6..26c85103e5 100644 --- a/stdlib/Cargo.toml +++ b/stdlib/Cargo.toml @@ -18,7 +18,7 @@ bz2 = ["bzip2"] sqlite = ["dep:libsqlite3-sys"] ssl = ["openssl", "openssl-sys", "foreign-types-shared", "openssl-probe"] ssl-vendor = ["ssl", "openssl/vendored"] -tkinter = ["dep:tk", "dep:tcl"] +tkinter = ["dep:tk-sys", "dep:tcl-sys"] [dependencies] # rustpython crates @@ -83,8 +83,8 @@ libz-sys = { package = "libz-rs-sys", version = "0.4" } bzip2 = { version = "0.4", optional = true } # tkinter -tk = { version = "0.1.10", optional = true } -tcl = { version = "0.1.9", optional = true } +tk-sys = { git = "https://github.com/arihant2math/tkinter.git", tag = "v0.1.0", optional = true } +tcl-sys = { git = "https://github.com/arihant2math/tkinter.git", tag = "v0.1.0", optional = true } # uuid [target.'cfg(not(any(target_os = "ios", target_os = "android", target_os = "windows", target_arch = "wasm32", target_os = "redox")))'.dependencies] diff --git a/stdlib/src/tkinter.rs b/stdlib/src/tkinter.rs index 907dc55002..242570b410 100644 --- a/stdlib/src/tkinter.rs +++ b/stdlib/src/tkinter.rs @@ -4,83 +4,478 @@ pub(crate) use self::_tkinter::make_module; #[pymodule] mod _tkinter { - use crate::builtins::PyTypeRef; - use rustpython_vm::function::{Either, FuncArgs}; - use rustpython_vm::{PyResult, VirtualMachine, function::OptionalArg}; + use rustpython_vm::types::Constructor; + use rustpython_vm::{PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine}; - use crate::common::lock::PyRwLock; - use std::sync::Arc; - use tk::cmd::*; - use tk::*; + use rustpython_vm::builtins::{PyInt, PyStr, PyType}; + use std::{ffi, ptr}; - #[pyattr] - const TK_VERSION: &str = "8.6"; - #[pyattr] - const TCL_VERSION: &str = "8.6"; - #[pyattr] - const READABLE: i32 = 2; - #[pyattr] - const WRITABLE: i32 = 4; - #[pyattr] - const EXCEPTION: i32 = 8; - - fn demo() -> tk::TkResult<()> { - let tk = make_tk!()?; - let root = tk.root(); - root.add_label(-text("constructs widgets and layout step by step"))? - .pack(())?; - let f = root.add_frame(())?.pack(())?; - let _btn = f - .add_button("btn" - text("quit") - command("destroy ."))? - .pack(())?; - Ok(main_loop()) + use crate::builtins::PyTypeRef; + use rustpython_common::atomic::AtomicBool; + use rustpython_common::atomic::Ordering; + + #[cfg(windows)] + fn _get_tcl_lib_path() -> String { + // TODO: fix packaging + String::from(r"C:\ActiveTcl\lib") } - #[pyattr(once, name = "TclError")] + #[pyattr(name = "TclError", once)] fn tcl_error(vm: &VirtualMachine) -> PyTypeRef { vm.ctx.new_exception_type( - "zlib", + "_tkinter", "TclError", Some(vec![vm.ctx.exceptions.exception_type.to_owned()]), ) } - #[pyfunction] - fn create(args: FuncArgs, _vm: &VirtualMachine) -> PyResult { - // TODO: handle arguments - // TODO: this means creating 2 tk instances is not possible. - let tk = Tk::new(()).unwrap(); - Ok(TkApp { - tk: Arc::new(PyRwLock::new(tk)), - }) + #[pyattr(name = "TkError", once)] + fn tk_error(vm: &VirtualMachine) -> PyTypeRef { + vm.ctx.new_exception_type( + "_tkinter", + "TkError", + Some(vec![vm.ctx.exceptions.exception_type.to_owned()]), + ) + } + + #[pyattr(once, name = "TK_VERSION")] + fn tk_version(_vm: &VirtualMachine) -> String { + format!("{}.{}", 8, 6) + } + + #[pyattr(once, name = "TCL_VERSION")] + fn tcl_version(_vm: &VirtualMachine) -> String { + format!( + "{}.{}", + tk_sys::TCL_MAJOR_VERSION, + tk_sys::TCL_MINOR_VERSION + ) + } + + #[pyattr] + #[pyclass(name = "TclObject")] + #[derive(PyPayload)] + struct TclObject { + value: *mut tk_sys::Tcl_Obj, } + impl std::fmt::Debug for TclObject { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "TclObject") + } + } + + unsafe impl Send for TclObject {} + unsafe impl Sync for TclObject {} + + #[pyclass] + impl TclObject {} + + static QUIT_MAIN_LOOP: AtomicBool = AtomicBool::new(false); + #[pyattr] #[pyclass(name = "tkapp")] #[derive(PyPayload)] struct TkApp { - tk: Arc>>, + // Tcl_Interp *interp; + interpreter: *mut tk_sys::Tcl_Interp, + // int wantobjects; + want_objects: bool, + // int threaded; /* True if tcl_platform[threaded] */ + threaded: bool, + // Tcl_ThreadId thread_id; + thread_id: Option, + // int dispatching; + dispatching: bool, + // PyObject *trace; + trace: Option<()>, + // /* We cannot include tclInt.h, as this is internal. + // So we cache interesting types here. */ + old_boolean_type: *const tk_sys::Tcl_ObjType, + boolean_type: *const tk_sys::Tcl_ObjType, + byte_array_type: *const tk_sys::Tcl_ObjType, + double_type: *const tk_sys::Tcl_ObjType, + int_type: *const tk_sys::Tcl_ObjType, + wide_int_type: *const tk_sys::Tcl_ObjType, + bignum_type: *const tk_sys::Tcl_ObjType, + list_type: *const tk_sys::Tcl_ObjType, + string_type: *const tk_sys::Tcl_ObjType, + utf32_string_type: *const tk_sys::Tcl_ObjType, + pixel_type: *const tk_sys::Tcl_ObjType, } unsafe impl Send for TkApp {} - unsafe impl Sync for TkApp {} impl std::fmt::Debug for TkApp { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("TkApp").finish() + write!(f, "TkApp") } } - #[pyclass] + #[derive(FromArgs, Debug)] + struct TkAppConstructorArgs { + #[pyarg(any)] + screen_name: Option, + #[pyarg(any)] + _base_name: Option, + #[pyarg(any)] + class_name: String, + #[pyarg(any)] + interactive: i32, + #[pyarg(any)] + wantobjects: i32, + #[pyarg(any, default = "true")] + want_tk: bool, + #[pyarg(any)] + sync: i32, + #[pyarg(any)] + use_: Option, + } + + impl Constructor for TkApp { + type Args = TkAppConstructorArgs; + + fn py_new( + _zelf: PyRef, + args: Self::Args, + vm: &VirtualMachine, + ) -> PyResult { + create(args, vm) + } + } + + fn varname_converter(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { + // if let Ok(bytes) = obj.bytes(vm) { + // todo!() + // } + + // str + if let Some(str) = obj.downcast_ref::() { + return Ok(str.as_str().to_string()); + } + + if let Some(_tcl_obj) = obj.downcast_ref::() { + // Assume that the Tcl object has a method to retrieve a string. + // return tcl_obj. + todo!(); + } + + // Construct an error message using the type name (truncated to 50 characters). + Err(vm.new_type_error(format!( + "must be str, bytes or Tcl_Obj, not {:.50}", + obj.obj_type().str(vm)?.as_str() + ))) + } + + // TODO: DISALLOW_INSTANTIATION + #[pyclass(with(Constructor))] impl TkApp { + fn from_bool(&self, obj: *mut tk_sys::Tcl_Obj) -> bool { + let mut res = -1; + unsafe { + if tk_sys::Tcl_GetBooleanFromObj(self.interpreter, obj, &mut res) + != tk_sys::TCL_OK as i32 + { + panic!("Tcl_GetBooleanFromObj failed"); + } + } + assert!(res == 0 || res == 1); + res != 0 + } + + fn from_object( + &self, + obj: *mut tk_sys::Tcl_Obj, + vm: &VirtualMachine, + ) -> PyResult { + let type_ptr = unsafe { (*obj).typePtr }; + if type_ptr == ptr::null() { + return self.unicode_from_object(obj, vm); + } else if type_ptr == self.old_boolean_type || type_ptr == self.boolean_type { + return Ok(vm.ctx.new_bool(self.from_bool(obj)).into()); + } else if type_ptr == self.string_type + || type_ptr == self.utf32_string_type + || type_ptr == self.pixel_type + { + return self.unicode_from_object(obj, vm); + } + // TODO: handle other types + + return Ok(TclObject { value: obj }.into_pyobject(vm)); + } + + fn unicode_from_string( + s: *mut ffi::c_char, + size: usize, + vm: &VirtualMachine, + ) -> PyResult { + // terribly unsafe + let s = unsafe { std::slice::from_raw_parts(s, size) } + .to_vec() + .into_iter() + .map(|c| c as u8) + .collect::>(); + let s = String::from_utf8(s).unwrap(); + Ok(PyObjectRef::from(vm.ctx.new_str(s))) + } + + fn unicode_from_object( + &self, + obj: *mut tk_sys::Tcl_Obj, + vm: &VirtualMachine, + ) -> PyResult { + let type_ptr = unsafe { (*obj).typePtr }; + if type_ptr != ptr::null() + && self.interpreter != ptr::null_mut() + && (type_ptr == self.string_type || type_ptr == self.utf32_string_type) + { + let len = ptr::null_mut(); + let data = unsafe { tk_sys::Tcl_GetUnicodeFromObj(obj, len) }; + return if size_of::() == 2 { + let v = unsafe { std::slice::from_raw_parts(data as *const u16, len as usize) }; + let s = String::from_utf16(v).unwrap(); + Ok(PyObjectRef::from(vm.ctx.new_str(s))) + } else { + let v = unsafe { std::slice::from_raw_parts(data as *const u32, len as usize) }; + let s = widestring::U32String::from_vec(v).to_string_lossy(); + Ok(PyObjectRef::from(vm.ctx.new_str(s))) + }; + } + let len = ptr::null_mut(); + let s = unsafe { tk_sys::Tcl_GetStringFromObj(obj, len) }; + Self::unicode_from_string(s, len as _, vm) + } + #[pymethod] - fn getvar(&self, name: &str) -> PyResult { - let tk = self.tk.read().unwrap(); - Ok(tk.getvar(name).unwrap()) + fn getvar(&self, arg: PyObjectRef, vm: &VirtualMachine) -> PyResult { + // TODO: technically not thread safe + let name = varname_converter(arg, vm)?; + + let res = unsafe { + tk_sys::Tcl_GetVar2Ex( + self.interpreter, + ptr::null(), + name.as_ptr() as _, + tk_sys::TCL_LEAVE_ERR_MSG as _, + ) + }; + if res == ptr::null_mut() { + todo!(); + } + let res = if self.want_objects { + self.from_object(res, vm) + } else { + self.unicode_from_object(res, vm) + }?; + Ok(res) + } + + #[pymethod] + fn getint(&self, arg: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if let Some(int) = arg.downcast_ref::() { + return Ok(PyObjectRef::from(vm.ctx.new_int(int.as_bigint().clone()))); + } + + if let Some(obj) = arg.downcast_ref::() { + let value = obj.value; + unsafe { tk_sys::Tcl_IncrRefCount(value) }; + } else { + todo!(); + } + todo!(); + } + // TODO: Fix arguments + #[pymethod] + fn mainloop(&self, threshold: Option) -> PyResult<()> { + let threshold = threshold.unwrap_or(0); + todo!(); } #[pymethod] - fn createcommand(&self, name: String, callback: PyObjectRef) {} + fn quit(&self) { + QUIT_MAIN_LOOP.store(true, Ordering::Relaxed); + } } + + #[pyfunction] + fn create(args: TkAppConstructorArgs, vm: &VirtualMachine) -> PyResult { + unsafe { + let interp = tk_sys::Tcl_CreateInterp(); + let want_objects = args.wantobjects != 0; + let threaded = { + let part1 = String::from("tcl_platform"); + let part2 = String::from("threaded"); + let part1_ptr = part1.as_ptr(); + let part2_ptr = part2.as_ptr(); + tk_sys::Tcl_GetVar2Ex( + interp, + part1_ptr as _, + part2_ptr as _, + tk_sys::TCL_GLOBAL_ONLY as ffi::c_int, + ) + } != ptr::null_mut(); + let thread_id = tk_sys::Tcl_GetCurrentThread(); + let dispatching = false; + let trace = None; + // TODO: Handle threaded build + let bool_str = String::from("oldBoolean"); + let old_boolean_type = tk_sys::Tcl_GetObjType(bool_str.as_ptr() as _); + let (boolean_type, byte_array_type) = { + let true_str = String::from("true"); + let mut value = *tk_sys::Tcl_NewStringObj(true_str.as_ptr() as _, -1); + let mut bool_value = 0; + tk_sys::Tcl_GetBooleanFromObj(interp, &mut value, &mut bool_value); + let boolean_type = value.typePtr; + tk_sys::Tcl_DecrRefCount(&mut value); + + let mut value = + *tk_sys::Tcl_NewByteArrayObj(&bool_value as *const i32 as *const u8, 1); + let byte_array_type = value.typePtr; + tk_sys::Tcl_DecrRefCount(&mut value); + (boolean_type, byte_array_type) + }; + let double_str = String::from("double"); + let double_type = tk_sys::Tcl_GetObjType(double_str.as_ptr() as _); + let int_str = String::from("int"); + let int_type = tk_sys::Tcl_GetObjType(int_str.as_ptr() as _); + let int_type = if int_type == ptr::null() { + let mut value = *tk_sys::Tcl_NewIntObj(0); + let res = value.typePtr; + tk_sys::Tcl_DecrRefCount(&mut value); + res + } else { + int_type + }; + let wide_int_str = String::from("wideInt"); + let wide_int_type = tk_sys::Tcl_GetObjType(wide_int_str.as_ptr() as _); + let bignum_str = String::from("bignum"); + let bignum_type = tk_sys::Tcl_GetObjType(bignum_str.as_ptr() as _); + let list_str = String::from("list"); + let list_type = tk_sys::Tcl_GetObjType(list_str.as_ptr() as _); + let string_str = String::from("string"); + let string_type = tk_sys::Tcl_GetObjType(string_str.as_ptr() as _); + let utf32_str = String::from("utf32"); + let utf32_string_type = tk_sys::Tcl_GetObjType(utf32_str.as_ptr() as _); + let pixel_str = String::from("pixel"); + let pixel_type = tk_sys::Tcl_GetObjType(pixel_str.as_ptr() as _); + + let exit_str = String::from("exit"); + tk_sys::Tcl_DeleteCommand(interp, exit_str.as_ptr() as _); + + if let Some(name) = args.screen_name { + tk_sys::Tcl_SetVar2( + interp, + "env".as_ptr() as _, + "DISPLAY".as_ptr() as _, + name.as_ptr() as _, + tk_sys::TCL_GLOBAL_ONLY as i32, + ); + } + + if args.interactive != 0 { + tk_sys::Tcl_SetVar( + interp, + "tcl_interactive".as_ptr() as _, + "1".as_ptr() as _, + tk_sys::TCL_GLOBAL_ONLY as i32, + ); + } else { + tk_sys::Tcl_SetVar( + interp, + "tcl_interactive".as_ptr() as _, + "0".as_ptr() as _, + tk_sys::TCL_GLOBAL_ONLY as i32, + ); + } + + let argv0 = args.class_name.clone().to_lowercase(); + tk_sys::Tcl_SetVar( + interp, + "argv0".as_ptr() as _, + argv0.as_ptr() as _, + tk_sys::TCL_GLOBAL_ONLY as i32, + ); + + if !args.want_tk { + tk_sys::Tcl_SetVar( + interp, + "_tkinter_skip_tk_init".as_ptr() as _, + "1".as_ptr() as _, + tk_sys::TCL_GLOBAL_ONLY as i32, + ); + } + + if args.sync != 0 || args.use_.is_some() { + let mut argv = String::with_capacity(4); + if args.sync != 0 { + argv.push_str("-sync"); + } + if args.use_.is_some() { + if args.sync != 0 { + argv.push(' '); + } + argv.push_str("-use "); + argv.push_str(&args.use_.unwrap()); + } + argv.push_str("\0"); + let argv_ptr = argv.as_ptr() as *mut *mut i8; + tk_sys::Tcl_SetVar( + interp, + "argv".as_ptr() as _, + argv_ptr as *const i8, + tk_sys::TCL_GLOBAL_ONLY as i32, + ); + } + + #[cfg(windows)] + { + let ret = std::env::var("TCL_LIBRARY"); + if ret.is_err() { + let loc = _get_tcl_lib_path(); + std::env::set_var("TCL_LIBRARY", loc); + } + } + + // Bindgen cannot handle Tcl_AppInit + if tk_sys::Tcl_Init(interp) != tk_sys::TCL_OK as ffi::c_int { + todo!("Tcl_Init failed"); + } + + Ok(TkApp { + interpreter: interp, + want_objects, + threaded, + thread_id: Some(thread_id), + dispatching, + trace, + old_boolean_type, + boolean_type, + byte_array_type, + double_type, + int_type, + wide_int_type, + bignum_type, + list_type, + string_type, + utf32_string_type, + pixel_type, + } + .into_pyobject(vm)) + } + } + + #[pyattr] + const READABLE: i32 = tk_sys::TCL_READABLE as i32; + #[pyattr] + const WRITABLE: i32 = tk_sys::TCL_WRITABLE as i32; + #[pyattr] + const EXCEPTION: i32 = tk_sys::TCL_EXCEPTION as i32; + + #[pyattr] + const TIMER_EVENTS: i32 = tk_sys::TCL_TIMER_EVENTS as i32; + #[pyattr] + const IDLE_EVENTS: i32 = tk_sys::TCL_IDLE_EVENTS as i32; + #[pyattr] + const DONT_WAIT: i32 = tk_sys::TCL_DONT_WAIT as i32; } From 3c6bc2cf9f83a77cdeeb850799bc19f160bdf145 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sun, 6 Apr 2025 01:22:26 -0700 Subject: [PATCH 181/295] Add _suggestions module (#5675) --- stdlib/src/lib.rs | 2 ++ stdlib/src/suggestions.rs | 20 ++++++++++++++++++++ vm/src/suggestion.rs | 2 +- 3 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 stdlib/src/suggestions.rs diff --git a/stdlib/src/lib.rs b/stdlib/src/lib.rs index 9ee8e3e81d..e9d10dfde4 100644 --- a/stdlib/src/lib.rs +++ b/stdlib/src/lib.rs @@ -33,6 +33,7 @@ mod pyexpat; mod pystruct; mod random; mod statistics; +mod suggestions; // TODO: maybe make this an extension module, if we ever get those // mod re; #[cfg(feature = "bz2")] @@ -133,6 +134,7 @@ pub fn get_module_inits() -> impl Iterator, StdlibInit "unicodedata" => unicodedata::make_module, "zlib" => zlib::make_module, "_statistics" => statistics::make_module, + "suggestions" => suggestions::make_module, // crate::vm::sysmodule::sysconfigdata_name() => sysconfigdata::make_module, } #[cfg(any(unix, target_os = "wasi"))] diff --git a/stdlib/src/suggestions.rs b/stdlib/src/suggestions.rs new file mode 100644 index 0000000000..e49e9dd4a4 --- /dev/null +++ b/stdlib/src/suggestions.rs @@ -0,0 +1,20 @@ +pub(crate) use _suggestions::make_module; + +#[pymodule] +mod _suggestions { + use rustpython_vm::VirtualMachine; + + use crate::vm::PyObjectRef; + + #[pyfunction] + fn _generate_suggestions( + candidates: Vec, + name: PyObjectRef, + vm: &VirtualMachine, + ) -> PyObjectRef { + match crate::vm::suggestion::calculate_suggestions(candidates.iter(), &name) { + Some(suggestion) => suggestion.into(), + None => vm.ctx.none(), + } + } +} diff --git a/vm/src/suggestion.rs b/vm/src/suggestion.rs index 01f53d70ca..3d075ee3bb 100644 --- a/vm/src/suggestion.rs +++ b/vm/src/suggestion.rs @@ -12,7 +12,7 @@ use std::iter::ExactSizeIterator; const MAX_CANDIDATE_ITEMS: usize = 750; -fn calculate_suggestions<'a>( +pub fn calculate_suggestions<'a>( dir_iter: impl ExactSizeIterator, name: &PyObjectRef, ) -> Option { From 861055f558779e307fae56a09274e7e482c5f770 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sun, 6 Apr 2025 01:23:56 -0700 Subject: [PATCH 182/295] Add nt constants (#5676) --- vm/src/stdlib/nt.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/vm/src/stdlib/nt.rs b/vm/src/stdlib/nt.rs index 428d3421fd..083824bcd0 100644 --- a/vm/src/stdlib/nt.rs +++ b/vm/src/stdlib/nt.rs @@ -37,7 +37,13 @@ pub(crate) mod module { use libc::{O_BINARY, O_TEMPORARY}; #[pyattr] - const _LOAD_LIBRARY_SEARCH_DEFAULT_DIRS: i32 = 4096; + use windows_sys::Win32::System::LibraryLoader::{ + LOAD_LIBRARY_SEARCH_APPLICATION_DIR as _LOAD_LIBRARY_SEARCH_APPLICATION_DIR, + LOAD_LIBRARY_SEARCH_DEFAULT_DIRS as _LOAD_LIBRARY_SEARCH_DEFAULT_DIRS, + LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR as _LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR, + LOAD_LIBRARY_SEARCH_SYSTEM32 as _LOAD_LIBRARY_SEARCH_SYSTEM32, + LOAD_LIBRARY_SEARCH_USER_DIRS as _LOAD_LIBRARY_SEARCH_USER_DIRS, + }; #[pyfunction] pub(super) fn access(path: OsPath, mode: u8, vm: &VirtualMachine) -> PyResult { From ad5ffb648f2c0cb73152bdb75befa6e4b4e6ab52 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Mon, 7 Apr 2025 21:37:47 -0700 Subject: [PATCH 183/295] Remove packaging from release (#5680) --- .github/workflows/release.yml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3b0a797e0c..f6a1ad3209 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -80,20 +80,6 @@ jobs: run: cp target/${{ matrix.platform.target }}/release/rustpython.exe target/rustpython-release-${{ runner.os }}-${{ matrix.platform.target }}.exe if: runner.os == 'Windows' - - name: Install cargo-packager - run: cargo binstall cargo-packager - - - name: Generate MSI - if: runner.os == 'Windows' - run: cargo packager -f wix --release -o installer - - - name: Upload MSI - if: runner.os == 'Windows' - uses: actions/upload-artifact@v4 - with: - name: rustpython-installer-msi-${{ runner.os }}-${{ matrix.platform.target }} - path: installer/*.msi - - name: Upload Binary Artifacts uses: actions/upload-artifact@v4 with: From 8dc17180029337676395154fc33b513678ea4a8f Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Wed, 9 Apr 2025 22:00:54 -0700 Subject: [PATCH 184/295] Match statements rewrite (#5628) --- Cargo.lock | 1 + compiler/codegen/Cargo.toml | 3 +- compiler/codegen/src/compile.rs | 1022 +++++++++++++++-- compiler/codegen/src/error.rs | 27 +- ...python_codegen__compile__tests__match.snap | 53 + compiler/core/src/bytecode.rs | 41 +- extra_tests/snippets/syntax_match.py | 50 + vm/src/frame.rs | 109 ++ 8 files changed, 1192 insertions(+), 114 deletions(-) create mode 100644 compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__match.snap create mode 100644 extra_tests/snippets/syntax_match.py diff --git a/Cargo.lock b/Cargo.lock index f3a8f59af3..60f0c44ce1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2187,6 +2187,7 @@ dependencies = [ "num-complex", "num-traits", "ruff_python_ast", + "ruff_python_parser", "ruff_source_file", "ruff_text_size", "rustpython-compiler-core", diff --git a/compiler/codegen/Cargo.toml b/compiler/codegen/Cargo.toml index 53469b9f6e..479b0b29f6 100644 --- a/compiler/codegen/Cargo.toml +++ b/compiler/codegen/Cargo.toml @@ -33,8 +33,7 @@ memchr = { workspace = true } unicode_names2 = { workspace = true } [dev-dependencies] -# rustpython-parser = { workspace = true } - +ruff_python_parser = { workspace = true } insta = { workspace = true } [lints] diff --git a/compiler/codegen/src/compile.rs b/compiler/codegen/src/compile.rs index 83e2f5cf44..c8474a537f 100644 --- a/compiler/codegen/src/compile.rs +++ b/compiler/codegen/src/compile.rs @@ -9,8 +9,8 @@ use crate::{ IndexSet, ToPythonName, - error::{CodegenError, CodegenErrorType}, - ir, + error::{CodegenError, CodegenErrorType, PatternUnreachableReason}, + ir::{self, BlockIdx}, symboltable::{self, SymbolFlags, SymbolScope, SymbolTable}, unparse::unparse_expr, }; @@ -22,10 +22,11 @@ use ruff_python_ast::{ Alias, Arguments, BoolOp, CmpOp, Comprehension, ConversionFlag, DebugText, Decorator, DictItem, ExceptHandler, ExceptHandlerExceptHandler, Expr, ExprAttribute, ExprBoolOp, ExprFString, ExprList, ExprName, ExprStarred, ExprSubscript, ExprTuple, ExprUnaryOp, FString, - FStringElement, FStringElements, FStringFlags, FStringPart, Int, Keyword, MatchCase, - ModExpression, ModModule, Operator, Parameters, Pattern, PatternMatchAs, PatternMatchValue, - Stmt, StmtExpr, TypeParam, TypeParamParamSpec, TypeParamTypeVar, TypeParamTypeVarTuple, - TypeParams, UnaryOp, WithItem, + FStringElement, FStringElements, FStringFlags, FStringPart, Identifier, Int, Keyword, + MatchCase, ModExpression, ModModule, Operator, Parameters, Pattern, PatternMatchAs, + PatternMatchClass, PatternMatchOr, PatternMatchSequence, PatternMatchSingleton, + PatternMatchStar, PatternMatchValue, Singleton, Stmt, StmtExpr, TypeParam, TypeParamParamSpec, + TypeParamTypeVar, TypeParamTypeVarTuple, TypeParams, UnaryOp, WithItem, }; use ruff_source_file::OneIndexed; use ruff_text_size::{Ranged, TextRange}; @@ -33,7 +34,10 @@ use rustpython_wtf8::Wtf8Buf; // use rustpython_ast::located::{self as located_ast, Located}; use rustpython_compiler_core::{ Mode, - bytecode::{self, Arg as OpArgMarker, CodeObject, ConstantData, Instruction, OpArg, OpArgType}, + bytecode::{ + self, Arg as OpArgMarker, BinaryOperator, CodeObject, ComparisonOperator, ConstantData, + Instruction, OpArg, OpArgType, UnpackExArgs, + }, }; use rustpython_compiler_source::SourceCode; // use rustpython_parser_core::source_code::{LineNumber, SourceLocation}; @@ -206,10 +210,43 @@ macro_rules! emit { }; } -struct PatternContext { - current_block: usize, - blocks: Vec, - allow_irrefutable: bool, +/// The pattern context holds information about captured names and jump targets. +#[derive(Clone)] +pub struct PatternContext { + /// A list of names captured by the pattern. + pub stores: Vec, + /// If false, then any name captures against our subject will raise. + pub allow_irrefutable: bool, + /// A list of jump target labels used on pattern failure. + pub fail_pop: Vec, + /// The number of items on top of the stack that should remain. + pub on_top: usize, +} + +impl Default for PatternContext { + fn default() -> Self { + Self::new() + } +} + +impl PatternContext { + pub fn new() -> Self { + PatternContext { + stores: Vec::new(), + allow_irrefutable: false, + fail_pop: Vec::new(), + on_top: 0, + } + } + + pub fn fail_pop_size(&self) -> usize { + self.fail_pop.len() + } +} + +enum JumpOp { + Jump, + PopJumpIfFalse, } impl<'src> Compiler<'src> { @@ -1800,73 +1837,824 @@ impl Compiler<'_> { Ok(()) } - fn compile_pattern_value( + fn forbidden_name(&mut self, name: &str, ctx: NameUsage) -> CompileResult { + if ctx == NameUsage::Store && name == "__debug__" { + return Err(self.error(CodegenErrorType::Assign("__debug__"))); + // return Ok(true); + } + if ctx == NameUsage::Delete && name == "__debug__" { + return Err(self.error(CodegenErrorType::Delete("__debug__"))); + // return Ok(true); + } + Ok(false) + } + + fn compile_error_forbidden_name(&mut self, name: &str) -> CodegenError { + // TODO: make into error (fine for now since it realistically errors out earlier) + panic!("Failing due to forbidden name {:?}", name); + } + + /// Ensures that `pc.fail_pop` has at least `n + 1` entries. + /// If not, new labels are generated and pushed until the required size is reached. + fn ensure_fail_pop(&mut self, pc: &mut PatternContext, n: usize) -> CompileResult<()> { + let required_size = n + 1; + if required_size <= pc.fail_pop.len() { + return Ok(()); + } + while pc.fail_pop.len() < required_size { + let new_block = self.new_block(); + pc.fail_pop.push(new_block); + } + Ok(()) + } + + fn jump_to_fail_pop(&mut self, pc: &mut PatternContext, op: JumpOp) -> CompileResult<()> { + // Compute the total number of items to pop: + // items on top plus the captured objects. + let pops = pc.on_top + pc.stores.len(); + // Ensure that the fail_pop vector has at least `pops + 1` elements. + self.ensure_fail_pop(pc, pops)?; + // Emit a jump using the jump target stored at index `pops`. + match op { + JumpOp::Jump => { + emit!( + self, + Instruction::Jump { + target: pc.fail_pop[pops] + } + ); + } + JumpOp::PopJumpIfFalse => { + emit!( + self, + Instruction::JumpIfFalse { + target: pc.fail_pop[pops] + } + ); + } + } + Ok(()) + } + + /// Emits the necessary POP instructions for all failure targets in the pattern context, + /// then resets the fail_pop vector. + fn emit_and_reset_fail_pop(&mut self, pc: &mut PatternContext) -> CompileResult<()> { + // If the fail_pop vector is empty, nothing needs to be done. + if pc.fail_pop.is_empty() { + debug_assert!(pc.fail_pop.is_empty()); + return Ok(()); + } + // Iterate over the fail_pop vector in reverse order, skipping the first label. + for &label in pc.fail_pop.iter().skip(1).rev() { + self.switch_to_block(label); + // Emit the POP instruction. + emit!(self, Instruction::Pop); + } + // Finally, use the first label. + self.switch_to_block(pc.fail_pop[0]); + pc.fail_pop.clear(); + // Free the memory used by the vector. + pc.fail_pop.shrink_to_fit(); + Ok(()) + } + + /// Duplicate the effect of Python 3.10's ROT_* instructions using SWAPs. + fn pattern_helper_rotate(&mut self, mut count: usize) -> CompileResult<()> { + while count > 1 { + // Emit a SWAP instruction with the current count. + emit!( + self, + Instruction::Swap { + index: u32::try_from(count).unwrap() + } + ); + count -= 1; + } + Ok(()) + } + + /// Helper to store a captured name for a star pattern. + /// + /// If `n` is `None`, it emits a POP_TOP instruction. Otherwise, it first + /// checks that the name is allowed and not already stored. Then it rotates + /// the object on the stack beneath any preserved items and appends the name + /// to the list of captured names. + fn pattern_helper_store_name( + &mut self, + n: Option<&Identifier>, + pc: &mut PatternContext, + ) -> CompileResult<()> { + // If no name is provided, simply pop the top of the stack. + if n.is_none() { + emit!(self, Instruction::Pop); + return Ok(()); + } + let name = n.unwrap(); + + // Check if the name is forbidden for storing. + if self.forbidden_name(name.as_str(), NameUsage::Store)? { + return Err(self.compile_error_forbidden_name(name.as_str())); + } + + // Ensure we don't store the same name twice. + if pc.stores.contains(&name.to_string()) { + return Err(self.error(CodegenErrorType::DuplicateStore(name.as_str().to_string()))); + } + + // Calculate how many items to rotate: + // the count is the number of items to preserve on top plus the current stored names, + // plus one for the new value. + let rotations = pc.on_top + pc.stores.len() + 1; + self.pattern_helper_rotate(rotations)?; + + // Append the name to the captured stores. + pc.stores.push(name.to_string()); + Ok(()) + } + + fn pattern_unpack_helper(&mut self, elts: &[Pattern]) -> CompileResult<()> { + let n = elts.len(); + let mut seen_star = false; + for (i, elt) in elts.iter().enumerate() { + if elt.is_match_star() { + if !seen_star { + if i >= (1 << 8) || (n - i - 1) >= ((i32::MAX as usize) >> 8) { + todo!(); + // return self.compiler_error(loc, "too many expressions in star-unpacking sequence pattern"); + } + let args = UnpackExArgs { + before: u8::try_from(i).unwrap(), + after: u8::try_from(n - i - 1).unwrap(), + }; + emit!(self, Instruction::UnpackEx { args }); + seen_star = true; + } else { + // TODO: Fix error msg + return Err(self.error(CodegenErrorType::MultipleStarArgs)); + // return self.compiler_error(loc, "multiple starred expressions in sequence pattern"); + } + } + } + if !seen_star { + emit!( + self, + Instruction::UnpackSequence { + size: u32::try_from(n).unwrap() + } + ); + } + Ok(()) + } + + fn pattern_helper_sequence_unpack( &mut self, - value: &PatternMatchValue, - _pattern_context: &mut PatternContext, + patterns: &[Pattern], + _star: Option, + pc: &mut PatternContext, ) -> CompileResult<()> { - use crate::compile::bytecode::ComparisonOperator::*; + // Unpack the sequence into individual subjects. + self.pattern_unpack_helper(patterns)?; + let size = patterns.len(); + // Increase the on_top counter for the newly unpacked subjects. + pc.on_top += size; + // For each unpacked subject, compile its subpattern. + for pattern in patterns { + // Decrement on_top for each subject as it is consumed. + pc.on_top -= 1; + self.compile_pattern_subpattern(pattern, pc)?; + } + Ok(()) + } - self.compile_expression(&value.value)?; - emit!(self, Instruction::CompareOperation { op: Equal }); + fn pattern_helper_sequence_subscr( + &mut self, + patterns: &[Pattern], + star: usize, + pc: &mut PatternContext, + ) -> CompileResult<()> { + // Keep the subject around for extracting elements. + pc.on_top += 1; + for (i, pattern) in patterns.iter().enumerate() { + // if pattern.is_wildcard() { + // continue; + // } + if i == star { + // This must be a starred wildcard. + // assert!(pattern.is_star_wildcard()); + continue; + } + // Duplicate the subject. + emit!(self, Instruction::CopyItem { index: 1_u32 }); + if i < star { + // For indices before the star, use a nonnegative index equal to i. + self.emit_load_const(ConstantData::Integer { value: i.into() }); + } else { + // For indices after the star, compute a nonnegative index: + // index = len(subject) - (size - i) + emit!(self, Instruction::GetLen); + self.emit_load_const(ConstantData::Integer { + value: (patterns.len() - 1).into(), + }); + // Subtract to compute the correct index. + emit!( + self, + Instruction::BinaryOperation { + op: BinaryOperator::Subtract + } + ); + } + // Use BINARY_OP/NB_SUBSCR to extract the element. + emit!(self, Instruction::BinarySubscript); + // Compile the subpattern in irrefutable mode. + self.compile_pattern_subpattern(pattern, pc)?; + } + // Pop the subject off the stack. + pc.on_top -= 1; + emit!(self, Instruction::Pop); + Ok(()) + } + + fn compile_pattern_subpattern( + &mut self, + p: &Pattern, + pc: &mut PatternContext, + ) -> CompileResult<()> { + // Save the current allow_irrefutable state. + let old_allow_irrefutable = pc.allow_irrefutable; + // Temporarily allow irrefutable patterns. + pc.allow_irrefutable = true; + // Compile the pattern. + self.compile_pattern(p, pc)?; + // Restore the original state. + pc.allow_irrefutable = old_allow_irrefutable; Ok(()) } fn compile_pattern_as( &mut self, - as_pattern: &PatternMatchAs, - pattern_context: &mut PatternContext, + p: &PatternMatchAs, + pc: &mut PatternContext, ) -> CompileResult<()> { - if as_pattern.pattern.is_none() && !pattern_context.allow_irrefutable { - // TODO: better error message - if let Some(_name) = as_pattern.name.as_ref() { - return Err(self.error_ranged(CodegenErrorType::InvalidMatchCase, as_pattern.range)); + // If there is no sub-pattern, then it's an irrefutable match. + if p.pattern.is_none() { + if !pc.allow_irrefutable { + if let Some(_name) = p.name.as_ref() { + // TODO: This error message does not match cpython exactly + // A name capture makes subsequent patterns unreachable. + return Err(self.error(CodegenErrorType::UnreachablePattern( + PatternUnreachableReason::NameCapture, + ))); + } else { + // A wildcard makes remaining patterns unreachable. + return Err(self.error(CodegenErrorType::UnreachablePattern( + PatternUnreachableReason::Wildcard, + ))); + } } - return Err(self.error_ranged(CodegenErrorType::InvalidMatchCase, as_pattern.range)); + // If irrefutable matches are allowed, store the name (if any). + return self.pattern_helper_store_name(p.name.as_ref(), pc); } - // Need to make a copy for (possibly) storing later: - emit!(self, Instruction::Duplicate); - if let Some(pattern) = &as_pattern.pattern { - self.compile_pattern_inner(pattern, pattern_context)?; + + // Otherwise, there is a sub-pattern. Duplicate the object on top of the stack. + pc.on_top += 1; + emit!(self, Instruction::CopyItem { index: 1_u32 }); + // Compile the sub-pattern. + self.compile_pattern(p.pattern.as_ref().unwrap(), pc)?; + // After success, decrement the on_top counter. + pc.on_top -= 1; + // Store the captured name (if any). + self.pattern_helper_store_name(p.name.as_ref(), pc)?; + Ok(()) + } + + fn compile_pattern_star( + &mut self, + p: &PatternMatchStar, + pc: &mut PatternContext, + ) -> CompileResult<()> { + self.pattern_helper_store_name(p.name.as_ref(), pc)?; + Ok(()) + } + + /// Validates that keyword attributes in a class pattern are allowed + /// and not duplicated. + fn validate_kwd_attrs( + &mut self, + attrs: &[Identifier], + _patterns: &[Pattern], + ) -> CompileResult<()> { + let nattrs = attrs.len(); + for i in 0..nattrs { + let attr = attrs[i].as_str(); + // Check if the attribute name is forbidden in a Store context. + if self.forbidden_name(attr, NameUsage::Store)? { + // Return an error if the name is forbidden. + return Err(self.compile_error_forbidden_name(attr)); + } + // Check for duplicates: compare with every subsequent attribute. + for ident in attrs.iter().take(nattrs).skip(i + 1) { + let other = ident.as_str(); + if attr == other { + todo!(); + // return Err(self.compiler_error( + // &format!("attribute name repeated in class pattern: {}", attr), + // )); + } + } } - if let Some(name) = as_pattern.name.as_ref() { - self.store_name(name.as_str())?; - } else { - emit!(self, Instruction::Pop); + Ok(()) + } + + fn compile_pattern_class( + &mut self, + p: &PatternMatchClass, + pc: &mut PatternContext, + ) -> CompileResult<()> { + // Extract components from the MatchClass pattern. + let match_class = p; + let patterns = &match_class.arguments.patterns; + + // Extract keyword attributes and patterns. + // Capacity is pre-allocated based on the number of keyword arguments. + let mut kwd_attrs = Vec::with_capacity(match_class.arguments.keywords.len()); + let mut kwd_patterns = Vec::with_capacity(match_class.arguments.keywords.len()); + for kwd in &match_class.arguments.keywords { + kwd_attrs.push(kwd.attr.clone()); + kwd_patterns.push(kwd.pattern.clone()); + } + + let nargs = patterns.len(); + let nattrs = kwd_attrs.len(); + let nkwd_patterns = kwd_patterns.len(); + + // Validate that keyword attribute names and patterns match in length. + if nattrs != nkwd_patterns { + let msg = format!( + "kwd_attrs ({}) / kwd_patterns ({}) length mismatch in class pattern", + nattrs, nkwd_patterns + ); + unreachable!("{}", msg); + } + + // Check for too many sub-patterns. + if nargs > u32::MAX as usize || (nargs + nattrs).saturating_sub(1) > i32::MAX as usize { + let msg = format!( + "too many sub-patterns in class pattern {:?}", + match_class.cls + ); + panic!("{}", msg); + // return self.compiler_error(&msg); + } + + // Validate keyword attributes if any. + if nattrs != 0 { + self.validate_kwd_attrs(&kwd_attrs, &kwd_patterns)?; + } + + // Compile the class expression. + self.compile_expression(&match_class.cls)?; + + // Create a new tuple of attribute names. + let mut attr_names = vec![]; + for name in kwd_attrs.iter() { + // Py_NewRef(name) is emulated by cloning the name into a PyObject. + attr_names.push(ConstantData::Str { + value: name.as_str().to_string().into(), + }); + } + + // Emit instructions: + // 1. Load the new tuple of attribute names. + self.emit_load_const(ConstantData::Tuple { + elements: attr_names, + }); + // 2. Emit MATCH_CLASS with nargs. + emit!(self, Instruction::MatchClass(u32::try_from(nargs).unwrap())); + // 3. Duplicate the top of the stack. + emit!(self, Instruction::CopyItem { index: 1_u32 }); + // 4. Load None. + self.emit_load_const(ConstantData::None); + // 5. Compare with IS_OP 1. + emit!(self, Instruction::IsOperation(true)); + + // At this point the TOS is a tuple of (nargs + nattrs) attributes (or None). + pc.on_top += 1; + self.jump_to_fail_pop(pc, JumpOp::PopJumpIfFalse)?; + + // Unpack the tuple into (nargs + nattrs) items. + let total = nargs + nattrs; + emit!( + self, + Instruction::UnpackSequence { + size: u32::try_from(total).unwrap() + } + ); + pc.on_top += total; + pc.on_top -= 1; + + // Process each sub-pattern. + for i in 0..total { + // Decrement the on_top counter as each sub-pattern is processed. + pc.on_top -= 1; + let subpattern = if i < nargs { + // Positional sub-pattern. + &patterns[i] + } else { + // Keyword sub-pattern. + &kwd_patterns[i - nargs] + }; + if subpattern.is_wildcard() { + // For wildcard patterns, simply pop the top of the stack. + emit!(self, Instruction::Pop); + continue; + } + // Compile the subpattern without irrefutability checks. + self.compile_pattern_subpattern(subpattern, pc)?; } Ok(()) } - fn compile_pattern_inner( + // fn compile_pattern_mapping(&mut self, p: &PatternMatchMapping, pc: &mut PatternContext) -> CompileResult<()> { + // // Ensure the pattern is a mapping pattern. + // let mapping = p; // Extract MatchMapping-specific data. + // let keys = &mapping.keys; + // let patterns = &mapping.patterns; + // let size = keys.len(); + // let npatterns = patterns.len(); + + // if size != npatterns { + // panic!("keys ({}) / patterns ({}) length mismatch in mapping pattern", size, npatterns); + // // return self.compiler_error( + // // &format!("keys ({}) / patterns ({}) length mismatch in mapping pattern", size, npatterns) + // // ); + // } + + // // A double-star target is present if `rest` is set. + // let star_target = mapping.rest; + + // // Keep the subject on top during the mapping and length checks. + // pc.on_top += 1; + // emit!(self, Instruction::MatchMapping); + // self.jump_to_fail_pop(pc, JumpOp::PopJumpIfFalse)?; + + // // If the pattern is just "{}" (empty mapping) and there's no star target, + // // we're done—pop the subject. + // if size == 0 && star_target.is_none() { + // pc.on_top -= 1; + // emit!(self, Instruction::Pop); + // return Ok(()); + // } + + // // If there are any keys, perform a length check. + // if size != 0 { + // emit!(self, Instruction::GetLen); + // self.emit_load_const(ConstantData::Integer { value: size.into() }); + // emit!(self, Instruction::CompareOperation { op: ComparisonOperator::GreaterOrEqual }); + // self.jump_to_fail_pop(pc, JumpOp::PopJumpIfFalse)?; + // } + + // // Check that the number of subpatterns is not absurd. + // if size.saturating_sub(1) > (i32::MAX as usize) { + // panic!("too many sub-patterns in mapping pattern"); + // // return self.compiler_error("too many sub-patterns in mapping pattern"); + // } + + // // Collect all keys into a set for duplicate checking. + // let mut seen = HashSet::new(); + + // // For each key, validate it and check for duplicates. + // for (i, key) in keys.iter().enumerate() { + // if let Some(key_val) = key.as_literal_expr() { + // let in_seen = seen.contains(&key_val); + // if in_seen { + // panic!("mapping pattern checks duplicate key: {:?}", key_val); + // // return self.compiler_error(format!("mapping pattern checks duplicate key: {:?}", key_val)); + // } + // seen.insert(key_val); + // } else if !key.is_attribute_expr() { + // panic!("mapping pattern keys may only match literals and attribute lookups"); + // // return self.compiler_error("mapping pattern keys may only match literals and attribute lookups"); + // } + + // // Visit the key expression. + // self.compile_expression(key)?; + // } + // // Drop the set (its resources will be freed automatically). + + // // Build a tuple of keys and emit MATCH_KEYS. + // emit!(self, Instruction::BuildTuple { size: size as u32 }); + // emit!(self, Instruction::MatchKeys); + // // Now, on top of the subject there are two new tuples: one of keys and one of values. + // pc.on_top += 2; + + // // Prepare for matching the values. + // emit!(self, Instruction::CopyItem { index: 1_u32 }); + // self.emit_load_const(ConstantData::None); + // // TODO: should be is + // emit!(self, Instruction::IsOperation(true)); + // self.jump_to_fail_pop(pc, JumpOp::PopJumpIfFalse)?; + + // // Unpack the tuple of values. + // emit!(self, Instruction::UnpackSequence { size: size as u32 }); + // pc.on_top += size.saturating_sub(1); + + // // Compile each subpattern in "subpattern" mode. + // for pattern in patterns { + // pc.on_top = pc.on_top.saturating_sub(1); + // self.compile_pattern_subpattern(pattern, pc)?; + // } + + // // Consume the tuple of keys and the subject. + // pc.on_top = pc.on_top.saturating_sub(2); + // if let Some(star_target) = star_target { + // // If we have a starred name, bind a dict of remaining items to it. + // // This sequence of instructions performs: + // // rest = dict(subject) + // // for key in keys: del rest[key] + // emit!(self, Instruction::BuildMap { size: 0 }); // Build an empty dict. + // emit!(self, Instruction::Swap(3)); // Rearrange stack: [empty, keys, subject] + // emit!(self, Instruction::DictUpdate { size: 2 }); // Update dict with subject. + // emit!(self, Instruction::UnpackSequence { size: size as u32 }); // Unpack keys. + // let mut remaining = size; + // while remaining > 0 { + // emit!(self, Instruction::CopyItem { index: 1 + remaining as u32 }); // Duplicate subject copy. + // emit!(self, Instruction::Swap { index: 2_u32 }); // Bring key to top. + // emit!(self, Instruction::DeleteSubscript); // Delete key from dict. + // remaining -= 1; + // } + // // Bind the dict to the starred target. + // self.pattern_helper_store_name(Some(&star_target), pc)?; + // } else { + // // No starred target: just pop the tuple of keys and the subject. + // emit!(self, Instruction::Pop); + // emit!(self, Instruction::Pop); + // } + // Ok(()) + // } + + fn compile_pattern_or( &mut self, - pattern_type: &Pattern, - pattern_context: &mut PatternContext, + p: &PatternMatchOr, + pc: &mut PatternContext, ) -> CompileResult<()> { - match &pattern_type { - Pattern::MatchValue(value) => self.compile_pattern_value(value, pattern_context), - Pattern::MatchAs(as_pattern) => self.compile_pattern_as(as_pattern, pattern_context), - _ => { - eprintln!("not implemented pattern type: {pattern_type:?}"); - Err(self.error(CodegenErrorType::NotImplementedYet)) + // Ensure the pattern is a MatchOr. + let end = self.new_block(); // Create a new jump target label. + let size = p.patterns.len(); + assert!(size > 1, "MatchOr must have more than one alternative"); + + // Save the current pattern context. + let old_pc = pc.clone(); + // Simulate Py_INCREF on pc.stores by cloning it. + pc.stores = pc.stores.clone(); + let mut control: Option> = None; // Will hold the capture list of the first alternative. + + // Process each alternative. + for (i, alt) in p.patterns.iter().enumerate() { + // Create a fresh empty store for this alternative. + pc.stores = Vec::new(); + // An irrefutable subpattern must be last (if allowed). + pc.allow_irrefutable = (i == size - 1) && old_pc.allow_irrefutable; + // Reset failure targets and the on_top counter. + pc.fail_pop.clear(); + pc.on_top = 0; + // Emit a COPY(1) instruction before compiling the alternative. + emit!(self, Instruction::CopyItem { index: 1_u32 }); + self.compile_pattern(alt, pc)?; + + let nstores = pc.stores.len(); + if i == 0 { + // Save the captured names from the first alternative. + control = Some(pc.stores.clone()); + } else { + let control_vec = control.as_ref().unwrap(); + if nstores != control_vec.len() { + todo!(); + // return self.compiler_error("alternative patterns bind different names"); + } else if nstores > 0 { + // Check that the names occur in the same order. + for icontrol in (0..nstores).rev() { + let name = &control_vec[icontrol]; + // Find the index of `name` in the current stores. + let istores = pc.stores.iter().position(|n| n == name).unwrap(); + // .ok_or_else(|| self.compiler_error("alternative patterns bind different names"))?; + if icontrol != istores { + // The orders differ; we must reorder. + assert!(istores < icontrol, "expected istores < icontrol"); + let rotations = istores + 1; + // Rotate pc.stores: take a slice of the first `rotations` items... + let rotated = pc.stores[0..rotations].to_vec(); + // Remove those elements. + for _ in 0..rotations { + pc.stores.remove(0); + } + // Insert the rotated slice at the appropriate index. + let insert_pos = icontrol - istores; + for (j, elem) in rotated.into_iter().enumerate() { + pc.stores.insert(insert_pos + j, elem); + } + // Also perform the same rotation on the evaluation stack. + for _ in 0..(istores + 1) { + self.pattern_helper_rotate(icontrol + 1)?; + } + } + } + } + } + // Emit a jump to the common end label and reset any failure jump targets. + emit!(self, Instruction::Jump { target: end }); + self.emit_and_reset_fail_pop(pc)?; + } + + // Restore the original pattern context. + *pc = old_pc.clone(); + // Simulate Py_INCREF on pc.stores. + pc.stores = pc.stores.clone(); + // In C, old_pc.fail_pop is set to NULL to avoid freeing it later. + // In Rust, old_pc is a local clone, so we need not worry about that. + + // No alternative matched: pop the subject and fail. + emit!(self, Instruction::Pop); + self.jump_to_fail_pop(pc, JumpOp::Jump)?; + + // Use the label "end". + self.switch_to_block(end); + + // Adjust the final captures. + let nstores = control.as_ref().unwrap().len(); + let nrots = nstores + 1 + pc.on_top + pc.stores.len(); + for i in 0..nstores { + // Rotate the capture to its proper place. + self.pattern_helper_rotate(nrots)?; + let name = &control.as_ref().unwrap()[i]; + // Check for duplicate binding. + if pc.stores.iter().any(|n| n == name) { + return Err(self.error(CodegenErrorType::DuplicateStore(name.to_string()))); } + pc.stores.push(name.clone()); } + + // Old context and control will be dropped automatically. + // Finally, pop the copy of the subject. + emit!(self, Instruction::Pop); + Ok(()) } - fn compile_pattern( + fn compile_pattern_sequence( &mut self, - pattern_type: &Pattern, - pattern_context: &mut PatternContext, + p: &PatternMatchSequence, + pc: &mut PatternContext, ) -> CompileResult<()> { - self.compile_pattern_inner(pattern_type, pattern_context)?; + // Ensure the pattern is a MatchSequence. + let patterns = &p.patterns; // a slice of Pattern + let size = patterns.len(); + let mut star: Option = None; + let mut only_wildcard = true; + let mut star_wildcard = false; + + // Find a starred pattern, if it exists. There may be at most one. + for (i, pattern) in patterns.iter().enumerate() { + if pattern.is_match_star() { + if star.is_some() { + // TODO: Fix error msg + return Err(self.error(CodegenErrorType::MultipleStarArgs)); + } + // star wildcard check + star_wildcard = pattern + .as_match_star() + .map(|m| m.name.is_none()) + .unwrap_or(false); + only_wildcard &= star_wildcard; + star = Some(i); + continue; + } + // wildcard check + only_wildcard &= pattern + .as_match_as() + .map(|m| m.name.is_none()) + .unwrap_or(false); + } + + // Keep the subject on top during the sequence and length checks. + pc.on_top += 1; + emit!(self, Instruction::MatchSequence); + self.jump_to_fail_pop(pc, JumpOp::PopJumpIfFalse)?; + + if star.is_none() { + // No star: len(subject) == size + emit!(self, Instruction::GetLen); + self.emit_load_const(ConstantData::Integer { value: size.into() }); + emit!( + self, + Instruction::CompareOperation { + op: ComparisonOperator::Equal + } + ); + self.jump_to_fail_pop(pc, JumpOp::PopJumpIfFalse)?; + } else if size > 1 { + // Star exists: len(subject) >= size - 1 + emit!(self, Instruction::GetLen); + self.emit_load_const(ConstantData::Integer { + value: (size - 1).into(), + }); + emit!( + self, + Instruction::CompareOperation { + op: ComparisonOperator::GreaterOrEqual + } + ); + self.jump_to_fail_pop(pc, JumpOp::PopJumpIfFalse)?; + } + + // Whatever comes next should consume the subject. + pc.on_top -= 1; + if only_wildcard { + // Patterns like: [] / [_] / [_, _] / [*_] / [_, *_] / [_, _, *_] / etc. + emit!(self, Instruction::Pop); + } else if star_wildcard { + self.pattern_helper_sequence_subscr(patterns, star.unwrap(), pc)?; + } else { + self.pattern_helper_sequence_unpack(patterns, star, pc)?; + } + Ok(()) + } + + fn compile_pattern_value( + &mut self, + p: &PatternMatchValue, + pc: &mut PatternContext, + ) -> CompileResult<()> { + // TODO: ensure literal or attribute lookup + self.compile_expression(&p.value)?; emit!( self, - Instruction::JumpIfFalse { - target: pattern_context.blocks[pattern_context.current_block + 1] + Instruction::CompareOperation { + op: bytecode::ComparisonOperator::Equal } ); + // emit!(self, Instruction::ToBool); + self.jump_to_fail_pop(pc, JumpOp::PopJumpIfFalse)?; Ok(()) } + fn compile_pattern_singleton( + &mut self, + p: &PatternMatchSingleton, + pc: &mut PatternContext, + ) -> CompileResult<()> { + // Load the singleton constant value. + self.emit_load_const(match p.value { + Singleton::None => ConstantData::None, + Singleton::False => ConstantData::Boolean { value: false }, + Singleton::True => ConstantData::Boolean { value: true }, + }); + // Compare using the "Is" operator. + emit!( + self, + Instruction::CompareOperation { + op: bytecode::ComparisonOperator::Equal + } + ); + // Jump to the failure label if the comparison is false. + self.jump_to_fail_pop(pc, JumpOp::PopJumpIfFalse)?; + Ok(()) + } + + fn compile_pattern( + &mut self, + pattern_type: &Pattern, + pattern_context: &mut PatternContext, + ) -> CompileResult<()> { + match &pattern_type { + Pattern::MatchValue(pattern_type) => { + self.compile_pattern_value(pattern_type, pattern_context) + } + Pattern::MatchSingleton(pattern_type) => { + self.compile_pattern_singleton(pattern_type, pattern_context) + } + Pattern::MatchSequence(pattern_type) => { + self.compile_pattern_sequence(pattern_type, pattern_context) + } + // Pattern::MatchMapping(pattern_type) => self.compile_pattern_mapping(pattern_type, pattern_context), + Pattern::MatchClass(pattern_type) => { + self.compile_pattern_class(pattern_type, pattern_context) + } + Pattern::MatchStar(pattern_type) => { + self.compile_pattern_star(pattern_type, pattern_context) + } + Pattern::MatchAs(pattern_type) => { + self.compile_pattern_as(pattern_type, pattern_context) + } + Pattern::MatchOr(pattern_type) => { + self.compile_pattern_or(pattern_type, pattern_context) + } + _ => { + // The eprintln gives context as to which pattern type is not implemented. + eprintln!("not implemented pattern type: {pattern_type:?}"); + Err(self.error(CodegenErrorType::NotImplementedYet)) + } + } + } + fn compile_match_inner( &mut self, subject: &Expr, @@ -1874,63 +2662,67 @@ impl Compiler<'_> { pattern_context: &mut PatternContext, ) -> CompileResult<()> { self.compile_expression(subject)?; - pattern_context.blocks = std::iter::repeat_with(|| self.new_block()) - .take(cases.len() + 1) - .collect::>(); - let end_block = *pattern_context.blocks.last().unwrap(); - - let _match_case_type = cases.last().expect("cases is not empty"); - // TODO: get proper check for default case - // let has_default = match_case_type.pattern.is_match_as() && 1 < cases.len(); - let has_default = false; - for i in 0..cases.len() - (has_default as usize) { - self.switch_to_block(pattern_context.blocks[i]); - pattern_context.current_block = i; - pattern_context.allow_irrefutable = cases[i].guard.is_some() || i == cases.len() - 1; - let m = &cases[i]; - // Only copy the subject if we're *not* on the last case: - if i != cases.len() - has_default as usize - 1 { - emit!(self, Instruction::Duplicate); + let end = self.new_block(); + + let num_cases = cases.len(); + assert!(num_cases > 0); + let has_default = cases.iter().last().unwrap().pattern.is_match_star() && num_cases > 1; + + let case_count = num_cases - if has_default { 1 } else { 0 }; + for (i, m) in cases.iter().enumerate().take(case_count) { + // Only copy the subject if not on the last case + if i != case_count - 1 { + emit!(self, Instruction::CopyItem { index: 1_u32 }); } + + pattern_context.stores = Vec::with_capacity(1); + pattern_context.allow_irrefutable = m.guard.is_some() || i == case_count - 1; + pattern_context.fail_pop.clear(); + pattern_context.on_top = 0; + self.compile_pattern(&m.pattern, pattern_context)?; + assert_eq!(pattern_context.on_top, 0); + + for name in &pattern_context.stores { + self.compile_name(name, NameUsage::Store)?; + } + + if let Some(ref _guard) = m.guard { + self.ensure_fail_pop(pattern_context, 0)?; + // TODO: Fix compile jump if call + return Err(self.error(CodegenErrorType::NotImplementedYet)); + // Jump if the guard fails. We assume that patter_context.fail_pop[0] is the jump target. + // self.compile_jump_if(&m.pattern, &guard, pattern_context.fail_pop[0])?; + } + + if i != case_count - 1 { + emit!(self, Instruction::Pop); + } + self.compile_statements(&m.body)?; - emit!(self, Instruction::Jump { target: end_block }); + emit!(self, Instruction::Jump { target: end }); + self.emit_and_reset_fail_pop(pattern_context)?; } - // TODO: below code is not called and does not work + if has_default { - // A trailing "case _" is common, and lets us save a bit of redundant - // pushing and popping in the loop above: - let m = &cases.last().unwrap(); - self.switch_to_block(*pattern_context.blocks.last().unwrap()); - if cases.len() == 1 { - // No matches. Done with the subject: + let m = &cases[num_cases - 1]; + if num_cases == 1 { emit!(self, Instruction::Pop); } else { - // Show line coverage for default case (it doesn't create bytecode) - // emit!(self, Instruction::Nop); + emit!(self, Instruction::Nop); + } + if let Some(ref _guard) = m.guard { + // TODO: Fix compile jump if call + return Err(self.error(CodegenErrorType::NotImplementedYet)); } self.compile_statements(&m.body)?; } - - self.switch_to_block(end_block); - - let code = self.current_code_info(); - pattern_context - .blocks - .iter() - .zip(pattern_context.blocks.iter().skip(1)) - .for_each(|(a, b)| { - code.blocks[a.0 as usize].next = *b; - }); + self.switch_to_block(end); Ok(()) } fn compile_match(&mut self, subject: &Expr, cases: &[MatchCase]) -> CompileResult<()> { - let mut pattern_context = PatternContext { - current_block: usize::MAX, - blocks: Vec::new(), - allow_irrefutable: false, - }; + let mut pattern_context = PatternContext::new(); self.compile_match_inner(subject, cases, &mut pattern_context)?; Ok(()) } @@ -3637,7 +4429,7 @@ impl ToU32 for usize { } #[cfg(test)] -mod tests { +mod ruff_tests { use super::*; use ruff_python_ast::name::Name; use ruff_python_ast::*; @@ -3740,26 +4532,26 @@ mod tests { } } -/* #[cfg(test)] mod tests { use super::*; - use rustpython_parser::Parse; - use rustpython_parser::ast::Suite; - use rustpython_parser_core::source_code::LinearLocator; fn compile_exec(source: &str) -> CodeObject { - let mut locator: LinearLocator<'_> = LinearLocator::new(source); - use rustpython_parser::ast::fold::Fold; - let mut compiler: Compiler = Compiler::new( - CompileOpts::default(), - "source_path".to_owned(), - "".to_owned(), - ); - let ast = Suite::parse(source, "").unwrap(); - let ast = locator.fold(ast).unwrap(); - let symbol_scope = SymbolTable::scan_program(&ast).unwrap(); - compiler.compile_program(&ast, symbol_scope).unwrap(); + let opts = CompileOpts::default(); + let source_code = SourceCode::new("source_path", source); + let parsed = + ruff_python_parser::parse(source_code.text, ruff_python_parser::Mode::Module.into()) + .unwrap(); + let ast = parsed.into_syntax(); + let ast = match ast { + ruff_python_ast::Mod::Module(stmts) => stmts, + _ => unreachable!(), + }; + let symbol_table = SymbolTable::scan_program(&ast, source_code.clone()) + .map_err(|e| e.into_codegen_error(source_code.path.to_owned())) + .unwrap(); + let mut compiler = Compiler::new(opts, source_code, "".to_owned()); + compiler.compile_program(&ast, symbol_table).unwrap(); compiler.pop_code_object() } @@ -3816,8 +4608,24 @@ for stop_exc in (StopIteration('spam'), StopAsyncIteration('ham')): self.assertIs(ex, stop_exc) else: self.fail(f'{stop_exc} was suppressed') +" + )); + } + + #[test] + fn test_match() { + assert_dis_snapshot!(compile_exec( + "\ +class Test: + pass + +t = Test() +match t: + case Test(): + assert True + case _: + assert False " )); } } -*/ diff --git a/compiler/codegen/src/error.rs b/compiler/codegen/src/error.rs index 8f38680de0..b1b4f9379f 100644 --- a/compiler/codegen/src/error.rs +++ b/compiler/codegen/src/error.rs @@ -1,7 +1,22 @@ use ruff_source_file::SourceLocation; -use std::fmt; +use std::fmt::{self, Display}; use thiserror::Error; +#[derive(Debug)] +pub enum PatternUnreachableReason { + NameCapture, + Wildcard, +} + +impl Display for PatternUnreachableReason { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::NameCapture => write!(f, "name capture"), + Self::Wildcard => write!(f, "wildcard"), + } + } +} + // pub type CodegenError = rustpython_parser_core::source_code::LocatedError; #[derive(Error, Debug)] @@ -47,8 +62,9 @@ pub enum CodegenErrorType { TooManyStarUnpack, EmptyWithItems, EmptyWithBody, + ForbiddenName, DuplicateStore(String), - InvalidMatchCase, + UnreachablePattern(PatternUnreachableReason), NotImplementedYet, // RustPython marker for unimplemented features } @@ -94,11 +110,14 @@ impl fmt::Display for CodegenErrorType { EmptyWithBody => { write!(f, "empty body on With") } + ForbiddenName => { + write!(f, "forbidden attribute name") + } DuplicateStore(s) => { write!(f, "duplicate store {s}") } - InvalidMatchCase => { - write!(f, "invalid match case") + UnreachablePattern(reason) => { + write!(f, "{reason} makes remaining patterns unreachable") } NotImplementedYet => { write!(f, "RustPython does not implement this feature yet") diff --git a/compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__match.snap b/compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__match.snap new file mode 100644 index 0000000000..f09f0f5eaf --- /dev/null +++ b/compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__match.snap @@ -0,0 +1,53 @@ +--- +source: compiler/codegen/src/compile.rs +expression: "compile_exec(\"\\\nclass Test:\n pass\n\nt = Test()\nmatch t:\n case Test():\n assert True\n case _:\n assert False\n\")" +--- + 2 0 LoadBuildClass + 1 LoadConst (): 1 0 LoadGlobal (0, __name__) + 1 StoreLocal (1, __module__) + 2 LoadConst ("Test") + 3 StoreLocal (2, __qualname__) + 4 LoadConst (None) + 5 StoreLocal (3, __doc__) + + 2 6 ReturnConst (None) + + 2 LoadConst ("Test") + 3 MakeFunction (MakeFunctionFlags(0x0)) + 4 LoadConst ("Test") + 5 CallFunctionPositional(2) + 6 StoreLocal (0, Test) + + 4 7 LoadNameAny (0, Test) + 8 CallFunctionPositional(0) + 9 StoreLocal (1, t) + + 5 10 LoadNameAny (1, t) + 11 CopyItem (1) + + 6 12 LoadNameAny (0, Test) + 13 LoadConst (()) + 14 MatchClass (0) + 15 CopyItem (1) + 16 LoadConst (None) + 17 IsOperation (true) + 18 JumpIfFalse (27) + 19 UnpackSequence (0) + 20 Pop + + 7 21 LoadConst (True) + 22 JumpIfTrue (26) + 23 LoadGlobal (2, AssertionError) + 24 CallFunctionPositional(0) + 25 Raise (Raise) + >> 26 Jump (35) + >> 27 Pop + 28 Pop + + 9 29 LoadConst (False) + 30 JumpIfTrue (34) + 31 LoadGlobal (2, AssertionError) + 32 CallFunctionPositional(0) + 33 Raise (Raise) + >> 34 Jump (35) + >> 35 ReturnConst (None) diff --git a/compiler/core/src/bytecode.rs b/compiler/core/src/bytecode.rs index 94d080ace4..81dd591ad1 100644 --- a/compiler/core/src/bytecode.rs +++ b/compiler/core/src/bytecode.rs @@ -381,6 +381,7 @@ pub type NameIdx = u32; #[derive(Debug, Copy, Clone, PartialEq, Eq)] #[repr(u8)] pub enum Instruction { + Nop, /// Importing by name ImportName { idx: Arg, @@ -429,21 +430,33 @@ pub enum Instruction { BinaryOperationInplace { op: Arg, }, + BinarySubscript, LoadAttr { idx: Arg, }, TestOperation { op: Arg, }, + /// If the argument is true, perform IS NOT. Otherwise perform the IS operation. + // TODO: duplication of TestOperator::{Is,IsNot}. Fix later. + IsOperation(Arg), CompareOperation { op: Arg, }, + CopyItem { + index: Arg, + }, Pop, + Swap { + index: Arg, + }, + // ToBool, Rotate2, Rotate3, Duplicate, Duplicate2, GetIter, + GetLen, Continue { target: Arg