From 9e310934d3f1ad197d41815dc7f859bfb728b7d1 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Mon, 3 Feb 2025 16:20:47 -0800 Subject: [PATCH 01/26] fix panic Signed-off-by: Ashwin Naren --- vm/src/codecs.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/src/codecs.rs b/vm/src/codecs.rs index ff7bc48915..61be9f9176 100644 --- a/vm/src/codecs.rs +++ b/vm/src/codecs.rs @@ -623,7 +623,7 @@ fn surrogatepass_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<(PyOb // Try decoding a single surrogate character. If there are more, // let the codec call us again. let p = &s.as_bytes()[range.start..]; - if p.len() - range.start >= byte_length { + if p.len().saturating_sub(range.start) >= byte_length { match standard_encoding { StandardEncoding::Utf8 => { if (p[0] as u32 & 0xf0) == 0xe0 From 6788010f7d4c8897fa9b818f5ee118cd5c37627c Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Mon, 10 Feb 2025 14:29:52 -0800 Subject: [PATCH 02/26] windows-rs upgrade to 0.59 --- Cargo.lock | 191 ++++++++++++++++++++++++++++++++-------- Cargo.toml | 2 +- common/src/fileutils.rs | 14 +-- vm/Cargo.toml | 4 +- vm/src/stdlib/nt.rs | 5 +- vm/src/stdlib/winapi.rs | 4 + vm/src/stdlib/winreg.rs | 3 +- 7 files changed, 173 insertions(+), 50 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a62a5dfced..115451adfa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -216,9 +216,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.2.13" +version = "1.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7777341816418c02e033934a09f20dc0ccaf65a5201ef8a450ae0105a573fda" +checksum = "0c3d1b2e905a3a7b00a6141adb0e4c0bb941d11caf55349d863942a1cc44e3c9" dependencies = [ "shlex", ] @@ -645,9 +645,9 @@ dependencies = [ [[package]] name = "equivalent" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "errno" @@ -903,7 +903,7 @@ dependencies = [ "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "windows-core", + "windows-core 0.52.0", ] [[package]] @@ -1406,9 +1406,9 @@ checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9" [[package]] name = "openssl" -version = "0.10.70" +version = "0.10.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61cfb4e166a8bb8c9b55c500bc2308550148ece889be90f609377e58140f42c6" +checksum = "5e14130c6a98cd258fdcb0fb6d744152343ff729cbfcb28c656a9d12b999fbcd" dependencies = [ "bitflags 2.8.0", "cfg-if", @@ -1447,9 +1447,9 @@ dependencies = [ [[package]] name = "openssl-sys" -version = "0.9.105" +version = "0.9.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b22d5b84be05a8d6947c7cb71f7c849aa0f112acd4bf51c2a7c1c988ac0a9dc" +checksum = "8bb61ea9811cc39e3c2069f40b8b8e2e70d8569b361f879786cc7ed48b777cdd" dependencies = [ "cc", "libc", @@ -1745,8 +1745,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" dependencies = [ "rand_chacha 0.9.0", - "rand_core 0.9.0", - "zerocopy 0.8.17", + "rand_core 0.9.1", + "zerocopy 0.8.18", ] [[package]] @@ -1766,7 +1766,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core 0.9.0", + "rand_core 0.9.1", ] [[package]] @@ -1780,12 +1780,12 @@ dependencies = [ [[package]] name = "rand_core" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b08f3c9802962f7e1b25113931d94f43ed9725bebc59db9d0c3e9a23b67e15ff" +checksum = "a88e0da7a2c97baa202165137c158d0a2e824ac465d13d81046727b34cb247d3" dependencies = [ "getrandom 0.3.1", - "zerocopy 0.8.17", + "zerocopy 0.8.18", ] [[package]] @@ -2012,7 +2012,7 @@ dependencies = [ "siphasher 0.3.11", "volatile", "widestring", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -2242,7 +2242,7 @@ dependencies = [ "uuid", "widestring", "winapi", - "windows-sys 0.52.0", + "windows-sys 0.59.0", "xml-rs", ] @@ -2320,7 +2320,7 @@ dependencies = [ "which", "widestring", "windows", - "windows-sys 0.52.0", + "windows-sys 0.59.0", "winreg", ] @@ -2524,9 +2524,9 @@ checksum = "826167069c09b99d56f31e9ae5c99049e932a98c9dc2dac47645b08dbbf76ba7" [[package]] name = "smallvec" -version = "1.13.2" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" +checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd" [[package]] name = "socket2" @@ -2552,15 +2552,15 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" [[package]] name = "strum" -version = "0.27.0" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce1475c515a4f03a8a7129bb5228b81a781a86cb0b3fbbc19e1c556d491a401f" +checksum = "f64def088c51c9510a8579e3c5d67c65349dcf755e5479ad3d010aa6454e2c32" [[package]] name = "strum_macros" -version = "0.27.0" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9688894b43459159c82bfa5a5fa0435c19cbe3c9b427fa1dd7b1ce0c279b18a7" +checksum = "c77a8c5abcaf0f9ce05d62342b7d298c346515365c36b673df4ebe3ced01fde8" dependencies = [ "heck", "proc-macro2", @@ -3185,12 +3185,12 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows" -version = "0.52.0" +version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be" +checksum = "7f919aee0a93304be7f62e8e5027811bbba96bcb1de84d6618be56e43f8a32a1" dependencies = [ - "windows-core", - "windows-targets 0.52.6", + "windows-core 0.59.0", + "windows-targets 0.53.0", ] [[package]] @@ -3202,6 +3202,59 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-core" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "810ce18ed2112484b0d4e15d022e5f598113e220c53e373fb31e67e21670c1ce" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-result", + "windows-strings", + "windows-targets 0.53.0", +] + +[[package]] +name = "windows-implement" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83577b051e2f49a058c308f17f273b570a6a758386fc291b5f6a934dd84e48c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.98", +] + +[[package]] +name = "windows-interface" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb26fd936d991781ea39e87c3a27285081e3c0da5ca0fcbc02d368cc6f52ff01" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.98", +] + +[[package]] +name = "windows-result" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d08106ce80268c4067c0571ca55a9b4e9516518eaa1a1fe9b37ca403ae1d1a34" +dependencies = [ + "windows-targets 0.53.0", +] + +[[package]] +name = "windows-strings" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b888f919960b42ea4e11c2f408fadb55f78a9f236d5eef084103c8ce52893491" +dependencies = [ + "windows-targets 0.53.0", +] + [[package]] name = "windows-sys" version = "0.36.1" @@ -3266,13 +3319,29 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm", + "windows_i686_gnullvm 0.52.6", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] +[[package]] +name = "windows-targets" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e4c7e8ceaaf9cb7d7507c974735728ab453b67ef8f18febdd7c11fe59dca8b" +dependencies = [ + "windows_aarch64_gnullvm 0.53.0", + "windows_aarch64_msvc 0.53.0", + "windows_i686_gnu 0.53.0", + "windows_i686_gnullvm 0.53.0", + "windows_i686_msvc 0.53.0", + "windows_x86_64_gnu 0.53.0", + "windows_x86_64_gnullvm 0.53.0", + "windows_x86_64_msvc 0.53.0", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -3285,6 +3354,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" + [[package]] name = "windows_aarch64_msvc" version = "0.36.1" @@ -3303,6 +3378,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" + [[package]] name = "windows_i686_gnu" version = "0.36.1" @@ -3321,12 +3402,24 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" + [[package]] name = "windows_i686_msvc" version = "0.36.1" @@ -3345,6 +3438,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_i686_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" + [[package]] name = "windows_x86_64_gnu" version = "0.36.1" @@ -3363,6 +3462,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -3375,6 +3480,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" + [[package]] name = "windows_x86_64_msvc" version = "0.36.1" @@ -3393,14 +3504,20 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" + [[package]] name = "winreg" -version = "0.52.0" +version = "0.55.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5" +checksum = "cb5a765337c50e9ec252c2069be9bf91c7df47afb103b642ba3a53bf8101be97" dependencies = [ "cfg-if", - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] @@ -3436,11 +3553,11 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.17" +version = "0.8.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa91407dacce3a68c56de03abe2760159582b846c6a4acd2f456618087f12713" +checksum = "79386d31a42a4996e3336b0919ddb90f81112af416270cff95b5f5af22b839c2" dependencies = [ - "zerocopy-derive 0.8.17", + "zerocopy-derive 0.8.18", ] [[package]] @@ -3456,9 +3573,9 @@ dependencies = [ [[package]] name = "zerocopy-derive" -version = "0.8.17" +version = "0.8.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06718a168365cad3d5ff0bb133aad346959a2074bd4a85c121255a11304a8626" +checksum = "76331675d372f91bf8d17e13afbd5fe639200b73d01f0fc748bb059f9cca2db7" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 958bc5eeff..f7d7d896c5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -181,7 +181,7 @@ thiserror = "2.0" thread_local = "1.1.8" unicode_names2 = "1.3.0" widestring = "1.1.0" -windows-sys = "0.52.0" +windows-sys = "0.59.0" wasm-bindgen = "0.2.100" # Lints diff --git a/common/src/fileutils.rs b/common/src/fileutils.rs index dcb78675d8..296bb6be7c 100644 --- a/common/src/fileutils.rs +++ b/common/src/fileutils.rs @@ -25,7 +25,7 @@ pub mod windows { use crate::suppress_iph; use crate::windows::ToWideString; use libc::{S_IFCHR, S_IFDIR, S_IFMT}; - use std::ffi::{CString, OsStr, OsString}; + use std::ffi::{c_int, c_void, CString, OsStr, OsString}; use std::os::windows::ffi::OsStrExt; use std::sync::OnceLock; use windows_sys::core::PCWSTR; @@ -113,10 +113,10 @@ pub mod windows { if h.is_err() { unsafe { SetLastError(ERROR_INVALID_HANDLE) }; } - let h = h?; + let mut h = c_int::from(h? as i32); // reset stat? - let file_type = unsafe { GetFileType(h) }; + let file_type = unsafe { GetFileType(&mut h as *mut c_int as *mut c_void) }; if file_type == FILE_TYPE_UNKNOWN { return Err(std::io::Error::last_os_error()); } @@ -138,10 +138,10 @@ pub mod windows { let mut basic_info: FILE_BASIC_INFO = unsafe { std::mem::zeroed() }; let mut id_info: FILE_ID_INFO = unsafe { std::mem::zeroed() }; - if unsafe { GetFileInformationByHandle(h, &mut info) } == 0 + if unsafe { GetFileInformationByHandle(&mut h as *mut c_int as *mut c_void, &mut info) } == 0 || unsafe { GetFileInformationByHandleEx( - h, + &mut h as *mut c_int as *mut c_void, FileBasicInfo, &mut basic_info as *mut _ as *mut _, std::mem::size_of_val(&basic_info) as u32, @@ -153,7 +153,7 @@ pub mod windows { let p_id_info = if unsafe { GetFileInformationByHandleEx( - h, + &mut h as *mut c_int as *mut c_void, FileIdInfo, &mut id_info as *mut _ as *mut _, std::mem::size_of_val(&id_info) as u32, @@ -320,7 +320,7 @@ pub mod windows { .get_or_init(|| { let library_name = OsString::from("api-ms-win-core-file-l2-1-4").to_wide_with_nul(); let module = unsafe { LoadLibraryW(library_name.as_ptr()) }; - if module == 0 { + if module == c_int::from(0) as *mut c_int as *mut c_void { return None; } let name = CString::new("GetFileInformationByName").unwrap(); diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 1468651aeb..49eaabf96a 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -108,10 +108,10 @@ num_cpus = "1.13.1" [target.'cfg(windows)'.dependencies] junction = { workspace = true } schannel = { workspace = true } -winreg = "0.52" +winreg = "0.55" [target.'cfg(windows)'.dependencies.windows] -version = "0.52.0" +version = "0.59.0" features = [ "Win32_Foundation", "Win32_System_LibraryLoader", diff --git a/vm/src/stdlib/nt.rs b/vm/src/stdlib/nt.rs index d2785f42dd..64c04adbd6 100644 --- a/vm/src/stdlib/nt.rs +++ b/vm/src/stdlib/nt.rs @@ -25,6 +25,7 @@ pub(crate) mod module { mem::MaybeUninit, os::windows::ffi::{OsStrExt, OsStringExt}, }; + use std::ffi::{c_int, c_void}; use windows_sys::Win32::{ Foundation::{self, INVALID_HANDLE_VALUE}, Storage::FileSystem, @@ -150,7 +151,7 @@ pub(crate) mod module { } let h = unsafe { Threading::OpenProcess(Threading::PROCESS_ALL_ACCESS, 0, pid) }; - if h == 0 { + if h == c_int::from(0) as *mut c_int as *mut c_void { return Err(errno_err(vm)); } let ret = unsafe { Threading::TerminateProcess(h, sig) }; @@ -172,7 +173,7 @@ pub(crate) mod module { _ => return Err(vm.new_value_error("bad file descriptor".to_owned())), }; let h = unsafe { Console::GetStdHandle(stdhandle) }; - if h == 0 { + if h == c_int::from(0) as *mut c_int as *mut c_void { return Err(vm.new_os_error("handle cannot be retrieved".to_owned())); } if h == INVALID_HANDLE_VALUE { diff --git a/vm/src/stdlib/winapi.rs b/vm/src/stdlib/winapi.rs index ad6db12474..45ad6d12a7 100644 --- a/vm/src/stdlib/winapi.rs +++ b/vm/src/stdlib/winapi.rs @@ -3,6 +3,7 @@ pub(crate) use _winapi::make_module; #[pymodule] mod _winapi { + use std::ffi::{c_int, c_void}; use crate::{ builtins::PyStrRef, common::windows::ToWideString, @@ -116,6 +117,8 @@ mod _winapi { .to_pyresult(vm)?; target.assume_init() }; + let target = c_int::from(target as i32); + Ok(HANDLE(target)) } @@ -436,6 +439,7 @@ mod _winapi { #[pyfunction] fn GetModuleFileName(handle: isize, vm: &VirtualMachine) -> PyResult { let mut path: Vec = vec![0; MAX_PATH as usize]; + let handle = c_int::from(handle as i32) as *mut c_int as *mut c_void; let handle = HINSTANCE(handle); let length = diff --git a/vm/src/stdlib/winreg.rs b/vm/src/stdlib/winreg.rs index b368c43a3e..10a1ebc49e 100644 --- a/vm/src/stdlib/winreg.rs +++ b/vm/src/stdlib/winreg.rs @@ -35,6 +35,7 @@ mod winreg { use ::winreg::{enums::RegType, RegKey, RegValue}; use std::mem::ManuallyDrop; use std::{ffi::OsStr, io}; + use std::ffi::{c_int, c_void}; use windows_sys::Win32::Foundation; // access rights @@ -98,7 +99,7 @@ mod winreg { #[pymethod(magic)] fn bool(&self) -> bool { - self.key().raw_handle() != 0 + self.key().raw_handle() != c_int::from(0) as *mut c_int as *mut c_void } #[pymethod(magic)] fn enter(zelf: PyRef) -> PyRef { From 7e1568a1ffef0b44c47e98e465269e00693c207b Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sun, 16 Feb 2025 21:30:41 -0800 Subject: [PATCH 03/26] Revert "windows-rs upgrade to 0.59" This reverts commit 547530724e77a592734d8cd396115c4124d7a9f9. --- Cargo.lock | 191 ++++++++-------------------------------- Cargo.toml | 2 +- common/src/fileutils.rs | 14 +-- vm/Cargo.toml | 4 +- vm/src/stdlib/nt.rs | 5 +- vm/src/stdlib/winapi.rs | 4 - vm/src/stdlib/winreg.rs | 3 +- 7 files changed, 50 insertions(+), 173 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 115451adfa..a62a5dfced 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -216,9 +216,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.2.14" +version = "1.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c3d1b2e905a3a7b00a6141adb0e4c0bb941d11caf55349d863942a1cc44e3c9" +checksum = "c7777341816418c02e033934a09f20dc0ccaf65a5201ef8a450ae0105a573fda" dependencies = [ "shlex", ] @@ -645,9 +645,9 @@ dependencies = [ [[package]] name = "equivalent" -version = "1.0.2" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" @@ -903,7 +903,7 @@ dependencies = [ "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "windows-core 0.52.0", + "windows-core", ] [[package]] @@ -1406,9 +1406,9 @@ checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9" [[package]] name = "openssl" -version = "0.10.71" +version = "0.10.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e14130c6a98cd258fdcb0fb6d744152343ff729cbfcb28c656a9d12b999fbcd" +checksum = "61cfb4e166a8bb8c9b55c500bc2308550148ece889be90f609377e58140f42c6" dependencies = [ "bitflags 2.8.0", "cfg-if", @@ -1447,9 +1447,9 @@ dependencies = [ [[package]] name = "openssl-sys" -version = "0.9.106" +version = "0.9.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bb61ea9811cc39e3c2069f40b8b8e2e70d8569b361f879786cc7ed48b777cdd" +checksum = "8b22d5b84be05a8d6947c7cb71f7c849aa0f112acd4bf51c2a7c1c988ac0a9dc" dependencies = [ "cc", "libc", @@ -1745,8 +1745,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" dependencies = [ "rand_chacha 0.9.0", - "rand_core 0.9.1", - "zerocopy 0.8.18", + "rand_core 0.9.0", + "zerocopy 0.8.17", ] [[package]] @@ -1766,7 +1766,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core 0.9.1", + "rand_core 0.9.0", ] [[package]] @@ -1780,12 +1780,12 @@ dependencies = [ [[package]] name = "rand_core" -version = "0.9.1" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a88e0da7a2c97baa202165137c158d0a2e824ac465d13d81046727b34cb247d3" +checksum = "b08f3c9802962f7e1b25113931d94f43ed9725bebc59db9d0c3e9a23b67e15ff" dependencies = [ "getrandom 0.3.1", - "zerocopy 0.8.18", + "zerocopy 0.8.17", ] [[package]] @@ -2012,7 +2012,7 @@ dependencies = [ "siphasher 0.3.11", "volatile", "widestring", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -2242,7 +2242,7 @@ dependencies = [ "uuid", "widestring", "winapi", - "windows-sys 0.59.0", + "windows-sys 0.52.0", "xml-rs", ] @@ -2320,7 +2320,7 @@ dependencies = [ "which", "widestring", "windows", - "windows-sys 0.59.0", + "windows-sys 0.52.0", "winreg", ] @@ -2524,9 +2524,9 @@ checksum = "826167069c09b99d56f31e9ae5c99049e932a98c9dc2dac47645b08dbbf76ba7" [[package]] name = "smallvec" -version = "1.14.0" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" [[package]] name = "socket2" @@ -2552,15 +2552,15 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" [[package]] name = "strum" -version = "0.27.1" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f64def088c51c9510a8579e3c5d67c65349dcf755e5479ad3d010aa6454e2c32" +checksum = "ce1475c515a4f03a8a7129bb5228b81a781a86cb0b3fbbc19e1c556d491a401f" [[package]] name = "strum_macros" -version = "0.27.1" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c77a8c5abcaf0f9ce05d62342b7d298c346515365c36b673df4ebe3ced01fde8" +checksum = "9688894b43459159c82bfa5a5fa0435c19cbe3c9b427fa1dd7b1ce0c279b18a7" dependencies = [ "heck", "proc-macro2", @@ -3185,12 +3185,12 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows" -version = "0.59.0" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f919aee0a93304be7f62e8e5027811bbba96bcb1de84d6618be56e43f8a32a1" +checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be" dependencies = [ - "windows-core 0.59.0", - "windows-targets 0.53.0", + "windows-core", + "windows-targets 0.52.6", ] [[package]] @@ -3202,59 +3202,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "windows-core" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "810ce18ed2112484b0d4e15d022e5f598113e220c53e373fb31e67e21670c1ce" -dependencies = [ - "windows-implement", - "windows-interface", - "windows-result", - "windows-strings", - "windows-targets 0.53.0", -] - -[[package]] -name = "windows-implement" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83577b051e2f49a058c308f17f273b570a6a758386fc291b5f6a934dd84e48c1" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.98", -] - -[[package]] -name = "windows-interface" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb26fd936d991781ea39e87c3a27285081e3c0da5ca0fcbc02d368cc6f52ff01" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.98", -] - -[[package]] -name = "windows-result" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d08106ce80268c4067c0571ca55a9b4e9516518eaa1a1fe9b37ca403ae1d1a34" -dependencies = [ - "windows-targets 0.53.0", -] - -[[package]] -name = "windows-strings" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b888f919960b42ea4e11c2f408fadb55f78a9f236d5eef084103c8ce52893491" -dependencies = [ - "windows-targets 0.53.0", -] - [[package]] name = "windows-sys" version = "0.36.1" @@ -3319,29 +3266,13 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", + "windows_i686_gnullvm", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] -[[package]] -name = "windows-targets" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1e4c7e8ceaaf9cb7d7507c974735728ab453b67ef8f18febdd7c11fe59dca8b" -dependencies = [ - "windows_aarch64_gnullvm 0.53.0", - "windows_aarch64_msvc 0.53.0", - "windows_i686_gnu 0.53.0", - "windows_i686_gnullvm 0.53.0", - "windows_i686_msvc 0.53.0", - "windows_x86_64_gnu 0.53.0", - "windows_x86_64_gnullvm 0.53.0", - "windows_x86_64_msvc 0.53.0", -] - [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -3354,12 +3285,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" - [[package]] name = "windows_aarch64_msvc" version = "0.36.1" @@ -3378,12 +3303,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" - [[package]] name = "windows_i686_gnu" version = "0.36.1" @@ -3402,24 +3321,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" -[[package]] -name = "windows_i686_gnu" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" - [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" - [[package]] name = "windows_i686_msvc" version = "0.36.1" @@ -3438,12 +3345,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_i686_msvc" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" - [[package]] name = "windows_x86_64_gnu" version = "0.36.1" @@ -3462,12 +3363,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" - [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -3480,12 +3375,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" - [[package]] name = "windows_x86_64_msvc" version = "0.36.1" @@ -3504,20 +3393,14 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" - [[package]] name = "winreg" -version = "0.55.0" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb5a765337c50e9ec252c2069be9bf91c7df47afb103b642ba3a53bf8101be97" +checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5" dependencies = [ "cfg-if", - "windows-sys 0.59.0", + "windows-sys 0.48.0", ] [[package]] @@ -3553,11 +3436,11 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.18" +version = "0.8.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79386d31a42a4996e3336b0919ddb90f81112af416270cff95b5f5af22b839c2" +checksum = "aa91407dacce3a68c56de03abe2760159582b846c6a4acd2f456618087f12713" dependencies = [ - "zerocopy-derive 0.8.18", + "zerocopy-derive 0.8.17", ] [[package]] @@ -3573,9 +3456,9 @@ dependencies = [ [[package]] name = "zerocopy-derive" -version = "0.8.18" +version = "0.8.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76331675d372f91bf8d17e13afbd5fe639200b73d01f0fc748bb059f9cca2db7" +checksum = "06718a168365cad3d5ff0bb133aad346959a2074bd4a85c121255a11304a8626" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index f7d7d896c5..958bc5eeff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -181,7 +181,7 @@ thiserror = "2.0" thread_local = "1.1.8" unicode_names2 = "1.3.0" widestring = "1.1.0" -windows-sys = "0.59.0" +windows-sys = "0.52.0" wasm-bindgen = "0.2.100" # Lints diff --git a/common/src/fileutils.rs b/common/src/fileutils.rs index 296bb6be7c..dcb78675d8 100644 --- a/common/src/fileutils.rs +++ b/common/src/fileutils.rs @@ -25,7 +25,7 @@ pub mod windows { use crate::suppress_iph; use crate::windows::ToWideString; use libc::{S_IFCHR, S_IFDIR, S_IFMT}; - use std::ffi::{c_int, c_void, CString, OsStr, OsString}; + use std::ffi::{CString, OsStr, OsString}; use std::os::windows::ffi::OsStrExt; use std::sync::OnceLock; use windows_sys::core::PCWSTR; @@ -113,10 +113,10 @@ pub mod windows { if h.is_err() { unsafe { SetLastError(ERROR_INVALID_HANDLE) }; } - let mut h = c_int::from(h? as i32); + let h = h?; // reset stat? - let file_type = unsafe { GetFileType(&mut h as *mut c_int as *mut c_void) }; + let file_type = unsafe { GetFileType(h) }; if file_type == FILE_TYPE_UNKNOWN { return Err(std::io::Error::last_os_error()); } @@ -138,10 +138,10 @@ pub mod windows { let mut basic_info: FILE_BASIC_INFO = unsafe { std::mem::zeroed() }; let mut id_info: FILE_ID_INFO = unsafe { std::mem::zeroed() }; - if unsafe { GetFileInformationByHandle(&mut h as *mut c_int as *mut c_void, &mut info) } == 0 + if unsafe { GetFileInformationByHandle(h, &mut info) } == 0 || unsafe { GetFileInformationByHandleEx( - &mut h as *mut c_int as *mut c_void, + h, FileBasicInfo, &mut basic_info as *mut _ as *mut _, std::mem::size_of_val(&basic_info) as u32, @@ -153,7 +153,7 @@ pub mod windows { let p_id_info = if unsafe { GetFileInformationByHandleEx( - &mut h as *mut c_int as *mut c_void, + h, FileIdInfo, &mut id_info as *mut _ as *mut _, std::mem::size_of_val(&id_info) as u32, @@ -320,7 +320,7 @@ pub mod windows { .get_or_init(|| { let library_name = OsString::from("api-ms-win-core-file-l2-1-4").to_wide_with_nul(); let module = unsafe { LoadLibraryW(library_name.as_ptr()) }; - if module == c_int::from(0) as *mut c_int as *mut c_void { + if module == 0 { return None; } let name = CString::new("GetFileInformationByName").unwrap(); diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 49eaabf96a..1468651aeb 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -108,10 +108,10 @@ num_cpus = "1.13.1" [target.'cfg(windows)'.dependencies] junction = { workspace = true } schannel = { workspace = true } -winreg = "0.55" +winreg = "0.52" [target.'cfg(windows)'.dependencies.windows] -version = "0.59.0" +version = "0.52.0" features = [ "Win32_Foundation", "Win32_System_LibraryLoader", diff --git a/vm/src/stdlib/nt.rs b/vm/src/stdlib/nt.rs index 64c04adbd6..d2785f42dd 100644 --- a/vm/src/stdlib/nt.rs +++ b/vm/src/stdlib/nt.rs @@ -25,7 +25,6 @@ pub(crate) mod module { mem::MaybeUninit, os::windows::ffi::{OsStrExt, OsStringExt}, }; - use std::ffi::{c_int, c_void}; use windows_sys::Win32::{ Foundation::{self, INVALID_HANDLE_VALUE}, Storage::FileSystem, @@ -151,7 +150,7 @@ pub(crate) mod module { } let h = unsafe { Threading::OpenProcess(Threading::PROCESS_ALL_ACCESS, 0, pid) }; - if h == c_int::from(0) as *mut c_int as *mut c_void { + if h == 0 { return Err(errno_err(vm)); } let ret = unsafe { Threading::TerminateProcess(h, sig) }; @@ -173,7 +172,7 @@ pub(crate) mod module { _ => return Err(vm.new_value_error("bad file descriptor".to_owned())), }; let h = unsafe { Console::GetStdHandle(stdhandle) }; - if h == c_int::from(0) as *mut c_int as *mut c_void { + if h == 0 { return Err(vm.new_os_error("handle cannot be retrieved".to_owned())); } if h == INVALID_HANDLE_VALUE { diff --git a/vm/src/stdlib/winapi.rs b/vm/src/stdlib/winapi.rs index 45ad6d12a7..ad6db12474 100644 --- a/vm/src/stdlib/winapi.rs +++ b/vm/src/stdlib/winapi.rs @@ -3,7 +3,6 @@ pub(crate) use _winapi::make_module; #[pymodule] mod _winapi { - use std::ffi::{c_int, c_void}; use crate::{ builtins::PyStrRef, common::windows::ToWideString, @@ -117,8 +116,6 @@ mod _winapi { .to_pyresult(vm)?; target.assume_init() }; - let target = c_int::from(target as i32); - Ok(HANDLE(target)) } @@ -439,7 +436,6 @@ mod _winapi { #[pyfunction] fn GetModuleFileName(handle: isize, vm: &VirtualMachine) -> PyResult { let mut path: Vec = vec![0; MAX_PATH as usize]; - let handle = c_int::from(handle as i32) as *mut c_int as *mut c_void; let handle = HINSTANCE(handle); let length = diff --git a/vm/src/stdlib/winreg.rs b/vm/src/stdlib/winreg.rs index 10a1ebc49e..b368c43a3e 100644 --- a/vm/src/stdlib/winreg.rs +++ b/vm/src/stdlib/winreg.rs @@ -35,7 +35,6 @@ mod winreg { use ::winreg::{enums::RegType, RegKey, RegValue}; use std::mem::ManuallyDrop; use std::{ffi::OsStr, io}; - use std::ffi::{c_int, c_void}; use windows_sys::Win32::Foundation; // access rights @@ -99,7 +98,7 @@ mod winreg { #[pymethod(magic)] fn bool(&self) -> bool { - self.key().raw_handle() != c_int::from(0) as *mut c_int as *mut c_void + self.key().raw_handle() != 0 } #[pymethod(magic)] fn enter(zelf: PyRef) -> PyRef { From 630c1ff8d18fee8fe4e8ca1cbf4d25e0f30d2889 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sun, 16 Feb 2025 22:36:05 -0800 Subject: [PATCH 04/26] simple part of the bump Signed-off-by: Ashwin Naren --- Cargo.lock | 57 ++++++++++++++++++++++++++++++++++++----- vm/Cargo.toml | 4 +-- vm/src/stdlib/winreg.rs | 2 +- 3 files changed, 53 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a62a5dfced..0d9f468106 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -903,7 +903,7 @@ dependencies = [ "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "windows-core", + "windows-core 0.52.0", ] [[package]] @@ -3185,11 +3185,11 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows" -version = "0.52.0" +version = "0.57.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be" +checksum = "12342cb4d8e3b046f3d80effd474a7a02447231330ef77d71daa6fbc40681143" dependencies = [ - "windows-core", + "windows-core 0.57.0", "windows-targets 0.52.6", ] @@ -3202,6 +3202,49 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-core" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2ed2439a290666cd67ecce2b0ffaad89c2a56b976b736e6ece670297897832d" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-result", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-implement" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.98", +] + +[[package]] +name = "windows-interface" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.98", +] + +[[package]] +name = "windows-result" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-sys" version = "0.36.1" @@ -3395,12 +3438,12 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winreg" -version = "0.52.0" +version = "0.55.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5" +checksum = "cb5a765337c50e9ec252c2069be9bf91c7df47afb103b642ba3a53bf8101be97" dependencies = [ "cfg-if", - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 1468651aeb..72a874a821 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -108,10 +108,10 @@ num_cpus = "1.13.1" [target.'cfg(windows)'.dependencies] junction = { workspace = true } schannel = { workspace = true } -winreg = "0.52" +winreg = "0.55" [target.'cfg(windows)'.dependencies.windows] -version = "0.52.0" +version = "0.57.0" features = [ "Win32_Foundation", "Win32_System_LibraryLoader", diff --git a/vm/src/stdlib/winreg.rs b/vm/src/stdlib/winreg.rs index b368c43a3e..a67fd02b38 100644 --- a/vm/src/stdlib/winreg.rs +++ b/vm/src/stdlib/winreg.rs @@ -98,7 +98,7 @@ mod winreg { #[pymethod(magic)] fn bool(&self) -> bool { - self.key().raw_handle() != 0 + self.key().raw_handle() != core::ptr::null_mut() } #[pymethod(magic)] fn enter(zelf: PyRef) -> PyRef { From 38a6a8d98480e00c7182f9f2d0b4b3fcf6c26f95 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sun, 16 Feb 2025 22:49:25 -0800 Subject: [PATCH 05/26] duplicate windows-sys Signed-off-by: Ashwin Naren --- vm/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 72a874a821..f092e153a0 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -111,7 +111,7 @@ schannel = { workspace = true } winreg = "0.55" [target.'cfg(windows)'.dependencies.windows] -version = "0.57.0" +version = "0.52.0" features = [ "Win32_Foundation", "Win32_System_LibraryLoader", From 517ffed40167f02d1fffd2b49f02f2ce2191e8ba Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Mon, 17 Feb 2025 11:44:25 -0800 Subject: [PATCH 06/26] fix clippy lint Signed-off-by: Ashwin Naren --- vm/src/stdlib/winreg.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/src/stdlib/winreg.rs b/vm/src/stdlib/winreg.rs index a67fd02b38..747c58cfd8 100644 --- a/vm/src/stdlib/winreg.rs +++ b/vm/src/stdlib/winreg.rs @@ -98,7 +98,7 @@ mod winreg { #[pymethod(magic)] fn bool(&self) -> bool { - self.key().raw_handle() != core::ptr::null_mut() + !self.key().raw_handle().is_null() } #[pymethod(magic)] fn enter(zelf: PyRef) -> PyRef { From 9856d94f2d560f0d684d44a3e3357d70852bfc5b Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Mon, 17 Feb 2025 21:15:58 -0800 Subject: [PATCH 07/26] function to retrieve tz info on windows Signed-off-by: Ashwin Naren --- vm/Cargo.toml | 1 + vm/src/stdlib/time.rs | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/vm/Cargo.toml b/vm/Cargo.toml index f092e153a0..853338c372 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -116,6 +116,7 @@ features = [ "Win32_Foundation", "Win32_System_LibraryLoader", "Win32_System_Threading", + "Win32_System_Time", "Win32_UI_Shell", ] diff --git a/vm/src/stdlib/time.rs b/vm/src/stdlib/time.rs index 566650d0f2..e6581ce847 100644 --- a/vm/src/stdlib/time.rs +++ b/vm/src/stdlib/time.rs @@ -152,6 +152,17 @@ mod decl { Ok(get_perf_time(vm)?.as_nanos()) } + #[cfg(target_env = "msvc")] + #[cfg(not(target_arch = "wasm32"))] + fn get_tz_info() -> Time::TIME_ZONE_INFORMATION { + let mut info = Time::TIME_ZONE_INFORMATION::default(); + let info_ptr = &mut info as *mut Time::TIME_ZONE_INFORMATION; + let _ = unsafe { + Time::GetTimeZoneInformation(info_ptr) + }; + info + } + // #[pyfunction] // fn tzset() { // unsafe { super::_tzset() }; From 72338d578b4e604530a8dbff240208b0ad1de972 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Mon, 17 Feb 2025 21:16:17 -0800 Subject: [PATCH 08/26] tzname on windows Signed-off-by: Ashwin Naren --- vm/src/stdlib/time.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/vm/src/stdlib/time.rs b/vm/src/stdlib/time.rs index e6581ce847..d087cd3b1c 100644 --- a/vm/src/stdlib/time.rs +++ b/vm/src/stdlib/time.rs @@ -43,6 +43,9 @@ mod decl { DateTime, Datelike, Timelike, }; use std::time::Duration; + #[cfg(target_env = "msvc")] + #[cfg(not(target_arch = "wasm32"))] + use windows::Win32::System::Time; #[allow(dead_code)] pub(super) const SEC_TO_MS: i64 = 1000; @@ -195,6 +198,18 @@ mod decl { unsafe { (to_str(super::c_tzname[0]), to_str(super::c_tzname[1])) }.into_pytuple(vm) } + #[cfg(target_env = "msvc")] + #[cfg(not(target_arch = "wasm32"))] + #[pyattr] + fn tzname(vm: &VirtualMachine) -> crate::builtins::PyTupleRef { + use crate::builtins::tuple::IntoPyTuple; + let info = get_tz_info(); + let standard = widestring::decode_utf16_lossy(info.StandardName).filter(|&c| c != '\0').collect::(); + let daylight = widestring::decode_utf16_lossy(info.DaylightName).filter(|&c| c != '\0').collect::(); + let tz_name = (&*standard, &*daylight); + tz_name.into_pytuple(vm) + } + fn pyobj_to_date_time( value: Either, vm: &VirtualMachine, From 175afd97d8a5589d71bc7b7c0a8f683271425bc3 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Mon, 17 Feb 2025 21:19:36 -0800 Subject: [PATCH 09/26] time.timezone for windows Signed-off-by: Ashwin Naren --- vm/src/stdlib/time.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/vm/src/stdlib/time.rs b/vm/src/stdlib/time.rs index d087cd3b1c..e5398d498a 100644 --- a/vm/src/stdlib/time.rs +++ b/vm/src/stdlib/time.rs @@ -178,6 +178,16 @@ mod decl { unsafe { super::c_timezone } } + #[cfg(target_env = "msvc")] + #[cfg(not(target_arch = "wasm32"))] + #[pyattr] + fn timezone() -> i32 { + let info = get_tz_info(); + // https://users.rust-lang.org/t/accessing-tzname-and-similar-constants-in-windows/125771/3 + (info.Bias + info.StandardBias) * 60 + } + + #[cfg(not(target_os = "freebsd"))] #[cfg(not(target_env = "msvc"))] #[cfg(not(target_arch = "wasm32"))] From 4ed735b424d5d502472f5dad32fd4f1ee9bd60ad Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Mon, 17 Feb 2025 21:19:55 -0800 Subject: [PATCH 10/26] time.daylight for windows Signed-off-by: Ashwin Naren --- vm/src/stdlib/time.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/vm/src/stdlib/time.rs b/vm/src/stdlib/time.rs index e5398d498a..1277b6a7e1 100644 --- a/vm/src/stdlib/time.rs +++ b/vm/src/stdlib/time.rs @@ -196,6 +196,15 @@ mod decl { unsafe { super::c_daylight } } + #[cfg(target_env = "msvc")] + #[cfg(not(target_arch = "wasm32"))] + #[pyattr] + fn daylight() -> i32 { + let info = get_tz_info(); + // https://users.rust-lang.org/t/accessing-tzname-and-similar-constants-in-windows/125771/3 + (info.StandardBias != info.DaylightBias) as i32 + } + #[cfg(not(target_env = "msvc"))] #[cfg(not(target_arch = "wasm32"))] #[pyattr] From 69b1a9910f0d3adc56537ad67795d6a92ed81a95 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Mon, 17 Feb 2025 21:20:15 -0800 Subject: [PATCH 11/26] formatting Signed-off-by: Ashwin Naren --- vm/src/stdlib/time.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/vm/src/stdlib/time.rs b/vm/src/stdlib/time.rs index 1277b6a7e1..51411889a1 100644 --- a/vm/src/stdlib/time.rs +++ b/vm/src/stdlib/time.rs @@ -160,9 +160,7 @@ mod decl { fn get_tz_info() -> Time::TIME_ZONE_INFORMATION { let mut info = Time::TIME_ZONE_INFORMATION::default(); let info_ptr = &mut info as *mut Time::TIME_ZONE_INFORMATION; - let _ = unsafe { - Time::GetTimeZoneInformation(info_ptr) - }; + let _ = unsafe { Time::GetTimeZoneInformation(info_ptr) }; info } @@ -187,7 +185,6 @@ mod decl { (info.Bias + info.StandardBias) * 60 } - #[cfg(not(target_os = "freebsd"))] #[cfg(not(target_env = "msvc"))] #[cfg(not(target_arch = "wasm32"))] @@ -223,8 +220,12 @@ mod decl { fn tzname(vm: &VirtualMachine) -> crate::builtins::PyTupleRef { use crate::builtins::tuple::IntoPyTuple; let info = get_tz_info(); - let standard = widestring::decode_utf16_lossy(info.StandardName).filter(|&c| c != '\0').collect::(); - let daylight = widestring::decode_utf16_lossy(info.DaylightName).filter(|&c| c != '\0').collect::(); + let standard = widestring::decode_utf16_lossy(info.StandardName) + .filter(|&c| c != '\0') + .collect::(); + let daylight = widestring::decode_utf16_lossy(info.DaylightName) + .filter(|&c| c != '\0') + .collect::(); let tz_name = (&*standard, &*daylight); tz_name.into_pytuple(vm) } From f46697131258fcf3dd77a8a64e97e3e9f5ceb244 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Mon, 17 Feb 2025 21:22:10 -0800 Subject: [PATCH 12/26] clippy Signed-off-by: Ashwin Naren --- vm/src/stdlib/time.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vm/src/stdlib/time.rs b/vm/src/stdlib/time.rs index 51411889a1..5a493c3472 100644 --- a/vm/src/stdlib/time.rs +++ b/vm/src/stdlib/time.rs @@ -179,7 +179,7 @@ mod decl { #[cfg(target_env = "msvc")] #[cfg(not(target_arch = "wasm32"))] #[pyattr] - fn timezone() -> i32 { + fn timezone(_vm: &VirtualMachine) -> i32 { let info = get_tz_info(); // https://users.rust-lang.org/t/accessing-tzname-and-similar-constants-in-windows/125771/3 (info.Bias + info.StandardBias) * 60 @@ -196,7 +196,7 @@ mod decl { #[cfg(target_env = "msvc")] #[cfg(not(target_arch = "wasm32"))] #[pyattr] - fn daylight() -> i32 { + fn daylight(_vm: &VirtualMachine) -> i32 { let info = get_tz_info(); // https://users.rust-lang.org/t/accessing-tzname-and-similar-constants-in-windows/125771/3 (info.StandardBias != info.DaylightBias) as i32 From a71c16f8cb36c086bdc822955c50294945a17e39 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Mon, 17 Feb 2025 22:25:40 -0800 Subject: [PATCH 13/26] test colorize on ci Signed-off-by: Ashwin Naren --- .github/workflows/ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7bbed2c3c2..ed91125a0f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -34,6 +34,7 @@ env: # PLATFORM_INDEPENDENT_TESTS are tests that do not depend on the underlying OS. They are currently # only run on Linux to speed up the CI. PLATFORM_INDEPENDENT_TESTS: >- + test__colorize test_argparse test_array test_asyncgen From fa2acd7cde461b7c8aff0fe93365e4e4c809fc8b Mon Sep 17 00:00:00 2001 From: Noa Date: Mon, 17 Feb 2025 11:48:50 -0600 Subject: [PATCH 14/26] Update rand to 0.9 --- .github/workflows/ci.yaml | 3 ++- Cargo.lock | 20 ++++++++++---------- Cargo.toml | 4 ++-- common/src/hash.rs | 6 +++--- stdlib/Cargo.toml | 5 ++--- stdlib/src/random.rs | 8 +------- stdlib/src/uuid.rs | 20 +++----------------- vm/Cargo.toml | 4 ++-- vm/src/import.rs | 3 +-- vm/src/stdlib/os.rs | 2 +- wasm/lib/.cargo/config.toml | 5 +++++ wasm/lib/Cargo.toml | 5 ++++- wasm/wasm-unknown-test/.cargo/config.toml | 5 +++++ wasm/wasm-unknown-test/Cargo.toml | 1 + wasm/wasm-unknown-test/src/lib.rs | 8 ++++++++ 15 files changed, 50 insertions(+), 49 deletions(-) create mode 100644 wasm/lib/.cargo/config.toml create mode 100644 wasm/wasm-unknown-test/.cargo/config.toml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ed91125a0f..dc2dcc6c7e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -379,7 +379,8 @@ jobs: with: { wabt-version: "1.0.30" } - name: check wasm32-unknown without js run: | - cargo build --release --manifest-path wasm/wasm-unknown-test/Cargo.toml --target wasm32-unknown-unknown --verbose + cd wasm/wasm-unknown-test + cargo build --release --verbose if wasm-objdump -xj Import target/wasm32-unknown-unknown/release/wasm_unknown_test.wasm; then echo "ERROR: wasm32-unknown module expects imports from the host environment" >2 fi diff --git a/Cargo.lock b/Cargo.lock index 0d9f468106..87174aeac6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -808,8 +808,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi 0.13.3+wasi-0.2.2", + "wasm-bindgen", "windows-targets 0.52.6", ] @@ -1306,11 +1308,11 @@ dependencies = [ [[package]] name = "mt19937" -version = "2.0.1" +version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12ca7f22ed370d5991a9caec16a83187e865bc8a532f889670337d5a5689e3a1" +checksum = "df7151a832e54d2d6b2c827a20e5bcdd80359281cd2c354e725d4b82e7c471de" dependencies = [ - "rand_core 0.6.4", + "rand_core 0.9.0", ] [[package]] @@ -2007,7 +2009,7 @@ dependencies = [ "once_cell", "parking_lot", "radium", - "rand 0.8.5", + "rand 0.9.0", "rustpython-format", "siphasher 0.3.11", "volatile", @@ -2216,8 +2218,7 @@ dependencies = [ "parking_lot", "paste", "puruspe", - "rand 0.8.5", - "rand_core 0.6.4", + "rand 0.9.0", "rustix", "rustpython-common", "rustpython-derive", @@ -2262,7 +2263,7 @@ dependencies = [ "exitcode", "flame", "flamer", - "getrandom 0.2.15", + "getrandom 0.3.1", "glob", "half 2.4.1", "hex", @@ -2285,7 +2286,7 @@ dependencies = [ "optional", "parking_lot", "paste", - "rand 0.8.5", + "rand 0.9.0", "result-like", "rustc_version", "rustix", @@ -2329,6 +2330,7 @@ name = "rustpython_wasm" version = "0.4.0" dependencies = [ "console_error_panic_hook", + "getrandom 0.2.15", "js-sys", "rustpython-common", "rustpython-parser", @@ -3000,8 +3002,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced87ca4be083373936a67f8de945faa23b6b42384bd5b64434850802c6dccd0" dependencies = [ "atomic", - "getrandom 0.3.1", - "rand 0.9.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 958bc5eeff..bda7286afd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -146,7 +146,7 @@ cfg-if = "1.0" chrono = "0.4.39" crossbeam-utils = "0.8.21" flame = "0.2.2" -getrandom = "0.2.15" +getrandom = "0.3" glob = "0.3" hex = "0.4.3" indexmap = { version = "2.2.6", features = ["std"] } @@ -168,7 +168,7 @@ num_enum = { version = "0.7", default-features = false } once_cell = "1.20.3" parking_lot = "0.12.3" paste = "1.0.15" -rand = "0.8.5" +rand = "0.9" rustix = { version = "0.38", features = ["event"] } rustyline = "15.0.0" serde = { version = "1.0.133", default-features = false } diff --git a/common/src/hash.rs b/common/src/hash.rs index 4e87eff799..700d2ceef5 100644 --- a/common/src/hash.rs +++ b/common/src/hash.rs @@ -37,11 +37,11 @@ impl BuildHasher for HashSecret { } } -impl rand::distributions::Distribution for rand::distributions::Standard { +impl rand::distr::Distribution for rand::distr::StandardUniform { fn sample(&self, rng: &mut R) -> HashSecret { HashSecret { - k0: rng.gen(), - k1: rng.gen(), + k0: rng.random(), + k1: rng.random(), } } } diff --git a/stdlib/Cargo.toml b/stdlib/Cargo.toml index 3d91b6fab9..31f5bd18d4 100644 --- a/stdlib/Cargo.toml +++ b/stdlib/Cargo.toml @@ -54,8 +54,7 @@ xml-rs = "0.8.14" # random rand = { workspace = true } -rand_core = "0.6.4" -mt19937 = "2.0.1" +mt19937 = "3.1" # Crypto: digest = "0.10.3" @@ -88,7 +87,7 @@ bzip2 = { version = "0.4", optional = true } # uuid [target.'cfg(not(any(target_os = "ios", target_os = "android", target_os = "windows", target_arch = "wasm32", target_os = "redox")))'.dependencies] mac_address = "1.1.3" -uuid = { version = "1.1.2", features = ["v1", "fast-rng"] } +uuid = { version = "1.1.2", features = ["v1"] } # mmap [target.'cfg(all(unix, not(target_arch = "wasm32")))'.dependencies] diff --git a/stdlib/src/random.rs b/stdlib/src/random.rs index 1dfc4fcc30..35cbd94457 100644 --- a/stdlib/src/random.rs +++ b/stdlib/src/random.rs @@ -23,7 +23,7 @@ mod _random { impl Default for PyRng { fn default() -> Self { - PyRng::Std(Box::new(StdRng::from_entropy())) + PyRng::Std(Box::new(StdRng::from_os_rng())) } } @@ -46,12 +46,6 @@ mod _random { Self::MT(m) => m.fill_bytes(dest), } } - fn try_fill_bytes(&mut self, dest: &mut [u8]) -> Result<(), rand::Error> { - match self { - Self::Std(s) => s.try_fill_bytes(dest), - Self::MT(m) => m.try_fill_bytes(dest), - } - } } #[pyattr] diff --git a/stdlib/src/uuid.rs b/stdlib/src/uuid.rs index e5434da0e1..4cadbb787b 100644 --- a/stdlib/src/uuid.rs +++ b/stdlib/src/uuid.rs @@ -5,33 +5,19 @@ mod _uuid { use crate::{builtins::PyNone, vm::VirtualMachine}; use mac_address::get_mac_address; use once_cell::sync::OnceCell; - use rand::Rng; - use std::time::{Duration, SystemTime}; - use uuid::{ - v1::{Context, Timestamp}, - Uuid, - }; + use uuid::{timestamp::Timestamp, Context, Uuid}; fn get_node_id() -> [u8; 6] { match get_mac_address() { Ok(Some(_ma)) => get_mac_address().unwrap().unwrap().bytes(), - _ => rand::thread_rng().gen::<[u8; 6]>(), + _ => rand::random::<[u8; 6]>(), } } - pub fn now_unix_duration() -> Duration { - use std::time::UNIX_EPOCH; - - let now = SystemTime::now(); - now.duration_since(UNIX_EPOCH) - .expect("SystemTime before UNIX EPOCH!") - } - #[pyfunction] fn generate_time_safe() -> (Vec, PyNone) { static CONTEXT: Context = Context::new(0); - let now = now_unix_duration(); - let ts = Timestamp::from_unix(&CONTEXT, now.as_secs(), now.subsec_nanos()); + let ts = Timestamp::now(&CONTEXT); static NODE_ID: OnceCell<[u8; 6]> = OnceCell::new(); let unique_node_id = NODE_ID.get_or_init(get_node_id); diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 853338c372..5c10223bfc 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -23,7 +23,7 @@ ast = ["rustpython-ast"] codegen = ["rustpython-codegen", "ast"] parser = ["rustpython-parser", "ast"] serde = ["dep:serde"] -wasmbind = ["chrono/wasmbind", "getrandom/js", "wasm-bindgen"] +wasmbind = ["chrono/wasmbind", "getrandom/wasm_js", "wasm-bindgen"] [dependencies] rustpython-compiler = { workspace = true, optional = true } @@ -145,7 +145,7 @@ features = [ [target.'cfg(all(target_arch = "wasm32", target_os = "unknown"))'.dependencies] wasm-bindgen = { workspace = true, optional = true } -getrandom = { workspace = true, features = ["custom"] } +getrandom = { workspace = true } [build-dependencies] glob = { workspace = true } diff --git a/vm/src/import.rs b/vm/src/import.rs index f14ae31ef4..fbd563dabb 100644 --- a/vm/src/import.rs +++ b/vm/src/import.rs @@ -8,7 +8,6 @@ use crate::{ vm::{thread, VirtualMachine}, AsObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, }; -use rand::Rng; pub(crate) fn init_importlib_base(vm: &mut VirtualMachine) -> PyResult { flame_guard!("init importlib"); @@ -50,7 +49,7 @@ pub(crate) fn init_importlib_package(vm: &VirtualMachine, importlib: PyObjectRef let mut magic = get_git_revision().into_bytes(); magic.truncate(4); if magic.len() != 4 { - magic = rand::thread_rng().gen::<[u8; 4]>().to_vec(); + magic = rand::random::<[u8; 4]>().to_vec(); } let magic: PyObjectRef = vm.ctx.new_bytes(magic).into(); importlib_external.set_attr("MAGIC_NUMBER", magic, vm)?; diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 0b3f617552..6519af094d 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -978,7 +978,7 @@ pub(super) mod _os { return Err(vm.new_value_error("negative argument not allowed".to_owned())); } let mut buf = vec![0u8; size as usize]; - getrandom::getrandom(&mut buf).map_err(|e| match e.raw_os_error() { + getrandom::fill(&mut buf).map_err(|e| match e.raw_os_error() { Some(errno) => io::Error::from_raw_os_error(errno).into_pyexception(vm), None => vm.new_os_error("Getting random failed".to_owned()), })?; diff --git a/wasm/lib/.cargo/config.toml b/wasm/lib/.cargo/config.toml new file mode 100644 index 0000000000..ce1e7c694a --- /dev/null +++ b/wasm/lib/.cargo/config.toml @@ -0,0 +1,5 @@ +[build] +target = "wasm32-unknown-unknown" + +[target.wasm32-unknown-unknown] +rustflags = ["--cfg=getrandom_backend=\"wasm_js\""] diff --git a/wasm/lib/Cargo.toml b/wasm/lib/Cargo.toml index 1e5c37f4ef..4703cb9f4a 100644 --- a/wasm/lib/Cargo.toml +++ b/wasm/lib/Cargo.toml @@ -28,6 +28,9 @@ rustpython-parser = { workspace = true } serde = { workspace = true } wasm-bindgen = { workspace = true } +# remove once getrandom 0.2 is no longer otherwise in the dependency tree +getrandom = { version = "0.2", features = ["js"] } + console_error_panic_hook = "0.1" js-sys = "0.3" serde-wasm-bindgen = "0.3.1" @@ -47,4 +50,4 @@ web-sys = { version = "0.3", features = [ wasm-opt = false#["-O1"] [lints] -workspace = true \ No newline at end of file +workspace = true diff --git a/wasm/wasm-unknown-test/.cargo/config.toml b/wasm/wasm-unknown-test/.cargo/config.toml new file mode 100644 index 0000000000..f86ad96761 --- /dev/null +++ b/wasm/wasm-unknown-test/.cargo/config.toml @@ -0,0 +1,5 @@ +[build] +target = "wasm32-unknown-unknown" + +[target.wasm32-unknown-unknown] +rustflags = ["--cfg=getrandom_backend=\"custom\""] diff --git a/wasm/wasm-unknown-test/Cargo.toml b/wasm/wasm-unknown-test/Cargo.toml index f5e0b55786..ca8b15cfc5 100644 --- a/wasm/wasm-unknown-test/Cargo.toml +++ b/wasm/wasm-unknown-test/Cargo.toml @@ -8,6 +8,7 @@ crate-type = ["cdylib"] [dependencies] getrandom = { version = "0.2.12", features = ["custom"] } +getrandom_03 = { package = "getrandom", version = "0.3" } rustpython-vm = { path = "../../vm", default-features = false, features = ["compiler"] } [workspace] diff --git a/wasm/wasm-unknown-test/src/lib.rs b/wasm/wasm-unknown-test/src/lib.rs index fd043aea3a..cfdc445574 100644 --- a/wasm/wasm-unknown-test/src/lib.rs +++ b/wasm/wasm-unknown-test/src/lib.rs @@ -14,3 +14,11 @@ fn getrandom_always_fail(_buf: &mut [u8]) -> Result<(), getrandom::Error> { } getrandom::register_custom_getrandom!(getrandom_always_fail); + +#[unsafe(no_mangle)] +unsafe extern "Rust" fn __getrandom_v03_custom( + _dest: *mut u8, + _len: usize, +) -> Result<(), getrandom_03::Error> { + Err(getrandom_03::Error::UNSUPPORTED) +} From e2b0fe4266febafdc9ab34d558bfa8aef91b59cc Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Wed, 19 Feb 2025 17:50:10 -0800 Subject: [PATCH 15/26] _ctypes pt. 2 (#5524) * add __version__ * add more types/constants * shared library and ExternalLibs implementation * FreeLibrary for windows * fixed PyCSimple * LoadLibrary and FreeLibrary for non-windows * fault-tolerant float equality Signed-off-by: Ashwin Naren --- Cargo.lock | 124 +++++++++--------------- extra_tests/snippets/builtins_ctypes.py | 28 ++++++ vm/Cargo.toml | 1 + vm/src/stdlib/ctypes.rs | 116 +++++++++++++++++++++- vm/src/stdlib/ctypes/array.rs | 5 + vm/src/stdlib/ctypes/base.rs | 91 ++++++++++++++--- vm/src/stdlib/ctypes/function.rs | 24 +++++ vm/src/stdlib/ctypes/library.rs | 115 ++++++++++++++++++++++ vm/src/stdlib/ctypes/pointer.rs | 5 + vm/src/stdlib/ctypes/structure.rs | 5 + vm/src/stdlib/ctypes/union.rs | 5 + 11 files changed, 424 insertions(+), 95 deletions(-) create mode 100644 vm/src/stdlib/ctypes/array.rs create mode 100644 vm/src/stdlib/ctypes/function.rs create mode 100644 vm/src/stdlib/ctypes/library.rs create mode 100644 vm/src/stdlib/ctypes/pointer.rs create mode 100644 vm/src/stdlib/ctypes/structure.rs create mode 100644 vm/src/stdlib/ctypes/union.rs diff --git a/Cargo.lock b/Cargo.lock index 87174aeac6..11992d8d8c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -216,9 +216,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.2.13" +version = "1.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7777341816418c02e033934a09f20dc0ccaf65a5201ef8a450ae0105a573fda" +checksum = "0c3d1b2e905a3a7b00a6141adb0e4c0bb941d11caf55349d863942a1cc44e3c9" dependencies = [ "shlex", ] @@ -645,9 +645,9 @@ dependencies = [ [[package]] name = "equivalent" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "errno" @@ -905,7 +905,7 @@ dependencies = [ "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "windows-core 0.52.0", + "windows-core", ] [[package]] @@ -1028,9 +1028,9 @@ checksum = "507460a910eb7b32ee961886ff48539633b788a36b65692b95f225b844c82553" [[package]] name = "lambert_w" -version = "1.0.14" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b0c98033daa8d13aa2171722fd201bae337924189091f7988bdaff5301fa8c9" +checksum = "45bf98425154bfe790a47b72ac452914f6df9ebfb202bc59e089e29db00258cf" [[package]] name = "lazy_static" @@ -1099,6 +1099,16 @@ dependencies = [ "cc", ] +[[package]] +name = "libloading" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" +dependencies = [ + "cfg-if", + "windows-targets 0.52.6", +] + [[package]] name = "libm" version = "0.2.11" @@ -1408,9 +1418,9 @@ checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9" [[package]] name = "openssl" -version = "0.10.70" +version = "0.10.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61cfb4e166a8bb8c9b55c500bc2308550148ece889be90f609377e58140f42c6" +checksum = "5e14130c6a98cd258fdcb0fb6d744152343ff729cbfcb28c656a9d12b999fbcd" dependencies = [ "bitflags 2.8.0", "cfg-if", @@ -1449,9 +1459,9 @@ dependencies = [ [[package]] name = "openssl-sys" -version = "0.9.105" +version = "0.9.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b22d5b84be05a8d6947c7cb71f7c849aa0f112acd4bf51c2a7c1c988ac0a9dc" +checksum = "8bb61ea9811cc39e3c2069f40b8b8e2e70d8569b361f879786cc7ed48b777cdd" dependencies = [ "cc", "libc", @@ -1747,8 +1757,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" dependencies = [ "rand_chacha 0.9.0", - "rand_core 0.9.0", - "zerocopy 0.8.17", + "rand_core 0.9.1", + "zerocopy 0.8.18", ] [[package]] @@ -1768,7 +1778,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core 0.9.0", + "rand_core 0.9.1", ] [[package]] @@ -1782,12 +1792,12 @@ dependencies = [ [[package]] name = "rand_core" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b08f3c9802962f7e1b25113931d94f43ed9725bebc59db9d0c3e9a23b67e15ff" +checksum = "a88e0da7a2c97baa202165137c158d0a2e824ac465d13d81046727b34cb247d3" dependencies = [ "getrandom 0.3.1", - "zerocopy 0.8.17", + "zerocopy 0.8.18", ] [[package]] @@ -2272,6 +2282,7 @@ dependencies = [ "itertools 0.14.0", "junction", "libc", + "libloading", "log", "malachite-bigint", "memchr", @@ -2526,9 +2537,9 @@ checksum = "826167069c09b99d56f31e9ae5c99049e932a98c9dc2dac47645b08dbbf76ba7" [[package]] name = "smallvec" -version = "1.13.2" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" +checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd" [[package]] name = "socket2" @@ -2554,15 +2565,15 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" [[package]] name = "strum" -version = "0.27.0" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce1475c515a4f03a8a7129bb5228b81a781a86cb0b3fbbc19e1c556d491a401f" +checksum = "f64def088c51c9510a8579e3c5d67c65349dcf755e5479ad3d010aa6454e2c32" [[package]] name = "strum_macros" -version = "0.27.0" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9688894b43459159c82bfa5a5fa0435c19cbe3c9b427fa1dd7b1ce0c279b18a7" +checksum = "c77a8c5abcaf0f9ce05d62342b7d298c346515365c36b673df4ebe3ced01fde8" dependencies = [ "heck", "proc-macro2", @@ -2781,9 +2792,9 @@ dependencies = [ [[package]] name = "typenum" -version = "1.17.0" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" +checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" [[package]] name = "ucd" @@ -2997,9 +3008,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.13.1" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ced87ca4be083373936a67f8de945faa23b6b42384bd5b64434850802c6dccd0" +checksum = "8c1f41ffb7cf259f1ecc2876861a17e7142e63ead296f671f81f6ae85903e0d6" dependencies = [ "atomic", ] @@ -3185,11 +3196,11 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows" -version = "0.57.0" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12342cb4d8e3b046f3d80effd474a7a02447231330ef77d71daa6fbc40681143" +checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be" dependencies = [ - "windows-core 0.57.0", + "windows-core", "windows-targets 0.52.6", ] @@ -3202,49 +3213,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "windows-core" -version = "0.57.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2ed2439a290666cd67ecce2b0ffaad89c2a56b976b736e6ece670297897832d" -dependencies = [ - "windows-implement", - "windows-interface", - "windows-result", - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-implement" -version = "0.57.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.98", -] - -[[package]] -name = "windows-interface" -version = "0.57.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.98", -] - -[[package]] -name = "windows-result" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8" -dependencies = [ - "windows-targets 0.52.6", -] - [[package]] name = "windows-sys" version = "0.36.1" @@ -3479,11 +3447,11 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.17" +version = "0.8.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa91407dacce3a68c56de03abe2760159582b846c6a4acd2f456618087f12713" +checksum = "79386d31a42a4996e3336b0919ddb90f81112af416270cff95b5f5af22b839c2" dependencies = [ - "zerocopy-derive 0.8.17", + "zerocopy-derive 0.8.18", ] [[package]] @@ -3499,9 +3467,9 @@ dependencies = [ [[package]] name = "zerocopy-derive" -version = "0.8.17" +version = "0.8.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06718a168365cad3d5ff0bb133aad346959a2074bd4a85c121255a11304a8626" +checksum = "76331675d372f91bf8d17e13afbd5fe639200b73d01f0fc748bb059f9cca2db7" dependencies = [ "proc-macro2", "quote", diff --git a/extra_tests/snippets/builtins_ctypes.py b/extra_tests/snippets/builtins_ctypes.py index 5df259cf82..5bd6e5ef25 100644 --- a/extra_tests/snippets/builtins_ctypes.py +++ b/extra_tests/snippets/builtins_ctypes.py @@ -1,7 +1,29 @@ +import os as _os, sys as _sys + from _ctypes import sizeof from _ctypes import _SimpleCData from struct import calcsize as _calcsize +def create_string_buffer(init, size=None): + """create_string_buffer(aBytes) -> character array + create_string_buffer(anInteger) -> character array + create_string_buffer(aBytes, anInteger) -> character array + """ + if isinstance(init, bytes): + if size is None: + size = len(init)+1 + _sys.audit("ctypes.create_string_buffer", init, size) + buftype = c_char * size + buf = buftype() + buf.value = init + return buf + elif isinstance(init, int): + _sys.audit("ctypes.create_string_buffer", None, init) + buftype = c_char * init + buf = buftype() + return buf + raise TypeError(init) + def _check_size(typ, typecode=None): # Check if sizeof(ctypes_type) against struct.calcsize. This # should protect somewhat against a misconfigured libffi. @@ -103,3 +125,9 @@ class c_void_p(_SimpleCData): class c_bool(_SimpleCData): _type_ = "?" _check_size(c_bool) + +i = c_int(42) +f = c_float(3.14) +# s = create_string_buffer(b'\000' * 32) +assert i.value == 42 +assert abs(f.value - 3.14) < 1e-06 diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 5c10223bfc..895b29aa50 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -100,6 +100,7 @@ uname = "0.1.1" rustyline = { workspace = true } which = "6" errno = "0.3" +libloading = "0.8" widestring = { workspace = true } [target.'cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))'.dependencies] diff --git a/vm/src/stdlib/ctypes.rs b/vm/src/stdlib/ctypes.rs index d1fc26869f..027a680951 100644 --- a/vm/src/stdlib/ctypes.rs +++ b/vm/src/stdlib/ctypes.rs @@ -1,11 +1,34 @@ +pub(crate) mod array; pub(crate) mod base; +pub(crate) mod function; +pub(crate) mod library; +pub(crate) mod pointer; +pub(crate) mod structure; +pub(crate) mod union; use crate::builtins::PyModule; -use crate::{PyRef, VirtualMachine}; +use crate::class::PyClassImpl; +use crate::stdlib::ctypes::base::{PyCData, PyCSimple, PySimpleMeta}; +use crate::{Py, PyRef, VirtualMachine}; + +pub fn extend_module_nodes(vm: &VirtualMachine, module: &Py) { + let ctx = &vm.ctx; + PySimpleMeta::make_class(ctx); + extend_module!(vm, module, { + "_CData" => PyCData::make_class(ctx), + "_SimpleCData" => PyCSimple::make_class(ctx), + "Array" => array::PyCArray::make_class(ctx), + "CFuncPtr" => function::PyCFuncPtr::make_class(ctx), + "_Pointer" => pointer::PyCPointer::make_class(ctx), + "_pointer_type_cache" => ctx.new_dict(), + "Structure" => structure::PyCStructure::make_class(ctx), + "Union" => union::PyCUnion::make_class(ctx), + }) +} pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { let module = _ctypes::make_module(vm); - base::extend_module_nodes(vm, &module); + extend_module_nodes(vm, &module); module } @@ -15,6 +38,7 @@ pub(crate) mod _ctypes { use crate::builtins::PyTypeRef; use crate::class::StaticType; use crate::function::Either; + use crate::stdlib::ctypes::library; use crate::{AsObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine}; use crossbeam_utils::atomic::AtomicCell; use std::ffi::{ @@ -24,6 +48,57 @@ pub(crate) mod _ctypes { use std::mem; use widestring::WideChar; + #[pyattr(name = "__version__")] + const __VERSION__: &str = "1.1.0"; + + // TODO: get properly + #[pyattr(name = "RTLD_LOCAL")] + const RTLD_LOCAL: i32 = 0; + + // TODO: get properly + #[pyattr(name = "RTLD_GLOBAL")] + const RTLD_GLOBAL: i32 = 0; + + #[cfg(target_os = "windows")] + #[pyattr(name = "SIZEOF_TIME_T")] + pub const SIZEOF_TIME_T: usize = 8; + #[cfg(not(target_os = "windows"))] + #[pyattr(name = "SIZEOF_TIME_T")] + pub const SIZEOF_TIME_T: usize = 4; + + #[pyattr(name = "CTYPES_MAX_ARGCOUNT")] + pub const CTYPES_MAX_ARGCOUNT: usize = 1024; + + #[pyattr] + pub const FUNCFLAG_STDCALL: u32 = 0x0; + #[pyattr] + pub const FUNCFLAG_CDECL: u32 = 0x1; + #[pyattr] + pub const FUNCFLAG_HRESULT: u32 = 0x2; + #[pyattr] + pub const FUNCFLAG_PYTHONAPI: u32 = 0x4; + #[pyattr] + pub const FUNCFLAG_USE_ERRNO: u32 = 0x8; + #[pyattr] + pub const FUNCFLAG_USE_LASTERROR: u32 = 0x10; + + #[pyattr] + pub const TYPEFLAG_ISPOINTER: u32 = 0x100; + #[pyattr] + pub const TYPEFLAG_HASPOINTER: u32 = 0x200; + + #[pyattr] + pub const DICTFLAG_FINAL: u32 = 0x1000; + + #[pyattr(once)] + fn error(vm: &VirtualMachine) -> PyTypeRef { + vm.ctx.new_exception_type( + "_ctypes", + "ArgumentError", + Some(vec![vm.ctx.exceptions.exception_type.to_owned()]), + ) + } + pub fn get_size(ty: &str) -> usize { match ty { "u" => mem::size_of::(), @@ -68,7 +143,7 @@ pub(crate) mod _ctypes { } else { Ok(PyCSimple { _type_: tp_str, - _value: AtomicCell::new(vm.ctx.none()), + value: AtomicCell::new(vm.ctx.none()), }) } } else { @@ -95,6 +170,41 @@ pub(crate) mod _ctypes { } } + #[pyfunction(name = "LoadLibrary")] + fn load_library(name: String, vm: &VirtualMachine) -> PyResult { + // TODO: audit functions first + let cache = library::libcache(); + let mut cache_write = cache.write(); + let lib_ref = cache_write.get_or_insert_lib(&name, vm).unwrap(); + Ok(lib_ref.get_pointer()) + } + + #[pyfunction(name = "FreeLibrary")] + fn free_library(handle: usize) -> PyResult<()> { + let cache = library::libcache(); + let mut cache_write = cache.write(); + cache_write.drop_lib(handle); + Ok(()) + } + + #[pyfunction(name = "POINTER")] + pub fn pointer(_cls: PyTypeRef) {} + + #[pyfunction] + pub fn pointer_fn(_inst: PyObjectRef) {} + + #[cfg(target_os = "windows")] + #[pyfunction(name = "_check_HRESULT")] + pub fn check_hresult(_self: PyObjectRef, hr: i32, _vm: &VirtualMachine) -> PyResult { + // TODO: fixme + if hr < 0 { + // vm.ctx.new_windows_error(hr) + todo!(); + } else { + Ok(hr) + } + } + #[pyfunction] fn get_errno() -> i32 { errno::errno().0 diff --git a/vm/src/stdlib/ctypes/array.rs b/vm/src/stdlib/ctypes/array.rs new file mode 100644 index 0000000000..8b023582c9 --- /dev/null +++ b/vm/src/stdlib/ctypes/array.rs @@ -0,0 +1,5 @@ +#[pyclass(name = "Array", module = "_ctypes")] +pub struct PyCArray {} + +#[pyclass(flags(BASETYPE, IMMUTABLETYPE))] +impl PyCArray {} diff --git a/vm/src/stdlib/ctypes/base.rs b/vm/src/stdlib/ctypes/base.rs index 0142c45345..5d0316352c 100644 --- a/vm/src/stdlib/ctypes/base.rs +++ b/vm/src/stdlib/ctypes/base.rs @@ -1,6 +1,10 @@ -use crate::builtins::{PyBytes, PyFloat, PyInt, PyModule, PyNone, PyStr}; -use crate::class::PyClassImpl; -use crate::{Py, PyObjectRef, PyResult, TryFromObject, VirtualMachine}; +use crate::builtins::PyType; +use crate::builtins::{PyBytes, PyFloat, PyInt, PyNone, PyStr, PyTypeRef}; +use crate::convert::ToPyObject; +use crate::function::{Either, OptionalArg}; +use crate::stdlib::ctypes::_ctypes::new_simple_type; +use crate::types::Constructor; +use crate::{AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine}; use crossbeam_utils::atomic::AtomicCell; use num_traits::ToPrimitive; use rustpython_common::lock::PyRwLock; @@ -129,16 +133,32 @@ pub struct PyCData { #[pyclass] impl PyCData {} +#[pyclass(module = "_ctypes", name = "PyCSimpleType", base = "PyType")] +pub struct PySimpleMeta {} + +#[pyclass(flags(BASETYPE))] +impl PySimpleMeta { + #[allow(clippy::new_ret_no_self)] + #[pymethod] + fn new(cls: PyTypeRef, _: OptionalArg, vm: &VirtualMachine) -> PyResult { + Ok(PyObjectRef::from( + new_simple_type(Either::B(&cls), vm)? + .into_ref_with_type(vm, cls)? + .clone(), + )) + } +} + #[pyclass( name = "_SimpleCData", base = "PyCData", - module = "_ctypes" - // TODO: metaclass + module = "_ctypes", + metaclass = "PySimpleMeta" )] #[derive(PyPayload)] pub struct PyCSimple { pub _type_: String, - pub _value: AtomicCell, + pub value: AtomicCell, } impl Debug for PyCSimple { @@ -149,13 +169,56 @@ impl Debug for PyCSimple { } } -#[pyclass(flags(BASETYPE))] -impl PyCSimple {} +impl Constructor for PyCSimple { + type Args = (OptionalArg,); -pub fn extend_module_nodes(vm: &VirtualMachine, module: &Py) { - let ctx = &vm.ctx; - extend_module!(vm, module, { - "_CData" => PyCData::make_class(ctx), - "_SimpleCData" => PyCSimple::make_class(ctx), - }) + fn py_new(cls: PyTypeRef, args: Self::Args, vm: &VirtualMachine) -> PyResult { + let attributes = cls.get_attributes(); + let _type_ = attributes + .iter() + .find(|(&k, _)| k.to_object().str(vm).unwrap().to_string() == *"_type_") + .unwrap() + .1 + .str(vm)? + .to_string(); + let value = if let Some(ref v) = args.0.into_option() { + set_primitive(_type_.as_str(), v, vm)? + } else { + match _type_.as_str() { + "c" | "u" => PyObjectRef::from(vm.ctx.new_bytes(vec![0])), + "b" | "B" | "h" | "H" | "i" | "I" | "l" | "q" | "L" | "Q" => { + PyObjectRef::from(vm.ctx.new_int(0)) + } + "f" | "d" | "g" => PyObjectRef::from(vm.ctx.new_float(0.0)), + "?" => PyObjectRef::from(vm.ctx.new_bool(false)), + _ => vm.ctx.none(), // "z" | "Z" | "P" + } + }; + Ok(PyCSimple { + _type_, + value: AtomicCell::new(value), + } + .to_pyobject(vm)) + } +} + +#[pyclass(flags(BASETYPE), with(Constructor))] +impl PyCSimple { + #[pygetset(name = "value")] + pub fn value(instance: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let zelf: &Py = instance + .downcast_ref() + .ok_or_else(|| vm.new_type_error("cannot get value of instance".to_string()))?; + Ok(unsafe { (*zelf.value.as_ptr()).clone() }) + } + + #[pygetset(name = "value", setter)] + fn set_value(instance: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let zelf: PyRef = instance + .downcast() + .map_err(|_| vm.new_type_error("cannot set value of instance".to_string()))?; + let content = set_primitive(zelf._type_.as_str(), &value, vm)?; + zelf.value.store(content); + Ok(()) + } } diff --git a/vm/src/stdlib/ctypes/function.rs b/vm/src/stdlib/ctypes/function.rs new file mode 100644 index 0000000000..bf7f0b9adf --- /dev/null +++ b/vm/src/stdlib/ctypes/function.rs @@ -0,0 +1,24 @@ +use crate::stdlib::ctypes::PyCData; +use crate::PyObjectRef; +use crossbeam_utils::atomic::AtomicCell; +use rustpython_common::lock::PyRwLock; +use std::ffi::c_void; + +#[derive(Debug)] +pub struct Function { + _pointer: *mut c_void, + _arguments: Vec<()>, + _return_type: Box<()>, +} + +#[pyclass(module = "_ctypes", name = "CFuncPtr", base = "PyCData")] +pub struct PyCFuncPtr { + pub _name_: String, + pub _argtypes_: AtomicCell>, + pub _restype_: AtomicCell, + _handle: PyObjectRef, + _f: PyRwLock, +} + +#[pyclass] +impl PyCFuncPtr {} diff --git a/vm/src/stdlib/ctypes/library.rs b/vm/src/stdlib/ctypes/library.rs new file mode 100644 index 0000000000..94b6327440 --- /dev/null +++ b/vm/src/stdlib/ctypes/library.rs @@ -0,0 +1,115 @@ +use crate::VirtualMachine; +use crossbeam_utils::atomic::AtomicCell; +use libloading::Library; +use rustpython_common::lock::PyRwLock; +use std::collections::HashMap; +use std::ffi::c_void; +use std::fmt; +use std::ptr::null; + +pub struct SharedLibrary { + lib: AtomicCell>, +} + +impl fmt::Debug for SharedLibrary { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "SharedLibrary") + } +} + +impl SharedLibrary { + pub fn new(name: &str) -> Result { + Ok(SharedLibrary { + lib: AtomicCell::new(Some(unsafe { Library::new(name)? })), + }) + } + + #[allow(dead_code)] + pub fn get_sym(&self, name: &str) -> Result<*mut c_void, String> { + if let Some(inner) = unsafe { &*self.lib.as_ptr() } { + unsafe { + inner + .get(name.as_bytes()) + .map(|f: libloading::Symbol<*mut c_void>| *f) + .map_err(|err| err.to_string()) + } + } else { + Err("The library has been closed".to_string()) + } + } + + pub fn get_pointer(&self) -> usize { + if let Some(l) = unsafe { &*self.lib.as_ptr() } { + l as *const Library as usize + } else { + null::() as usize + } + } + + pub fn is_closed(&self) -> bool { + unsafe { &*self.lib.as_ptr() }.is_none() + } + + pub fn close(&self) { + let old = self.lib.take(); + self.lib.store(None); + drop(old); + } +} + +impl Drop for SharedLibrary { + fn drop(&mut self) { + self.close(); + } +} + +pub struct ExternalLibs { + libraries: HashMap, +} + +impl ExternalLibs { + pub fn new() -> Self { + Self { + libraries: HashMap::new(), + } + } + + #[allow(dead_code)] + pub fn get_lib(&self, key: usize) -> Option<&SharedLibrary> { + self.libraries.get(&key) + } + + pub fn get_or_insert_lib( + &mut self, + library_path: &str, + _vm: &VirtualMachine, + ) -> Result<&SharedLibrary, libloading::Error> { + let nlib = SharedLibrary::new(library_path)?; + let key = nlib.get_pointer(); + + match self.libraries.get(&key) { + Some(l) => { + if l.is_closed() { + self.libraries.insert(key, nlib); + } + } + _ => { + self.libraries.insert(key, nlib); + } + }; + + Ok(self.libraries.get(&key).unwrap()) + } + + pub fn drop_lib(&mut self, key: usize) { + self.libraries.remove(&key); + } +} + +rustpython_common::static_cell! { + static LIBCACHE: PyRwLock; +} + +pub fn libcache() -> &'static PyRwLock { + LIBCACHE.get_or_init(|| PyRwLock::new(ExternalLibs::new())) +} diff --git a/vm/src/stdlib/ctypes/pointer.rs b/vm/src/stdlib/ctypes/pointer.rs new file mode 100644 index 0000000000..d1360f9862 --- /dev/null +++ b/vm/src/stdlib/ctypes/pointer.rs @@ -0,0 +1,5 @@ +#[pyclass(name = "Pointer", module = "_ctypes")] +pub struct PyCPointer {} + +#[pyclass(flags(BASETYPE, IMMUTABLETYPE))] +impl PyCPointer {} diff --git a/vm/src/stdlib/ctypes/structure.rs b/vm/src/stdlib/ctypes/structure.rs new file mode 100644 index 0000000000..13cca6c260 --- /dev/null +++ b/vm/src/stdlib/ctypes/structure.rs @@ -0,0 +1,5 @@ +#[pyclass(name = "Structure", module = "_ctypes")] +pub struct PyCStructure {} + +#[pyclass(flags(BASETYPE, IMMUTABLETYPE))] +impl PyCStructure {} diff --git a/vm/src/stdlib/ctypes/union.rs b/vm/src/stdlib/ctypes/union.rs new file mode 100644 index 0000000000..5a39d9062e --- /dev/null +++ b/vm/src/stdlib/ctypes/union.rs @@ -0,0 +1,5 @@ +#[pyclass(name = "Union", module = "_ctypes")] +pub struct PyCUnion {} + +#[pyclass(flags(BASETYPE, IMMUTABLETYPE))] +impl PyCUnion {} From 65dcf1ce1c0999a5db9c1bc322ee3bf69b3d509f Mon Sep 17 00:00:00 2001 From: Hanif Ariffin Date: Thu, 20 Feb 2025 10:21:12 +0800 Subject: [PATCH 16/26] Updating test_math.py to CPython 3.12.9 (#5507) * Fixed implementation against CPython 3.12.9 Lib/test/test_math.py tests --------- Signed-off-by: Hanif Ariffin Co-authored-by: Jeong YunWon --- Lib/test/ieee754.txt | 183 ++++++++++++++++++ Lib/test/test_math.py | 402 +++++++++++++++++++++++++++++++++++++-- common/src/float_ops.rs | 63 ++++++ stdlib/src/math.rs | 121 ++++++++---- vm/src/builtins/float.rs | 4 +- 5 files changed, 727 insertions(+), 46 deletions(-) create mode 100644 Lib/test/ieee754.txt diff --git a/Lib/test/ieee754.txt b/Lib/test/ieee754.txt new file mode 100644 index 0000000000..3e986cdb10 --- /dev/null +++ b/Lib/test/ieee754.txt @@ -0,0 +1,183 @@ +====================================== +Python IEEE 754 floating point support +====================================== + +>>> from sys import float_info as FI +>>> from math import * +>>> PI = pi +>>> E = e + +You must never compare two floats with == because you are not going to get +what you expect. We treat two floats as equal if the difference between them +is small than epsilon. +>>> EPS = 1E-15 +>>> def equal(x, y): +... """Almost equal helper for floats""" +... return abs(x - y) < EPS + + +NaNs and INFs +============= + +In Python 2.6 and newer NaNs (not a number) and infinity can be constructed +from the strings 'inf' and 'nan'. + +>>> INF = float('inf') +>>> NINF = float('-inf') +>>> NAN = float('nan') + +>>> INF +inf +>>> NINF +-inf +>>> NAN +nan + +The math module's ``isnan`` and ``isinf`` functions can be used to detect INF +and NAN: +>>> isinf(INF), isinf(NINF), isnan(NAN) +(True, True, True) +>>> INF == -NINF +True + +Infinity +-------- + +Ambiguous operations like ``0 * inf`` or ``inf - inf`` result in NaN. +>>> INF * 0 +nan +>>> INF - INF +nan +>>> INF / INF +nan + +However unambiguous operations with inf return inf: +>>> INF * INF +inf +>>> 1.5 * INF +inf +>>> 0.5 * INF +inf +>>> INF / 1000 +inf + +Not a Number +------------ + +NaNs are never equal to another number, even itself +>>> NAN == NAN +False +>>> NAN < 0 +False +>>> NAN >= 0 +False + +All operations involving a NaN return a NaN except for nan**0 and 1**nan. +>>> 1 + NAN +nan +>>> 1 * NAN +nan +>>> 0 * NAN +nan +>>> 1 ** NAN +1.0 +>>> NAN ** 0 +1.0 +>>> 0 ** NAN +nan +>>> (1.0 + FI.epsilon) * NAN +nan + +Misc Functions +============== + +The power of 1 raised to x is always 1.0, even for special values like 0, +infinity and NaN. + +>>> pow(1, 0) +1.0 +>>> pow(1, INF) +1.0 +>>> pow(1, -INF) +1.0 +>>> pow(1, NAN) +1.0 + +The power of 0 raised to x is defined as 0, if x is positive. Negative +finite values are a domain error or zero division error and NaN result in a +silent NaN. + +>>> pow(0, 0) +1.0 +>>> pow(0, INF) +0.0 +>>> pow(0, -INF) +inf +>>> 0 ** -1 +Traceback (most recent call last): +... +ZeroDivisionError: 0.0 cannot be raised to a negative power +>>> pow(0, NAN) +nan + + +Trigonometric Functions +======================= + +>>> sin(INF) +Traceback (most recent call last): +... +ValueError: math domain error +>>> sin(NINF) +Traceback (most recent call last): +... +ValueError: math domain error +>>> sin(NAN) +nan +>>> cos(INF) +Traceback (most recent call last): +... +ValueError: math domain error +>>> cos(NINF) +Traceback (most recent call last): +... +ValueError: math domain error +>>> cos(NAN) +nan +>>> tan(INF) +Traceback (most recent call last): +... +ValueError: math domain error +>>> tan(NINF) +Traceback (most recent call last): +... +ValueError: math domain error +>>> tan(NAN) +nan + +Neither pi nor tan are exact, but you can assume that tan(pi/2) is a large value +and tan(pi) is a very small value: +>>> tan(PI/2) > 1E10 +True +>>> -tan(-PI/2) > 1E10 +True +>>> tan(PI) < 1E-15 +True + +>>> asin(NAN), acos(NAN), atan(NAN) +(nan, nan, nan) +>>> asin(INF), asin(NINF) +Traceback (most recent call last): +... +ValueError: math domain error +>>> acos(INF), acos(NINF) +Traceback (most recent call last): +... +ValueError: math domain error +>>> equal(atan(INF), PI/2), equal(atan(NINF), -PI/2) +(True, True) + + +Hyberbolic Functions +==================== + diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py index 2f64180652..fa79456ed4 100644 --- a/Lib/test/test_math.py +++ b/Lib/test/test_math.py @@ -4,6 +4,7 @@ from test.support import verbose, requires_IEEE_754 from test import support import unittest +import fractions import itertools import decimal import math @@ -186,6 +187,9 @@ def result_check(expected, got, ulp_tol=5, abs_tol=0.0): # Check exactly equal (applies also to strings representing exceptions) if got == expected: + if not got and not expected: + if math.copysign(1, got) != math.copysign(1, expected): + return f"expected {expected}, got {got} (zero has wrong sign)" return None failure = "not equal" @@ -234,6 +238,10 @@ def __init__(self, value): def __index__(self): return self.value +class BadDescr: + def __get__(self, obj, objtype=None): + raise ValueError + class MathTests(unittest.TestCase): def ftest(self, name, got, expected, ulp_tol=5, abs_tol=0.0): @@ -323,6 +331,7 @@ def testAtan2(self): self.ftest('atan2(0, 1)', math.atan2(0, 1), 0) self.ftest('atan2(1, 1)', math.atan2(1, 1), math.pi/4) self.ftest('atan2(1, 0)', math.atan2(1, 0), math.pi/2) + self.ftest('atan2(1, -1)', math.atan2(1, -1), 3*math.pi/4) # math.atan2(0, x) self.ftest('atan2(0., -inf)', math.atan2(0., NINF), math.pi) @@ -416,16 +425,22 @@ def __ceil__(self): return 42 class TestNoCeil: pass + class TestBadCeil: + __ceil__ = BadDescr() self.assertEqual(math.ceil(TestCeil()), 42) self.assertEqual(math.ceil(FloatCeil()), 42) self.assertEqual(math.ceil(FloatLike(42.5)), 43) self.assertRaises(TypeError, math.ceil, TestNoCeil()) + self.assertRaises(ValueError, math.ceil, TestBadCeil()) t = TestNoCeil() t.__ceil__ = lambda *args: args self.assertRaises(TypeError, math.ceil, t) self.assertRaises(TypeError, math.ceil, t, 0) + self.assertEqual(math.ceil(FloatLike(+1.0)), +1.0) + self.assertEqual(math.ceil(FloatLike(-1.0)), -1.0) + @requires_IEEE_754 def testCopysign(self): self.assertEqual(math.copysign(1, 42), 1.0) @@ -566,16 +581,22 @@ def __floor__(self): return 42 class TestNoFloor: pass + class TestBadFloor: + __floor__ = BadDescr() self.assertEqual(math.floor(TestFloor()), 42) self.assertEqual(math.floor(FloatFloor()), 42) self.assertEqual(math.floor(FloatLike(41.9)), 41) self.assertRaises(TypeError, math.floor, TestNoFloor()) + self.assertRaises(ValueError, math.floor, TestBadFloor()) t = TestNoFloor() t.__floor__ = lambda *args: args self.assertRaises(TypeError, math.floor, t) self.assertRaises(TypeError, math.floor, t, 0) + self.assertEqual(math.floor(FloatLike(+1.0)), +1.0) + self.assertEqual(math.floor(FloatLike(-1.0)), -1.0) + def testFmod(self): self.assertRaises(TypeError, math.fmod) self.ftest('fmod(10, 1)', math.fmod(10, 1), 0.0) @@ -597,6 +618,7 @@ def testFmod(self): self.assertEqual(math.fmod(-3.0, NINF), -3.0) self.assertEqual(math.fmod(0.0, 3.0), 0.0) self.assertEqual(math.fmod(0.0, NINF), 0.0) + self.assertRaises(ValueError, math.fmod, INF, INF) def testFrexp(self): self.assertRaises(TypeError, math.frexp) @@ -638,7 +660,7 @@ def testFsum(self): def msum(iterable): """Full precision summation. Compute sum(iterable) without any intermediate accumulation of error. Based on the 'lsum' function - at http://code.activestate.com/recipes/393090/ + at https://code.activestate.com/recipes/393090-binary-floating-point-summation-accurate-to-full-p/ """ tmant, texp = 0, 0 @@ -666,6 +688,7 @@ def msum(iterable): ([], 0.0), ([0.0], 0.0), ([1e100, 1.0, -1e100, 1e-100, 1e50, -1.0, -1e50], 1e-100), + ([1e100, 1.0, -1e100, 1e-100, 1e50, -1, -1e50], 1e-100), ([2.0**53, -0.5, -2.0**-54], 2.0**53-1.0), ([2.0**53, 1.0, 2.0**-100], 2.0**53+2.0), ([2.0**53+10.0, 1.0, 2.0**-100], 2.0**53+12.0), @@ -713,6 +736,22 @@ def msum(iterable): s = msum(vals) self.assertEqual(msum(vals), math.fsum(vals)) + self.assertEqual(math.fsum([1.0, math.inf]), math.inf) + self.assertTrue(math.isnan(math.fsum([math.nan, 1.0]))) + self.assertEqual(math.fsum([1e100, FloatLike(1.0), -1e100, 1e-100, + 1e50, FloatLike(-1.0), -1e50]), 1e-100) + self.assertRaises(OverflowError, math.fsum, [1e+308, 1e+308]) + self.assertRaises(ValueError, math.fsum, [math.inf, -math.inf]) + self.assertRaises(TypeError, math.fsum, ['spam']) + self.assertRaises(TypeError, math.fsum, 1) + self.assertRaises(OverflowError, math.fsum, [10**1000]) + + def bad_iter(): + yield 1.0 + raise ZeroDivisionError + + self.assertRaises(ZeroDivisionError, math.fsum, bad_iter()) + def testGcd(self): gcd = math.gcd self.assertEqual(gcd(0, 0), 0) @@ -773,9 +812,13 @@ def testHypot(self): # Test allowable types (those with __float__) self.assertEqual(hypot(12.0, 5.0), 13.0) self.assertEqual(hypot(12, 5), 13) + self.assertEqual(hypot(0.75, -1), 1.25) + self.assertEqual(hypot(-1, 0.75), 1.25) + self.assertEqual(hypot(0.75, FloatLike(-1.)), 1.25) + self.assertEqual(hypot(FloatLike(-1.), 0.75), 1.25) self.assertEqual(hypot(Decimal(12), Decimal(5)), 13) self.assertEqual(hypot(Fraction(12, 32), Fraction(5, 32)), Fraction(13, 32)) - self.assertEqual(hypot(bool(1), bool(0), bool(1), bool(1)), math.sqrt(3)) + self.assertEqual(hypot(True, False, True, True, True), 2.0) # Test corner cases self.assertEqual(hypot(0.0, 0.0), 0.0) # Max input is zero @@ -830,6 +873,8 @@ def testHypot(self): scale = FLOAT_MIN / 2.0 ** exp self.assertEqual(math.hypot(4*scale, 3*scale), 5*scale) + self.assertRaises(TypeError, math.hypot, *([1.0]*18), 'spam') + @requires_IEEE_754 @unittest.skipIf(HAVE_DOUBLE_ROUNDING, "hypot() loses accuracy on machines with double rounding") @@ -922,12 +967,16 @@ def testDist(self): # Test allowable types (those with __float__) self.assertEqual(dist((14.0, 1.0), (2.0, -4.0)), 13.0) self.assertEqual(dist((14, 1), (2, -4)), 13) + self.assertEqual(dist((FloatLike(14.), 1), (2, -4)), 13) + self.assertEqual(dist((11, 1), (FloatLike(-1.), -4)), 13) + self.assertEqual(dist((14, FloatLike(-1.)), (2, -6)), 13) + self.assertEqual(dist((14, -1), (2, -6)), 13) self.assertEqual(dist((D(14), D(1)), (D(2), D(-4))), D(13)) self.assertEqual(dist((F(14, 32), F(1, 32)), (F(2, 32), F(-4, 32))), F(13, 32)) - self.assertEqual(dist((True, True, False, True, False), - (True, False, True, True, False)), - sqrt(2.0)) + self.assertEqual(dist((True, True, False, False, True, True), + (True, False, True, False, False, False)), + 2.0) # Test corner cases self.assertEqual(dist((13.25, 12.5, -3.25), @@ -965,6 +1014,8 @@ class T(tuple): dist((1, 2, 3, 4), (5, 6, 7)) with self.assertRaises(ValueError): # Check dimension agree dist((1, 2, 3), (4, 5, 6, 7)) + with self.assertRaises(TypeError): + dist((1,)*17 + ("spam",), (1,)*18) with self.assertRaises(TypeError): # Rejects invalid types dist("abc", "xyz") int_too_big_for_float = 10 ** (sys.float_info.max_10_exp + 5) @@ -972,6 +1023,16 @@ class T(tuple): dist((1, int_too_big_for_float), (2, 3)) with self.assertRaises((ValueError, OverflowError)): dist((2, 3), (1, int_too_big_for_float)) + with self.assertRaises(TypeError): + dist((1,), 2) + with self.assertRaises(TypeError): + dist([1], 2) + + class BadFloat: + __float__ = BadDescr() + + with self.assertRaises(ValueError): + dist([1], [BadFloat()]) # Verify that the one dimensional case is equivalent to abs() for i in range(20): @@ -1110,6 +1171,7 @@ def test_lcm(self): def testLdexp(self): self.assertRaises(TypeError, math.ldexp) + self.assertRaises(TypeError, math.ldexp, 2.0, 1.1) self.ftest('ldexp(0,1)', math.ldexp(0,1), 0) self.ftest('ldexp(1,1)', math.ldexp(1,1), 2) self.ftest('ldexp(1,-1)', math.ldexp(1,-1), 0.5) @@ -1142,6 +1204,7 @@ def testLdexp(self): def testLog(self): self.assertRaises(TypeError, math.log) + self.assertRaises(TypeError, math.log, 1, 2, 3) self.ftest('log(1/e)', math.log(1/math.e), -1) self.ftest('log(1)', math.log(1), 0) self.ftest('log(e)', math.log(math.e), 1) @@ -1152,6 +1215,7 @@ def testLog(self): 2302.5850929940457) self.assertRaises(ValueError, math.log, -1.5) self.assertRaises(ValueError, math.log, -10**1000) + self.assertRaises(ValueError, math.log, 10, -10) self.assertRaises(ValueError, math.log, NINF) self.assertEqual(math.log(INF), INF) self.assertTrue(math.isnan(math.log(NAN))) @@ -1202,6 +1266,277 @@ def testLog10(self): self.assertEqual(math.log(INF), INF) self.assertTrue(math.isnan(math.log10(NAN))) + def testSumProd(self): + sumprod = math.sumprod + Decimal = decimal.Decimal + Fraction = fractions.Fraction + + # Core functionality + self.assertEqual(sumprod(iter([10, 20, 30]), (1, 2, 3)), 140) + self.assertEqual(sumprod([1.5, 2.5], [3.5, 4.5]), 16.5) + self.assertEqual(sumprod([], []), 0) + self.assertEqual(sumprod([-1], [1.]), -1) + self.assertEqual(sumprod([1.], [-1]), -1) + + # Type preservation and coercion + for v in [ + (10, 20, 30), + (1.5, -2.5), + (Fraction(3, 5), Fraction(4, 5)), + (Decimal(3.5), Decimal(4.5)), + (2.5, 10), # float/int + (2.5, Fraction(3, 5)), # float/fraction + (25, Fraction(3, 5)), # int/fraction + (25, Decimal(4.5)), # int/decimal + ]: + for p, q in [(v, v), (v, v[::-1])]: + with self.subTest(p=p, q=q): + expected = sum(p_i * q_i for p_i, q_i in zip(p, q, strict=True)) + actual = sumprod(p, q) + self.assertEqual(expected, actual) + self.assertEqual(type(expected), type(actual)) + + # Bad arguments + self.assertRaises(TypeError, sumprod) # No args + self.assertRaises(TypeError, sumprod, []) # One arg + self.assertRaises(TypeError, sumprod, [], [], []) # Three args + self.assertRaises(TypeError, sumprod, None, [10]) # Non-iterable + self.assertRaises(TypeError, sumprod, [10], None) # Non-iterable + self.assertRaises(TypeError, sumprod, ['x'], [1.0]) + + # Uneven lengths + self.assertRaises(ValueError, sumprod, [10, 20], [30]) + self.assertRaises(ValueError, sumprod, [10], [20, 30]) + + # Overflows + self.assertEqual(sumprod([10**20], [1]), 10**20) + self.assertEqual(sumprod([1], [10**20]), 10**20) + self.assertEqual(sumprod([10**10], [10**10]), 10**20) + self.assertEqual(sumprod([10**7]*10**5, [10**7]*10**5), 10**19) + self.assertRaises(OverflowError, sumprod, [10**1000], [1.0]) + self.assertRaises(OverflowError, sumprod, [1.0], [10**1000]) + + # Error in iterator + def raise_after(n): + for i in range(n): + yield i + raise RuntimeError + with self.assertRaises(RuntimeError): + sumprod(range(10), raise_after(5)) + with self.assertRaises(RuntimeError): + sumprod(raise_after(5), range(10)) + + from test.test_iter import BasicIterClass + + self.assertEqual(sumprod(BasicIterClass(1), [1]), 0) + self.assertEqual(sumprod([1], BasicIterClass(1)), 0) + + # Error in multiplication + class BadMultiply: + def __mul__(self, other): + raise RuntimeError + def __rmul__(self, other): + raise RuntimeError + with self.assertRaises(RuntimeError): + sumprod([10, BadMultiply(), 30], [1, 2, 3]) + with self.assertRaises(RuntimeError): + sumprod([1, 2, 3], [10, BadMultiply(), 30]) + + # Error in addition + with self.assertRaises(TypeError): + sumprod(['abc', 3], [5, 10]) + with self.assertRaises(TypeError): + sumprod([5, 10], ['abc', 3]) + + # Special values should give the same as the pure python recipe + self.assertEqual(sumprod([10.1, math.inf], [20.2, 30.3]), math.inf) + self.assertEqual(sumprod([10.1, math.inf], [math.inf, 30.3]), math.inf) + self.assertEqual(sumprod([10.1, math.inf], [math.inf, math.inf]), math.inf) + self.assertEqual(sumprod([10.1, -math.inf], [20.2, 30.3]), -math.inf) + self.assertTrue(math.isnan(sumprod([10.1, math.inf], [-math.inf, math.inf]))) + self.assertTrue(math.isnan(sumprod([10.1, math.nan], [20.2, 30.3]))) + self.assertTrue(math.isnan(sumprod([10.1, math.inf], [math.nan, 30.3]))) + self.assertTrue(math.isnan(sumprod([10.1, math.inf], [20.3, math.nan]))) + + # Error cases that arose during development + args = ((-5, -5, 10), (1.5, 4611686018427387904, 2305843009213693952)) + self.assertEqual(sumprod(*args), 0.0) + + + @requires_IEEE_754 + @unittest.skipIf(HAVE_DOUBLE_ROUNDING, + "sumprod() accuracy not guaranteed on machines with double rounding") + @support.cpython_only # Other implementations may choose a different algorithm + def test_sumprod_accuracy(self): + sumprod = math.sumprod + self.assertEqual(sumprod([0.1] * 10, [1]*10), 1.0) + self.assertEqual(sumprod([0.1] * 20, [True, False] * 10), 1.0) + self.assertEqual(sumprod([True, False] * 10, [0.1] * 20), 1.0) + self.assertEqual(sumprod([1.0, 10E100, 1.0, -10E100], [1.0]*4), 2.0) + + @support.requires_resource('cpu') + def test_sumprod_stress(self): + sumprod = math.sumprod + product = itertools.product + Decimal = decimal.Decimal + Fraction = fractions.Fraction + + class Int(int): + def __add__(self, other): + return Int(int(self) + int(other)) + def __mul__(self, other): + return Int(int(self) * int(other)) + __radd__ = __add__ + __rmul__ = __mul__ + def __repr__(self): + return f'Int({int(self)})' + + class Flt(float): + def __add__(self, other): + return Int(int(self) + int(other)) + def __mul__(self, other): + return Int(int(self) * int(other)) + __radd__ = __add__ + __rmul__ = __mul__ + def __repr__(self): + return f'Flt({int(self)})' + + def baseline_sumprod(p, q): + """This defines the target behavior including exceptions and special values. + However, it is subject to rounding errors, so float inputs should be exactly + representable with only a few bits. + """ + total = 0 + for p_i, q_i in zip(p, q, strict=True): + total += p_i * q_i + return total + + def run(func, *args): + "Make comparing functions easier. Returns error status, type, and result." + try: + result = func(*args) + except (AssertionError, NameError): + raise + except Exception as e: + return type(e), None, 'None' + return None, type(result), repr(result) + + pools = [ + (-5, 10, -2**20, 2**31, 2**40, 2**61, 2**62, 2**80, 1.5, Int(7)), + (5.25, -3.5, 4.75, 11.25, 400.5, 0.046875, 0.25, -1.0, -0.078125), + (-19.0*2**500, 11*2**1000, -3*2**1500, 17*2*333, + 5.25, -3.25, -3.0*2**(-333), 3, 2**513), + (3.75, 2.5, -1.5, float('inf'), -float('inf'), float('NaN'), 14, + 9, 3+4j, Flt(13), 0.0), + (13.25, -4.25, Decimal('10.5'), Decimal('-2.25'), Fraction(13, 8), + Fraction(-11, 16), 4.75 + 0.125j, 97, -41, Int(3)), + (Decimal('6.125'), Decimal('12.375'), Decimal('-2.75'), Decimal(0), + Decimal('Inf'), -Decimal('Inf'), Decimal('NaN'), 12, 13.5), + (-2.0 ** -1000, 11*2**1000, 3, 7, -37*2**32, -2*2**-537, -2*2**-538, + 2*2**-513), + (-7 * 2.0 ** -510, 5 * 2.0 ** -520, 17, -19.0, -6.25), + (11.25, -3.75, -0.625, 23.375, True, False, 7, Int(5)), + ] + + for pool in pools: + for size in range(4): + for args1 in product(pool, repeat=size): + for args2 in product(pool, repeat=size): + args = (args1, args2) + self.assertEqual( + run(baseline_sumprod, *args), + run(sumprod, *args), + args, + ) + + @requires_IEEE_754 + @unittest.skipIf(HAVE_DOUBLE_ROUNDING, + "sumprod() accuracy not guaranteed on machines with double rounding") + @support.cpython_only # Other implementations may choose a different algorithm + @support.requires_resource('cpu') + def test_sumprod_extended_precision_accuracy(self): + import operator + from fractions import Fraction + from itertools import starmap + from collections import namedtuple + from math import log2, exp2, fabs + from random import choices, uniform, shuffle + from statistics import median + + DotExample = namedtuple('DotExample', ('x', 'y', 'target_sumprod', 'condition')) + + def DotExact(x, y): + vec1 = map(Fraction, x) + vec2 = map(Fraction, y) + return sum(starmap(operator.mul, zip(vec1, vec2, strict=True))) + + def Condition(x, y): + return 2.0 * DotExact(map(abs, x), map(abs, y)) / abs(DotExact(x, y)) + + def linspace(lo, hi, n): + width = (hi - lo) / (n - 1) + return [lo + width * i for i in range(n)] + + def GenDot(n, c): + """ Algorithm 6.1 (GenDot) works as follows. The condition number (5.7) of + the dot product xT y is proportional to the degree of cancellation. In + order to achieve a prescribed cancellation, we generate the first half of + the vectors x and y randomly within a large exponent range. This range is + chosen according to the anticipated condition number. The second half of x + and y is then constructed choosing xi randomly with decreasing exponent, + and calculating yi such that some cancellation occurs. Finally, we permute + the vectors x, y randomly and calculate the achieved condition number. + """ + + assert n >= 6 + n2 = n // 2 + x = [0.0] * n + y = [0.0] * n + b = log2(c) + + # First half with exponents from 0 to |_b/2_| and random ints in between + e = choices(range(int(b/2)), k=n2) + e[0] = int(b / 2) + 1 + e[-1] = 0.0 + + x[:n2] = [uniform(-1.0, 1.0) * exp2(p) for p in e] + y[:n2] = [uniform(-1.0, 1.0) * exp2(p) for p in e] + + # Second half + e = list(map(round, linspace(b/2, 0.0 , n-n2))) + for i in range(n2, n): + x[i] = uniform(-1.0, 1.0) * exp2(e[i - n2]) + y[i] = (uniform(-1.0, 1.0) * exp2(e[i - n2]) - DotExact(x, y)) / x[i] + + # Shuffle + pairs = list(zip(x, y)) + shuffle(pairs) + x, y = zip(*pairs) + + return DotExample(x, y, DotExact(x, y), Condition(x, y)) + + def RelativeError(res, ex): + x, y, target_sumprod, condition = ex + n = DotExact(list(x) + [-res], list(y) + [1]) + return fabs(n / target_sumprod) + + def Trial(dotfunc, c, n): + ex = GenDot(10, c) + res = dotfunc(ex.x, ex.y) + return RelativeError(res, ex) + + times = 1000 # Number of trials + n = 20 # Length of vectors + c = 1e30 # Target condition number + + # If the following test fails, it means that the C math library + # implementation of fma() is not compliant with the C99 standard + # and is inaccurate. To solve this problem, make a new build + # with the symbol UNRELIABLE_FMA defined. That will enable a + # slower but accurate code path that avoids the fma() call. + relative_err = median(Trial(math.sumprod, c, n) for i in range(times)) + self.assertLess(relative_err, 1e-16) + def testModf(self): self.assertRaises(TypeError, math.modf) @@ -1235,6 +1570,7 @@ def testPow(self): self.assertTrue(math.isnan(math.pow(2, NAN))) self.assertTrue(math.isnan(math.pow(0, NAN))) self.assertEqual(math.pow(1, NAN), 1) + self.assertRaises(OverflowError, math.pow, 1e+100, 1e+100) # pow(0., x) self.assertEqual(math.pow(0., INF), 0.) @@ -1550,7 +1886,7 @@ def testTan(self): try: self.assertTrue(math.isnan(math.tan(INF))) self.assertTrue(math.isnan(math.tan(NINF))) - except: + except ValueError: self.assertRaises(ValueError, math.tan, INF) self.assertRaises(ValueError, math.tan, NINF) self.assertTrue(math.isnan(math.tan(NAN))) @@ -1591,6 +1927,8 @@ def __trunc__(self): return 23 class TestNoTrunc: pass + class TestBadTrunc: + __trunc__ = BadDescr() self.assertEqual(math.trunc(TestTrunc()), 23) self.assertEqual(math.trunc(FloatTrunc()), 23) @@ -1599,6 +1937,7 @@ class TestNoTrunc: self.assertRaises(TypeError, math.trunc, 1, 2) self.assertRaises(TypeError, math.trunc, FloatLike(23.5)) self.assertRaises(TypeError, math.trunc, TestNoTrunc()) + self.assertRaises(ValueError, math.trunc, TestBadTrunc()) def testIsfinite(self): self.assertTrue(math.isfinite(0.0)) @@ -1626,11 +1965,11 @@ def testIsinf(self): self.assertFalse(math.isinf(0.)) self.assertFalse(math.isinf(1.)) - @requires_IEEE_754 def test_nan_constant(self): + # `math.nan` must be a quiet NaN with positive sign bit self.assertTrue(math.isnan(math.nan)) + self.assertEqual(math.copysign(1., math.nan), 1.) - @requires_IEEE_754 def test_inf_constant(self): self.assertTrue(math.isinf(math.inf)) self.assertGreater(math.inf, 0.0) @@ -1719,6 +2058,13 @@ def test_testfile(self): except OverflowError: result = 'OverflowError' + # C99+ says for math.h's sqrt: If the argument is +∞ or ±0, it is + # returned, unmodified. On another hand, for csqrt: If z is ±0+0i, + # the result is +0+0i. Lets correct zero sign of er to follow + # first convention. + if id in ['sqrt0002', 'sqrt0003', 'sqrt1001', 'sqrt1023']: + er = math.copysign(er, ar) + # Default tolerances ulp_tol, abs_tol = 5, 0.0 @@ -1802,6 +2148,8 @@ def test_mtestfile(self): '\n '.join(failures)) def test_prod(self): + from fractions import Fraction as F + prod = math.prod self.assertEqual(prod([]), 1) self.assertEqual(prod([], start=5), 5) @@ -1813,6 +2161,14 @@ def test_prod(self): self.assertEqual(prod([1.0, 2.0, 3.0, 4.0, 5.0]), 120.0) self.assertEqual(prod([1, 2, 3, 4.0, 5.0]), 120.0) self.assertEqual(prod([1.0, 2.0, 3.0, 4, 5]), 120.0) + self.assertEqual(prod([1., F(3, 2)]), 1.5) + + # Error in multiplication + class BadMultiply: + def __rmul__(self, other): + raise RuntimeError + with self.assertRaises(RuntimeError): + prod([10., BadMultiply()]) # Test overflow in fast-path for integers self.assertEqual(prod([1, 1, 2**32, 1, 1]), 2**32) @@ -2044,11 +2400,20 @@ def test_nextafter(self): float.fromhex('0x1.fffffffffffffp-1')) self.assertEqual(math.nextafter(1.0, INF), float.fromhex('0x1.0000000000001p+0')) + self.assertEqual(math.nextafter(1.0, -INF, steps=1), + float.fromhex('0x1.fffffffffffffp-1')) + self.assertEqual(math.nextafter(1.0, INF, steps=1), + float.fromhex('0x1.0000000000001p+0')) + self.assertEqual(math.nextafter(1.0, -INF, steps=3), + float.fromhex('0x1.ffffffffffffdp-1')) + self.assertEqual(math.nextafter(1.0, INF, steps=3), + float.fromhex('0x1.0000000000003p+0')) # x == y: y is returned - self.assertEqual(math.nextafter(2.0, 2.0), 2.0) - self.assertEqualSign(math.nextafter(-0.0, +0.0), +0.0) - self.assertEqualSign(math.nextafter(+0.0, -0.0), -0.0) + for steps in range(1, 5): + self.assertEqual(math.nextafter(2.0, 2.0, steps=steps), 2.0) + self.assertEqualSign(math.nextafter(-0.0, +0.0, steps=steps), +0.0) + self.assertEqualSign(math.nextafter(+0.0, -0.0, steps=steps), -0.0) # around 0.0 smallest_subnormal = sys.float_info.min * sys.float_info.epsilon @@ -2073,6 +2438,11 @@ def test_nextafter(self): self.assertIsNaN(math.nextafter(1.0, NAN)) self.assertIsNaN(math.nextafter(NAN, NAN)) + self.assertEqual(1.0, math.nextafter(1.0, INF, steps=0)) + with self.assertRaises(ValueError): + math.nextafter(1.0, INF, steps=-1) + + @requires_IEEE_754 def test_ulp(self): self.assertEqual(math.ulp(1.0), sys.float_info.epsilon) @@ -2112,6 +2482,14 @@ def __float__(self): # argument to a float. self.assertFalse(getattr(y, "converted", False)) + def test_input_exceptions(self): + self.assertRaises(TypeError, math.exp, "spam") + self.assertRaises(TypeError, math.erf, "spam") + self.assertRaises(TypeError, math.atan2, "spam", 1.0) + self.assertRaises(TypeError, math.atan2, 1.0, "spam") + self.assertRaises(TypeError, math.atan2, 1.0) + self.assertRaises(TypeError, math.atan2, 1.0, 2.0, 3.0) + # Custom assertions. def assertIsNaN(self, value): @@ -2252,7 +2630,7 @@ def test_fractions(self): def load_tests(loader, tests, pattern): from doctest import DocFileSuite - # tests.addTest(DocFileSuite("ieee754.txt")) + tests.addTest(DocFileSuite("ieee754.txt")) return tests if __name__ == '__main__': diff --git a/common/src/float_ops.rs b/common/src/float_ops.rs index 69ae8833a2..8e16e71c14 100644 --- a/common/src/float_ops.rs +++ b/common/src/float_ops.rs @@ -133,6 +133,69 @@ pub fn nextafter(x: f64, y: f64) -> f64 { } } +#[allow(clippy::float_cmp)] +pub fn nextafter_with_steps(x: f64, y: f64, steps: u64) -> f64 { + if x == y { + y + } else if x.is_nan() || y.is_nan() { + f64::NAN + } else if x >= f64::INFINITY { + f64::MAX + } else if x <= f64::NEG_INFINITY { + f64::MIN + } else if x == 0.0 { + f64::from_bits(1).copysign(y) + } else { + if steps == 0 { + return x; + } + + if x.is_nan() { + return x; + } + + if y.is_nan() { + return y; + } + + let sign_bit: u64 = 1 << 63; + + let mut ux = x.to_bits(); + let uy = y.to_bits(); + + let ax = ux & !sign_bit; + let ay = uy & !sign_bit; + + // If signs are different + if ((ux ^ uy) & sign_bit) != 0 { + return if ax + ay <= steps { + f64::from_bits(uy) + } else if ax < steps { + let result = (uy & sign_bit) | (steps - ax); + f64::from_bits(result) + } else { + ux -= steps; + f64::from_bits(ux) + }; + } + + // If signs are the same + if ax > ay { + if ax - ay >= steps { + ux -= steps; + f64::from_bits(ux) + } else { + f64::from_bits(uy) + } + } else if ay - ax >= steps { + ux += steps; + f64::from_bits(ux) + } else { + f64::from_bits(uy) + } + } +} + pub fn ulp(x: f64) -> f64 { if x.is_nan() { return x; diff --git a/stdlib/src/math.rs b/stdlib/src/math.rs index 21451203b5..68930cfd0f 100644 --- a/stdlib/src/math.rs +++ b/stdlib/src/math.rs @@ -629,7 +629,7 @@ mod math { #[pyfunction] fn fsum(seq: ArgIterable, vm: &VirtualMachine) -> PyResult { - let mut partials = vec![]; + let mut partials = Vec::with_capacity(32); let mut special_sum = 0.0; let mut inf_sum = 0.0; @@ -637,11 +637,11 @@ mod math { let mut x = *obj?; let xsave = x; - let mut j = 0; + let mut i = 0; // This inner loop applies `hi`/`lo` summation to each // partial so that the list of partial sums remains exact. - for i in 0..partials.len() { - let mut y: f64 = partials[i]; + for j in 0..partials.len() { + let mut y: f64 = partials[j]; if x.abs() < y.abs() { std::mem::swap(&mut x, &mut y); } @@ -650,33 +650,33 @@ mod math { let hi = x + y; let lo = y - (hi - x); if lo != 0.0 { - partials[j] = lo; - j += 1; + partials[i] = lo; + i += 1; } x = hi; } - if !x.is_finite() { - // a nonfinite x could arise either as - // a result of intermediate overflow, or - // as a result of a nan or inf in the - // summands - if xsave.is_finite() { - return Err(vm.new_overflow_error("intermediate overflow in fsum".to_owned())); - } - if xsave.is_infinite() { - inf_sum += xsave; + partials.truncate(i); + if x != 0.0 { + if !x.is_finite() { + // a nonfinite x could arise either as + // a result of intermediate overflow, or + // as a result of a nan or inf in the + // summands + if xsave.is_finite() { + return Err( + vm.new_overflow_error("intermediate overflow in fsum".to_owned()) + ); + } + if xsave.is_infinite() { + inf_sum += xsave; + } + special_sum += xsave; + // reset partials + partials.clear(); + } else { + partials.push(x); } - special_sum += xsave; - // reset partials - partials.clear(); - } - - if j >= partials.len() { - partials.push(x); - } else { - partials[j] = x; - partials.truncate(j + 1); } } if special_sum != 0.0 { @@ -831,9 +831,38 @@ mod math { (x.fract(), x.trunc()) } - #[pyfunction] - fn nextafter(x: ArgIntoFloat, y: ArgIntoFloat) -> f64 { - float_ops::nextafter(*x, *y) + #[derive(FromArgs)] + struct NextAfterArgs { + #[pyarg(positional)] + x: ArgIntoFloat, + #[pyarg(positional)] + y: ArgIntoFloat, + #[pyarg(named, optional)] + steps: OptionalArg, + } + + #[pyfunction] + fn nextafter(arg: NextAfterArgs, vm: &VirtualMachine) -> PyResult { + let steps: Option = arg + .steps + .map(|v| v.try_to_primitive(vm)) + .transpose()? + .into_option(); + match steps { + Some(steps) => { + if steps < 0 { + return Err( + vm.new_value_error("steps must be a non-negative integer".to_string()) + ); + } + Ok(float_ops::nextafter_with_steps( + *arg.x, + *arg.y, + steps as u64, + )) + } + None => Ok(float_ops::nextafter(*arg.x, *arg.y)), + } } #[pyfunction] @@ -917,10 +946,38 @@ mod math { // refer: https://github.com/python/cpython/blob/main/Modules/mathmodule.c#L3093-L3193 for obj in iter.iter(vm)? { let obj = obj?; + result = vm._mul(&result, &obj)?; + } + + Ok(result) + } - result = vm - ._mul(&result, &obj) - .map_err(|_| vm.new_type_error("math type error".to_owned()))?; + #[pyfunction] + fn sumprod( + p: ArgIterable, + q: ArgIterable, + vm: &VirtualMachine, + ) -> PyResult { + let mut p_iter = p.iter(vm)?; + let mut q_iter = q.iter(vm)?; + // We cannot just create a float because the iterator may contain + // anything as long as it supports __add__ and __mul__. + let mut result = vm.new_pyobj(0); + loop { + let m_p = p_iter.next(); + let m_q = q_iter.next(); + match (m_p, m_q) { + (Some(r_p), Some(r_q)) => { + let p = r_p?; + let q = r_q?; + let tmp = vm._mul(&p, &q)?; + result = vm._add(&result, &tmp)?; + } + (None, None) => break, + _ => { + return Err(vm.new_value_error("Inputs are not the same length".to_string())); + } + } } Ok(result) diff --git a/vm/src/builtins/float.rs b/vm/src/builtins/float.rs index 1cd041b7b9..27fdff12ee 100644 --- a/vm/src/builtins/float.rs +++ b/vm/src/builtins/float.rs @@ -124,8 +124,8 @@ fn inner_divmod(v1: f64, v2: f64, vm: &VirtualMachine) -> PyResult<(f64, f64)> { pub fn float_pow(v1: f64, v2: f64, vm: &VirtualMachine) -> PyResult { if v1.is_zero() && v2.is_sign_negative() { - let msg = format!("{v1} cannot be raised to a negative power"); - Err(vm.new_zero_division_error(msg)) + let msg = "0.0 cannot be raised to a negative power"; + Err(vm.new_zero_division_error(msg.to_owned())) } else if v1.is_sign_negative() && (v2.floor() - v2).abs() > f64::EPSILON { let v1 = Complex64::new(v1, 0.); let v2 = Complex64::new(v2, 0.); From a9331bb34df1ce1b61b156044087886e2a182bf3 Mon Sep 17 00:00:00 2001 From: Noa Date: Mon, 10 Feb 2025 23:47:27 -0600 Subject: [PATCH 17/26] Fix warnings for rust 1.85 --- Cargo.lock | 2 +- Cargo.toml | 2 +- common/src/hash.rs | 4 +- compiler/core/src/bytecode.rs | 2 +- src/shell/helper.rs | 2 +- stdlib/src/binascii.rs | 14 ++-- vm/src/builtins/object.rs | 2 +- vm/src/builtins/str.rs | 148 +++++++++++++++++----------------- vm/src/codecs.rs | 4 +- vm/src/stdlib/io.rs | 2 +- vm/src/vm/vm_ops.rs | 4 +- vm/sre_engine/src/string.rs | 4 +- 12 files changed, 95 insertions(+), 95 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 11992d8d8c..edf28dc7f4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1322,7 +1322,7 @@ version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df7151a832e54d2d6b2c827a20e5bcdd80359281cd2c354e725d4b82e7c471de" dependencies = [ - "rand_core 0.9.0", + "rand_core 0.9.1", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index bda7286afd..83007606c6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -104,7 +104,7 @@ members = [ version = "0.4.0" authors = ["RustPython Team"] edition = "2021" -rust-version = "1.83.0" +rust-version = "1.85.0" repository = "https://github.com/RustPython/RustPython" license = "MIT" diff --git a/common/src/hash.rs b/common/src/hash.rs index 700d2ceef5..5b5ac003cb 100644 --- a/common/src/hash.rs +++ b/common/src/hash.rs @@ -114,7 +114,7 @@ pub fn hash_float(value: f64) -> Option { let mut e = frexp.1; let mut x: PyUHash = 0; while m != 0.0 { - x = ((x << 28) & MODULUS) | x >> (BITS - 28); + x = ((x << 28) & MODULUS) | (x >> (BITS - 28)); m *= 268_435_456.0; // 2**28 e -= 28; let y = m as PyUHash; // pull out integer part @@ -132,7 +132,7 @@ pub fn hash_float(value: f64) -> Option { } else { BITS32 - 1 - ((-1 - e) % BITS32) }; - x = ((x << e) & MODULUS) | x >> (BITS32 - e); + x = ((x << e) & MODULUS) | (x >> (BITS32 - e)); Some(fix_sentinel(x as PyHash * value.signum() as PyHash)) } diff --git a/compiler/core/src/bytecode.rs b/compiler/core/src/bytecode.rs index 46839c2695..21d342b541 100644 --- a/compiler/core/src/bytecode.rs +++ b/compiler/core/src/bytecode.rs @@ -197,7 +197,7 @@ impl OpArgState { } #[inline(always)] pub fn extend(&mut self, arg: OpArgByte) -> OpArg { - self.state = self.state << 8 | u32::from(arg.0); + self.state = (self.state << 8) | u32::from(arg.0); OpArg(self.state) } #[inline(always)] diff --git a/src/shell/helper.rs b/src/shell/helper.rs index c228dbdf65..4d663b9424 100644 --- a/src/shell/helper.rs +++ b/src/shell/helper.rs @@ -107,7 +107,7 @@ impl<'vm> ShellHelper<'vm> { .filter(|res| { res.as_ref() .ok() - .map_or(true, |s| s.as_str().starts_with(word_start)) + .is_none_or(|s| s.as_str().starts_with(word_start)) }) .collect::, _>>() .ok()?; diff --git a/stdlib/src/binascii.rs b/stdlib/src/binascii.rs index d348049f4c..eb11645879 100644 --- a/stdlib/src/binascii.rs +++ b/stdlib/src/binascii.rs @@ -76,7 +76,7 @@ mod decl { let mut unhex = Vec::::with_capacity(hex_bytes.len() / 2); for (n1, n2) in hex_bytes.iter().tuples() { if let (Some(n1), Some(n2)) = (unhex_nibble(*n1), unhex_nibble(*n2)) { - unhex.push(n1 << 4 | n2); + unhex.push((n1 << 4) | n2); } else { return Err(super::new_binascii_error( "Non-hexadecimal digit found".to_owned(), @@ -343,7 +343,7 @@ mod decl { if let (Some(ch1), Some(ch2)) = (unhex_nibble(buffer[idx]), unhex_nibble(buffer[idx + 1])) { - out_data.push(ch1 << 4 | ch2); + out_data.push((ch1 << 4) | ch2); } idx += 2; } else { @@ -661,19 +661,19 @@ mod decl { }; if res.len() < length { - res.push(char_a << 2 | char_b >> 4); + res.push((char_a << 2) | (char_b >> 4)); } else if char_a != 0 || char_b != 0 { return trailing_garbage_error(); } if res.len() < length { - res.push((char_b & 0xf) << 4 | char_c >> 2); + res.push(((char_b & 0xf) << 4) | (char_c >> 2)); } else if char_c != 0 { return trailing_garbage_error(); } if res.len() < length { - res.push((char_c & 0x3) << 6 | char_d); + res.push(((char_c & 0x3) << 6) | char_d); } else if char_d != 0 { return trailing_garbage_error(); } @@ -725,8 +725,8 @@ mod decl { let char_c = *chunk.get(2).unwrap_or(&0); res.push(uu_b2a(char_a >> 2, backtick)); - res.push(uu_b2a((char_a & 0x3) << 4 | char_b >> 4, backtick)); - res.push(uu_b2a((char_b & 0xf) << 2 | char_c >> 6, backtick)); + res.push(uu_b2a(((char_a & 0x3) << 4) | (char_b >> 4), backtick)); + res.push(uu_b2a(((char_b & 0xf) << 2) | (char_c >> 6), backtick)); res.push(uu_b2a(char_c & 0x3f, backtick)); } diff --git a/vm/src/builtins/object.rs b/vm/src/builtins/object.rs index f783ee017c..aeb9e93cc6 100644 --- a/vm/src/builtins/object.rs +++ b/vm/src/builtins/object.rs @@ -113,7 +113,7 @@ fn object_getstate_default(obj: &PyObject, required: bool, vm: &VirtualMachine) // )); // } - let state = if obj.dict().map_or(true, |d| d.is_empty()) { + let state = if obj.dict().is_none_or(|d| d.is_empty()) { vm.ctx.none() } else { // let state = object_get_dict(obj.clone(), obj.ctx()).unwrap(); diff --git a/vm/src/builtins/str.rs b/vm/src/builtins/str.rs index 52112f371d..f5a340836c 100644 --- a/vm/src/builtins/str.rs +++ b/vm/src/builtins/str.rs @@ -1565,80 +1565,6 @@ impl AsRef for PyExact { } } -#[cfg(test)] -mod tests { - use super::*; - use crate::Interpreter; - - #[test] - fn str_title() { - let tests = vec![ - (" Hello ", " hello "), - ("Hello ", "hello "), - ("Hello ", "Hello "), - ("Format This As Title String", "fOrMaT thIs aS titLe String"), - ("Format,This-As*Title;String", "fOrMaT,thIs-aS*titLe;String"), - ("Getint", "getInt"), - ("Greek Ωppercases ...", "greek ωppercases ..."), - ("Greek ῼitlecases ...", "greek ῳitlecases ..."), - ]; - for (title, input) in tests { - assert_eq!(PyStr::from(input).title().as_str(), title); - } - } - - #[test] - fn str_istitle() { - let pos = vec![ - "A", - "A Titlecased Line", - "A\nTitlecased Line", - "A Titlecased, Line", - "Greek Ωppercases ...", - "Greek ῼitlecases ...", - ]; - - for s in pos { - assert!(PyStr::from(s).istitle()); - } - - let neg = vec![ - "", - "a", - "\n", - "Not a capitalized String", - "Not\ta Titlecase String", - "Not--a Titlecase String", - "NOT", - ]; - for s in neg { - assert!(!PyStr::from(s).istitle()); - } - } - - #[test] - fn str_maketrans_and_translate() { - Interpreter::without_stdlib(Default::default()).enter(|vm| { - let table = vm.ctx.new_dict(); - table - .set_item("a", vm.ctx.new_str("🎅").into(), vm) - .unwrap(); - table.set_item("b", vm.ctx.none(), vm).unwrap(); - table - .set_item("c", vm.ctx.new_str(ascii!("xda")).into(), vm) - .unwrap(); - let translated = - PyStr::maketrans(table.into(), OptionalArg::Missing, OptionalArg::Missing, vm) - .unwrap(); - let text = PyStr::from("abc"); - let translated = text.translate(translated, vm).unwrap(); - assert_eq!(translated, "🎅xda".to_owned()); - let translated = text.translate(vm.ctx.new_int(3).into(), vm); - assert_eq!("TypeError", &*translated.unwrap_err().class().name(),); - }) - } -} - impl AnyStrWrapper for PyStrRef { type Str = str; fn as_ref(&self) -> &str { @@ -1780,3 +1706,77 @@ impl AsRef for PyStrInterned { self.as_str() } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::Interpreter; + + #[test] + fn str_title() { + let tests = vec![ + (" Hello ", " hello "), + ("Hello ", "hello "), + ("Hello ", "Hello "), + ("Format This As Title String", "fOrMaT thIs aS titLe String"), + ("Format,This-As*Title;String", "fOrMaT,thIs-aS*titLe;String"), + ("Getint", "getInt"), + ("Greek Ωppercases ...", "greek ωppercases ..."), + ("Greek ῼitlecases ...", "greek ῳitlecases ..."), + ]; + for (title, input) in tests { + assert_eq!(PyStr::from(input).title().as_str(), title); + } + } + + #[test] + fn str_istitle() { + let pos = vec![ + "A", + "A Titlecased Line", + "A\nTitlecased Line", + "A Titlecased, Line", + "Greek Ωppercases ...", + "Greek ῼitlecases ...", + ]; + + for s in pos { + assert!(PyStr::from(s).istitle()); + } + + let neg = vec![ + "", + "a", + "\n", + "Not a capitalized String", + "Not\ta Titlecase String", + "Not--a Titlecase String", + "NOT", + ]; + for s in neg { + assert!(!PyStr::from(s).istitle()); + } + } + + #[test] + fn str_maketrans_and_translate() { + Interpreter::without_stdlib(Default::default()).enter(|vm| { + let table = vm.ctx.new_dict(); + table + .set_item("a", vm.ctx.new_str("🎅").into(), vm) + .unwrap(); + table.set_item("b", vm.ctx.none(), vm).unwrap(); + table + .set_item("c", vm.ctx.new_str(ascii!("xda")).into(), vm) + .unwrap(); + let translated = + PyStr::maketrans(table.into(), OptionalArg::Missing, OptionalArg::Missing, vm) + .unwrap(); + let text = PyStr::from("abc"); + let translated = text.translate(translated, vm).unwrap(); + assert_eq!(translated, "🎅xda".to_owned()); + let translated = text.translate(vm.ctx.new_int(3).into(), vm); + assert_eq!("TypeError", &*translated.unwrap_err().class().name(),); + }) + } +} diff --git a/vm/src/codecs.rs b/vm/src/codecs.rs index 61be9f9176..898198f3d3 100644 --- a/vm/src/codecs.rs +++ b/vm/src/codecs.rs @@ -637,10 +637,10 @@ fn surrogatepass_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<(PyOb } } StandardEncoding::Utf16Le => { - c = (p[1] as u32) << 8 | p[0] as u32; + c = ((p[1] as u32) << 8) | p[0] as u32; } StandardEncoding::Utf16Be => { - c = (p[0] as u32) << 8 | p[1] as u32; + c = ((p[0] as u32) << 8) | p[1] as u32; } StandardEncoding::Utf32Le => { c = ((p[3] as u32) << 24) diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index 3c35b05e1a..c4f67cd6a6 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -482,7 +482,7 @@ mod _io { let size = size.to_usize(); let read = instance.get_attr("read", vm)?; let mut res = Vec::new(); - while size.map_or(true, |s| res.len() < s) { + while size.is_none_or(|s| res.len() < s) { let read_res = ArgBytesLike::try_from_object(vm, read.call((1,), vm)?)?; if read_res.with_ref(|b| b.is_empty()) { break; diff --git a/vm/src/vm/vm_ops.rs b/vm/src/vm/vm_ops.rs index 56c32d2927..3aa4144190 100644 --- a/vm/src/vm/vm_ops.rs +++ b/vm/src/vm/vm_ops.rs @@ -298,8 +298,8 @@ impl VirtualMachine { } if let Some(slot_c) = class_c.slots.as_number.left_ternary_op(op_slot) { - if slot_a.is_some_and(|slot_a| slot_a != slot_c) - && slot_b.is_some_and(|slot_b| slot_b != slot_c) + if slot_a.is_some_and(|slot_a| !std::ptr::fn_addr_eq(slot_a, slot_c)) + && slot_b.is_some_and(|slot_b| !std::ptr::fn_addr_eq(slot_b, slot_c)) { let ret = slot_c(a, b, c, self)?; if !ret.is(&self.ctx.not_implemented) { diff --git a/vm/sre_engine/src/string.rs b/vm/sre_engine/src/string.rs index ca8b3179dc..b5a2470f52 100644 --- a/vm/sre_engine/src/string.rs +++ b/vm/sre_engine/src/string.rs @@ -181,7 +181,7 @@ unsafe fn next_code_point(ptr: &mut *const u8) -> u32 { let z = **ptr; *ptr = ptr.offset(1); let y_z = utf8_acc_cont_byte((y & CONT_MASK) as u32, z); - ch = init << 12 | y_z; + ch = (init << 12) | y_z; if x >= 0xF0 { // [x y z w] case // use only the lower 3 bits of `init` @@ -189,7 +189,7 @@ unsafe fn next_code_point(ptr: &mut *const u8) -> u32 { // so the iterator must produce a value here. let w = **ptr; *ptr = ptr.offset(1); - ch = (init & 7) << 18 | utf8_acc_cont_byte(y_z, w); + ch = ((init & 7) << 18) | utf8_acc_cont_byte(y_z, w); } } From 23236aa8c7b0b0227919581cc86d0c9d02798246 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Thu, 20 Feb 2025 10:50:50 -0800 Subject: [PATCH 18/26] test_datetime now works on windows Signed-off-by: Ashwin Naren --- .github/workflows/ci.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index dc2dcc6c7e..9c0f736928 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -18,7 +18,6 @@ env: CARGO_ARGS: --no-default-features --features stdlib,zlib,importlib,encodings,sqlite,ssl # Skip additional tests on Windows. They are checked on Linux and MacOS. WINDOWS_SKIPS: >- - test_datetime test_glob test_importlib test_io From d698b28ce5af90b7c033e07f5c8b5130dad2799b Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sat, 22 Feb 2025 00:22:31 -0800 Subject: [PATCH 19/26] Ensure pymethod cannot be both magic and named simultaneously + macro documentation (#5538) Signed-off-by: Ashwin Naren --- derive-impl/src/pyclass.rs | 7 ++ derive/src/lib.rs | 183 +++++++++++++++++++++++++++++++++++++ 2 files changed, 190 insertions(+) diff --git a/derive-impl/src/pyclass.rs b/derive-impl/src/pyclass.rs index dedb3eb77a..c42069c31c 100644 --- a/derive-impl/src/pyclass.rs +++ b/derive-impl/src/pyclass.rs @@ -1229,6 +1229,13 @@ impl MethodItemMeta { let inner = self.inner(); let name = inner._optional_str("name")?; let magic = inner._bool("magic")?; + if magic && name.is_some() { + bail_span!( + &inner.meta_ident, + "A #[{}] method cannot be magic and have a specified name, choose one.", + inner.meta_name() + ); + } Ok(if let Some(name) = name { name } else { diff --git a/derive/src/lib.rs b/derive/src/lib.rs index 76b7e23488..e59db12f90 100644 --- a/derive/src/lib.rs +++ b/derive/src/lib.rs @@ -12,6 +12,124 @@ pub fn derive_from_args(input: TokenStream) -> TokenStream { derive_impl::derive_from_args(input).into() } +/// The attribute can be applied either to a struct, trait, or impl. +/// # Struct +/// This implements `MaybeTraverse`, `PyClassDef`, and `StaticType` for the struct. +/// Consider deriving `Traverse` to implement it. +/// ## Arguments +/// - `module`: the module which contains the class -- can be omitted if in a `#[pymodule]`. +/// - `name`: the name of the Python class, by default it is the name of the struct. +/// - `base`: the base class of the Python class. +/// This does not cause inheritance of functions or attributes that must be done by a separate trait. +/// # Impl +/// This part implements `PyClassImpl` for the struct. +/// This includes methods, getters/setters, etc.; only annotated methods will be included. +/// Common functions and abilities like instantiation and `__call__` are often implemented by +/// traits rather than in the `impl` itself; see `Constructor` and `Callable` respectively for those. +/// ## Arguments +/// - `name`: the name of the Python class, when no name is provided the struct name is used. +/// - `flags`: the flags of the class, see `PyTypeFlags`. +/// - `BASETYPE`: allows the class to be inheritable. +/// - `IMMUTABLETYPE`: class attributes are immutable. +/// - `with`: which trait implementations are to be included in the python class. +/// ```rust, ignore +/// #[pyclass(module = "mymodule", name = "MyClass", base = "BaseClass")] +/// struct MyStruct { +/// x: i32, +/// } +/// +/// impl Constructor for MyStruct { +/// ... +/// } +/// +/// #[pyclass(with(Constructor))] +/// impl MyStruct { +/// ... +/// } +/// ``` +/// ## Inner markers +/// ### pymethod/pyclassmethod/pystaticmethod +/// `pymethod` is used to mark a method of the Python class. +/// `pyclassmethod` is used to mark a class method. +/// `pystaticmethod` is used to mark a static method. +/// #### Method signature +/// The first parameter can be either `&self` or `: PyRef` for `pymethod`. +/// The first parameter can be `cls: PyTypeRef` for `pyclassmethod`. +/// There is no mandatory parameter for `pystaticmethod`. +/// Both are valid and essentially the same, but the latter can yield more control. +/// The last parameter can optionally be of the type `&VirtualMachine` to access the VM. +/// All other values must implement `IntoPyResult`. +/// Numeric types, `String`, `bool`, and `PyObjectRef` implement this trait, +/// but so does any object that implements `PyValue`. +/// Consider using `OptionalArg` for optional arguments. +/// #### Arguments +/// - `magic`: marks the method as a magic method: the method name is surrounded with double underscores. +/// ```rust, ignore +/// #[pyclass] +/// impl MyStruct { +/// // This will be called as the `__add__` method in Python. +/// #[pymethod(magic)] +/// fn add(&self, other: &Self) -> PyResult { +/// ... +/// } +/// } +/// ``` +/// - `name`: the name of the method in Python, +/// by default it is the same as the Rust method, or surrounded by double underscores if magic is present. +/// This overrides `magic` and the default name and cannot be used with `magic` to prevent ambiguity. +/// ### pygetset +/// This is used to mark a getter/setter pair. +/// #### Arguments +/// - `setter`: marks the method as a setter, it acts as a getter by default. +/// Setter method names should be prefixed with `set_`. +/// - `name`: the name of the attribute in Python, by default it is the same as the Rust method. +/// - `magic`: marks the method as a magic method: the method name is surrounded with double underscores. +/// This cannot be used with `name` to prevent ambiguity. +/// +/// Ensure both the getter and setter are marked with `name` and `magic` in the same manner. +/// #### Examples +/// ```rust, ignore +/// #[pyclass] +/// impl MyStruct { +/// #[pygetset] +/// fn x(&self) -> PyResult { +/// Ok(self.x.lock()) +/// } +/// #[pygetset(setter)] +/// fn set_x(&mut self, value: i32) -> PyResult<()> { +/// self.x.set(value); +/// Ok(()) +/// } +/// } +/// ``` +/// ### pyslot +/// This is used to mark a slot method it should be marked by prefixing the method in rust with `slot_`. +/// #### Arguments +/// - name: the name of the slot method. +/// ### pyattr +/// ### extend_class +/// This helps inherit attributes from a parent class. +/// The method this is applied on should be called `extend_class_with_fields`. +/// #### Examples +/// ```rust, ignore +/// #[extend_class] +/// fn extend_class_with_fields(ctx: &Context, class: &'static Py) { +/// class.set_attr( +/// identifier!(ctx, _fields), +/// ctx.new_tuple(vec![ +/// ctx.new_str(ascii!("body")).into(), +/// ctx.new_str(ascii!("type_ignores")).into(), +/// ]) +/// .into(), +/// ); +/// class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); +/// } +/// ``` +/// ### pymember +/// # Trait +/// `#[pyclass]` on traits functions a lot like `#[pyclass]` on `impl` blocks. +/// Note that associated functions that are annotated with `#[pymethod]` or similar **must** +/// have a body, abstract functions should be wrapped before applying an annotation. #[proc_macro_attribute] pub fn pyclass(attr: TokenStream, item: TokenStream) -> TokenStream { let attr = parse_macro_input!(attr); @@ -34,6 +152,71 @@ pub fn pyexception(attr: TokenStream, item: TokenStream) -> TokenStream { derive_impl::pyexception(attr, item).into() } +/// This attribute must be applied to an inline module. +/// It defines a Python module in the form a `make_module` function in the module; +/// this has to be used in a `get_module_inits` to properly register the module. +/// Additionally, this macro defines 'MODULE_NAME' and 'DOC' in the module. +/// # Arguments +/// - `name`: the name of the python module, +/// by default, it is the name of the module, but this can be configured. +/// ```rust, ignore +/// // This will create a module named `mymodule` +/// #[pymodule(name = "mymodule")] +/// mod module { +/// } +/// ``` +/// - `sub`: declare the module as a submodule of another module. +/// ```rust, ignore +/// #[pymodule(sub)] +/// mod submodule { +/// } +/// +/// #[pymodule(with(submodule))] +/// mod mymodule { +/// } +/// ``` +/// - `with`: declare the list of submodules that this module contains (see `sub` for example). +/// ## Inner markers +/// ### pyattr +/// `pyattr` is a multipurpose marker that can be used in a pymodule. +/// The most common use is to mark a function or class as a part of the module. +/// This can be done by applying it to a function or struct prior to the `#[pyfunction]` or `#[pyclass]` macro. +/// If applied to a constant, it will be added to the module as an attribute. +/// If applied to a function not marked with `pyfunction`, +/// it will also be added to the module as an attribute but the value is the result of the function. +/// If `#[pyattr(once)]` is used in this case, the function will be called once +/// and the result will be stored using a `static_cell`. +/// #### Examples +/// ```rust, ignore +/// #[pymodule] +/// mod mymodule { +/// #[pyattr] +/// const MY_CONSTANT: i32 = 42; +/// #[pyattr] +/// fn another_constant() -> PyResult { +/// Ok(42) +/// } +/// #[pyattr(once)] +/// fn once() -> PyResult { +/// // This will only be called once and the result will be stored. +/// Ok(2 ** 24) +/// } +/// +/// #[pyattr] +/// #[pyfunction] +/// fn my_function(vm: &VirtualMachine) -> PyResult<()> { +/// ... +/// } +/// } +/// ``` +/// ### pyfunction +/// This is used to create a python function. +/// #### Function signature +/// The last argument can optionally be of the type `&VirtualMachine` to access the VM. +/// Refer to the `pymethod` documentation (located in the `pyclass` macro documentation) +/// for more information on what regular argument types are permitted. +/// #### Arguments +/// - `name`: the name of the function in Python, by default it is the same as the associated Rust function. #[proc_macro_attribute] pub fn pymodule(attr: TokenStream, item: TokenStream) -> TokenStream { let attr = parse_macro_input!(attr); From 331a3c108f1fe44806961607ae96c6798677f44b Mon Sep 17 00:00:00 2001 From: Noa Date: Thu, 20 Feb 2025 14:57:40 -0600 Subject: [PATCH 20/26] Switch to criterion in sre_engine benchmarks --- Cargo.lock | 1 + Cargo.toml | 3 +- vm/sre_engine/Cargo.toml | 7 +++ vm/sre_engine/benches/benches.rs | 83 +++++++++++++++++--------------- vm/sre_engine/generate_tests.py | 31 +++++++++--- vm/sre_engine/tests/tests.rs | 34 +++++++------ 6 files changed, 97 insertions(+), 62 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index edf28dc7f4..3711b6f16c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2178,6 +2178,7 @@ name = "rustpython-sre_engine" version = "0.4.0" dependencies = [ "bitflags 2.8.0", + "criterion", "num_enum", "optional", ] diff --git a/Cargo.toml b/Cargo.toml index 83007606c6..446733d515 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,7 +47,7 @@ libc = { workspace = true } rustyline = { workspace = true } [dev-dependencies] -criterion = { version = "0.3.5", features = ["html_reports"] } +criterion = { workspace = true } pyo3 = { version = "0.22", features = ["auto-initialize"] } [[bench]] @@ -144,6 +144,7 @@ bitflags = "2.4.2" bstr = "1" cfg-if = "1.0" chrono = "0.4.39" +criterion = { version = "0.3.5", features = ["html_reports"] } crossbeam-utils = "0.8.21" flame = "0.2.2" getrandom = "0.3" diff --git a/vm/sre_engine/Cargo.toml b/vm/sre_engine/Cargo.toml index 28f98a3212..ac0e7b5f16 100644 --- a/vm/sre_engine/Cargo.toml +++ b/vm/sre_engine/Cargo.toml @@ -10,7 +10,14 @@ rust-version.workspace = true repository.workspace = true license.workspace = true +[[bench]] +name = "benches" +harness = false + [dependencies] num_enum = { workspace = true } bitflags = { workspace = true } optional = "0.5" + +[dev-dependencies] +criterion = { workspace = true } diff --git a/vm/sre_engine/benches/benches.rs b/vm/sre_engine/benches/benches.rs index e31c73b0d0..915d568333 100644 --- a/vm/sre_engine/benches/benches.rs +++ b/vm/sre_engine/benches/benches.rs @@ -1,11 +1,9 @@ -#![feature(test)] - -extern crate test; -use test::Bencher; - use rustpython_sre_engine::{Request, State, StrDrive}; +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion}; + struct Pattern { + pattern: &'static str, code: &'static [u32], } @@ -25,52 +23,51 @@ impl Pattern { } } -#[bench] -fn benchmarks(b: &mut Bencher) { +fn basic(c: &mut Criterion) { // # test common prefix // pattern p1 = re.compile('Python|Perl') # , 'Perl'), # Alternation // START GENERATED by generate_tests.py - #[rustfmt::skip] let p1 = Pattern { code: &[14, 8, 1, 4, 6, 1, 1, 80, 0, 16, 80, 7, 13, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 15, 11, 9, 16, 101, 16, 114, 16, 108, 15, 2, 0, 1] }; + #[rustfmt::skip] let p1 = Pattern { pattern: "Python|Perl", code: &[14, 8, 1, 4, 6, 1, 1, 80, 0, 16, 80, 7, 13, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 15, 11, 9, 16, 101, 16, 114, 16, 108, 15, 2, 0, 1] }; // END GENERATED // pattern p2 = re.compile('(Python|Perl)') #, 'Perl'), # Grouped alternation // START GENERATED by generate_tests.py - #[rustfmt::skip] let p2 = Pattern { code: &[14, 8, 1, 4, 6, 1, 0, 80, 0, 17, 0, 16, 80, 7, 13, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 15, 11, 9, 16, 101, 16, 114, 16, 108, 15, 2, 0, 17, 1, 1] }; + #[rustfmt::skip] let p2 = Pattern { pattern: "(Python|Perl)", code: &[14, 8, 1, 4, 6, 1, 0, 80, 0, 17, 0, 16, 80, 7, 13, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 15, 11, 9, 16, 101, 16, 114, 16, 108, 15, 2, 0, 17, 1, 1] }; // END GENERATED // pattern p3 = re.compile('Python|Perl|Tcl') #, 'Perl'), # Alternation // START GENERATED by generate_tests.py - #[rustfmt::skip] let p3 = Pattern { code: &[14, 9, 4, 3, 6, 16, 80, 16, 84, 0, 7, 15, 16, 80, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 15, 22, 11, 16, 80, 16, 101, 16, 114, 16, 108, 15, 11, 9, 16, 84, 16, 99, 16, 108, 15, 2, 0, 1] }; + #[rustfmt::skip] let p3 = Pattern { pattern: "Python|Perl|Tcl", code: &[14, 9, 4, 3, 6, 16, 80, 16, 84, 0, 7, 15, 16, 80, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 15, 22, 11, 16, 80, 16, 101, 16, 114, 16, 108, 15, 11, 9, 16, 84, 16, 99, 16, 108, 15, 2, 0, 1] }; // END GENERATED // pattern p4 = re.compile('(Python|Perl|Tcl)') #, 'Perl'), # Grouped alternation // START GENERATED by generate_tests.py - #[rustfmt::skip] let p4 = Pattern { code: &[14, 9, 4, 3, 6, 16, 80, 16, 84, 0, 17, 0, 7, 15, 16, 80, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 15, 22, 11, 16, 80, 16, 101, 16, 114, 16, 108, 15, 11, 9, 16, 84, 16, 99, 16, 108, 15, 2, 0, 17, 1, 1] }; + #[rustfmt::skip] let p4 = Pattern { pattern: "(Python|Perl|Tcl)", code: &[14, 9, 4, 3, 6, 16, 80, 16, 84, 0, 17, 0, 7, 15, 16, 80, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 15, 22, 11, 16, 80, 16, 101, 16, 114, 16, 108, 15, 11, 9, 16, 84, 16, 99, 16, 108, 15, 2, 0, 17, 1, 1] }; // END GENERATED // pattern p5 = re.compile('(Python)\\1') #, 'PythonPython'), # Backreference // START GENERATED by generate_tests.py - #[rustfmt::skip] let p5 = Pattern { code: &[14, 18, 1, 12, 12, 6, 0, 80, 121, 116, 104, 111, 110, 0, 0, 0, 0, 0, 0, 17, 0, 16, 80, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 17, 1, 11, 0, 1] }; + #[rustfmt::skip] let p5 = Pattern { pattern: "(Python)\\1", code: &[14, 18, 1, 12, 12, 6, 0, 80, 121, 116, 104, 111, 110, 0, 0, 0, 0, 0, 0, 17, 0, 16, 80, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 17, 1, 11, 0, 1] }; // END GENERATED // pattern p6 = re.compile('([0a-z][a-z0-9]*,)+') #, 'a5,b7,c9,'), # Disable the fastmap optimization // START GENERATED by generate_tests.py - #[rustfmt::skip] let p6 = Pattern { code: &[14, 4, 0, 2, 4294967295, 23, 31, 1, 4294967295, 17, 0, 13, 7, 16, 48, 22, 97, 122, 0, 24, 13, 0, 4294967295, 13, 8, 22, 97, 122, 22, 48, 57, 0, 1, 16, 44, 17, 1, 18, 1] }; + #[rustfmt::skip] let p6 = Pattern { pattern: "([0a-z][a-z0-9]*,)+", code: &[14, 4, 0, 2, 4294967295, 23, 31, 1, 4294967295, 17, 0, 13, 7, 16, 48, 22, 97, 122, 0, 24, 13, 0, 4294967295, 13, 8, 22, 97, 122, 22, 48, 57, 0, 1, 16, 44, 17, 1, 18, 1] }; // END GENERATED // pattern p7 = re.compile('([a-z][a-z0-9]*,)+') #, 'a5,b7,c9,'), # A few sets // START GENERATED by generate_tests.py - #[rustfmt::skip] let p7 = Pattern { code: &[14, 4, 0, 2, 4294967295, 23, 29, 1, 4294967295, 17, 0, 13, 5, 22, 97, 122, 0, 24, 13, 0, 4294967295, 13, 8, 22, 97, 122, 22, 48, 57, 0, 1, 16, 44, 17, 1, 18, 1] }; + #[rustfmt::skip] let p7 = Pattern { pattern: "([a-z][a-z0-9]*,)+", code: &[14, 4, 0, 2, 4294967295, 23, 29, 1, 4294967295, 17, 0, 13, 5, 22, 97, 122, 0, 24, 13, 0, 4294967295, 13, 8, 22, 97, 122, 22, 48, 57, 0, 1, 16, 44, 17, 1, 18, 1] }; // END GENERATED // pattern p8 = re.compile('Python') #, 'Python'), # Simple text literal // START GENERATED by generate_tests.py - #[rustfmt::skip] let p8 = Pattern { code: &[14, 18, 3, 6, 6, 6, 6, 80, 121, 116, 104, 111, 110, 0, 0, 0, 0, 0, 0, 16, 80, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 1] }; + #[rustfmt::skip] let p8 = Pattern { pattern: "Python", code: &[14, 18, 3, 6, 6, 6, 6, 80, 121, 116, 104, 111, 110, 0, 0, 0, 0, 0, 0, 16, 80, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 1] }; // END GENERATED // pattern p9 = re.compile('.*Python') #, 'Python'), # Bad text literal // START GENERATED by generate_tests.py - #[rustfmt::skip] let p9 = Pattern { code: &[14, 4, 0, 6, 4294967295, 24, 5, 0, 4294967295, 2, 1, 16, 80, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 1] }; + #[rustfmt::skip] let p9 = Pattern { pattern: ".*Python", code: &[14, 4, 0, 6, 4294967295, 24, 5, 0, 4294967295, 2, 1, 16, 80, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 1] }; // END GENERATED // pattern p10 = re.compile('.*Python.*') #, 'Python'), # Worse text literal // START GENERATED by generate_tests.py - #[rustfmt::skip] let p10 = Pattern { code: &[14, 4, 0, 6, 4294967295, 24, 5, 0, 4294967295, 2, 1, 16, 80, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 24, 5, 0, 4294967295, 2, 1, 1] }; + #[rustfmt::skip] let p10 = Pattern { pattern: ".*Python.*", code: &[14, 4, 0, 6, 4294967295, 24, 5, 0, 4294967295, 2, 1, 16, 80, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 24, 5, 0, 4294967295, 2, 1, 1] }; // END GENERATED // pattern p11 = re.compile('.*(Python)') #, 'Python'), # Bad text literal with grouping // START GENERATED by generate_tests.py - #[rustfmt::skip] let p11 = Pattern { code: &[14, 4, 0, 6, 4294967295, 24, 5, 0, 4294967295, 2, 1, 17, 0, 16, 80, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 17, 1, 1] }; + #[rustfmt::skip] let p11 = Pattern { pattern: ".*(Python)", code: &[14, 4, 0, 6, 4294967295, 24, 5, 0, 4294967295, 2, 1, 17, 0, 16, 80, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 17, 1, 1] }; // END GENERATED let tests = [ @@ -87,25 +84,33 @@ fn benchmarks(b: &mut Bencher) { (p11, "Python"), ]; - b.iter(move || { - for (p, s) in &tests { - let (req, mut state) = p.state(s.clone()); - assert!(state.search(req)); - let (req, mut state) = p.state(s.clone()); - assert!(state.pymatch(&req)); - let (mut req, mut state) = p.state(s.clone()); - req.match_all = true; - assert!(state.pymatch(&req)); - let s2 = format!("{}{}{}", " ".repeat(10000), s, " ".repeat(10000)); - let (req, mut state) = p.state_range(s2.as_str(), 0..usize::MAX); - assert!(state.search(req)); - let (req, mut state) = p.state_range(s2.as_str(), 10000..usize::MAX); - assert!(state.pymatch(&req)); - let (req, mut state) = p.state_range(s2.as_str(), 10000..10000 + s.len()); - assert!(state.pymatch(&req)); - let (mut req, mut state) = p.state_range(s2.as_str(), 10000..10000 + s.len()); - req.match_all = true; - assert!(state.pymatch(&req)); - } - }) + let mut group = c.benchmark_group("basic"); + + for (p, s) in tests { + group.bench_with_input(BenchmarkId::new(p.pattern, s), s, |b, s| { + b.iter(|| { + let (req, mut state) = p.state(s); + assert!(state.search(req)); + let (req, mut state) = p.state(s); + assert!(state.pymatch(&req)); + let (mut req, mut state) = p.state(s); + req.match_all = true; + assert!(state.pymatch(&req)); + let s2 = format!("{}{}{}", " ".repeat(10000), s, " ".repeat(10000)); + let (req, mut state) = p.state_range(s2.as_str(), 0..usize::MAX); + assert!(state.search(req)); + let (req, mut state) = p.state_range(s2.as_str(), 10000..usize::MAX); + assert!(state.pymatch(&req)); + let (req, mut state) = p.state_range(s2.as_str(), 10000..10000 + s.len()); + assert!(state.pymatch(&req)); + let (mut req, mut state) = p.state_range(s2.as_str(), 10000..10000 + s.len()); + req.match_all = true; + assert!(state.pymatch(&req)); + }); + }); + } } + +criterion_group!(benches, basic); + +criterion_main!(benches); diff --git a/vm/sre_engine/generate_tests.py b/vm/sre_engine/generate_tests.py index 8adf043f29..6621c56813 100644 --- a/vm/sre_engine/generate_tests.py +++ b/vm/sre_engine/generate_tests.py @@ -1,9 +1,6 @@ import os from pathlib import Path import re -import sre_constants -import sre_compile -import sre_parse import json from itertools import chain @@ -11,13 +8,13 @@ sre_engine_magic = int(m.group(1)) del m -assert sre_constants.MAGIC == sre_engine_magic +assert re._constants.MAGIC == sre_engine_magic class CompiledPattern: @classmethod def compile(cls, pattern, flags=0): - p = sre_parse.parse(pattern) - code = sre_compile._code(p, flags) + p = re._parser.parse(pattern) + code = re._compiler._code(p, flags) self = cls() self.pattern = pattern self.code = code @@ -28,12 +25,32 @@ def compile(cls, pattern, flags=0): setattr(CompiledPattern, k, v) +class EscapeRustStr: + hardcoded = { + ord('\r'): '\\r', + ord('\t'): '\\t', + ord('\r'): '\\r', + ord('\n'): '\\n', + ord('\\'): '\\\\', + ord('\''): '\\\'', + ord('\"'): '\\\"', + } + @classmethod + def __class_getitem__(cls, ch): + if (rpl := cls.hardcoded.get(ch)) is not None: + return rpl + if ch in range(0x20, 0x7f): + return ch + return f"\\u{{{ch:x}}}" +def rust_str(s): + return '"' + s.translate(EscapeRustStr) + '"' + # matches `// pattern {varname} = re.compile(...)` pattern_pattern = re.compile(r"^((\s*)\/\/\s*pattern\s+(\w+)\s+=\s+(.+?))$(?:.+?END GENERATED)?", re.M | re.S) def replace_compiled(m): line, indent, varname, pattern = m.groups() pattern = eval(pattern, {"re": CompiledPattern}) - pattern = f"Pattern {{ code: &{json.dumps(pattern.code)} }}" + pattern = f"Pattern {{ pattern: {rust_str(pattern.pattern)}, code: &{json.dumps(pattern.code)} }}" return f'''{line} {indent}// START GENERATED by generate_tests.py {indent}#[rustfmt::skip] let {varname} = {pattern}; diff --git a/vm/sre_engine/tests/tests.rs b/vm/sre_engine/tests/tests.rs index 53494c5e3d..0946fd64ca 100644 --- a/vm/sre_engine/tests/tests.rs +++ b/vm/sre_engine/tests/tests.rs @@ -1,6 +1,7 @@ use rustpython_sre_engine::{Request, State, StrDrive}; struct Pattern { + pattern: &'static str, code: &'static [u32], } @@ -16,7 +17,7 @@ impl Pattern { fn test_2427() { // pattern lookbehind = re.compile(r'(?x)++x') // START GENERATED by generate_tests.py - #[rustfmt::skip] let p = Pattern { code: &[14, 4, 0, 2, 4294967295, 28, 8, 1, 4294967295, 27, 4, 16, 120, 1, 1, 16, 120, 1] }; + #[rustfmt::skip] let p = Pattern { pattern: "(?>x)++x", code: &[14, 4, 0, 2, 4294967295, 28, 8, 1, 4294967295, 27, 4, 16, 120, 1, 1, 16, 120, 1] }; // END GENERATED let (req, mut state) = p.state("xxx"); assert!(!state.pymatch(&req)); @@ -156,7 +157,7 @@ fn test_possessive_atomic_group() { fn test_bug_20998() { // pattern p = re.compile('[a-c]+', re.I) // START GENERATED by generate_tests.py - #[rustfmt::skip] let p = Pattern { code: &[14, 4, 0, 1, 4294967295, 24, 10, 1, 4294967295, 39, 5, 22, 97, 99, 0, 1, 1] }; + #[rustfmt::skip] let p = Pattern { pattern: "[a-c]+", code: &[14, 4, 0, 1, 4294967295, 24, 10, 1, 4294967295, 39, 5, 22, 97, 99, 0, 1, 1] }; // END GENERATED let (mut req, mut state) = p.state("ABC"); req.match_all = true; @@ -168,7 +169,7 @@ fn test_bug_20998() { fn test_bigcharset() { // pattern p = re.compile('[a-z]*', re.I) // START GENERATED by generate_tests.py - #[rustfmt::skip] let p = Pattern { code: &[14, 4, 0, 0, 4294967295, 24, 97, 0, 4294967295, 39, 92, 10, 3, 33685760, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 0, 0, 0, 134217726, 0, 0, 0, 0, 0, 131072, 0, 2147483648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1] }; + #[rustfmt::skip] let p = Pattern { pattern: "[a-z]*", code: &[14, 4, 0, 0, 4294967295, 24, 97, 0, 4294967295, 39, 92, 10, 3, 33685760, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 0, 0, 0, 134217726, 0, 0, 0, 0, 0, 131072, 0, 2147483648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1] }; // END GENERATED let (req, mut state) = p.state("x "); assert!(state.pymatch(&req)); @@ -178,4 +179,7 @@ fn test_bigcharset() { #[test] fn test_search_nonascii() { // pattern p = re.compile('\xe0+') + // START GENERATED by generate_tests.py + #[rustfmt::skip] let p = Pattern { pattern: "\u{e0}+", code: &[14, 4, 0, 1, 4294967295, 24, 6, 1, 4294967295, 16, 224, 1, 1] }; + // END GENERATED } From b4f0a589ed16db087ca3e5a7cde2159b2a554de9 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sat, 22 Feb 2025 16:48:02 -0800 Subject: [PATCH 21/26] platform-dependent Windows testing (#5536) * disable test_argparse on windows * fix test_exceptions and mark it as platform dependent * test importlib on windows * explain why windows tests fail * mark test_argparse as non platform-independent Signed-off-by: Ashwin Naren --- .github/workflows/ci.yaml | 15 +++++++++++---- Lib/test/test_importlib/frozen/test_loader.py | 4 ++++ Lib/test/test_importlib/resources/test_files.py | 7 +++++++ Lib/test/test_importlib/source/test_finder.py | 1 - vm/src/exceptions.rs | 4 +++- 5 files changed, 25 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9c0f736928..3d1d32d12d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -17,13 +17,23 @@ concurrency: env: CARGO_ARGS: --no-default-features --features stdlib,zlib,importlib,encodings,sqlite,ssl # Skip additional tests on Windows. They are checked on Linux and MacOS. + # test_argparse: UnicodeDecodeError + # test_glob: many failing tests + # test_io: many failing tests + # test_os: many failing tests + # test_pathlib: support.rmtree() failing + # test_posixpath: OSError: (22, 'The filename, directory name, or volume label syntax is incorrect. (os error 123)') + # test_unicode: AttributeError: module '_winapi' has no attribute 'GetACP' + # test_venv: couple of failing tests WINDOWS_SKIPS: >- + test_argparse test_glob - test_importlib test_io test_os + test_rlcompleter test_pathlib test_posixpath + test_unicode test_venv # configparser: https://github.com/RustPython/RustPython/issues/4995#issuecomment-1582397417 # socketserver: seems related to configparser crash. @@ -34,7 +44,6 @@ env: # only run on Linux to speed up the CI. PLATFORM_INDEPENDENT_TESTS: >- test__colorize - test_argparse test_array test_asyncgen test_binop @@ -59,7 +68,6 @@ env: test_dis test_enumerate test_exception_variations - test_exceptions test_float test_format test_fractions @@ -100,7 +108,6 @@ env: test_tuple test_types test_unary - test_unicode test_unpack test_weakref test_yield_from diff --git a/Lib/test/test_importlib/frozen/test_loader.py b/Lib/test/test_importlib/frozen/test_loader.py index 4f1af454b5..b1eb399d93 100644 --- a/Lib/test/test_importlib/frozen/test_loader.py +++ b/Lib/test/test_importlib/frozen/test_loader.py @@ -7,6 +7,7 @@ import contextlib import marshal import os.path +import sys import types import unittest import warnings @@ -77,6 +78,7 @@ def test_module(self): self.assertTrue(hasattr(module, '__spec__')) self.assertEqual(module.__spec__.loader_state.origname, name) + @unittest.skipIf(sys.platform == "win32", "TODO: RUSTPYTHON") def test_package(self): name = '__phello__' module, output = self.exec_module(name) @@ -90,6 +92,7 @@ def test_package(self): self.assertEqual(output, 'Hello world!\n') self.assertEqual(module.__spec__.loader_state.origname, name) + @unittest.skipIf(sys.platform == 'win32', "TODO:RUSTPYTHON Flaky on Windows") def test_lacking_parent(self): name = '__phello__.spam' with util.uncache('__phello__'): @@ -147,6 +150,7 @@ def test_get_source(self): result = self.machinery.FrozenImporter.get_source('__hello__') self.assertIsNone(result) + @unittest.skipIf(sys.platform == "win32", "TODO: RUSTPYTHON") def test_is_package(self): # Should be able to tell what is a package. test_for = (('__hello__', False), ('__phello__', True), diff --git a/Lib/test/test_importlib/resources/test_files.py b/Lib/test/test_importlib/resources/test_files.py index 1450cfb310..1d04cda1a8 100644 --- a/Lib/test/test_importlib/resources/test_files.py +++ b/Lib/test/test_importlib/resources/test_files.py @@ -52,6 +52,10 @@ class OpenDiskTests(FilesTests, unittest.TestCase): def setUp(self): self.data = data01 + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") + def test_read_bytes(self): + super().test_read_bytes() + class OpenZipTests(FilesTests, util.ZipSetup, unittest.TestCase): pass @@ -63,6 +67,9 @@ def setUp(self): self.data = namespacedata01 + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") + def test_read_bytes(self): + super().test_read_bytes() class SiteDir: def setUp(self): diff --git a/Lib/test/test_importlib/source/test_finder.py b/Lib/test/test_importlib/source/test_finder.py index 17d09d4cee..12db7c7d35 100644 --- a/Lib/test/test_importlib/source/test_finder.py +++ b/Lib/test/test_importlib/source/test_finder.py @@ -168,7 +168,6 @@ def test_no_read_directory(self): found = self._find(finder, 'doesnotexist') self.assertEqual(found, self.NOT_FOUND) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_ignore_file(self): # If a directory got changed to a file from underneath us, then don't # worry about looking for submodules. diff --git a/vm/src/exceptions.rs b/vm/src/exceptions.rs index 0b61c90174..94a16f912e 100644 --- a/vm/src/exceptions.rs +++ b/vm/src/exceptions.rs @@ -374,7 +374,9 @@ fn write_traceback_entry( writeln!( output, r##" File "{}", line {}, in {}"##, - filename, tb_entry.lineno, tb_entry.frame.code.obj_name + filename.trim_start_matches(r"\\?\"), + tb_entry.lineno, + tb_entry.frame.code.obj_name )?; print_source_line(output, filename, tb_entry.lineno.to_usize())?; From 429754fd3364492f87b0924b73736c5adc20148c Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sat, 22 Feb 2025 23:07:22 -0800 Subject: [PATCH 22/26] Fix unicode decode bug on surrogate error mode (#5546) * subtract with overflow to check for whether to use surrogate * enable test_argparse for windows on ci ------ Signed-off-by: Ashwin Naren --- .github/workflows/ci.yaml | 2 -- vm/src/codecs.rs | 7 ++++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 3d1d32d12d..1eeecbd2a7 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -17,7 +17,6 @@ concurrency: env: CARGO_ARGS: --no-default-features --features stdlib,zlib,importlib,encodings,sqlite,ssl # Skip additional tests on Windows. They are checked on Linux and MacOS. - # test_argparse: UnicodeDecodeError # test_glob: many failing tests # test_io: many failing tests # test_os: many failing tests @@ -26,7 +25,6 @@ env: # test_unicode: AttributeError: module '_winapi' has no attribute 'GetACP' # test_venv: couple of failing tests WINDOWS_SKIPS: >- - test_argparse test_glob test_io test_os diff --git a/vm/src/codecs.rs b/vm/src/codecs.rs index 898198f3d3..767bd873b6 100644 --- a/vm/src/codecs.rs +++ b/vm/src/codecs.rs @@ -619,11 +619,16 @@ fn surrogatepass_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<(PyOb // Not supported, fail with original exception return Err(err.downcast().unwrap()); } + + debug_assert!(range.start <= 0.max(s.len() - 1)); + debug_assert!(range.end >= 1.min(s.len())); + debug_assert!(range.end <= s.len()); + let mut c: u32 = 0; // Try decoding a single surrogate character. If there are more, // let the codec call us again. let p = &s.as_bytes()[range.start..]; - if p.len().saturating_sub(range.start) >= byte_length { + if p.len().overflowing_sub(range.start).0 >= byte_length { match standard_encoding { StandardEncoding::Utf8 => { if (p[0] as u32 & 0xf0) == 0xe0 From 7fada8b97ea1fc015e98d5d6e51ca64c9777ef74 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sat, 22 Feb 2025 20:41:13 -0800 Subject: [PATCH 23/26] fix _ctypes error names Signed-off-by: Ashwin Naren --- vm/src/stdlib/ctypes.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/vm/src/stdlib/ctypes.rs b/vm/src/stdlib/ctypes.rs index 027a680951..2580939b62 100644 --- a/vm/src/stdlib/ctypes.rs +++ b/vm/src/stdlib/ctypes.rs @@ -90,8 +90,8 @@ pub(crate) mod _ctypes { #[pyattr] pub const DICTFLAG_FINAL: u32 = 0x1000; - #[pyattr(once)] - fn error(vm: &VirtualMachine) -> PyTypeRef { + #[pyattr(name = "ArgumentError", once)] + fn argument_error(vm: &VirtualMachine) -> PyTypeRef { vm.ctx.new_exception_type( "_ctypes", "ArgumentError", @@ -99,6 +99,15 @@ pub(crate) mod _ctypes { ) } + #[pyattr(name = "FormatError", once)] + fn format_error(vm: &VirtualMachine) -> PyTypeRef { + vm.ctx.new_exception_type( + "_ctypes", + "FormatError", + Some(vec![vm.ctx.exceptions.exception_type.to_owned()]), + ) + } + pub fn get_size(ty: &str) -> usize { match ty { "u" => mem::size_of::(), From 31c5c3eb9d62f1895a66245f2ea8c06ecadb48e6 Mon Sep 17 00:00:00 2001 From: Axect Date: Mon, 24 Feb 2025 10:44:57 +0900 Subject: [PATCH 24/26] Update `puruspe` version to `0.4.0` To resolve the issue (#5496) --- stdlib/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/Cargo.toml b/stdlib/Cargo.toml index 31f5bd18d4..b5cfab98b4 100644 --- a/stdlib/Cargo.toml +++ b/stdlib/Cargo.toml @@ -49,7 +49,7 @@ base64 = "0.13.0" csv-core = "0.1.11" dyn-clone = "1.0.10" libz-sys = { version = "1.1", default-features = false, optional = true } -puruspe = "0.3.0" +puruspe = "0.4.0" xml-rs = "0.8.14" # random From 1f3a9672c3599e0f3985aa0d04153a0ae2a96dc0 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sun, 23 Feb 2025 20:21:02 -0800 Subject: [PATCH 25/26] Add _winapi.GetACP and enable test_unicode on windows (#5547) Signed-off-by: Ashwin Naren --- .github/workflows/ci.yaml | 2 -- Lib/test/test_charmapcodec.py | 3 --- Lib/test/test_codecs.py | 11 ----------- Lib/test/test_tokenize.py | 2 -- vm/Cargo.toml | 1 + vm/src/stdlib/winapi.rs | 5 +++++ 6 files changed, 6 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1eeecbd2a7..e96c2f26a6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -22,7 +22,6 @@ env: # test_os: many failing tests # test_pathlib: support.rmtree() failing # test_posixpath: OSError: (22, 'The filename, directory name, or volume label syntax is incorrect. (os error 123)') - # test_unicode: AttributeError: module '_winapi' has no attribute 'GetACP' # test_venv: couple of failing tests WINDOWS_SKIPS: >- test_glob @@ -31,7 +30,6 @@ env: test_rlcompleter test_pathlib test_posixpath - test_unicode test_venv # configparser: https://github.com/RustPython/RustPython/issues/4995#issuecomment-1582397417 # socketserver: seems related to configparser crash. diff --git a/Lib/test/test_charmapcodec.py b/Lib/test/test_charmapcodec.py index e69f1c6e4b..8ea75d9129 100644 --- a/Lib/test/test_charmapcodec.py +++ b/Lib/test/test_charmapcodec.py @@ -26,7 +26,6 @@ def codec_search_function(encoding): codecname = 'testcodec' class CharmapCodecTest(unittest.TestCase): - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_constructorx(self): self.assertEqual(str(b'abc', codecname), 'abc') self.assertEqual(str(b'xdef', codecname), 'abcdef') @@ -43,14 +42,12 @@ def test_encodex(self): self.assertEqual('dxf'.encode(codecname), b'dabcf') self.assertEqual('dxfx'.encode(codecname), b'dabcfabc') - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_constructory(self): self.assertEqual(str(b'ydef', codecname), 'def') self.assertEqual(str(b'defy', codecname), 'def') self.assertEqual(str(b'dyf', codecname), 'df') self.assertEqual(str(b'dyfy', codecname), 'df') - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_maptoundefined(self): self.assertRaises(UnicodeError, str, b'abc\001', codecname) diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 085b800b6d..f29e91e088 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -1827,7 +1827,6 @@ def test_decode(self): self.assertEqual(codecs.decode(b'[\xff]', 'ascii', errors='ignore'), '[]') - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_encode(self): self.assertEqual(codecs.encode('\xe4\xf6\xfc', 'latin-1'), b'\xe4\xf6\xfc') @@ -1846,7 +1845,6 @@ def test_register(self): self.assertRaises(TypeError, codecs.register) self.assertRaises(TypeError, codecs.register, 42) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; AttributeError: module '_winapi' has no attribute 'GetACP'") def test_unregister(self): name = "nonexistent_codec_name" search_function = mock.Mock() @@ -1859,28 +1857,23 @@ def test_unregister(self): self.assertRaises(LookupError, codecs.lookup, name) search_function.assert_not_called() - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_lookup(self): self.assertRaises(TypeError, codecs.lookup) self.assertRaises(LookupError, codecs.lookup, "__spam__") self.assertRaises(LookupError, codecs.lookup, " ") - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_getencoder(self): self.assertRaises(TypeError, codecs.getencoder) self.assertRaises(LookupError, codecs.getencoder, "__spam__") - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_getdecoder(self): self.assertRaises(TypeError, codecs.getdecoder) self.assertRaises(LookupError, codecs.getdecoder, "__spam__") - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_getreader(self): self.assertRaises(TypeError, codecs.getreader) self.assertRaises(LookupError, codecs.getreader, "__spam__") - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_getwriter(self): self.assertRaises(TypeError, codecs.getwriter) self.assertRaises(LookupError, codecs.getwriter, "__spam__") @@ -1939,7 +1932,6 @@ def test_undefined(self): self.assertRaises(UnicodeError, codecs.decode, b'abc', 'undefined', errors) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_file_closes_if_lookup_error_raised(self): mock_open = mock.mock_open() with mock.patch('builtins.open', mock_open) as file: @@ -3287,7 +3279,6 @@ def test_multiple_args(self): self.check_note(RuntimeError('a', 'b', 'c'), msg_re) # http://bugs.python.org/issue19609 - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_codec_lookup_failure(self): msg = "^unknown encoding: {}$".format(self.codec_name) with self.assertRaisesRegex(LookupError, msg): @@ -3523,8 +3514,6 @@ def test_incremental(self): False) self.assertEqual(decoded, ('abc', 3)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_mbcs_alias(self): # Check that looking up our 'default' codepage will return # mbcs when we don't have a more specific one available diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index e2d2f89454..44ef4e2416 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -1237,7 +1237,6 @@ def test_utf8_normalization(self): found, consumed_lines = detect_encoding(rl) self.assertEqual(found, "utf-8") - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_short_files(self): readline = self.get_readline((b'print(something)\n',)) encoding, consumed_lines = detect_encoding(readline) @@ -1316,7 +1315,6 @@ def readline(self): ins = Bunk(lines, path) detect_encoding(ins.readline) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_open_error(self): # Issue #23840: open() must close the binary file on error m = BytesIO(b'#coding:xxx') diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 895b29aa50..acc645bb74 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -125,6 +125,7 @@ features = [ workspace = true features = [ "Win32_Foundation", + "Win32_Globalization", "Win32_Networking_WinSock", "Win32_Security", "Win32_Storage_FileSystem", diff --git a/vm/src/stdlib/winapi.rs b/vm/src/stdlib/winapi.rs index ad6db12474..8feef61b14 100644 --- a/vm/src/stdlib/winapi.rs +++ b/vm/src/stdlib/winapi.rs @@ -119,6 +119,11 @@ mod _winapi { Ok(HANDLE(target)) } + #[pyfunction] + fn GetACP() -> u32 { + unsafe { windows_sys::Win32::Globalization::GetACP() } + } + #[pyfunction] fn GetCurrentProcess() -> HANDLE { unsafe { windows::Win32::System::Threading::GetCurrentProcess() } From d7a72b57553473a46fe2bfcf12a6a585f75838d4 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sun, 23 Feb 2025 20:02:26 -0800 Subject: [PATCH 26/26] add constants and implement functions Signed-off-by: Ashwin Naren --- Cargo.lock | 5 +-- vm/src/stdlib/winapi.rs | 68 +++++++++++++++++++++++++++++++++++++++-- vm/src/stdlib/winreg.rs | 10 ++++-- 3 files changed, 75 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3711b6f16c..2075ea3cbd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1644,11 +1644,12 @@ dependencies = [ [[package]] name = "puruspe" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5666f1226a41ebb7191b702947674f4814c9919da58a9f91a60090412664bbce" +checksum = "d76c522e44709f541a403db419a7e34d6fbbc8e6b208589ae29a030cddeefd96" dependencies = [ "lambert_w", + "num-complex", ] [[package]] diff --git a/vm/src/stdlib/winapi.rs b/vm/src/stdlib/winapi.rs index 8feef61b14..6cf8c45bfe 100644 --- a/vm/src/stdlib/winapi.rs +++ b/vm/src/stdlib/winapi.rs @@ -28,10 +28,20 @@ mod _winapi { ERROR_PIPE_CONNECTED, ERROR_SEM_TIMEOUT, GENERIC_READ, GENERIC_WRITE, STILL_ACTIVE, WAIT_ABANDONED, WAIT_ABANDONED_0, WAIT_OBJECT_0, WAIT_TIMEOUT, }, + Globalization::{ + LCMAP_FULLWIDTH, LCMAP_HALFWIDTH, LCMAP_HIRAGANA, LCMAP_KATAKANA, + LCMAP_LINGUISTIC_CASING, LCMAP_LOWERCASE, LCMAP_SIMPLIFIED_CHINESE, LCMAP_TITLECASE, + LCMAP_TRADITIONAL_CHINESE, LCMAP_UPPERCASE, + }, Storage::FileSystem::{ - FILE_FLAG_FIRST_PIPE_INSTANCE, FILE_FLAG_OVERLAPPED, FILE_GENERIC_READ, - FILE_GENERIC_WRITE, FILE_TYPE_CHAR, FILE_TYPE_DISK, FILE_TYPE_PIPE, FILE_TYPE_REMOTE, - FILE_TYPE_UNKNOWN, OPEN_EXISTING, PIPE_ACCESS_DUPLEX, PIPE_ACCESS_INBOUND, SYNCHRONIZE, + COPYFILE2_CALLBACK_CHUNK_FINISHED, COPYFILE2_CALLBACK_CHUNK_STARTED, + COPYFILE2_CALLBACK_ERROR, COPYFILE2_CALLBACK_POLL_CONTINUE, + COPYFILE2_CALLBACK_STREAM_FINISHED, COPYFILE2_CALLBACK_STREAM_STARTED, + COPYFILE2_PROGRESS_CANCEL, COPYFILE2_PROGRESS_CONTINUE, COPYFILE2_PROGRESS_PAUSE, + COPYFILE2_PROGRESS_QUIET, COPYFILE2_PROGRESS_STOP, FILE_FLAG_FIRST_PIPE_INSTANCE, + FILE_FLAG_OVERLAPPED, FILE_GENERIC_READ, FILE_GENERIC_WRITE, FILE_TYPE_CHAR, + FILE_TYPE_DISK, FILE_TYPE_PIPE, FILE_TYPE_REMOTE, FILE_TYPE_UNKNOWN, OPEN_EXISTING, + PIPE_ACCESS_DUPLEX, PIPE_ACCESS_INBOUND, SYNCHRONIZE, }, System::{ Console::{STD_ERROR_HANDLE, STD_INPUT_HANDLE, STD_OUTPUT_HANDLE}, @@ -53,6 +63,13 @@ mod _winapi { IDLE_PRIORITY_CLASS, INFINITE, NORMAL_PRIORITY_CLASS, PROCESS_DUP_HANDLE, REALTIME_PRIORITY_CLASS, STARTF_USESHOWWINDOW, STARTF_USESTDHANDLES, }, + WindowsProgramming::{ + COPY_FILE_ALLOW_DECRYPTED_DESTINATION, COPY_FILE_COPY_SYMLINK, + COPY_FILE_FAIL_IF_EXISTS, COPY_FILE_NO_BUFFERING, COPY_FILE_NO_OFFLOAD, + COPY_FILE_OPEN_SOURCE_FOR_WRITE, COPY_FILE_REQUEST_COMPRESSED_TRAFFIC, + COPY_FILE_REQUEST_SECURITY_PRIVILEGES, COPY_FILE_RESTARTABLE, + COPY_FILE_RESUME_FROM_PAUSE, + }, }, UI::WindowsAndMessaging::SW_HIDE, }; @@ -142,6 +159,16 @@ mod _winapi { } } + #[pyfunction] + fn GetLastError() -> u32 { + unsafe { windows_sys::Win32::Foundation::GetLastError() } + } + + #[pyfunction] + fn GetVersion() -> u32 { + unsafe { windows_sys::Win32::System::SystemInformation::GetVersion() } + } + #[derive(FromArgs)] struct CreateProcessArgs { #[pyarg(positional)] @@ -254,6 +281,21 @@ mod _winapi { )) } + #[pyfunction] + fn OpenProcess( + desired_access: u32, + inherit_handle: bool, + process_id: u32, + ) -> windows_sys::Win32::Foundation::HANDLE { + unsafe { + windows_sys::Win32::System::Threading::OpenProcess( + desired_access, + BOOL::from(inherit_handle), + process_id, + ) + } + } + #[pyfunction] fn NeedCurrentDirectoryForExePath(exe_name: PyStrRef) -> bool { let exe_name = exe_name.as_str().to_wide_with_nul(); @@ -452,4 +494,24 @@ mod _winapi { let (path, _) = path.split_at(length as usize); Ok(String::from_utf16(path).unwrap()) } + + #[pyfunction] + fn OpenMutexW(desired_access: u32, inherit_handle: bool, name: u16) -> PyResult { + let handle = unsafe { + windows_sys::Win32::System::Threading::OpenMutexW( + desired_access, + BOOL::from(inherit_handle), + windows_sys::core::PCWSTR::from(name as _), + ) + }; + // if handle.is_invalid() { + // return Err(errno_err(vm)); + // } + Ok(handle) + } + + #[pyfunction] + fn ReleaseMutex(handle: isize) -> WindowsSysResult { + WindowsSysResult(unsafe { windows_sys::Win32::System::Threading::ReleaseMutex(handle) }) + } } diff --git a/vm/src/stdlib/winreg.rs b/vm/src/stdlib/winreg.rs index 747c58cfd8..f25ca0b960 100644 --- a/vm/src/stdlib/winreg.rs +++ b/vm/src/stdlib/winreg.rs @@ -47,9 +47,13 @@ mod winreg { // value types #[pyattr] pub use windows_sys::Win32::System::Registry::{ - REG_BINARY, REG_DWORD, REG_DWORD_BIG_ENDIAN, REG_DWORD_LITTLE_ENDIAN, REG_EXPAND_SZ, - REG_FULL_RESOURCE_DESCRIPTOR, REG_LINK, REG_MULTI_SZ, REG_NONE, REG_QWORD, - REG_QWORD_LITTLE_ENDIAN, REG_RESOURCE_LIST, REG_RESOURCE_REQUIREMENTS_LIST, REG_SZ, + REG_BINARY, REG_CREATED_NEW_KEY, REG_DWORD, REG_DWORD_BIG_ENDIAN, REG_DWORD_LITTLE_ENDIAN, + REG_EXPAND_SZ, REG_FULL_RESOURCE_DESCRIPTOR, REG_LINK, REG_MULTI_SZ, REG_NONE, + REG_NOTIFY_CHANGE_ATTRIBUTES, REG_NOTIFY_CHANGE_LAST_SET, REG_NOTIFY_CHANGE_NAME, + REG_NOTIFY_CHANGE_SECURITY, REG_OPENED_EXISTING_KEY, REG_OPTION_BACKUP_RESTORE, + REG_OPTION_CREATE_LINK, REG_OPTION_NON_VOLATILE, REG_OPTION_OPEN_LINK, REG_OPTION_RESERVED, + REG_OPTION_VOLATILE, REG_QWORD, REG_QWORD_LITTLE_ENDIAN, REG_RESOURCE_LIST, + REG_RESOURCE_REQUIREMENTS_LIST, REG_SZ, REG_WHOLE_HIVE_VOLATILE, }; #[pyattr]