From defcadafbb1c627384936ba193ff995628dae6bf Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sun, 2 Mar 2025 19:21:01 -0800 Subject: [PATCH 01/16] publish demo on weekly release --- .github/workflows/release.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 99035d04fc..0d4b3a270c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -106,6 +106,28 @@ jobs: name: rustpython-release-wasm32-wasip1 path: target/rustpython-release-wasm32-wasip1.wasm + - name: install wasm-pack + run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh + - uses: actions/setup-node@v4 + - uses: mwilliamson/setup-wabt-action@v3 + with: { wabt-version: "1.0.30" } + - name: build notebook demo + if: github.ref == 'refs/heads/release' + run: | + npm install + npm run dist + mv dist ../demo/dist/notebook + env: + NODE_OPTIONS: "--openssl-legacy-provider" + working-directory: ./wasm/notebook + - name: Deploy demo to Github Pages + uses: peaceiris/actions-gh-pages@v4 + env: + ACTIONS_DEPLOY_KEY: ${{ secrets.ACTIONS_DEMO_DEPLOY_KEY }} + PUBLISH_DIR: ./wasm/demo/dist + EXTERNAL_REPOSITORY: RustPython/demo + PUBLISH_BRANCH: master + release: runs-on: ubuntu-latest needs: [build, build-wasm] From 6804dd4363b4368113750b2f29814f231ec24375 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sun, 2 Mar 2025 19:26:29 -0800 Subject: [PATCH 02/16] use rust-toolchain targets options instead of using rustup --- .github/workflows/release.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0d4b3a270c..98b7de4823 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -89,10 +89,8 @@ jobs: steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable - - - name: Set up Environment - shell: bash - run: rustup target add wasm32-wasip1 + with: + targets: wasm32-wasip1 - name: Build RustPython run: cargo build --target wasm32-wasip1 --no-default-features --features freeze-stdlib,stdlib --release @@ -112,7 +110,6 @@ jobs: - uses: mwilliamson/setup-wabt-action@v3 with: { wabt-version: "1.0.30" } - name: build notebook demo - if: github.ref == 'refs/heads/release' run: | npm install npm run dist From 3ae1160868a5aa4b341eb5956935f0e03b4169e7 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Mon, 3 Mar 2025 01:09:07 -0800 Subject: [PATCH 03/16] Remove winapi dependency --- Cargo.lock | 1 - stdlib/Cargo.toml | 6 ---- stdlib/src/socket.rs | 66 ++++++++++++++++++++++++++++++++------------ 3 files changed, 48 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ec6e5f0295..7ff4313548 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2252,7 +2252,6 @@ dependencies = [ "unicode_names2", "uuid", "widestring", - "winapi", "windows-sys 0.52.0", "xml-rs", ] diff --git a/stdlib/Cargo.toml b/stdlib/Cargo.toml index 1086dea090..f2ace7b26a 100644 --- a/stdlib/Cargo.toml +++ b/stdlib/Cargo.toml @@ -117,12 +117,6 @@ paste = { workspace = true } schannel = { workspace = true } widestring = { workspace = true } -[target.'cfg(windows)'.dependencies.winapi] -version = "0.3.9" -features = [ - "winsock2", "ifdef", "netioapi", "ws2tcpip", -] - [target.'cfg(windows)'.dependencies.windows-sys] workspace = true features = [ diff --git a/stdlib/src/socket.rs b/stdlib/src/socket.rs index 533a0a0b24..940fa3299e 100644 --- a/stdlib/src/socket.rs +++ b/stdlib/src/socket.rs @@ -35,17 +35,27 @@ mod _socket { use libc as c; #[cfg(windows)] mod c { - pub use winapi::shared::netioapi::{if_indextoname, if_nametoindex}; - pub use winapi::shared::ws2def::{ - INADDR_ANY, INADDR_BROADCAST, INADDR_LOOPBACK, INADDR_NONE, - }; - pub use winapi::um::winsock2::{ - SO_EXCLUSIVEADDRUSE, getprotobyname, getservbyname, getservbyport, getsockopt, + pub use windows_sys::Win32::NetworkManagement::IpHelper::{if_indextoname, if_nametoindex}; + + pub const INADDR_ANY: u32 = 0x00000000; + pub const INADDR_LOOPBACK: u32 = 0x7f000001; + pub const INADDR_BROADCAST: u32 = 0xffffffff; + pub const INADDR_NONE: u32 = 0xffffffff; + + pub use windows_sys::Win32::Networking::WinSock::{ + SO_REUSEADDR as SO_EXCLUSIVEADDRUSE, getprotobyname, getservbyname, getservbyport, getsockopt, setsockopt, }; - pub use winapi::um::ws2tcpip::{ - EAI_AGAIN, EAI_BADFLAGS, EAI_FAIL, EAI_FAMILY, EAI_MEMORY, EAI_NODATA, EAI_NONAME, - EAI_SERVICE, EAI_SOCKTYPE, + pub use windows_sys::Win32::Networking::WinSock::{ + WSATRY_AGAIN as EAI_AGAIN, + WSAEINVAL as EAI_BADFLAGS, + WSANO_RECOVERY as EAI_FAIL, + WSAEAFNOSUPPORT as EAI_FAMILY, + WSA_NOT_ENOUGH_MEMORY as EAI_MEMORY, + WSAHOST_NOT_FOUND as EAI_NODATA, + WSAHOST_NOT_FOUND as EAI_NONAME, + WSATYPE_NOT_FOUND as EAI_SERVICE, + WSAESOCKTNOSUPPORT as EAI_SOCKTYPE, }; pub use windows_sys::Win32::Networking::WinSock::{ AF_APPLETALK, AF_DECnet, AF_IPX, AF_LINK, AI_ADDRCONFIG, AI_ALL, AI_CANONNAME, @@ -755,7 +765,7 @@ mod _socket { } #[cfg(windows)] - use winapi::shared::netioapi; + use windows_sys::Win32::NetworkManagement::IpHelper; fn get_raw_sock(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { #[cfg(unix)] @@ -1755,6 +1765,9 @@ mod _socket { .map(|s| s.to_cstring(vm)) .transpose()?; let cstr_proto = cstr_opt_as_ptr(&cstr_proto); + #[cfg(windows)] + let serv = unsafe { c::getservbyname(cstr_name.as_ptr() as windows_sys::core::PCSTR, cstr_proto as windows_sys::core::PCSTR) }; + #[cfg(not(windows))] let serv = unsafe { c::getservbyname(cstr_name.as_ptr(), cstr_proto) }; if serv.is_null() { return Err(vm.new_os_error("service/proto not found".to_owned())); @@ -1777,10 +1790,16 @@ mod _socket { .map(|s| s.to_cstring(vm)) .transpose()?; let cstr_proto = cstr_opt_as_ptr(&cstr_proto); + #[cfg(windows)] + let serv = unsafe { c::getservbyport(port.to_be() as _, cstr_proto as windows_sys::core::PCSTR) }; + #[cfg(not(windows))] let serv = unsafe { c::getservbyport(port.to_be() as _, cstr_proto) }; if serv.is_null() { return Err(vm.new_os_error("port/proto not found".to_owned())); } + #[cfg(windows)] + let s = unsafe { ffi::CStr::from_ptr((*serv).s_name as *const i8) }; + #[cfg(not(windows))] let s = unsafe { ffi::CStr::from_ptr((*serv).s_name) }; Ok(s.to_string_lossy().into_owned()) } @@ -2033,6 +2052,9 @@ mod _socket { #[pyfunction] fn getprotobyname(name: PyStrRef, vm: &VirtualMachine) -> PyResult { let cstr = name.to_cstring(vm)?; + #[cfg(windows)] + let proto = unsafe { c::getprotobyname(cstr.as_ptr() as *const u8) }; + #[cfg(not(windows))] let proto = unsafe { c::getprotobyname(cstr.as_ptr()) }; if proto.is_null() { return Err(vm.new_os_error("protocol not found".to_owned())); @@ -2111,13 +2133,16 @@ mod _socket { #[cfg(all(unix, not(target_os = "redox")))] type IfIndex = c::c_uint; #[cfg(windows)] - type IfIndex = winapi::shared::ifdef::NET_IFINDEX; + type IfIndex = u32; #[cfg(not(target_os = "redox"))] #[pyfunction] fn if_nametoindex(name: FsPath, vm: &VirtualMachine) -> PyResult { let name = name.to_cstring(vm)?; + #[cfg(windows)] + let ret = unsafe { c::if_nametoindex(name.as_ptr() as *const u8) }; + #[cfg(not(windows))] let ret = unsafe { c::if_nametoindex(name.as_ptr()) }; if ret == 0 { Err(vm.new_os_error("no interface with this name".to_owned())) @@ -2134,6 +2159,9 @@ mod _socket { if ret.is_null() { Err(crate::vm::stdlib::os::errno_err(vm)) } else { + #[cfg(windows)] + let buf = unsafe { ffi::CStr::from_ptr(buf.as_ptr() as *const i8) }; + #[cfg(not(windows))] let buf = unsafe { ffi::CStr::from_ptr(buf.as_ptr()) }; Ok(buf.to_string_lossy().into_owned()) } @@ -2152,6 +2180,8 @@ mod _socket { ))] #[pyfunction] fn if_nameindex(vm: &VirtualMachine) -> PyResult> { + use windows_sys::Win32::NetworkManagement::Ndis::NET_LUID_LH; + #[cfg(not(windows))] { let list = nix::net::if_::if_nameindex() @@ -2182,11 +2212,11 @@ mod _socket { return Ok(list); fn get_name( - luid: &winapi::shared::ifdef::NET_LUID, + luid: &NET_LUID_LH, ) -> io::Result { let mut buf = [0; c::IF_NAMESIZE + 1]; let ret = unsafe { - netioapi::ConvertInterfaceLuidToNameW(luid, buf.as_mut_ptr(), buf.len()) + IpHelper::ConvertInterfaceLuidToNameW(luid, buf.as_mut_ptr(), buf.len()) }; if ret == 0 { Ok(widestring::WideCString::from_ustr_truncate( @@ -2197,12 +2227,12 @@ mod _socket { } } struct MibTable { - ptr: ptr::NonNull, + ptr: ptr::NonNull, } impl MibTable { fn get_raw() -> io::Result { let mut ptr = ptr::null_mut(); - let ret = unsafe { netioapi::GetIfTable2Ex(netioapi::MibIfTableRaw, &mut ptr) }; + let ret = unsafe { IpHelper::GetIfTable2Ex(IpHelper::MibIfTableRaw, &mut ptr) }; if ret == 0 { let ptr = unsafe { ptr::NonNull::new_unchecked(ptr) }; Ok(Self { ptr }) @@ -2212,17 +2242,17 @@ mod _socket { } } impl MibTable { - fn as_slice(&self) -> &[netioapi::MIB_IF_ROW2] { + fn as_slice(&self) -> &[IpHelper::MIB_IF_ROW2] { unsafe { let p = self.ptr.as_ptr(); - let ptr = &raw const (*p).Table as *const netioapi::MIB_IF_ROW2; + let ptr = &raw const (*p).Table as *const IpHelper::MIB_IF_ROW2; std::slice::from_raw_parts(ptr, (*p).NumEntries as usize) } } } impl Drop for MibTable { fn drop(&mut self) { - unsafe { netioapi::FreeMibTable(self.ptr.as_ptr() as *mut _) } + unsafe { IpHelper::FreeMibTable(self.ptr.as_ptr() as *mut _) }; } } } From 4d9804f188c9604ac4ce63f89a1f7befd7453562 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Mon, 3 Mar 2025 01:37:27 -0800 Subject: [PATCH 04/16] formatting --- stdlib/src/socket.rs | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/stdlib/src/socket.rs b/stdlib/src/socket.rs index 940fa3299e..cfdf4c8a4e 100644 --- a/stdlib/src/socket.rs +++ b/stdlib/src/socket.rs @@ -36,27 +36,12 @@ mod _socket { #[cfg(windows)] mod c { pub use windows_sys::Win32::NetworkManagement::IpHelper::{if_indextoname, if_nametoindex}; - + pub const INADDR_ANY: u32 = 0x00000000; pub const INADDR_LOOPBACK: u32 = 0x7f000001; pub const INADDR_BROADCAST: u32 = 0xffffffff; pub const INADDR_NONE: u32 = 0xffffffff; - - pub use windows_sys::Win32::Networking::WinSock::{ - SO_REUSEADDR as SO_EXCLUSIVEADDRUSE, getprotobyname, getservbyname, getservbyport, getsockopt, - setsockopt, - }; - pub use windows_sys::Win32::Networking::WinSock::{ - WSATRY_AGAIN as EAI_AGAIN, - WSAEINVAL as EAI_BADFLAGS, - WSANO_RECOVERY as EAI_FAIL, - WSAEAFNOSUPPORT as EAI_FAMILY, - WSA_NOT_ENOUGH_MEMORY as EAI_MEMORY, - WSAHOST_NOT_FOUND as EAI_NODATA, - WSAHOST_NOT_FOUND as EAI_NONAME, - WSATYPE_NOT_FOUND as EAI_SERVICE, - WSAESOCKTNOSUPPORT as EAI_SOCKTYPE, - }; + pub use windows_sys::Win32::Networking::WinSock::{ AF_APPLETALK, AF_DECnet, AF_IPX, AF_LINK, AI_ADDRCONFIG, AI_ALL, AI_CANONNAME, AI_NUMERICSERV, AI_V4MAPPED, IP_ADD_MEMBERSHIP, IP_DROP_MEMBERSHIP, IP_HDRINCL, @@ -78,6 +63,17 @@ mod _socket { SOL_SOCKET, SOMAXCONN, TCP_NODELAY, WSAEBADF, WSAECONNRESET, WSAENOTSOCK, WSAEWOULDBLOCK, }; + pub use windows_sys::Win32::Networking::WinSock::{ + SO_REUSEADDR as SO_EXCLUSIVEADDRUSE, getprotobyname, getservbyname, getservbyport, + getsockopt, setsockopt, + }; + pub use windows_sys::Win32::Networking::WinSock::{ + WSA_NOT_ENOUGH_MEMORY as EAI_MEMORY, WSAEAFNOSUPPORT as EAI_FAMILY, + WSAEINVAL as EAI_BADFLAGS, WSAESOCKTNOSUPPORT as EAI_SOCKTYPE, + WSAHOST_NOT_FOUND as EAI_NODATA, WSAHOST_NOT_FOUND as EAI_NONAME, + WSANO_RECOVERY as EAI_FAIL, WSATRY_AGAIN as EAI_AGAIN, + WSATYPE_NOT_FOUND as EAI_SERVICE, + }; pub const IF_NAMESIZE: usize = windows_sys::Win32::NetworkManagement::Ndis::IF_MAX_STRING_SIZE as _; pub const AF_UNSPEC: i32 = windows_sys::Win32::Networking::WinSock::AF_UNSPEC as _; @@ -1766,7 +1762,12 @@ mod _socket { .transpose()?; let cstr_proto = cstr_opt_as_ptr(&cstr_proto); #[cfg(windows)] - let serv = unsafe { c::getservbyname(cstr_name.as_ptr() as windows_sys::core::PCSTR, cstr_proto as windows_sys::core::PCSTR) }; + let serv = unsafe { + c::getservbyname( + cstr_name.as_ptr() as windows_sys::core::PCSTR, + cstr_proto as windows_sys::core::PCSTR, + ) + }; #[cfg(not(windows))] let serv = unsafe { c::getservbyname(cstr_name.as_ptr(), cstr_proto) }; if serv.is_null() { @@ -1791,7 +1792,8 @@ mod _socket { .transpose()?; let cstr_proto = cstr_opt_as_ptr(&cstr_proto); #[cfg(windows)] - let serv = unsafe { c::getservbyport(port.to_be() as _, cstr_proto as windows_sys::core::PCSTR) }; + let serv = + unsafe { c::getservbyport(port.to_be() as _, cstr_proto as windows_sys::core::PCSTR) }; #[cfg(not(windows))] let serv = unsafe { c::getservbyport(port.to_be() as _, cstr_proto) }; if serv.is_null() { @@ -2211,9 +2213,7 @@ mod _socket { let list = list.collect::>()?; return Ok(list); - fn get_name( - luid: &NET_LUID_LH, - ) -> io::Result { + fn get_name(luid: &NET_LUID_LH) -> io::Result { let mut buf = [0; c::IF_NAMESIZE + 1]; let ret = unsafe { IpHelper::ConvertInterfaceLuidToNameW(luid, buf.as_mut_ptr(), buf.len()) From 8d2c6807d28000c8ee3dc4b7d65d86c8f63bcb52 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Mon, 3 Mar 2025 10:50:19 -0800 Subject: [PATCH 05/16] fix non-windows build --- stdlib/src/socket.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/stdlib/src/socket.rs b/stdlib/src/socket.rs index cfdf4c8a4e..b46514eb70 100644 --- a/stdlib/src/socket.rs +++ b/stdlib/src/socket.rs @@ -2182,8 +2182,6 @@ mod _socket { ))] #[pyfunction] fn if_nameindex(vm: &VirtualMachine) -> PyResult> { - use windows_sys::Win32::NetworkManagement::Ndis::NET_LUID_LH; - #[cfg(not(windows))] { let list = nix::net::if_::if_nameindex() @@ -2202,6 +2200,7 @@ mod _socket { #[cfg(windows)] { use std::ptr; + use windows_sys::Win32::NetworkManagement::Ndis::NET_LUID_LH; let table = MibTable::get_raw().map_err(|err| err.into_pyexception(vm))?; let list = table.as_slice().iter().map(|entry| { From 05cb8c0b73918f12ca6e86f432f5b4b11dc523b5 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Tue, 4 Mar 2025 01:28:53 -0800 Subject: [PATCH 06/16] Update socket.rs Co-authored-by: Jeong, YunWon <69878+youknowone@users.noreply.github.com> --- stdlib/src/socket.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/src/socket.rs b/stdlib/src/socket.rs index b46514eb70..149408a6f3 100644 --- a/stdlib/src/socket.rs +++ b/stdlib/src/socket.rs @@ -2135,7 +2135,7 @@ mod _socket { #[cfg(all(unix, not(target_os = "redox")))] type IfIndex = c::c_uint; #[cfg(windows)] - type IfIndex = u32; + type IfIndex = u32; // NET_IFINDEX but windows-sys 0.59 doesn't have it #[cfg(not(target_os = "redox"))] #[pyfunction] From 33940726a8ad2f91cf7b19c8d24989f74298091b Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Wed, 5 Mar 2025 10:36:04 -0800 Subject: [PATCH 07/16] upgrade to windows-sys 0.59.0 --- Cargo.lock | 6 +++--- Cargo.toml | 2 +- common/src/fileutils.rs | 10 +++++----- stdlib/Cargo.toml | 1 + stdlib/src/overlapped.rs | 42 ++++++++++++++++++++++------------------ stdlib/src/socket.rs | 8 ++------ vm/Cargo.toml | 1 + vm/src/stdlib/nt.rs | 4 ++-- vm/src/stdlib/winapi.rs | 34 ++++++++++++++++---------------- vm/src/windows.rs | 2 +- 10 files changed, 56 insertions(+), 54 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7ff4313548..1c428c2d73 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2023,7 +2023,7 @@ dependencies = [ "siphasher 0.3.11", "volatile", "widestring", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -2252,7 +2252,7 @@ dependencies = [ "unicode_names2", "uuid", "widestring", - "windows-sys 0.52.0", + "windows-sys 0.59.0", "xml-rs", ] @@ -2332,7 +2332,7 @@ dependencies = [ "which", "widestring", "windows", - "windows-sys 0.52.0", + "windows-sys 0.59.0", "winreg", ] diff --git a/Cargo.toml b/Cargo.toml index 807e821ca7..14e66b39eb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -181,7 +181,7 @@ thiserror = "2.0" thread_local = "1.1.8" unicode_names2 = "1.3.0" widestring = "1.1.0" -windows-sys = "0.52.0" +windows-sys = "0.59.0" wasm-bindgen = "0.2.100" # Lints diff --git a/common/src/fileutils.rs b/common/src/fileutils.rs index 7d5ff01942..ce4aa8250a 100644 --- a/common/src/fileutils.rs +++ b/common/src/fileutils.rs @@ -116,7 +116,7 @@ pub mod windows { let h = h?; // reset stat? - let file_type = unsafe { GetFileType(h) }; + let file_type = unsafe { GetFileType(h as _) }; if file_type == FILE_TYPE_UNKNOWN { return Err(std::io::Error::last_os_error()); } @@ -138,10 +138,10 @@ pub mod windows { let mut basic_info: FILE_BASIC_INFO = unsafe { std::mem::zeroed() }; let mut id_info: FILE_ID_INFO = unsafe { std::mem::zeroed() }; - if unsafe { GetFileInformationByHandle(h, &mut info) } == 0 + if unsafe { GetFileInformationByHandle(h as _, &mut info) } == 0 || unsafe { GetFileInformationByHandleEx( - h, + h as _, FileBasicInfo, &mut basic_info as *mut _ as *mut _, std::mem::size_of_val(&basic_info) as u32, @@ -153,7 +153,7 @@ pub mod windows { let p_id_info = if unsafe { GetFileInformationByHandleEx( - h, + h as _, FileIdInfo, &mut id_info as *mut _ as *mut _, std::mem::size_of_val(&id_info) as u32, @@ -320,7 +320,7 @@ pub mod windows { .get_or_init(|| { let library_name = OsString::from("api-ms-win-core-file-l2-1-4").to_wide_with_nul(); let module = unsafe { LoadLibraryW(library_name.as_ptr()) }; - if module == 0 { + if module == std::ptr::null_mut() { return None; } let name = CString::new("GetFileInformationByName").unwrap(); diff --git a/stdlib/Cargo.toml b/stdlib/Cargo.toml index f2ace7b26a..d7552844be 100644 --- a/stdlib/Cargo.toml +++ b/stdlib/Cargo.toml @@ -125,6 +125,7 @@ features = [ "Win32_NetworkManagement_Ndis", "Win32_Security_Cryptography", "Win32_System_Environment", + "Win32_System_IO" ] [target.'cfg(target_os = "macos")'.dependencies] diff --git a/stdlib/src/overlapped.rs b/stdlib/src/overlapped.rs index f40494a432..d3c46cad0a 100644 --- a/stdlib/src/overlapped.rs +++ b/stdlib/src/overlapped.rs @@ -24,13 +24,17 @@ mod _overlapped { use windows_sys::Win32::{ Foundation::{ ERROR_IO_PENDING, ERROR_NETNAME_DELETED, ERROR_OPERATION_ABORTED, ERROR_PIPE_BUSY, - ERROR_PORT_UNREACHABLE, ERROR_SEM_TIMEOUT, INVALID_HANDLE_VALUE, + ERROR_PORT_UNREACHABLE, ERROR_SEM_TIMEOUT, }, Networking::WinSock::{ SO_UPDATE_ACCEPT_CONTEXT, SO_UPDATE_CONNECT_CONTEXT, TF_REUSE_SOCKET, }, System::Threading::INFINITE, }; + #[pyattr(once)] + fn INVALID_HANDLE_VALUE(_vm: &VirtualMachine) -> isize { + windows_sys::Win32::Foundation::INVALID_HANDLE_VALUE as isize + } #[pyattr] const NULL: isize = 0; @@ -126,7 +130,7 @@ mod _overlapped { fn mark_as_completed(ov: &mut OVERLAPPED) { ov.Internal = 0; - if ov.hEvent != 0 { + if ov.hEvent != std::ptr::null_mut() { unsafe { windows_sys::Win32::System::Threading::SetEvent(ov.hEvent) }; } } @@ -164,7 +168,7 @@ mod _overlapped { fn WSARecv_inner( inner: &mut OverlappedInner, - handle: HANDLE, + handle: isize, buf: &[u8], mut flags: u32, vm: &VirtualMachine, @@ -209,7 +213,7 @@ mod _overlapped { #[pymethod] fn WSARecv( zelf: &Py, - handle: HANDLE, + handle: isize, size: u32, flags: u32, vm: &VirtualMachine, @@ -224,9 +228,9 @@ mod _overlapped { let buf = vec![0u8; std::cmp::max(size, 1) as usize]; let buf = vm.ctx.new_bytes(buf); - inner.handle = handle; + inner.handle = handle as _; - let r = Self::WSARecv_inner(&mut inner, handle, buf.as_bytes(), flags, vm); + let r = Self::WSARecv_inner(&mut inner, handle as _, buf.as_bytes(), flags, vm); inner.data = OverlappedData::Read(buf); r } @@ -256,30 +260,30 @@ mod _overlapped { } impl Constructor for Overlapped { - type Args = (HANDLE,); + type Args = (isize,); fn py_new(cls: PyTypeRef, (mut event,): Self::Args, vm: &VirtualMachine) -> PyResult { - if event == INVALID_HANDLE_VALUE { + if event as isize == INVALID_HANDLE_VALUE as isize { event = unsafe { windows_sys::Win32::System::Threading::CreateEventA( std::ptr::null(), Foundation::TRUE, Foundation::FALSE, std::ptr::null(), - ) + ) as isize }; - if event == NULL { + if event as isize == NULL { return Err(errno_err(vm)); } } let mut overlapped: OVERLAPPED = unsafe { std::mem::zeroed() }; if event != NULL { - overlapped.hEvent = event; + overlapped.hEvent = event as _; } let inner = OverlappedInner { overlapped, - handle: NULL, + handle: NULL as _, error: 0, data: OverlappedData::None, }; @@ -292,29 +296,29 @@ mod _overlapped { #[pyfunction] fn CreateIoCompletionPort( - handle: HANDLE, - port: HANDLE, + handle: isize, + port: isize, key: usize, concurrency: u32, vm: &VirtualMachine, - ) -> PyResult { + ) -> PyResult { let r = unsafe { - windows_sys::Win32::System::IO::CreateIoCompletionPort(handle, port, key, concurrency) + windows_sys::Win32::System::IO::CreateIoCompletionPort(handle as _, port as _, key, concurrency) as isize }; - if r == 0 { + if r as usize == 0 { return Err(errno_err(vm)); } Ok(r) } #[pyfunction] - fn GetQueuedCompletionStatus(port: HANDLE, msecs: u32, vm: &VirtualMachine) -> PyResult { + fn GetQueuedCompletionStatus(port: isize, msecs: u32, vm: &VirtualMachine) -> PyResult { let mut bytes_transferred = 0; let mut completion_key = 0; let mut overlapped: *mut OVERLAPPED = std::ptr::null_mut(); let ret = unsafe { windows_sys::Win32::System::IO::GetQueuedCompletionStatus( - port, + port as _, &mut bytes_transferred, &mut completion_key, &mut overlapped, diff --git a/stdlib/src/socket.rs b/stdlib/src/socket.rs index 149408a6f3..25d8dcf5aa 100644 --- a/stdlib/src/socket.rs +++ b/stdlib/src/socket.rs @@ -36,11 +36,7 @@ mod _socket { #[cfg(windows)] mod c { pub use windows_sys::Win32::NetworkManagement::IpHelper::{if_indextoname, if_nametoindex}; - - pub const INADDR_ANY: u32 = 0x00000000; - pub const INADDR_LOOPBACK: u32 = 0x7f000001; - pub const INADDR_BROADCAST: u32 = 0xffffffff; - pub const INADDR_NONE: u32 = 0xffffffff; + pub use windows_sys::Win32::Networking::WinSock::{INADDR_ANY, INADDR_LOOPBACK, INADDR_BROADCAST, INADDR_NONE}; pub use windows_sys::Win32::Networking::WinSock::{ AF_APPLETALK, AF_DECnet, AF_IPX, AF_LINK, AI_ADDRCONFIG, AI_ALL, AI_CANONNAME, @@ -2135,7 +2131,7 @@ mod _socket { #[cfg(all(unix, not(target_os = "redox")))] type IfIndex = c::c_uint; #[cfg(windows)] - type IfIndex = u32; // NET_IFINDEX but windows-sys 0.59 doesn't have it + type IfIndex = u32; // NET_IFINDEX but windows-sys 0.59 doesn't have it #[cfg(not(target_os = "redox"))] #[pyfunction] diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 330a2beab5..a0e357baa9 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -144,6 +144,7 @@ features = [ "Win32_System_SystemInformation", "Win32_System_SystemServices", "Win32_System_Threading", + "Win32_System_WindowsProgramming", "Win32_UI_Shell", "Win32_UI_WindowsAndMessaging", ] diff --git a/vm/src/stdlib/nt.rs b/vm/src/stdlib/nt.rs index 34fa8792d5..bf7dca8b05 100644 --- a/vm/src/stdlib/nt.rs +++ b/vm/src/stdlib/nt.rs @@ -150,7 +150,7 @@ pub(crate) mod module { } let h = unsafe { Threading::OpenProcess(Threading::PROCESS_ALL_ACCESS, 0, pid) }; - if h == 0 { + if h == std::ptr::null_mut() { return Err(errno_err(vm)); } let ret = unsafe { Threading::TerminateProcess(h, sig) }; @@ -172,7 +172,7 @@ pub(crate) mod module { _ => return Err(vm.new_value_error("bad file descriptor".to_owned())), }; let h = unsafe { Console::GetStdHandle(stdhandle) }; - if h == 0 { + if h == std::ptr::null_mut() { return Err(vm.new_os_error("handle cannot be retrieved".to_owned())); } if h == INVALID_HANDLE_VALUE { diff --git a/vm/src/stdlib/winapi.rs b/vm/src/stdlib/winapi.rs index c1edb2739e..4a87a0f70f 100644 --- a/vm/src/stdlib/winapi.rs +++ b/vm/src/stdlib/winapi.rs @@ -79,7 +79,7 @@ mod _winapi { #[pyfunction] fn CloseHandle(handle: HANDLE) -> WindowsSysResult { - WindowsSysResult(unsafe { windows_sys::Win32::Foundation::CloseHandle(handle.0) }) + WindowsSysResult(unsafe { windows_sys::Win32::Foundation::CloseHandle(handle.0 as _) }) } #[pyfunction] @@ -99,8 +99,8 @@ mod _winapi { let mut read = std::mem::MaybeUninit::::uninit(); let mut write = std::mem::MaybeUninit::::uninit(); WindowsSysResult(windows_sys::Win32::System::Pipes::CreatePipe( - read.as_mut_ptr(), - write.as_mut_ptr(), + read.as_mut_ptr() as _, + write.as_mut_ptr() as _, std::ptr::null(), size, )) @@ -122,10 +122,10 @@ mod _winapi { let target = unsafe { let mut target = std::mem::MaybeUninit::::uninit(); WindowsSysResult(windows_sys::Win32::Foundation::DuplicateHandle( - src_process.0, - src.0, - target_process.0, - target.as_mut_ptr(), + src_process.0 as _, + src.0 as _, + target_process.0 as _, + target.as_mut_ptr() as _, access, inherit, options.unwrap_or(0), @@ -151,7 +151,7 @@ mod _winapi { h: HANDLE, vm: &VirtualMachine, ) -> PyResult { - let file_type = unsafe { windows_sys::Win32::Storage::FileSystem::GetFileType(h.0) }; + let file_type = unsafe { windows_sys::Win32::Storage::FileSystem::GetFileType(h.0 as _) }; if file_type == 0 && unsafe { windows_sys::Win32::Foundation::GetLastError() } != 0 { Err(errno_err(vm)) } else { @@ -274,8 +274,8 @@ mod _winapi { }; Ok(( - HANDLE(procinfo.hProcess), - HANDLE(procinfo.hThread), + HANDLE(procinfo.hProcess as _), + HANDLE(procinfo.hThread as _), procinfo.dwProcessId, procinfo.dwThreadId, )) @@ -286,13 +286,13 @@ mod _winapi { desired_access: u32, inherit_handle: bool, process_id: u32, - ) -> windows_sys::Win32::Foundation::HANDLE { + ) -> isize { unsafe { windows_sys::Win32::System::Threading::OpenProcess( desired_access, BOOL::from(inherit_handle), process_id, - ) + ) as _ } } @@ -438,7 +438,7 @@ mod _winapi { #[pyfunction] fn WaitForSingleObject(h: HANDLE, ms: u32, vm: &VirtualMachine) -> PyResult { - let ret = unsafe { windows_sys::Win32::System::Threading::WaitForSingleObject(h.0, ms) }; + let ret = unsafe { windows_sys::Win32::System::Threading::WaitForSingleObject(h.0 as _, ms) }; if ret == windows_sys::Win32::Foundation::WAIT_FAILED { Err(errno_err(vm)) } else { @@ -451,7 +451,7 @@ mod _winapi { unsafe { let mut ec = std::mem::MaybeUninit::uninit(); WindowsSysResult(windows_sys::Win32::System::Threading::GetExitCodeProcess( - h.0, + h.0 as _, ec.as_mut_ptr(), )) .to_pyresult(vm)?; @@ -462,7 +462,7 @@ mod _winapi { #[pyfunction] fn TerminateProcess(h: HANDLE, exit_code: u32) -> WindowsSysResult { WindowsSysResult(unsafe { - windows_sys::Win32::System::Threading::TerminateProcess(h.0, exit_code) + windows_sys::Win32::System::Threading::TerminateProcess(h.0 as _, exit_code) }) } @@ -507,11 +507,11 @@ mod _winapi { // if handle.is_invalid() { // return Err(errno_err(vm)); // } - Ok(handle) + Ok(handle as _) } #[pyfunction] fn ReleaseMutex(handle: isize) -> WindowsSysResult { - WindowsSysResult(unsafe { windows_sys::Win32::System::Threading::ReleaseMutex(handle) }) + WindowsSysResult(unsafe { windows_sys::Win32::System::Threading::ReleaseMutex(handle as _) }) } } diff --git a/vm/src/windows.rs b/vm/src/windows.rs index f4f4dad0b3..a14216768e 100644 --- a/vm/src/windows.rs +++ b/vm/src/windows.rs @@ -23,7 +23,7 @@ impl WindowsSysResultValue for RAW_HANDLE { *self == INVALID_HANDLE_VALUE } fn into_ok(self) -> Self::Ok { - HANDLE(self) + HANDLE(self as _) } } From ddf2e591c6f2933db520f265ec727c87769f76ec Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Wed, 5 Mar 2025 10:52:31 -0800 Subject: [PATCH 08/16] resolve comments --- stdlib/src/socket.rs | 33 +++++++-------------------------- 1 file changed, 7 insertions(+), 26 deletions(-) diff --git a/stdlib/src/socket.rs b/stdlib/src/socket.rs index 25d8dcf5aa..39c21da544 100644 --- a/stdlib/src/socket.rs +++ b/stdlib/src/socket.rs @@ -1757,15 +1757,12 @@ mod _socket { .map(|s| s.to_cstring(vm)) .transpose()?; let cstr_proto = cstr_opt_as_ptr(&cstr_proto); - #[cfg(windows)] let serv = unsafe { c::getservbyname( - cstr_name.as_ptr() as windows_sys::core::PCSTR, - cstr_proto as windows_sys::core::PCSTR, + cstr_name.as_ptr() as _, + cstr_proto as _, ) }; - #[cfg(not(windows))] - let serv = unsafe { c::getservbyname(cstr_name.as_ptr(), cstr_proto) }; if serv.is_null() { return Err(vm.new_os_error("service/proto not found".to_owned())); } @@ -1787,18 +1784,11 @@ mod _socket { .map(|s| s.to_cstring(vm)) .transpose()?; let cstr_proto = cstr_opt_as_ptr(&cstr_proto); - #[cfg(windows)] - let serv = - unsafe { c::getservbyport(port.to_be() as _, cstr_proto as windows_sys::core::PCSTR) }; - #[cfg(not(windows))] - let serv = unsafe { c::getservbyport(port.to_be() as _, cstr_proto) }; + let serv = unsafe { c::getservbyport(port.to_be() as _, cstr_proto as _) }; if serv.is_null() { return Err(vm.new_os_error("port/proto not found".to_owned())); } - #[cfg(windows)] - let s = unsafe { ffi::CStr::from_ptr((*serv).s_name as *const i8) }; - #[cfg(not(windows))] - let s = unsafe { ffi::CStr::from_ptr((*serv).s_name) }; + let s = unsafe { ffi::CStr::from_ptr((*serv).s_name as _) }; Ok(s.to_string_lossy().into_owned()) } @@ -2050,10 +2040,7 @@ mod _socket { #[pyfunction] fn getprotobyname(name: PyStrRef, vm: &VirtualMachine) -> PyResult { let cstr = name.to_cstring(vm)?; - #[cfg(windows)] - let proto = unsafe { c::getprotobyname(cstr.as_ptr() as *const u8) }; - #[cfg(not(windows))] - let proto = unsafe { c::getprotobyname(cstr.as_ptr()) }; + let proto = unsafe { c::getprotobyname(cstr.as_ptr() as _) }; if proto.is_null() { return Err(vm.new_os_error("protocol not found".to_owned())); } @@ -2138,10 +2125,7 @@ mod _socket { fn if_nametoindex(name: FsPath, vm: &VirtualMachine) -> PyResult { let name = name.to_cstring(vm)?; - #[cfg(windows)] - let ret = unsafe { c::if_nametoindex(name.as_ptr() as *const u8) }; - #[cfg(not(windows))] - let ret = unsafe { c::if_nametoindex(name.as_ptr()) }; + let ret = unsafe { c::if_nametoindex(name.as_ptr() as _) }; if ret == 0 { Err(vm.new_os_error("no interface with this name".to_owned())) } else { @@ -2157,10 +2141,7 @@ mod _socket { if ret.is_null() { Err(crate::vm::stdlib::os::errno_err(vm)) } else { - #[cfg(windows)] - let buf = unsafe { ffi::CStr::from_ptr(buf.as_ptr() as *const i8) }; - #[cfg(not(windows))] - let buf = unsafe { ffi::CStr::from_ptr(buf.as_ptr()) }; + let buf = unsafe { ffi::CStr::from_ptr(buf.as_ptr() as _) }; Ok(buf.to_string_lossy().into_owned()) } } From b4929d258da5bbe8d74873b978c552b39702851d Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Wed, 5 Mar 2025 11:07:20 -0800 Subject: [PATCH 09/16] formatting --- common/src/fileutils.rs | 2 +- stdlib/src/overlapped.rs | 7 ++++++- stdlib/src/socket.rs | 11 ++++------- vm/src/stdlib/winapi.rs | 15 +++++++-------- 4 files changed, 18 insertions(+), 17 deletions(-) diff --git a/common/src/fileutils.rs b/common/src/fileutils.rs index ce4aa8250a..cabeb833c1 100644 --- a/common/src/fileutils.rs +++ b/common/src/fileutils.rs @@ -153,7 +153,7 @@ pub mod windows { let p_id_info = if unsafe { GetFileInformationByHandleEx( - h as _, + h as _, FileIdInfo, &mut id_info as *mut _ as *mut _, std::mem::size_of_val(&id_info) as u32, diff --git a/stdlib/src/overlapped.rs b/stdlib/src/overlapped.rs index d3c46cad0a..8877e5befb 100644 --- a/stdlib/src/overlapped.rs +++ b/stdlib/src/overlapped.rs @@ -303,7 +303,12 @@ mod _overlapped { vm: &VirtualMachine, ) -> PyResult { let r = unsafe { - windows_sys::Win32::System::IO::CreateIoCompletionPort(handle as _, port as _, key, concurrency) as isize + windows_sys::Win32::System::IO::CreateIoCompletionPort( + handle as _, + port as _, + key, + concurrency, + ) as isize }; if r as usize == 0 { return Err(errno_err(vm)); diff --git a/stdlib/src/socket.rs b/stdlib/src/socket.rs index 39c21da544..39bfde4bee 100644 --- a/stdlib/src/socket.rs +++ b/stdlib/src/socket.rs @@ -36,7 +36,9 @@ mod _socket { #[cfg(windows)] mod c { pub use windows_sys::Win32::NetworkManagement::IpHelper::{if_indextoname, if_nametoindex}; - pub use windows_sys::Win32::Networking::WinSock::{INADDR_ANY, INADDR_LOOPBACK, INADDR_BROADCAST, INADDR_NONE}; + pub use windows_sys::Win32::Networking::WinSock::{ + INADDR_ANY, INADDR_BROADCAST, INADDR_LOOPBACK, INADDR_NONE, + }; pub use windows_sys::Win32::Networking::WinSock::{ AF_APPLETALK, AF_DECnet, AF_IPX, AF_LINK, AI_ADDRCONFIG, AI_ALL, AI_CANONNAME, @@ -1757,12 +1759,7 @@ mod _socket { .map(|s| s.to_cstring(vm)) .transpose()?; let cstr_proto = cstr_opt_as_ptr(&cstr_proto); - let serv = unsafe { - c::getservbyname( - cstr_name.as_ptr() as _, - cstr_proto as _, - ) - }; + let serv = unsafe { c::getservbyname(cstr_name.as_ptr() as _, cstr_proto as _) }; if serv.is_null() { return Err(vm.new_os_error("service/proto not found".to_owned())); } diff --git a/vm/src/stdlib/winapi.rs b/vm/src/stdlib/winapi.rs index 4a87a0f70f..7ffc4227e6 100644 --- a/vm/src/stdlib/winapi.rs +++ b/vm/src/stdlib/winapi.rs @@ -275,18 +275,14 @@ mod _winapi { Ok(( HANDLE(procinfo.hProcess as _), - HANDLE(procinfo.hThread as _), + HANDLE(procinfo.hThread as _), procinfo.dwProcessId, procinfo.dwThreadId, )) } #[pyfunction] - fn OpenProcess( - desired_access: u32, - inherit_handle: bool, - process_id: u32, - ) -> isize { + fn OpenProcess(desired_access: u32, inherit_handle: bool, process_id: u32) -> isize { unsafe { windows_sys::Win32::System::Threading::OpenProcess( desired_access, @@ -438,7 +434,8 @@ mod _winapi { #[pyfunction] fn WaitForSingleObject(h: HANDLE, ms: u32, vm: &VirtualMachine) -> PyResult { - let ret = unsafe { windows_sys::Win32::System::Threading::WaitForSingleObject(h.0 as _, ms) }; + let ret = + unsafe { windows_sys::Win32::System::Threading::WaitForSingleObject(h.0 as _, ms) }; if ret == windows_sys::Win32::Foundation::WAIT_FAILED { Err(errno_err(vm)) } else { @@ -512,6 +509,8 @@ mod _winapi { #[pyfunction] fn ReleaseMutex(handle: isize) -> WindowsSysResult { - WindowsSysResult(unsafe { windows_sys::Win32::System::Threading::ReleaseMutex(handle as _) }) + WindowsSysResult(unsafe { + windows_sys::Win32::System::Threading::ReleaseMutex(handle as _) + }) } } From d2bf31724f05ad1f6b32c2fa623ab2e111cc89ea Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Wed, 5 Mar 2025 11:15:42 -0800 Subject: [PATCH 10/16] fix clippy --- common/src/fileutils.rs | 2 +- stdlib/src/overlapped.rs | 6 +++--- vm/src/stdlib/nt.rs | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/common/src/fileutils.rs b/common/src/fileutils.rs index cabeb833c1..67713c0148 100644 --- a/common/src/fileutils.rs +++ b/common/src/fileutils.rs @@ -320,7 +320,7 @@ pub mod windows { .get_or_init(|| { let library_name = OsString::from("api-ms-win-core-file-l2-1-4").to_wide_with_nul(); let module = unsafe { LoadLibraryW(library_name.as_ptr()) }; - if module == std::ptr::null_mut() { + if module.is_null() { return None; } let name = CString::new("GetFileInformationByName").unwrap(); diff --git a/stdlib/src/overlapped.rs b/stdlib/src/overlapped.rs index 8877e5befb..234e3be53a 100644 --- a/stdlib/src/overlapped.rs +++ b/stdlib/src/overlapped.rs @@ -130,7 +130,7 @@ mod _overlapped { fn mark_as_completed(ov: &mut OVERLAPPED) { ov.Internal = 0; - if ov.hEvent != std::ptr::null_mut() { + if !ov.hEvent.is_null() { unsafe { windows_sys::Win32::System::Threading::SetEvent(ov.hEvent) }; } } @@ -263,7 +263,7 @@ mod _overlapped { type Args = (isize,); fn py_new(cls: PyTypeRef, (mut event,): Self::Args, vm: &VirtualMachine) -> PyResult { - if event as isize == INVALID_HANDLE_VALUE as isize { + if event == INVALID_HANDLE_VALUE(vm) { event = unsafe { windows_sys::Win32::System::Threading::CreateEventA( std::ptr::null(), @@ -272,7 +272,7 @@ mod _overlapped { std::ptr::null(), ) as isize }; - if event as isize == NULL { + if event == NULL { return Err(errno_err(vm)); } } diff --git a/vm/src/stdlib/nt.rs b/vm/src/stdlib/nt.rs index bf7dca8b05..ecc63e0aa6 100644 --- a/vm/src/stdlib/nt.rs +++ b/vm/src/stdlib/nt.rs @@ -150,7 +150,7 @@ pub(crate) mod module { } let h = unsafe { Threading::OpenProcess(Threading::PROCESS_ALL_ACCESS, 0, pid) }; - if h == std::ptr::null_mut() { + if h.is_null() { return Err(errno_err(vm)); } let ret = unsafe { Threading::TerminateProcess(h, sig) }; @@ -172,7 +172,7 @@ pub(crate) mod module { _ => return Err(vm.new_value_error("bad file descriptor".to_owned())), }; let h = unsafe { Console::GetStdHandle(stdhandle) }; - if h == std::ptr::null_mut() { + if h.is_null() { return Err(vm.new_os_error("handle cannot be retrieved".to_owned())); } if h == INVALID_HANDLE_VALUE { From 7fea1e1b4a92bec3a9addc4e2e64e161a8b6a0c7 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Fri, 28 Feb 2025 21:46:28 -0800 Subject: [PATCH 11/16] fix what is left data upload to website and trigger cron-ci on workflow update --- .github/workflows/cron-ci.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cron-ci.yaml b/.github/workflows/cron-ci.yaml index 4b8e701ced..b5590f145f 100644 --- a/.github/workflows/cron-ci.yaml +++ b/.github/workflows/cron-ci.yaml @@ -2,6 +2,9 @@ on: schedule: - cron: '0 0 * * 6' workflow_dispatch: + push: + paths: + - .github/workflows/cron-ci.yaml name: Periodic checks/tasks @@ -97,8 +100,8 @@ jobs: cd website [ -f ./_data/whats_left.temp ] && cp ./_data/whats_left.temp ./_data/whats_left_lastrun.temp cp ../whats_left.temp ./_data/whats_left.temp - rm _data/whats_left/modules.csv - cat _data/whats_left.temp | grep "(entire module)" | cut -d ' ' -f 1 | sort >> ../_data/whats_left/modules.csv + rm ./_data/whats_left/modules.csv + cat ./_data/whats_left.temp | grep "(entire module)" | cut -d ' ' -f 1 | sort >> ./_data/whats_left/modules.csv git add -A if git -c user.name="Github Actions" -c user.email="actions@github.com" commit -m "Update what is left results" --author="$GITHUB_ACTOR"; then From 58ebf04bac4a29359ecccbec930072e4146bd4d9 Mon Sep 17 00:00:00 2001 From: Daniel O'Hear <149127239+dohear@users.noreply.github.com> Date: Wed, 5 Mar 2025 20:41:45 +0000 Subject: [PATCH 12/16] Add JIT compilation support for integer multiplication, division, and exponents (#5561) * Initial commit for power function * Float power jit developed * Addded support for Floats and Ints * Integration Testing for JITPower implementation. * Update instructions.rs cranelift more like painlift * Update instructions.rs * Update instructions.rs * initial commit for making stable PR ready features * fixed final edge case for compile_ipow * fixed final edge case for compile_ipow * commenting out compile_ipow * fixed spelling errors * removed unused tests * forgot to run clippy --------- Co-authored-by: Nicholas Paulick Co-authored-by: Nick Co-authored-by: JoeLoparco Co-authored-by: Nathan Rusch Co-authored-by: dohear --- jit/src/instructions.rs | 163 ++++++++++++++++++++++++++++++++++++++++ jit/tests/int_tests.rs | 70 ++++++++++++++++- 2 files changed, 232 insertions(+), 1 deletion(-) diff --git a/jit/src/instructions.rs b/jit/src/instructions.rs index b495c6e1f6..1b74760dc0 100644 --- a/jit/src/instructions.rs +++ b/jit/src/instructions.rs @@ -425,12 +425,25 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> { (BinaryOperator::Subtract, JitValue::Int(a), JitValue::Int(b)) => { JitValue::Int(self.compile_sub(a, b)) } + (BinaryOperator::Multiply, JitValue::Int(a), JitValue::Int(b)) => { + JitValue::Int(self.builder.ins().imul(a, b)) + } (BinaryOperator::FloorDivide, JitValue::Int(a), JitValue::Int(b)) => { JitValue::Int(self.builder.ins().sdiv(a, b)) } + (BinaryOperator::Divide, JitValue::Int(a), JitValue::Int(b)) => { + // Convert to float for regular division + let a_float = self.builder.ins().fcvt_from_sint(types::F64, a); + let b_float = self.builder.ins().fcvt_from_sint(types::F64, b); + JitValue::Float(self.builder.ins().fdiv(a_float, b_float)) + } (BinaryOperator::Modulo, JitValue::Int(a), JitValue::Int(b)) => { JitValue::Int(self.builder.ins().srem(a, b)) } + // Todo: This should return int when possible + (BinaryOperator::Power, JitValue::Int(a), JitValue::Int(b)) => { + JitValue::Float(self.compile_ipow(a, b)) + } ( BinaryOperator::Lshift | BinaryOperator::Rshift, JitValue::Int(a), @@ -562,4 +575,154 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> { .trapif(IntCC::Overflow, carry, TrapCode::IntegerOverflow); out } + fn compile_ipow(&mut self, a: Value, b: Value) -> Value { + // Convert base to float since result might not always be a Int + let float_base = self.builder.ins().fcvt_from_sint(types::F64, a); + + // Create code blocks + let check_block1 = self.builder.create_block(); + let check_block2 = self.builder.create_block(); + let check_block3 = self.builder.create_block(); + let handle_neg_exp = self.builder.create_block(); + let loop_block = self.builder.create_block(); + let continue_block = self.builder.create_block(); + let exit_block = self.builder.create_block(); + + // Set code block params + // Set code block params + self.builder.append_block_param(check_block1, types::F64); + self.builder.append_block_param(check_block1, types::I64); + + self.builder.append_block_param(check_block2, types::F64); + self.builder.append_block_param(check_block2, types::I64); + + self.builder.append_block_param(check_block3, types::F64); + self.builder.append_block_param(check_block3, types::I64); + + self.builder.append_block_param(handle_neg_exp, types::F64); + self.builder.append_block_param(handle_neg_exp, types::I64); + + self.builder.append_block_param(loop_block, types::F64); //base + self.builder.append_block_param(loop_block, types::F64); //result + self.builder.append_block_param(loop_block, types::I64); //exponent + + self.builder.append_block_param(continue_block, types::F64); //base + self.builder.append_block_param(continue_block, types::F64); //result + self.builder.append_block_param(continue_block, types::I64); //exponent + + self.builder.append_block_param(exit_block, types::F64); + + // Begin evaluating by jumping to first check block + self.builder.ins().jump(check_block1, &[float_base, b]); + + // Check block one: + // Checks if input is O ** n where n > 0 + // Jumps to exit_block as 0 if true + self.builder.switch_to_block(check_block1); + let paramsc1 = self.builder.block_params(check_block1); + let basec1 = paramsc1[0]; + let expc1 = paramsc1[1]; + let zero_f64 = self.builder.ins().f64const(0.0); + let zero_i64 = self.builder.ins().iconst(types::I64, 0); + let is_base_zero = self.builder.ins().fcmp(FloatCC::Equal, zero_f64, basec1); + let is_exp_positive = self + .builder + .ins() + .icmp(IntCC::SignedGreaterThan, expc1, zero_i64); + let is_zero_to_positive = self.builder.ins().band(is_base_zero, is_exp_positive); + self.builder + .ins() + .brnz(is_zero_to_positive, exit_block, &[zero_f64]); + self.builder.ins().jump(check_block2, &[basec1, expc1]); + + // Check block two: + // Checks if exponent is negative + // Jumps to a special handle_neg_exponent block if true + self.builder.switch_to_block(check_block2); + let paramsc2 = self.builder.block_params(check_block2); + let basec2 = paramsc2[0]; + let expc2 = paramsc2[1]; + let zero_i64 = self.builder.ins().iconst(types::I64, 0); + let is_neg = self + .builder + .ins() + .icmp(IntCC::SignedLessThan, expc2, zero_i64); + self.builder + .ins() + .brnz(is_neg, handle_neg_exp, &[basec2, expc2]); + self.builder.ins().jump(check_block3, &[basec2, expc2]); + + // Check block three: + // Checks if exponent is one + // jumps to exit block with the base of the exponents value + self.builder.switch_to_block(check_block3); + let paramsc3 = self.builder.block_params(check_block3); + let basec3 = paramsc3[0]; + let expc3 = paramsc3[1]; + let resc3 = self.builder.ins().f64const(1.0); + let one_i64 = self.builder.ins().iconst(types::I64, 1); + let is_one = self.builder.ins().icmp(IntCC::Equal, expc3, one_i64); + self.builder.ins().brnz(is_one, exit_block, &[basec3]); + self.builder.ins().jump(loop_block, &[basec3, resc3, expc3]); + + // Handles negative Exponents + // calculates x^(-n) = (1/x)^n + // then proceeds to the loop to evaluate + self.builder.switch_to_block(handle_neg_exp); + let paramshn = self.builder.block_params(handle_neg_exp); + let basehn = paramshn[0]; + let exphn = paramshn[1]; + let one_f64 = self.builder.ins().f64const(1.0); + let base_inverse = self.builder.ins().fdiv(one_f64, basehn); + let pos_exp = self.builder.ins().ineg(exphn); + self.builder + .ins() + .jump(loop_block, &[base_inverse, one_f64, pos_exp]); + + // Main loop block + // checks loop condition (exp > 0) + // Jumps to continue block if true, exit block if false + self.builder.switch_to_block(loop_block); + let paramslb = self.builder.block_params(loop_block); + let baselb = paramslb[0]; + let reslb = paramslb[1]; + let explb = paramslb[2]; + let zero = self.builder.ins().iconst(types::I64, 0); + let is_zero = self.builder.ins().icmp(IntCC::Equal, explb, zero); + self.builder.ins().brnz(is_zero, exit_block, &[reslb]); + self.builder + .ins() + .jump(continue_block, &[baselb, reslb, explb]); + + // Continue block + // Main math logic + // Always jumps back to loob_block + self.builder.switch_to_block(continue_block); + let paramscb = self.builder.block_params(continue_block); + let basecb = paramscb[0]; + let rescb = paramscb[1]; + let expcb = paramscb[2]; + let is_odd = self.builder.ins().band_imm(expcb, 1); + let is_odd = self.builder.ins().icmp_imm(IntCC::Equal, is_odd, 1); + let mul_result = self.builder.ins().fmul(rescb, basecb); + let new_result = self.builder.ins().select(is_odd, mul_result, rescb); + let squared_base = self.builder.ins().fmul(basecb, basecb); + let new_exp = self.builder.ins().sshr_imm(expcb, 1); + self.builder + .ins() + .jump(loop_block, &[squared_base, new_result, new_exp]); + + self.builder.switch_to_block(exit_block); + let result = self.builder.block_params(exit_block)[0]; + + self.builder.seal_block(check_block1); + self.builder.seal_block(check_block2); + self.builder.seal_block(check_block3); + self.builder.seal_block(handle_neg_exp); + self.builder.seal_block(loop_block); + self.builder.seal_block(continue_block); + self.builder.seal_block(exit_block); + + result + } } diff --git a/jit/tests/int_tests.rs b/jit/tests/int_tests.rs index 9ce3f3b4a6..353052df00 100644 --- a/jit/tests/int_tests.rs +++ b/jit/tests/int_tests.rs @@ -1,3 +1,5 @@ +use core::f64; + #[test] fn test_add() { let add = jit_function! { add(a:i64, b:i64) -> i64 => r##" @@ -23,6 +25,51 @@ fn test_sub() { assert_eq!(sub(-3, -10), Ok(7)); } +#[test] +fn test_mul() { + let mul = jit_function! { mul(a:i64, b:i64) -> i64 => r##" + def mul(a: int, b: int): + return a * b + "## }; + + assert_eq!(mul(5, 10), Ok(50)); + assert_eq!(mul(0, 5), Ok(0)); + assert_eq!(mul(5, 0), Ok(0)); + assert_eq!(mul(0, 0), Ok(0)); + assert_eq!(mul(-5, 10), Ok(-50)); + assert_eq!(mul(5, -10), Ok(-50)); + assert_eq!(mul(-5, -10), Ok(50)); + assert_eq!(mul(999999, 999999), Ok(999998000001)); + assert_eq!(mul(i64::MAX, 1), Ok(i64::MAX)); + assert_eq!(mul(1, i64::MAX), Ok(i64::MAX)); +} + +#[test] + +fn test_div() { + let div = jit_function! { div(a:i64, b:i64) -> f64 => r##" + def div(a: int, b: int): + return a / b + "## }; + + assert_eq!(div(0, 1), Ok(0.0)); + assert_eq!(div(5, 1), Ok(5.0)); + assert_eq!(div(5, 10), Ok(0.5)); + assert_eq!(div(5, 2), Ok(2.5)); + assert_eq!(div(12, 10), Ok(1.2)); + assert_eq!(div(7, 10), Ok(0.7)); + assert_eq!(div(-3, -1), Ok(3.0)); + assert_eq!(div(-3, 1), Ok(-3.0)); + assert_eq!(div(1, 1000), Ok(0.001)); + assert_eq!(div(1, 100000), Ok(0.00001)); + assert_eq!(div(2, 3), Ok(0.6666666666666666)); + assert_eq!(div(1, 3), Ok(0.3333333333333333)); + assert_eq!(div(i64::MAX, 2), Ok(4611686018427387904.0)); + assert_eq!(div(i64::MIN, 2), Ok(-4611686018427387904.0)); + assert_eq!(div(i64::MIN, -1), Ok(9223372036854775808.0)); // Overflow case + assert_eq!(div(i64::MIN, i64::MAX), Ok(-1.0)); +} + #[test] fn test_floor_div() { let floor_div = jit_function! { floor_div(a:i64, b:i64) -> i64 => r##" @@ -35,7 +82,28 @@ fn test_floor_div() { assert_eq!(floor_div(12, 10), Ok(1)); assert_eq!(floor_div(7, 10), Ok(0)); assert_eq!(floor_div(-3, -1), Ok(3)); - assert_eq!(floor_div(-3, 1), Ok(-3)); +} + +#[test] + +fn test_exp() { + let exp = jit_function! { exp(a: i64, b: i64) -> f64 => r##" + def exp(a: int, b: int): + return a ** b + "## }; + + assert_eq!(exp(2, 3), Ok(8.0)); + assert_eq!(exp(3, 2), Ok(9.0)); + assert_eq!(exp(5, 0), Ok(1.0)); + assert_eq!(exp(0, 0), Ok(1.0)); + assert_eq!(exp(-5, 0), Ok(1.0)); + assert_eq!(exp(0, 1), Ok(0.0)); + assert_eq!(exp(0, 5), Ok(0.0)); + assert_eq!(exp(-2, 2), Ok(4.0)); + assert_eq!(exp(-3, 4), Ok(81.0)); + assert_eq!(exp(-2, 3), Ok(-8.0)); + assert_eq!(exp(-3, 3), Ok(-27.0)); + assert_eq!(exp(1000, 2), Ok(1000000.0)); } #[test] From cc0a1ce9e269062da91086e15089cff53d422ecf Mon Sep 17 00:00:00 2001 From: Noa Date: Wed, 5 Mar 2025 16:15:14 -0600 Subject: [PATCH 13/16] Update webpack (#5585) * Update webpack * Build demo before notebook * Use with instead of env for actions-gh-pages --- .github/workflows/release.yml | 19 +++++++++++++------ wasm/notebook/package.json | 20 +++++++++----------- wasm/notebook/webpack.config.js | 12 +++++++----- 3 files changed, 29 insertions(+), 22 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 98b7de4823..4be6b4eaf0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -109,6 +109,13 @@ jobs: - uses: actions/setup-node@v4 - uses: mwilliamson/setup-wabt-action@v3 with: { wabt-version: "1.0.30" } + - name: build demo + run: | + npm install + npm run dist + env: + NODE_OPTIONS: "--openssl-legacy-provider" + working-directory: ./wasm/demo - name: build notebook demo run: | npm install @@ -119,11 +126,11 @@ jobs: working-directory: ./wasm/notebook - name: Deploy demo to Github Pages uses: peaceiris/actions-gh-pages@v4 - env: - ACTIONS_DEPLOY_KEY: ${{ secrets.ACTIONS_DEMO_DEPLOY_KEY }} - PUBLISH_DIR: ./wasm/demo/dist - EXTERNAL_REPOSITORY: RustPython/demo - PUBLISH_BRANCH: master + with: + deploy_key: ${{ secrets.ACTIONS_DEMO_DEPLOY_KEY }} + publish_dir: ./wasm/demo/dist + external_repository: RustPython/demo + publish_branch: master release: runs-on: ubuntu-latest @@ -161,4 +168,4 @@ jobs: --target="$tag" \ --generate-notes \ $PRERELEASE_ARG \ - bin/rustpython-release-* \ No newline at end of file + bin/rustpython-release-* diff --git a/wasm/notebook/package.json b/wasm/notebook/package.json index 2a730258cd..64517331c4 100644 --- a/wasm/notebook/package.json +++ b/wasm/notebook/package.json @@ -12,19 +12,17 @@ "xterm": "^3.8.0" }, "devDependencies": { - "@wasm-tool/wasm-pack-plugin": "^1.1.0", - "clean-webpack-plugin": "^3.0.0", - "css-loader": "^3.4.1", - "html-webpack-plugin": "^3.2.0", - "mini-css-extract-plugin": "^0.9.0", - "raw-loader": "^4.0.0", - "serve": "^11.0.2", - "webpack": "^4.16.3", - "webpack-cli": "^3.1.0", - "webpack-dev-server": "^3.1.5" + "@wasm-tool/wasm-pack-plugin": "^1.7.0", + "css-loader": "^7.1.2", + "html-webpack-plugin": "^5.6.3", + "lezer-loader": "^0.3.0", + "mini-css-extract-plugin": "^2.9.2", + "webpack": "^5.97.1", + "webpack-cli": "^6.0.1", + "webpack-dev-server": "^5.2.0" }, "scripts": { - "dev": "webpack-dev-server -d", + "dev": "webpack serve", "build": "webpack", "dist": "webpack --mode production", "test": "webpack --mode production && cd ../tests && pytest" diff --git a/wasm/notebook/webpack.config.js b/wasm/notebook/webpack.config.js index 9fda3cf4aa..ca19f3384a 100644 --- a/wasm/notebook/webpack.config.js +++ b/wasm/notebook/webpack.config.js @@ -1,7 +1,6 @@ const HtmlWebpackPlugin = require('html-webpack-plugin'); const MiniCssExtractPlugin = require('mini-css-extract-plugin'); const WasmPackPlugin = require('@wasm-tool/wasm-pack-plugin'); -const { CleanWebpackPlugin } = require('clean-webpack-plugin'); const path = require('path'); const fs = require('fs'); @@ -12,6 +11,7 @@ module.exports = (env = {}) => { output: { path: path.join(__dirname, 'dist'), filename: 'index.js', + clean: true, }, mode: 'development', resolve: { @@ -30,15 +30,14 @@ module.exports = (env = {}) => { }, { test: /\.(woff(2)?|ttf)$/, - use: { - loader: 'file-loader', - options: { name: 'fonts/[name].[ext]' }, + type: 'asset/resource', + generator: { + filename: 'fonts/[name].[ext]', }, }, ], }, plugins: [ - new CleanWebpackPlugin(), new HtmlWebpackPlugin({ filename: 'index.html', template: 'src/index.ejs', @@ -58,6 +57,9 @@ module.exports = (env = {}) => { filename: 'styles.css', }), ], + experiments: { + asyncWebAssembly: true, + }, }; if (!env.noWasmPack) { config.plugins.push( From 97853bf0f1edbef92fc758badf102564653ed73a Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Wed, 5 Mar 2025 20:20:04 -0800 Subject: [PATCH 14/16] Fix module.csv generation in cron ci (#5586) --- .github/workflows/cron-ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cron-ci.yaml b/.github/workflows/cron-ci.yaml index b5590f145f..5c14502643 100644 --- a/.github/workflows/cron-ci.yaml +++ b/.github/workflows/cron-ci.yaml @@ -101,8 +101,8 @@ jobs: [ -f ./_data/whats_left.temp ] && cp ./_data/whats_left.temp ./_data/whats_left_lastrun.temp cp ../whats_left.temp ./_data/whats_left.temp rm ./_data/whats_left/modules.csv + echo -e "modules\n" > ./_data/whats_left/modules.csv cat ./_data/whats_left.temp | grep "(entire module)" | cut -d ' ' -f 1 | sort >> ./_data/whats_left/modules.csv - git add -A if git -c user.name="Github Actions" -c user.email="actions@github.com" commit -m "Update what is left results" --author="$GITHUB_ACTOR"; then git push From bae0ad3aeb2cccb86132ddf34436e346bd6ba53d Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sun, 9 Mar 2025 19:43:26 -0700 Subject: [PATCH 15/16] Fix extra newline in module.csv generation in cron ci (#5591) --- .github/workflows/cron-ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cron-ci.yaml b/.github/workflows/cron-ci.yaml index 5c14502643..6ae118cb56 100644 --- a/.github/workflows/cron-ci.yaml +++ b/.github/workflows/cron-ci.yaml @@ -101,7 +101,7 @@ jobs: [ -f ./_data/whats_left.temp ] && cp ./_data/whats_left.temp ./_data/whats_left_lastrun.temp cp ../whats_left.temp ./_data/whats_left.temp rm ./_data/whats_left/modules.csv - echo -e "modules\n" > ./_data/whats_left/modules.csv + echo -e "modules" > ./_data/whats_left/modules.csv cat ./_data/whats_left.temp | grep "(entire module)" | cut -d ' ' -f 1 | sort >> ./_data/whats_left/modules.csv git add -A if git -c user.name="Github Actions" -c user.email="actions@github.com" commit -m "Update what is left results" --author="$GITHUB_ACTOR"; then From bf28152a3251e6ebe9ccbbe8a4858c51c3c97e27 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sun, 9 Mar 2025 16:55:03 -0700 Subject: [PATCH 16/16] add os support modules --- Lib/_android_support.py | 181 ++++++++++++++++++++++++++++++++++++++++ Lib/_apple_support.py | 66 +++++++++++++++ Lib/_ios_support.py | 71 ++++++++++++++++ 3 files changed, 318 insertions(+) create mode 100644 Lib/_android_support.py create mode 100644 Lib/_apple_support.py create mode 100644 Lib/_ios_support.py diff --git a/Lib/_android_support.py b/Lib/_android_support.py new file mode 100644 index 0000000000..ae506f6a4b --- /dev/null +++ b/Lib/_android_support.py @@ -0,0 +1,181 @@ +import io +import sys +from threading import RLock +from time import sleep, time + +# The maximum length of a log message in bytes, including the level marker and +# tag, is defined as LOGGER_ENTRY_MAX_PAYLOAD at +# https://cs.android.com/android/platform/superproject/+/android-14.0.0_r1:system/logging/liblog/include/log/log.h;l=71. +# Messages longer than this will be truncated by logcat. This limit has already +# been reduced at least once in the history of Android (from 4076 to 4068 between +# API level 23 and 26), so leave some headroom. +MAX_BYTES_PER_WRITE = 4000 + +# UTF-8 uses a maximum of 4 bytes per character, so limiting text writes to this +# size ensures that we can always avoid exceeding MAX_BYTES_PER_WRITE. +# However, if the actual number of bytes per character is smaller than that, +# then we may still join multiple consecutive text writes into binary +# writes containing a larger number of characters. +MAX_CHARS_PER_WRITE = MAX_BYTES_PER_WRITE // 4 + + +# When embedded in an app on current versions of Android, there's no easy way to +# monitor the C-level stdout and stderr. The testbed comes with a .c file to +# redirect them to the system log using a pipe, but that wouldn't be convenient +# or appropriate for all apps. So we redirect at the Python level instead. +def init_streams(android_log_write, stdout_prio, stderr_prio): + if sys.executable: + return # Not embedded in an app. + + global logcat + logcat = Logcat(android_log_write) + + sys.stdout = TextLogStream( + stdout_prio, "python.stdout", sys.stdout.fileno()) + sys.stderr = TextLogStream( + stderr_prio, "python.stderr", sys.stderr.fileno()) + + +class TextLogStream(io.TextIOWrapper): + def __init__(self, prio, tag, fileno=None, **kwargs): + # The default is surrogateescape for stdout and backslashreplace for + # stderr, but in the context of an Android log, readability is more + # important than reversibility. + kwargs.setdefault("encoding", "UTF-8") + kwargs.setdefault("errors", "backslashreplace") + + super().__init__(BinaryLogStream(prio, tag, fileno), **kwargs) + self._lock = RLock() + self._pending_bytes = [] + self._pending_bytes_count = 0 + + def __repr__(self): + return f"" + + def write(self, s): + if not isinstance(s, str): + raise TypeError( + f"write() argument must be str, not {type(s).__name__}") + + # In case `s` is a str subclass that writes itself to stdout or stderr + # when we call its methods, convert it to an actual str. + s = str.__str__(s) + + # We want to emit one log message per line wherever possible, so split + # the string into lines first. Note that "".splitlines() == [], so + # nothing will be logged for an empty string. + with self._lock: + for line in s.splitlines(keepends=True): + while line: + chunk = line[:MAX_CHARS_PER_WRITE] + line = line[MAX_CHARS_PER_WRITE:] + self._write_chunk(chunk) + + return len(s) + + # The size and behavior of TextIOWrapper's buffer is not part of its public + # API, so we handle buffering ourselves to avoid truncation. + def _write_chunk(self, s): + b = s.encode(self.encoding, self.errors) + if self._pending_bytes_count + len(b) > MAX_BYTES_PER_WRITE: + self.flush() + + self._pending_bytes.append(b) + self._pending_bytes_count += len(b) + if ( + self.write_through + or b.endswith(b"\n") + or self._pending_bytes_count > MAX_BYTES_PER_WRITE + ): + self.flush() + + def flush(self): + with self._lock: + self.buffer.write(b"".join(self._pending_bytes)) + self._pending_bytes.clear() + self._pending_bytes_count = 0 + + # Since this is a line-based logging system, line buffering cannot be turned + # off, i.e. a newline always causes a flush. + @property + def line_buffering(self): + return True + + +class BinaryLogStream(io.RawIOBase): + def __init__(self, prio, tag, fileno=None): + self.prio = prio + self.tag = tag + self._fileno = fileno + + def __repr__(self): + return f"" + + def writable(self): + return True + + def write(self, b): + if type(b) is not bytes: + try: + b = bytes(memoryview(b)) + except TypeError: + raise TypeError( + f"write() argument must be bytes-like, not {type(b).__name__}" + ) from None + + # Writing an empty string to the stream should have no effect. + if b: + logcat.write(self.prio, self.tag, b) + return len(b) + + # This is needed by the test suite --timeout option, which uses faulthandler. + def fileno(self): + if self._fileno is None: + raise io.UnsupportedOperation("fileno") + return self._fileno + + +# When a large volume of data is written to logcat at once, e.g. when a test +# module fails in --verbose3 mode, there's a risk of overflowing logcat's own +# buffer and losing messages. We avoid this by imposing a rate limit using the +# token bucket algorithm, based on a conservative estimate of how fast `adb +# logcat` can consume data. +MAX_BYTES_PER_SECOND = 1024 * 1024 + +# The logcat buffer size of a device can be determined by running `logcat -g`. +# We set the token bucket size to half of the buffer size of our current minimum +# API level, because other things on the system will be producing messages as +# well. +BUCKET_SIZE = 128 * 1024 + +# https://cs.android.com/android/platform/superproject/+/android-14.0.0_r1:system/logging/liblog/include/log/log_read.h;l=39 +PER_MESSAGE_OVERHEAD = 28 + + +class Logcat: + def __init__(self, android_log_write): + self.android_log_write = android_log_write + self._lock = RLock() + self._bucket_level = 0 + self._prev_write_time = time() + + def write(self, prio, tag, message): + # Encode null bytes using "modified UTF-8" to avoid them truncating the + # message. + message = message.replace(b"\x00", b"\xc0\x80") + + with self._lock: + now = time() + self._bucket_level += ( + (now - self._prev_write_time) * MAX_BYTES_PER_SECOND) + + # If the bucket level is still below zero, the clock must have gone + # backwards, so reset it to zero and continue. + self._bucket_level = max(0, min(self._bucket_level, BUCKET_SIZE)) + self._prev_write_time = now + + self._bucket_level -= PER_MESSAGE_OVERHEAD + len(tag) + len(message) + if self._bucket_level < 0: + sleep(-self._bucket_level / MAX_BYTES_PER_SECOND) + + self.android_log_write(prio, tag, message) diff --git a/Lib/_apple_support.py b/Lib/_apple_support.py new file mode 100644 index 0000000000..92febdcf58 --- /dev/null +++ b/Lib/_apple_support.py @@ -0,0 +1,66 @@ +import io +import sys + + +def init_streams(log_write, stdout_level, stderr_level): + # Redirect stdout and stderr to the Apple system log. This method is + # invoked by init_apple_streams() (initconfig.c) if config->use_system_logger + # is enabled. + sys.stdout = SystemLog(log_write, stdout_level, errors=sys.stderr.errors) + sys.stderr = SystemLog(log_write, stderr_level, errors=sys.stderr.errors) + + +class SystemLog(io.TextIOWrapper): + def __init__(self, log_write, level, **kwargs): + kwargs.setdefault("encoding", "UTF-8") + kwargs.setdefault("line_buffering", True) + super().__init__(LogStream(log_write, level), **kwargs) + + def __repr__(self): + return f"" + + def write(self, s): + if not isinstance(s, str): + raise TypeError( + f"write() argument must be str, not {type(s).__name__}") + + # In case `s` is a str subclass that writes itself to stdout or stderr + # when we call its methods, convert it to an actual str. + s = str.__str__(s) + + # We want to emit one log message per line, so split + # the string before sending it to the superclass. + for line in s.splitlines(keepends=True): + super().write(line) + + return len(s) + + +class LogStream(io.RawIOBase): + def __init__(self, log_write, level): + self.log_write = log_write + self.level = level + + def __repr__(self): + return f"" + + def writable(self): + return True + + def write(self, b): + if type(b) is not bytes: + try: + b = bytes(memoryview(b)) + except TypeError: + raise TypeError( + f"write() argument must be bytes-like, not {type(b).__name__}" + ) from None + + # Writing an empty string to the stream should have no effect. + if b: + # Encode null bytes using "modified UTF-8" to avoid truncating the + # message. This should not affect the return value, as the caller + # may be expecting it to match the length of the input. + self.log_write(self.level, b.replace(b"\x00", b"\xc0\x80")) + + return len(b) diff --git a/Lib/_ios_support.py b/Lib/_ios_support.py new file mode 100644 index 0000000000..20467a7c2b --- /dev/null +++ b/Lib/_ios_support.py @@ -0,0 +1,71 @@ +import sys +try: + from ctypes import cdll, c_void_p, c_char_p, util +except ImportError: + # ctypes is an optional module. If it's not present, we're limited in what + # we can tell about the system, but we don't want to prevent the module + # from working. + print("ctypes isn't available; iOS system calls will not be available", file=sys.stderr) + objc = None +else: + # ctypes is available. Load the ObjC library, and wrap the objc_getClass, + # sel_registerName methods + lib = util.find_library("objc") + if lib is None: + # Failed to load the objc library + raise ImportError("ObjC runtime library couldn't be loaded") + + objc = cdll.LoadLibrary(lib) + objc.objc_getClass.restype = c_void_p + objc.objc_getClass.argtypes = [c_char_p] + objc.sel_registerName.restype = c_void_p + objc.sel_registerName.argtypes = [c_char_p] + + +def get_platform_ios(): + # Determine if this is a simulator using the multiarch value + is_simulator = sys.implementation._multiarch.endswith("simulator") + + # We can't use ctypes; abort + if not objc: + return None + + # Most of the methods return ObjC objects + objc.objc_msgSend.restype = c_void_p + # All the methods used have no arguments. + objc.objc_msgSend.argtypes = [c_void_p, c_void_p] + + # Equivalent of: + # device = [UIDevice currentDevice] + UIDevice = objc.objc_getClass(b"UIDevice") + SEL_currentDevice = objc.sel_registerName(b"currentDevice") + device = objc.objc_msgSend(UIDevice, SEL_currentDevice) + + # Equivalent of: + # device_systemVersion = [device systemVersion] + SEL_systemVersion = objc.sel_registerName(b"systemVersion") + device_systemVersion = objc.objc_msgSend(device, SEL_systemVersion) + + # Equivalent of: + # device_systemName = [device systemName] + SEL_systemName = objc.sel_registerName(b"systemName") + device_systemName = objc.objc_msgSend(device, SEL_systemName) + + # Equivalent of: + # device_model = [device model] + SEL_model = objc.sel_registerName(b"model") + device_model = objc.objc_msgSend(device, SEL_model) + + # UTF8String returns a const char*; + SEL_UTF8String = objc.sel_registerName(b"UTF8String") + objc.objc_msgSend.restype = c_char_p + + # Equivalent of: + # system = [device_systemName UTF8String] + # release = [device_systemVersion UTF8String] + # model = [device_model UTF8String] + system = objc.objc_msgSend(device_systemName, SEL_UTF8String).decode() + release = objc.objc_msgSend(device_systemVersion, SEL_UTF8String).decode() + model = objc.objc_msgSend(device_model, SEL_UTF8String).decode() + + return system, release, model, is_simulator