From 572f3ba50bb11f29046c45ad82fb51c0a15ce032 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Wed, 12 Feb 2025 20:48:46 -0800 Subject: [PATCH 1/3] initial _ctypes implementation with _CData, get_errno, and set_errno Signed-off-by: Ashwin Naren --- Cargo.lock | 5 +++-- vm/Cargo.toml | 1 + vm/src/stdlib/ctypes.rs | 34 ++++++++++++++++++++++++++++++++++ vm/src/stdlib/mod.rs | 6 ++++++ 4 files changed, 44 insertions(+), 2 deletions(-) create mode 100644 vm/src/stdlib/ctypes.rs diff --git a/Cargo.lock b/Cargo.lock index 8f4c765e02..97236f95ec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1265,9 +1265,9 @@ checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "memmap2" -version = "0.9.5" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd3f7eed9d3848f8b98834af67102b720745c4ec028fcd0aa0239277e7de374f" +checksum = "83faa42c0a078c393f6b29d5db232d8be22776a891f8f56e5284faee4a20b327" dependencies = [ "libc", ] @@ -2205,6 +2205,7 @@ dependencies = [ "cfg-if", "chrono", "crossbeam-utils", + "errno", "exitcode", "flame", "flamer", diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 6eaca281f8..0c992605ff 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -99,6 +99,7 @@ uname = "0.1.1" [target.'cfg(not(target_arch = "wasm32"))'.dependencies] rustyline = { workspace = true } which = "6" +errno = "0.3" [target.'cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))'.dependencies] num_cpus = "1.13.1" diff --git a/vm/src/stdlib/ctypes.rs b/vm/src/stdlib/ctypes.rs new file mode 100644 index 0000000000..e0976fa4a6 --- /dev/null +++ b/vm/src/stdlib/ctypes.rs @@ -0,0 +1,34 @@ +pub(crate) use _ctypes::make_module; + +#[pymodule] +mod _ctypes { + use crate::{common::lock::PyRwLock, PyObjectRef}; + use crossbeam_utils::atomic::AtomicCell; + + pub struct RawBuffer { + #[allow(dead_code)] + pub inner: Box<[u8]>, + #[allow(dead_code)] + pub size: usize, + } + + #[pyattr] + #[pyclass(name = "_CData")] + pub struct PyCData { + _objects: AtomicCell>, + _buffer: PyRwLock, + } + + #[pyclass] + impl PyCData {} + + #[pyfunction] + fn get_errno() -> i32 { + errno::errno().0 + } + + #[pyfunction] + fn set_errno(value: i32) { + errno::set_errno(errno::Errno(value)); + } +} diff --git a/vm/src/stdlib/mod.rs b/vm/src/stdlib/mod.rs index 12baee11f7..ca8f8f0bfd 100644 --- a/vm/src/stdlib/mod.rs +++ b/vm/src/stdlib/mod.rs @@ -37,6 +37,8 @@ pub mod posix; #[path = "posix_compat.rs"] pub mod posix; +#[cfg(any(target_family = "unix", target_family = "windows"))] +mod ctypes; #[cfg(windows)] pub(crate) mod msvcrt; #[cfg(all(unix, not(any(target_os = "android", target_os = "redox"))))] @@ -124,5 +126,9 @@ pub fn get_module_inits() -> StdlibMap { "_winapi" => winapi::make_module, "winreg" => winreg::make_module, } + #[cfg(any(target_family = "unix", target_family = "windows"))] + { + "_ctypes" => ctypes::make_module, + } } } From fc7aa2361c935085c83800d8100ba596d6e56646 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Thu, 13 Feb 2025 16:27:23 -0800 Subject: [PATCH 2/3] Refactor of ctypes into module and more implementations Adds sizeof and PyCSimple Signed-off-by: Ashwin Naren --- vm/Cargo.toml | 2 +- vm/src/stdlib/ctypes.rs | 103 ++++++++++++++++++---- vm/src/stdlib/ctypes/base.rs | 161 +++++++++++++++++++++++++++++++++++ 3 files changed, 250 insertions(+), 16 deletions(-) create mode 100644 vm/src/stdlib/ctypes/base.rs diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 0c992605ff..1468651aeb 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -100,6 +100,7 @@ uname = "0.1.1" rustyline = { workspace = true } which = "6" errno = "0.3" +widestring = { workspace = true } [target.'cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))'.dependencies] num_cpus = "1.13.1" @@ -107,7 +108,6 @@ num_cpus = "1.13.1" [target.'cfg(windows)'.dependencies] junction = { workspace = true } schannel = { workspace = true } -widestring = { workspace = true } winreg = "0.52" [target.'cfg(windows)'.dependencies.windows] diff --git a/vm/src/stdlib/ctypes.rs b/vm/src/stdlib/ctypes.rs index e0976fa4a6..d1fc26869f 100644 --- a/vm/src/stdlib/ctypes.rs +++ b/vm/src/stdlib/ctypes.rs @@ -1,26 +1,99 @@ -pub(crate) use _ctypes::make_module; +pub(crate) mod base; + +use crate::builtins::PyModule; +use crate::{PyRef, VirtualMachine}; + +pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { + let module = _ctypes::make_module(vm); + base::extend_module_nodes(vm, &module); + module +} #[pymodule] -mod _ctypes { - use crate::{common::lock::PyRwLock, PyObjectRef}; +pub(crate) mod _ctypes { + use super::base::PyCSimple; + use crate::builtins::PyTypeRef; + use crate::class::StaticType; + use crate::function::Either; + use crate::{AsObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine}; use crossbeam_utils::atomic::AtomicCell; + use std::ffi::{ + c_double, c_float, c_int, c_long, c_longlong, c_schar, c_short, c_uchar, c_uint, c_ulong, + c_ulonglong, + }; + use std::mem; + use widestring::WideChar; - pub struct RawBuffer { - #[allow(dead_code)] - pub inner: Box<[u8]>, - #[allow(dead_code)] - pub size: usize, + pub fn get_size(ty: &str) -> usize { + match ty { + "u" => mem::size_of::(), + "c" | "b" => mem::size_of::(), + "h" => mem::size_of::(), + "H" => mem::size_of::(), + "i" => mem::size_of::(), + "I" => mem::size_of::(), + "l" => mem::size_of::(), + "q" => mem::size_of::(), + "L" => mem::size_of::(), + "Q" => mem::size_of::(), + "f" => mem::size_of::(), + "d" | "g" => mem::size_of::(), + "?" | "B" => mem::size_of::(), + "P" | "z" | "Z" => mem::size_of::(), + _ => unreachable!(), + } } - #[pyattr] - #[pyclass(name = "_CData")] - pub struct PyCData { - _objects: AtomicCell>, - _buffer: PyRwLock, + const SIMPLE_TYPE_CHARS: &str = "cbBhHiIlLdfguzZPqQ?"; + + pub fn new_simple_type( + cls: Either<&PyObjectRef, &PyTypeRef>, + vm: &VirtualMachine, + ) -> PyResult { + let cls = match cls { + Either::A(obj) => obj, + Either::B(typ) => typ.as_object(), + }; + + if let Ok(_type_) = cls.get_attr("_type_", vm) { + if _type_.is_instance((&vm.ctx.types.str_type).as_ref(), vm)? { + let tp_str = _type_.str(vm)?.to_string(); + + if tp_str.len() != 1 { + Err(vm.new_value_error( + format!("class must define a '_type_' attribute which must be a string of length 1, str: {tp_str}"), + )) + } else if !SIMPLE_TYPE_CHARS.contains(tp_str.as_str()) { + Err(vm.new_attribute_error(format!("class must define a '_type_' attribute which must be\n a single character string containing one of {SIMPLE_TYPE_CHARS}, currently it is {tp_str}."))) + } else { + Ok(PyCSimple { + _type_: tp_str, + _value: AtomicCell::new(vm.ctx.none()), + }) + } + } else { + Err(vm.new_type_error("class must define a '_type_' string attribute".to_string())) + } + } else { + Err(vm.new_attribute_error("class must define a '_type_' attribute".to_string())) + } } - #[pyclass] - impl PyCData {} + #[pyfunction(name = "sizeof")] + pub fn size_of(tp: Either, vm: &VirtualMachine) -> PyResult { + match tp { + Either::A(type_) if type_.fast_issubclass(PyCSimple::static_type()) => { + let zelf = new_simple_type(Either::B(&type_), vm)?; + Ok(get_size(zelf._type_.as_str())) + } + Either::B(obj) if obj.has_attr("size_of_instances", vm)? => { + let size_of_method = obj.get_attr("size_of_instances", vm)?; + let size_of_return = size_of_method.call(vec![], vm)?; + Ok(usize::try_from_object(vm, size_of_return)?) + } + _ => Err(vm.new_type_error("this type has no size".to_string())), + } + } #[pyfunction] fn get_errno() -> i32 { diff --git a/vm/src/stdlib/ctypes/base.rs b/vm/src/stdlib/ctypes/base.rs new file mode 100644 index 0000000000..0142c45345 --- /dev/null +++ b/vm/src/stdlib/ctypes/base.rs @@ -0,0 +1,161 @@ +use crate::builtins::{PyBytes, PyFloat, PyInt, PyModule, PyNone, PyStr}; +use crate::class::PyClassImpl; +use crate::{Py, PyObjectRef, PyResult, TryFromObject, VirtualMachine}; +use crossbeam_utils::atomic::AtomicCell; +use num_traits::ToPrimitive; +use rustpython_common::lock::PyRwLock; +use std::fmt::Debug; + +#[allow(dead_code)] +fn set_primitive(_type_: &str, value: &PyObjectRef, vm: &VirtualMachine) -> PyResult { + match _type_ { + "c" => { + if value + .clone() + .downcast_exact::(vm) + .is_ok_and(|v| v.len() == 1) + || value + .clone() + .downcast_exact::(vm) + .is_ok_and(|v| v.len() == 1) + || value + .clone() + .downcast_exact::(vm) + .map_or(Ok(false), |v| { + let n = v.as_bigint().to_i64(); + if let Some(n) = n { + Ok((0..=255).contains(&n)) + } else { + Ok(false) + } + })? + { + Ok(value.clone()) + } else { + Err(vm.new_type_error( + "one character bytes, bytearray or integer expected".to_string(), + )) + } + } + "u" => { + if let Ok(b) = value.str(vm).map(|v| v.to_string().chars().count() == 1) { + if b { + Ok(value.clone()) + } else { + Err(vm.new_type_error("one character unicode string expected".to_string())) + } + } else { + Err(vm.new_type_error(format!( + "unicode string expected instead of {} instance", + value.class().name() + ))) + } + } + "b" | "h" | "H" | "i" | "I" | "l" | "q" | "L" | "Q" => { + if value.clone().downcast_exact::(vm).is_ok() { + Ok(value.clone()) + } else { + Err(vm.new_type_error(format!( + "an integer is required (got type {})", + value.class().name() + ))) + } + } + "f" | "d" | "g" => { + if value.clone().downcast_exact::(vm).is_ok() { + Ok(value.clone()) + } else { + Err(vm.new_type_error(format!("must be real number, not {}", value.class().name()))) + } + } + "?" => Ok(PyObjectRef::from( + vm.ctx.new_bool(value.clone().try_to_bool(vm)?), + )), + "B" => { + if value.clone().downcast_exact::(vm).is_ok() { + Ok(vm.new_pyobj(u8::try_from_object(vm, value.clone())?)) + } else { + Err(vm.new_type_error(format!("int expected instead of {}", value.class().name()))) + } + } + "z" => { + if value.clone().downcast_exact::(vm).is_ok() + || value.clone().downcast_exact::(vm).is_ok() + { + Ok(value.clone()) + } else { + Err(vm.new_type_error(format!( + "bytes or integer address expected instead of {} instance", + value.class().name() + ))) + } + } + "Z" => { + if value.clone().downcast_exact::(vm).is_ok() { + Ok(value.clone()) + } else { + Err(vm.new_type_error(format!( + "unicode string or integer address expected instead of {} instance", + value.class().name() + ))) + } + } + _ => { + // "P" + if value.clone().downcast_exact::(vm).is_ok() + || value.clone().downcast_exact::(vm).is_ok() + { + Ok(value.clone()) + } else { + Err(vm.new_type_error("cannot be converted to pointer".to_string())) + } + } + } +} + +pub struct RawBuffer { + #[allow(dead_code)] + pub inner: Box<[u8]>, + #[allow(dead_code)] + pub size: usize, +} + +#[pyclass(name = "_CData", module = "_ctypes")] +pub struct PyCData { + _objects: AtomicCell>, + _buffer: PyRwLock, +} + +#[pyclass] +impl PyCData {} + +#[pyclass( + name = "_SimpleCData", + base = "PyCData", + module = "_ctypes" + // TODO: metaclass +)] +#[derive(PyPayload)] +pub struct PyCSimple { + pub _type_: String, + pub _value: AtomicCell, +} + +impl Debug for PyCSimple { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PyCSimple") + .field("_type_", &self._type_) + .finish() + } +} + +#[pyclass(flags(BASETYPE))] +impl PyCSimple {} + +pub fn extend_module_nodes(vm: &VirtualMachine, module: &Py) { + let ctx = &vm.ctx; + extend_module!(vm, module, { + "_CData" => PyCData::make_class(ctx), + "_SimpleCData" => PyCSimple::make_class(ctx), + }) +} From 9b6d5bb9dc05ce95576c2f238499c1b26a16d3bc Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Fri, 14 Feb 2025 19:30:41 -0800 Subject: [PATCH 3/3] add extra test for ctypes Signed-off-by: Ashwin Naren --- extra_tests/snippets/builtins_ctypes.py | 105 ++++++++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 extra_tests/snippets/builtins_ctypes.py diff --git a/extra_tests/snippets/builtins_ctypes.py b/extra_tests/snippets/builtins_ctypes.py new file mode 100644 index 0000000000..5df259cf82 --- /dev/null +++ b/extra_tests/snippets/builtins_ctypes.py @@ -0,0 +1,105 @@ +from _ctypes import sizeof +from _ctypes import _SimpleCData +from struct import calcsize as _calcsize + +def _check_size(typ, typecode=None): + # Check if sizeof(ctypes_type) against struct.calcsize. This + # should protect somewhat against a misconfigured libffi. + from struct import calcsize + if typecode is None: + # Most _type_ codes are the same as used in struct + typecode = typ._type_ + actual, required = sizeof(typ), calcsize(typecode) + if actual != required: + raise SystemError("sizeof(%s) wrong: %d instead of %d" % \ + (typ, actual, required)) + +class c_short(_SimpleCData): + _type_ = "h" +_check_size(c_short) + +class c_ushort(_SimpleCData): + _type_ = "H" +_check_size(c_ushort) + +class c_long(_SimpleCData): + _type_ = "l" +_check_size(c_long) + +class c_ulong(_SimpleCData): + _type_ = "L" +_check_size(c_ulong) + +if _calcsize("i") == _calcsize("l"): + # if int and long have the same size, make c_int an alias for c_long + c_int = c_long + c_uint = c_ulong +else: + class c_int(_SimpleCData): + _type_ = "i" + _check_size(c_int) + + class c_uint(_SimpleCData): + _type_ = "I" + _check_size(c_uint) + +class c_float(_SimpleCData): + _type_ = "f" +_check_size(c_float) + +class c_double(_SimpleCData): + _type_ = "d" +_check_size(c_double) + +class c_longdouble(_SimpleCData): + _type_ = "g" +if sizeof(c_longdouble) == sizeof(c_double): + c_longdouble = c_double + +if _calcsize("l") == _calcsize("q"): + # if long and long long have the same size, make c_longlong an alias for c_long + c_longlong = c_long + c_ulonglong = c_ulong +else: + class c_longlong(_SimpleCData): + _type_ = "q" + _check_size(c_longlong) + + class c_ulonglong(_SimpleCData): + _type_ = "Q" + ## def from_param(cls, val): + ## return ('d', float(val), val) + ## from_param = classmethod(from_param) + _check_size(c_ulonglong) + +class c_ubyte(_SimpleCData): + _type_ = "B" +c_ubyte.__ctype_le__ = c_ubyte.__ctype_be__ = c_ubyte +# backward compatibility: +##c_uchar = c_ubyte +_check_size(c_ubyte) + +class c_byte(_SimpleCData): + _type_ = "b" +c_byte.__ctype_le__ = c_byte.__ctype_be__ = c_byte +_check_size(c_byte) + +class c_char(_SimpleCData): + _type_ = "c" +c_char.__ctype_le__ = c_char.__ctype_be__ = c_char +_check_size(c_char) + +class c_char_p(_SimpleCData): + _type_ = "z" + def __repr__(self): + return "%s(%s)" % (self.__class__.__name__, c_void_p.from_buffer(self).value) +_check_size(c_char_p, "P") + +class c_void_p(_SimpleCData): + _type_ = "P" +c_voidp = c_void_p # backwards compatibility (to a bug) +_check_size(c_void_p) + +class c_bool(_SimpleCData): + _type_ = "?" +_check_size(c_bool)