From 49d6c79b0fc254cfd146d611576f3624f87c0fcb Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 25 Dec 2025 10:25:52 +0000 Subject: [PATCH 1/6] Initial plan From 77c3af913a9a581350d408f809720c9ebde55307 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 25 Dec 2025 10:39:52 +0000 Subject: [PATCH 2/6] Implement freeze stdlib excludes Co-authored-by: youknowone <69878+youknowone@users.noreply.github.com> --- Cargo.toml | 1 + README.md | 1 + crates/derive-impl/src/compile_bytecode.rs | 83 ++++++++++++++++++++-- crates/pylib/Cargo.toml | 3 +- crates/pylib/src/lib.rs | 9 +++ crates/wasm/Cargo.toml | 1 + 6 files changed, 90 insertions(+), 8 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 44f9d3190f7..7578d63983b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,7 @@ stdio = ["rustpython-vm/stdio"] stdlib = ["rustpython-stdlib", "rustpython-pylib", "encodings"] flame-it = ["rustpython-vm/flame-it", "flame", "flamescope"] freeze-stdlib = ["stdlib", "rustpython-vm/freeze-stdlib", "rustpython-pylib?/freeze-stdlib"] +freeze-stdlib-full = ["freeze-stdlib", "rustpython-pylib?/freeze-stdlib-full"] jit = ["rustpython-vm/jit"] threading = ["rustpython-vm/threading", "rustpython-stdlib/threading"] sqlite = ["rustpython-stdlib/sqlite"] diff --git a/README.md b/README.md index 86d0738ec8e..f3dba8ca740 100644 --- a/README.md +++ b/README.md @@ -123,6 +123,7 @@ cargo build --release --target wasm32-wasip1 --features="freeze-stdlib" ``` > Note: we use the `freeze-stdlib` to include the standard library inside the binary. You also have to run once `rustup target add wasm32-wasip1`. +> Enable the `freeze-stdlib-full` feature if you need the frozen `Lib/test` or `Lib/ensurepip` modules, for example when running tests in a WASM build. ### JIT (Just in time) compiler diff --git a/crates/derive-impl/src/compile_bytecode.rs b/crates/derive-impl/src/compile_bytecode.rs index cdcc89b9984..f6771f6cd43 100644 --- a/crates/derive-impl/src/compile_bytecode.rs +++ b/crates/derive-impl/src/compile_bytecode.rs @@ -83,14 +83,22 @@ impl CompilationSource { mode: Mode, module_name: String, compiler: &dyn Compiler, + exclude: &[PathBuf], ) -> Result, Diagnostic> { match &self.kind { - CompilationSourceKind::Dir(rel_path) => self.compile_dir( - &CARGO_MANIFEST_DIR.join(rel_path), - String::new(), - mode, - compiler, - ), + CompilationSourceKind::Dir(rel_path) => { + let path = CARGO_MANIFEST_DIR.join(rel_path); + let resolved_root = Self::resolve_root(&path); + self.compile_dir( + &path, + resolved_root.as_deref(), + &path, + String::new(), + mode, + compiler, + exclude, + ) + } _ => Ok(hashmap! { module_name.clone() => CompiledModule { code: self.compile_single(mode, module_name, compiler)?, @@ -130,12 +138,49 @@ impl CompilationSource { } } + fn resolve_root(root: &Path) -> Option { + if cfg!(windows) && root.is_file() { + fs::read_to_string(root) + .ok() + .map(|path| PathBuf::from(path.trim())) + } else { + None + } + } + + fn should_exclude( + path: &Path, + root: &Path, + resolved_root: Option<&Path>, + exclude: &[PathBuf], + ) -> bool { + let matches_root = |base: &Path| match path.strip_prefix(base) { + Ok(rel_path) => exclude.iter().any(|e| rel_path.starts_with(e)), + Err(_) => false, + }; + + if matches_root(root) { + return true; + } + + if let Some(real_root) = resolved_root { + if matches_root(real_root) { + return true; + } + } + + false + } + fn compile_dir( &self, path: &Path, + resolved_root: Option<&Path>, + root: &Path, parent: String, mode: Mode, compiler: &dyn Compiler, + exclude: &[PathBuf], ) -> Result, Diagnostic> { let mut code_map = HashMap::new(); let paths = fs::read_dir(path) @@ -155,12 +200,17 @@ impl CompilationSource { Diagnostic::spans_error(self.span, format!("Failed to list file: {err}")) })?; let path = path.path(); + if Self::should_exclude(&path, root, resolved_root, exclude) { + continue; + } let file_name = path.file_name().unwrap().to_str().ok_or_else(|| { Diagnostic::spans_error(self.span, format!("Invalid UTF-8 in file name {path:?}")) })?; if path.is_dir() { code_map.extend(self.compile_dir( &path, + resolved_root, + root, if parent.is_empty() { file_name.to_string() } else { @@ -168,6 +218,7 @@ impl CompilationSource { }, mode, compiler, + exclude, )?); } else if file_name.ends_with(".py") { let stem = path.file_stem().unwrap().to_str().unwrap(); @@ -239,6 +290,7 @@ impl PyCompileArgs { let mut mode = None; let mut source: Option = None; let mut crate_name = None; + let mut exclude = Vec::new(); fn assert_source_empty(source: &Option) -> Result<(), syn::Error> { if let Some(source) = source { @@ -293,6 +345,12 @@ impl PyCompileArgs { } else if ident == "crate_name" { let name = check_str()?.parse()?; crate_name = Some(name); + } else if ident == "exclude" { + if !allow_dir { + bail_span!(ident, "py_compile doesn't accept exclude") + } + let path = check_str()?.value().into(); + exclude.push(path); } else { return Err(meta.error("unknown attr")); } @@ -307,11 +365,19 @@ impl PyCompileArgs { ) })?; + if !exclude.is_empty() && !matches!(source.kind, CompilationSourceKind::Dir(_)) { + return Err(Diagnostic::spans_error( + source.span, + "exclude is only supported with dir source", + )); + } + Ok(Self { source, mode: mode.unwrap_or(Mode::Exec), module_name: module_name.unwrap_or_else(|| "frozen".to_owned()), crate_name: crate_name.unwrap_or_else(|| syn::parse_quote!(::rustpython_vm)), + exclude, }) } } @@ -332,6 +398,7 @@ struct PyCompileArgs { mode: Mode, module_name: String, crate_name: syn::Path, + exclude: Vec, } pub fn impl_py_compile( @@ -362,7 +429,9 @@ pub fn impl_py_freeze( let args = PyCompileArgs::parse(input, true)?; let crate_name = args.crate_name; - let code_map = args.source.compile(args.mode, args.module_name, compiler)?; + let code_map = args + .source + .compile(args.mode, args.module_name, compiler, &args.exclude)?; let data = frozen::FrozenLib::encode(code_map.iter().map(|(k, v)| { let v = frozen::FrozenModule { diff --git a/crates/pylib/Cargo.toml b/crates/pylib/Cargo.toml index dcbb5928599..4315f74a449 100644 --- a/crates/pylib/Cargo.toml +++ b/crates/pylib/Cargo.toml @@ -11,6 +11,7 @@ repository.workspace = true [features] freeze-stdlib = ["dep:rustpython-compiler-core", "dep:rustpython-derive"] +freeze-stdlib-full = ["freeze-stdlib"] [dependencies] rustpython-compiler-core = { workspace = true, optional = true } @@ -20,4 +21,4 @@ rustpython-derive = { workspace = true, optional = true } glob = { workspace = true } [lints] -workspace = true \ No newline at end of file +workspace = true diff --git a/crates/pylib/src/lib.rs b/crates/pylib/src/lib.rs index f8a47ba67da..b4acc18dde4 100644 --- a/crates/pylib/src/lib.rs +++ b/crates/pylib/src/lib.rs @@ -10,5 +10,14 @@ pub const LIB_PATH: &str = match option_env!("win_lib_path") { }; #[cfg(feature = "freeze-stdlib")] +#[cfg(not(feature = "freeze-stdlib-full"))] +pub const FROZEN_STDLIB: &rustpython_compiler_core::frozen::FrozenLib = rustpython_derive::py_freeze!( + dir = "./Lib", + crate_name = "rustpython_compiler_core", + exclude = "test", + exclude = "ensurepip", +); + +#[cfg(feature = "freeze-stdlib-full")] pub const FROZEN_STDLIB: &rustpython_compiler_core::frozen::FrozenLib = rustpython_derive::py_freeze!(dir = "./Lib", crate_name = "rustpython_compiler_core"); diff --git a/crates/wasm/Cargo.toml b/crates/wasm/Cargo.toml index a2bb1a9f948..430fe7f7edc 100644 --- a/crates/wasm/Cargo.toml +++ b/crates/wasm/Cargo.toml @@ -14,6 +14,7 @@ crate-type = ["cdylib", "rlib"] [features] default = ["freeze-stdlib"] freeze-stdlib = ["rustpython-vm/freeze-stdlib", "rustpython-pylib/freeze-stdlib", "rustpython-stdlib"] +freeze-stdlib-full = ["freeze-stdlib", "rustpython-pylib/freeze-stdlib-full"] no-start-func = [] [dependencies] From 28644864d44b1d6a0eee51b17082d6968fe2dca4 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 25 Dec 2025 10:41:36 +0000 Subject: [PATCH 3/6] Document py_freeze exclude handling Co-authored-by: youknowone <69878+youknowone@users.noreply.github.com> --- crates/derive-impl/src/compile_bytecode.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/crates/derive-impl/src/compile_bytecode.rs b/crates/derive-impl/src/compile_bytecode.rs index f6771f6cd43..7cd928a3c6a 100644 --- a/crates/derive-impl/src/compile_bytecode.rs +++ b/crates/derive-impl/src/compile_bytecode.rs @@ -138,6 +138,9 @@ impl CompilationSource { } } + /// Returns an alternate base path when `root` is a git-symlink text file on + /// Windows so exclusions still apply to the real directory; otherwise returns + /// `None` to use `root` directly. fn resolve_root(root: &Path) -> Option { if cfg!(windows) && root.is_file() { fs::read_to_string(root) @@ -148,12 +151,16 @@ impl CompilationSource { } } + /// Check whether `path` should be excluded. The `resolved_root` argument is used + /// on Windows when `root` is a file that stores the real library path created + /// from a git symlink; otherwise exclusions are evaluated relative to `root`. fn should_exclude( path: &Path, root: &Path, resolved_root: Option<&Path>, exclude: &[PathBuf], ) -> bool { + // Return true when the entry is under `base` and has an excluded prefix. let matches_root = |base: &Path| match path.strip_prefix(base) { Ok(rel_path) => exclude.iter().any(|e| rel_path.starts_with(e)), Err(_) => false, From 5a5733ad61252236f39c5e0a85e224acafefec75 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 25 Dec 2025 12:03:10 +0000 Subject: [PATCH 4/6] Remove freeze-stdlib-full option Co-authored-by: youknowone <69878+youknowone@users.noreply.github.com> --- Cargo.toml | 1 - README.md | 1 - crates/pylib/Cargo.toml | 1 - crates/pylib/src/lib.rs | 5 ----- crates/wasm/Cargo.toml | 1 - 5 files changed, 9 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 7578d63983b..44f9d3190f7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,6 @@ stdio = ["rustpython-vm/stdio"] stdlib = ["rustpython-stdlib", "rustpython-pylib", "encodings"] flame-it = ["rustpython-vm/flame-it", "flame", "flamescope"] freeze-stdlib = ["stdlib", "rustpython-vm/freeze-stdlib", "rustpython-pylib?/freeze-stdlib"] -freeze-stdlib-full = ["freeze-stdlib", "rustpython-pylib?/freeze-stdlib-full"] jit = ["rustpython-vm/jit"] threading = ["rustpython-vm/threading", "rustpython-stdlib/threading"] sqlite = ["rustpython-stdlib/sqlite"] diff --git a/README.md b/README.md index f3dba8ca740..86d0738ec8e 100644 --- a/README.md +++ b/README.md @@ -123,7 +123,6 @@ cargo build --release --target wasm32-wasip1 --features="freeze-stdlib" ``` > Note: we use the `freeze-stdlib` to include the standard library inside the binary. You also have to run once `rustup target add wasm32-wasip1`. -> Enable the `freeze-stdlib-full` feature if you need the frozen `Lib/test` or `Lib/ensurepip` modules, for example when running tests in a WASM build. ### JIT (Just in time) compiler diff --git a/crates/pylib/Cargo.toml b/crates/pylib/Cargo.toml index 4315f74a449..6c8e1c50ade 100644 --- a/crates/pylib/Cargo.toml +++ b/crates/pylib/Cargo.toml @@ -11,7 +11,6 @@ repository.workspace = true [features] freeze-stdlib = ["dep:rustpython-compiler-core", "dep:rustpython-derive"] -freeze-stdlib-full = ["freeze-stdlib"] [dependencies] rustpython-compiler-core = { workspace = true, optional = true } diff --git a/crates/pylib/src/lib.rs b/crates/pylib/src/lib.rs index b4acc18dde4..fa73ba5b38b 100644 --- a/crates/pylib/src/lib.rs +++ b/crates/pylib/src/lib.rs @@ -10,14 +10,9 @@ pub const LIB_PATH: &str = match option_env!("win_lib_path") { }; #[cfg(feature = "freeze-stdlib")] -#[cfg(not(feature = "freeze-stdlib-full"))] pub const FROZEN_STDLIB: &rustpython_compiler_core::frozen::FrozenLib = rustpython_derive::py_freeze!( dir = "./Lib", crate_name = "rustpython_compiler_core", exclude = "test", exclude = "ensurepip", ); - -#[cfg(feature = "freeze-stdlib-full")] -pub const FROZEN_STDLIB: &rustpython_compiler_core::frozen::FrozenLib = - rustpython_derive::py_freeze!(dir = "./Lib", crate_name = "rustpython_compiler_core"); diff --git a/crates/wasm/Cargo.toml b/crates/wasm/Cargo.toml index 430fe7f7edc..a2bb1a9f948 100644 --- a/crates/wasm/Cargo.toml +++ b/crates/wasm/Cargo.toml @@ -14,7 +14,6 @@ crate-type = ["cdylib", "rlib"] [features] default = ["freeze-stdlib"] freeze-stdlib = ["rustpython-vm/freeze-stdlib", "rustpython-pylib/freeze-stdlib", "rustpython-stdlib"] -freeze-stdlib-full = ["freeze-stdlib", "rustpython-pylib/freeze-stdlib-full"] no-start-func = [] [dependencies] From 01dc9e62fb191758246c2e1c56828de7c3b1c878 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 25 Dec 2025 12:55:38 +0000 Subject: [PATCH 5/6] Fix clippy warnings Co-authored-by: youknowone <69878+youknowone@users.noreply.github.com> --- crates/derive-impl/src/compile_bytecode.rs | 52 ++++++++++------------ 1 file changed, 23 insertions(+), 29 deletions(-) diff --git a/crates/derive-impl/src/compile_bytecode.rs b/crates/derive-impl/src/compile_bytecode.rs index 7cd928a3c6a..40847f1254d 100644 --- a/crates/derive-impl/src/compile_bytecode.rs +++ b/crates/derive-impl/src/compile_bytecode.rs @@ -47,6 +47,12 @@ struct CompiledModule { package: bool, } +struct ExcludeCtx<'a> { + root: &'a Path, + resolved_root: Option<&'a Path>, + exclude: &'a [PathBuf], +} + struct CompilationSource { kind: CompilationSourceKind, span: (Span, Span), @@ -89,15 +95,12 @@ impl CompilationSource { CompilationSourceKind::Dir(rel_path) => { let path = CARGO_MANIFEST_DIR.join(rel_path); let resolved_root = Self::resolve_root(&path); - self.compile_dir( - &path, - resolved_root.as_deref(), - &path, - String::new(), - mode, - compiler, + let exclude_ctx = ExcludeCtx { + root: &path, + resolved_root: resolved_root.as_deref(), exclude, - ) + }; + self.compile_dir(&path, String::new(), mode, compiler, &exclude_ctx) } _ => Ok(hashmap! { module_name.clone() => CompiledModule { @@ -151,29 +154,24 @@ impl CompilationSource { } } - /// Check whether `path` should be excluded. The `resolved_root` argument is used - /// on Windows when `root` is a file that stores the real library path created + /// Check whether `path` should be excluded. The `resolved_root` is used on + /// Windows when `root` is a file that stores the real library path created /// from a git symlink; otherwise exclusions are evaluated relative to `root`. - fn should_exclude( - path: &Path, - root: &Path, - resolved_root: Option<&Path>, - exclude: &[PathBuf], - ) -> bool { + fn should_exclude(path: &Path, ctx: &ExcludeCtx<'_>) -> bool { // Return true when the entry is under `base` and has an excluded prefix. let matches_root = |base: &Path| match path.strip_prefix(base) { - Ok(rel_path) => exclude.iter().any(|e| rel_path.starts_with(e)), + Ok(rel_path) => ctx.exclude.iter().any(|e| rel_path.starts_with(e)), Err(_) => false, }; - if matches_root(root) { + if matches_root(ctx.root) { return true; } - if let Some(real_root) = resolved_root { - if matches_root(real_root) { - return true; - } + if let Some(real_root) = ctx.resolved_root + && matches_root(real_root) + { + return true; } false @@ -182,12 +180,10 @@ impl CompilationSource { fn compile_dir( &self, path: &Path, - resolved_root: Option<&Path>, - root: &Path, parent: String, mode: Mode, compiler: &dyn Compiler, - exclude: &[PathBuf], + exclude_ctx: &ExcludeCtx<'_>, ) -> Result, Diagnostic> { let mut code_map = HashMap::new(); let paths = fs::read_dir(path) @@ -207,7 +203,7 @@ impl CompilationSource { Diagnostic::spans_error(self.span, format!("Failed to list file: {err}")) })?; let path = path.path(); - if Self::should_exclude(&path, root, resolved_root, exclude) { + if Self::should_exclude(&path, exclude_ctx) { continue; } let file_name = path.file_name().unwrap().to_str().ok_or_else(|| { @@ -216,8 +212,6 @@ impl CompilationSource { if path.is_dir() { code_map.extend(self.compile_dir( &path, - resolved_root, - root, if parent.is_empty() { file_name.to_string() } else { @@ -225,7 +219,7 @@ impl CompilationSource { }, mode, compiler, - exclude, + exclude_ctx, )?); } else if file_name.ends_with(".py") { let stem = path.file_stem().unwrap().to_str().unwrap(); From ebcfae25849ed128f74d64342412a200b7f32d9a Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" <69878+youknowone@users.noreply.github.com> Date: Wed, 31 Dec 2025 02:22:12 +0900 Subject: [PATCH 6/6] Apply suggestions from code review --- crates/pylib/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/pylib/src/lib.rs b/crates/pylib/src/lib.rs index fa73ba5b38b..8196866c2d2 100644 --- a/crates/pylib/src/lib.rs +++ b/crates/pylib/src/lib.rs @@ -13,6 +13,6 @@ pub const LIB_PATH: &str = match option_env!("win_lib_path") { pub const FROZEN_STDLIB: &rustpython_compiler_core::frozen::FrozenLib = rustpython_derive::py_freeze!( dir = "./Lib", crate_name = "rustpython_compiler_core", - exclude = "test", + // exclude = "test", exclude = "ensurepip", );