Skip to content
Closed
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ stdio = ["rustpython-vm/stdio"]
stdlib = ["rustpython-stdlib", "rustpython-pylib", "encodings"]
flame-it = ["rustpython-vm/flame-it", "flame", "flamescope"]
freeze-stdlib = ["stdlib", "rustpython-vm/freeze-stdlib", "rustpython-pylib?/freeze-stdlib"]
freeze-stdlib-full = ["freeze-stdlib", "rustpython-pylib?/freeze-stdlib-full"]
jit = ["rustpython-vm/jit"]
threading = ["rustpython-vm/threading", "rustpython-stdlib/threading"]
sqlite = ["rustpython-stdlib/sqlite"]
Expand Down
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,7 @@ cargo build --release --target wasm32-wasip1 --features="freeze-stdlib"
```

> Note: we use the `freeze-stdlib` to include the standard library inside the binary. You also have to run once `rustup target add wasm32-wasip1`.
> Enable the `freeze-stdlib-full` feature if you need the frozen `Lib/test` or `Lib/ensurepip` modules, for example when running tests in a WASM build.

### JIT (Just in time) compiler

Expand Down
90 changes: 83 additions & 7 deletions crates/derive-impl/src/compile_bytecode.rs
Original file line number Diff line number Diff line change
Expand Up @@ -83,14 +83,22 @@ impl CompilationSource {
mode: Mode,
module_name: String,
compiler: &dyn Compiler,
exclude: &[PathBuf],
) -> Result<HashMap<String, CompiledModule>, Diagnostic> {
match &self.kind {
CompilationSourceKind::Dir(rel_path) => self.compile_dir(
&CARGO_MANIFEST_DIR.join(rel_path),
String::new(),
mode,
compiler,
),
CompilationSourceKind::Dir(rel_path) => {
let path = CARGO_MANIFEST_DIR.join(rel_path);
let resolved_root = Self::resolve_root(&path);
self.compile_dir(
&path,
resolved_root.as_deref(),
&path,
String::new(),
mode,
compiler,
exclude,
)
}
_ => Ok(hashmap! {
module_name.clone() => CompiledModule {
code: self.compile_single(mode, module_name, compiler)?,
Expand Down Expand Up @@ -130,12 +138,56 @@ impl CompilationSource {
}
}

/// Returns an alternate base path when `root` is a git-symlink text file on
/// Windows so exclusions still apply to the real directory; otherwise returns
/// `None` to use `root` directly.
fn resolve_root(root: &Path) -> Option<PathBuf> {
if cfg!(windows) && root.is_file() {
fs::read_to_string(root)
.ok()
.map(|path| PathBuf::from(path.trim()))
} else {
None
}
}

/// Check whether `path` should be excluded. The `resolved_root` argument is used
/// on Windows when `root` is a file that stores the real library path created
/// from a git symlink; otherwise exclusions are evaluated relative to `root`.
fn should_exclude(
path: &Path,
root: &Path,
resolved_root: Option<&Path>,
exclude: &[PathBuf],
) -> bool {
// Return true when the entry is under `base` and has an excluded prefix.
let matches_root = |base: &Path| match path.strip_prefix(base) {
Ok(rel_path) => exclude.iter().any(|e| rel_path.starts_with(e)),
Err(_) => false,
};

if matches_root(root) {
return true;
}

if let Some(real_root) = resolved_root {
if matches_root(real_root) {
return true;
}
}

false
}

fn compile_dir(
&self,
path: &Path,
resolved_root: Option<&Path>,
root: &Path,
parent: String,
mode: Mode,
compiler: &dyn Compiler,
exclude: &[PathBuf],
) -> Result<HashMap<String, CompiledModule>, Diagnostic> {
let mut code_map = HashMap::new();
let paths = fs::read_dir(path)
Expand All @@ -155,19 +207,25 @@ impl CompilationSource {
Diagnostic::spans_error(self.span, format!("Failed to list file: {err}"))
})?;
let path = path.path();
if Self::should_exclude(&path, root, resolved_root, exclude) {
continue;
}
let file_name = path.file_name().unwrap().to_str().ok_or_else(|| {
Diagnostic::spans_error(self.span, format!("Invalid UTF-8 in file name {path:?}"))
})?;
if path.is_dir() {
code_map.extend(self.compile_dir(
&path,
resolved_root,
root,
if parent.is_empty() {
file_name.to_string()
} else {
format!("{parent}.{file_name}")
},
mode,
compiler,
exclude,
)?);
} else if file_name.ends_with(".py") {
let stem = path.file_stem().unwrap().to_str().unwrap();
Expand Down Expand Up @@ -239,6 +297,7 @@ impl PyCompileArgs {
let mut mode = None;
let mut source: Option<CompilationSource> = None;
let mut crate_name = None;
let mut exclude = Vec::new();

fn assert_source_empty(source: &Option<CompilationSource>) -> Result<(), syn::Error> {
if let Some(source) = source {
Expand Down Expand Up @@ -293,6 +352,12 @@ impl PyCompileArgs {
} else if ident == "crate_name" {
let name = check_str()?.parse()?;
crate_name = Some(name);
} else if ident == "exclude" {
if !allow_dir {
bail_span!(ident, "py_compile doesn't accept exclude")
}
let path = check_str()?.value().into();
exclude.push(path);
} else {
return Err(meta.error("unknown attr"));
}
Expand All @@ -307,11 +372,19 @@ impl PyCompileArgs {
)
})?;

if !exclude.is_empty() && !matches!(source.kind, CompilationSourceKind::Dir(_)) {
return Err(Diagnostic::spans_error(
source.span,
"exclude is only supported with dir source",
));
}

Ok(Self {
source,
mode: mode.unwrap_or(Mode::Exec),
module_name: module_name.unwrap_or_else(|| "frozen".to_owned()),
crate_name: crate_name.unwrap_or_else(|| syn::parse_quote!(::rustpython_vm)),
exclude,
})
}
}
Expand All @@ -332,6 +405,7 @@ struct PyCompileArgs {
mode: Mode,
module_name: String,
crate_name: syn::Path,
exclude: Vec<PathBuf>,
}

pub fn impl_py_compile(
Expand Down Expand Up @@ -362,7 +436,9 @@ pub fn impl_py_freeze(
let args = PyCompileArgs::parse(input, true)?;

let crate_name = args.crate_name;
let code_map = args.source.compile(args.mode, args.module_name, compiler)?;
let code_map = args
.source
.compile(args.mode, args.module_name, compiler, &args.exclude)?;

let data = frozen::FrozenLib::encode(code_map.iter().map(|(k, v)| {
let v = frozen::FrozenModule {
Expand Down
3 changes: 2 additions & 1 deletion crates/pylib/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ repository.workspace = true

[features]
freeze-stdlib = ["dep:rustpython-compiler-core", "dep:rustpython-derive"]
freeze-stdlib-full = ["freeze-stdlib"]
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Change the default. we don't need this feature. blocklisted libs are useless

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Implemented: removed the freeze-stdlib-full feature and kept the default frozen stdlib excluding Lib/test and Lib/ensurepip (commit 5a5733a).


[dependencies]
rustpython-compiler-core = { workspace = true, optional = true }
Expand All @@ -20,4 +21,4 @@ rustpython-derive = { workspace = true, optional = true }
glob = { workspace = true }

[lints]
workspace = true
workspace = true
9 changes: 9 additions & 0 deletions crates/pylib/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,5 +10,14 @@ pub const LIB_PATH: &str = match option_env!("win_lib_path") {
};

#[cfg(feature = "freeze-stdlib")]
#[cfg(not(feature = "freeze-stdlib-full"))]
pub const FROZEN_STDLIB: &rustpython_compiler_core::frozen::FrozenLib = rustpython_derive::py_freeze!(
dir = "./Lib",
crate_name = "rustpython_compiler_core",
exclude = "test",
exclude = "ensurepip",
);

#[cfg(feature = "freeze-stdlib-full")]
pub const FROZEN_STDLIB: &rustpython_compiler_core::frozen::FrozenLib =
rustpython_derive::py_freeze!(dir = "./Lib", crate_name = "rustpython_compiler_core");
1 change: 1 addition & 0 deletions crates/wasm/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ crate-type = ["cdylib", "rlib"]
[features]
default = ["freeze-stdlib"]
freeze-stdlib = ["rustpython-vm/freeze-stdlib", "rustpython-pylib/freeze-stdlib", "rustpython-stdlib"]
freeze-stdlib-full = ["freeze-stdlib", "rustpython-pylib/freeze-stdlib-full"]
no-start-func = []

[dependencies]
Expand Down