diff --git a/Cargo.lock b/Cargo.lock index 5a58545ad0c44..224f0303be97a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5865,6 +5865,7 @@ dependencies = [ "uv-normalize", "uv-pep440", "uv-pep508", + "uv-platform", "uv-platform-tags", "uv-static", ] @@ -5876,6 +5877,29 @@ dependencies = [ "console 0.16.1", ] +[[package]] +name = "uv-delocate" +version = "0.0.11" +dependencies = [ + "base64 0.22.1", + "fs-err", + "goblin", + "pathdiff", + "sha2", + "tempfile", + "thiserror 2.0.17", + "tracing", + "uv-distribution-filename", + "uv-extract", + "uv-fs", + "uv-install-wheel", + "uv-platform", + "uv-platform-tags", + "uv-static", + "walkdir", + "zip", +] + [[package]] name = "uv-dev" version = "0.0.13" @@ -5983,9 +6007,11 @@ dependencies = [ "nanoid", "owo-colors", "reqwest", + "rkyv", "rmp-serde", "rustc-hash", "serde", + "serde_json", "tempfile", "thiserror 2.0.17", "tokio", @@ -5996,6 +6022,7 @@ dependencies = [ "uv-auth", "uv-cache", "uv-cache-info", + "uv-cache-key", "uv-client", "uv-configuration", "uv-distribution-filename", @@ -6012,6 +6039,7 @@ dependencies = [ "uv-platform-tags", "uv-pypi-types", "uv-redacted", + "uv-static", "uv-types", "uv-workspace", "walkdir", diff --git a/Cargo.toml b/Cargo.toml index 49afc2bd381a4..f4437b678404c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,9 @@ repository = "https://github.com/astral-sh/uv" authors = ["uv"] license = "MIT OR Apache-2.0" +[workspace.metadata.cargo-shear] +ignored = ["uv-delocate"] + [workspace.dependencies] uv-auth = { version = "0.0.13", path = "crates/uv-auth" } uv-bin-install = { version = "0.0.13", path = "crates/uv-bin-install" } @@ -27,6 +30,7 @@ uv-cli = { version = "0.0.13", path = "crates/uv-cli" } uv-client = { version = "0.0.13", path = "crates/uv-client" } uv-configuration = { version = "0.0.13", path = "crates/uv-configuration" } uv-console = { version = "0.0.13", path = "crates/uv-console" } +uv-delocate = { version = "0.0.13", path = "crates/uv-delocate" } uv-dirs = { version = "0.0.13", path = "crates/uv-dirs" } uv-dispatch = { version = "0.0.13", path = "crates/uv-dispatch" } uv-distribution = { version = "0.0.13", path = "crates/uv-distribution" } @@ -117,7 +121,7 @@ futures = { version = "0.3.30" } glob = { version = "0.3.1" } globset = { version = "0.4.15" } globwalk = { version = "0.9.1" } -goblin = { version = "0.10.0", default-features = false, features = ["std", "elf32", "elf64", "endian_fd"] } +goblin = { version = "0.10.0", default-features = false, features = ["std", "elf32", "elf64", "endian_fd", "mach32", "mach64"] } h2 = { version = "0.4.7" } hashbrown = { version = "0.16.0" } hex = { version = "0.4.3" } diff --git a/README.md b/README.md index 511778054abf9..50101ffc881a2 100644 --- a/README.md +++ b/README.md @@ -302,6 +302,8 @@ their support. uv's Git implementation is based on [Cargo](https://github.com/rust-lang/cargo). +uv's macOS wheel delocating is based on [delocate](https://github.com/matthew-brett/delocate). + Some of uv's optimizations are inspired by the great work we've seen in [pnpm](https://pnpm.io/), [Orogene](https://github.com/orogene/orogene), and [Bun](https://github.com/oven-sh/bun). We've also learned a lot from Nathaniel J. Smith's [Posy](https://github.com/njsmith/posy) and adapted its diff --git a/_typos.toml b/_typos.toml index b7b51e6aabaeb..0be61331c9fac 100644 --- a/_typos.toml +++ b/_typos.toml @@ -21,3 +21,7 @@ extend-ignore-re = [ [default.extend-identifiers] seeked = "seeked" # special term used for streams + +[default.extend-words] +Delocate = "Delocate" +delocate = "delocate" diff --git a/crates/uv-client/src/base_client.rs b/crates/uv-client/src/base_client.rs index 6c73dca3b8520..5a09520191f0c 100644 --- a/crates/uv-client/src/base_client.rs +++ b/crates/uv-client/src/base_client.rs @@ -1046,6 +1046,12 @@ impl<'a> RequestBuilder<'a> { self } + /// Set the request body. + pub fn body>(mut self, body: T) -> Self { + self.builder = self.builder.body(body); + self + } + /// Build a `Request`. pub fn build(self) -> reqwest::Result { self.builder.build() diff --git a/crates/uv-configuration/Cargo.toml b/crates/uv-configuration/Cargo.toml index 0cc74f350a2f1..a90271256e7d6 100644 --- a/crates/uv-configuration/Cargo.toml +++ b/crates/uv-configuration/Cargo.toml @@ -24,6 +24,7 @@ uv-git = { workspace = true } uv-normalize = { workspace = true } uv-pep440 = { workspace = true } uv-pep508 = { workspace = true, features = ["schemars"] } +uv-platform = { workspace = true } uv-platform-tags = { workspace = true } uv-static = { workspace = true } clap = { workspace = true, features = ["derive"], optional = true } diff --git a/crates/uv-configuration/src/target_triple.rs b/crates/uv-configuration/src/target_triple.rs index 479de159f04ad..bc937e1b23a35 100644 --- a/crates/uv-configuration/src/target_triple.rs +++ b/crates/uv-configuration/src/target_triple.rs @@ -1,6 +1,7 @@ use tracing::debug; use uv_pep508::MarkerEnvironment; +use uv_platform::MacOSVersion; use uv_platform_tags::{Arch, Os, Platform}; use uv_static::EnvVars; @@ -297,18 +298,36 @@ impl TargetTriple { Arch::X86_64, ), Self::Macos | Self::Aarch64AppleDarwin => { - let (major, minor) = macos_deployment_target().map_or((13, 0), |(major, minor)| { - debug!("Found macOS deployment target: {}.{}", major, minor); - (major, minor) - }); + let MacOSVersion { major, minor } = macos_deployment_target().map_or( + MacOSVersion { + major: 13, + minor: 0, + }, + |version| { + debug!( + "Found macOS deployment target: {}.{}", + version.major, version.minor + ); + version + }, + ); Platform::new(Os::Macos { major, minor }, Arch::Aarch64) } Self::I686PcWindowsMsvc => Platform::new(Os::Windows, Arch::X86), Self::X8664AppleDarwin => { - let (major, minor) = macos_deployment_target().map_or((13, 0), |(major, minor)| { - debug!("Found macOS deployment target: {}.{}", major, minor); - (major, minor) - }); + let MacOSVersion { major, minor } = macos_deployment_target().map_or( + MacOSVersion { + major: 13, + minor: 0, + }, + |version| { + debug!( + "Found macOS deployment target: {}.{}", + version.major, version.minor + ); + version + }, + ); Platform::new(Os::Macos { major, minor }, Arch::X86_64) } Self::Aarch64UnknownLinuxGnu => Platform::new( @@ -937,17 +956,9 @@ impl TargetTriple { } /// Return the macOS deployment target as parsed from the environment. -fn macos_deployment_target() -> Option<(u16, u16)> { +fn macos_deployment_target() -> Option { let version = std::env::var(EnvVars::MACOSX_DEPLOYMENT_TARGET).ok()?; - let mut parts = version.split('.'); - - // Parse the major version (e.g., `12` in `12.0`). - let major = parts.next()?.parse::().ok()?; - - // Parse the minor version (e.g., `0` in `12.0`), with a default of `0`. - let minor = parts.next().unwrap_or("0").parse::().ok()?; - - Some((major, minor)) + MacOSVersion::parse(&version) } /// Return the iOS deployment target as parsed from the environment. diff --git a/crates/uv-delocate/Cargo.toml b/crates/uv-delocate/Cargo.toml new file mode 100644 index 0000000000000..b67601ed33185 --- /dev/null +++ b/crates/uv-delocate/Cargo.toml @@ -0,0 +1,36 @@ +[package] +name = "uv-delocate" +version = "0.0.11" +description = "Mach-O delocate functionality for Python wheels" +edition = { workspace = true } +rust-version = { workspace = true } +homepage = { workspace = true } +repository = { workspace = true } +authors = { workspace = true } +license = { workspace = true } + +[lib] +doctest = false + +[lints] +workspace = true + +[dependencies] +uv-distribution-filename = { workspace = true } +uv-extract = { workspace = true } +uv-fs = { workspace = true } +uv-install-wheel = { workspace = true } +uv-platform = { workspace = true } +uv-platform-tags = { workspace = true } +uv-static = { workspace = true } + +base64 = { workspace = true } +fs-err = { workspace = true } +goblin = { workspace = true } +pathdiff = { workspace = true } +sha2 = { workspace = true } +tempfile = { workspace = true } +thiserror = { workspace = true } +tracing = { workspace = true } +walkdir = { workspace = true } +zip = { workspace = true } diff --git a/crates/uv-delocate/src/delocate.rs b/crates/uv-delocate/src/delocate.rs new file mode 100644 index 0000000000000..036714c8b9ebb --- /dev/null +++ b/crates/uv-delocate/src/delocate.rs @@ -0,0 +1,824 @@ +//! Delocate operations for macOS wheels. + +use std::collections::{HashMap, HashSet}; +use std::env; +use std::path::{Path, PathBuf}; +use std::str::FromStr; + +use fs_err as fs; +use tracing::{debug, trace}; +use walkdir::WalkDir; + +use uv_distribution_filename::WheelFilename; +use uv_platform_tags::PlatformTag; +use uv_static::EnvVars; + +use uv_platform::MacOSVersion; + +use crate::error::DelocateError; +use crate::macho::{self, Arch}; +use crate::wheel; + +/// Options for delocating a wheel. +#[derive(Debug, Clone)] +pub struct DelocateOptions { + /// Subdirectory within the package to store copied libraries. + /// Defaults to ".dylibs". + pub lib_sdir: String, + /// Required architectures to validate. + pub require_archs: Vec, + /// Libraries to exclude from delocating (by name pattern). + pub exclude: Vec, + /// Remove absolute rpaths from binaries. + /// This prevents issues when wheels are installed in different locations. + pub sanitize_rpaths: bool, + /// Check that bundled libraries don't require a newer macOS version than + /// declared in the wheel's platform tag. + pub check_version_compatibility: bool, + /// Target macOS version. If set, overrides the version from the wheel's platform tag. + /// This can also be set via the `MACOSX_DEPLOYMENT_TARGET` environment variable. + pub target_macos_version: Option, +} + +impl Default for DelocateOptions { + fn default() -> Self { + let target_macos_version = env::var(EnvVars::MACOSX_DEPLOYMENT_TARGET) + .ok() + .and_then(|value| MacOSVersion::parse(&value)); + + Self { + lib_sdir: ".dylibs".to_string(), + require_archs: Vec::new(), + exclude: Vec::new(), + sanitize_rpaths: true, + check_version_compatibility: true, + target_macos_version, + } + } +} + +/// System library prefixes that should not be bundled. +/// +/// See: +const SYSTEM_PREFIXES: &[&str] = &["/usr/lib", "/System"]; + +/// Check if a path is a system library that shouldn't be bundled. +fn is_system_library(path: &str) -> bool { + SYSTEM_PREFIXES + .iter() + .any(|prefix| path.starts_with(prefix)) +} + +/// Search for a library in DYLD environment paths. +fn search_dyld_paths(lib_name: &str) -> Option { + const DEFAULT_FALLBACK_PATHS: &[&str] = &["/usr/local/lib", "/usr/lib"]; + + // Search DYLD_LIBRARY_PATH first. + if let Ok(paths) = env::var(EnvVars::DYLD_LIBRARY_PATH) { + for dir in paths.split(':') { + let candidate = Path::new(dir).join(lib_name); + if let Ok(path) = candidate.canonicalize() { + return Some(path); + } + } + } + + // Then search DYLD_FALLBACK_LIBRARY_PATH. + if let Ok(paths) = env::var(EnvVars::DYLD_FALLBACK_LIBRARY_PATH) { + for dir in paths.split(':') { + let candidate = Path::new(dir).join(lib_name); + if let Ok(path) = candidate.canonicalize() { + return Some(path); + } + } + } + + // Default fallback paths. + for dir in DEFAULT_FALLBACK_PATHS { + let candidate = Path::new(dir).join(lib_name); + if let Ok(path) = candidate.canonicalize() { + return Some(path); + } + } + + None +} + +/// Resolve a dynamic path token (`@loader_path`, `@rpath`, `@executable_path`). +fn resolve_dynamic_path( + install_name: &str, + binary_path: &Path, + rpaths: &[String], +) -> Option { + if install_name.starts_with("@loader_path/") { + // @loader_path is relative to the binary containing the reference. + let relative = install_name.strip_prefix("@loader_path/").unwrap(); + let parent = binary_path.parent()?; + let resolved = parent.join(relative); + if let Ok(path) = resolved.canonicalize() { + return Some(path); + } + } else if install_name.starts_with("@executable_path/") { + // @executable_path; for wheels, we treat this similarly to @loader_path. + let relative = install_name.strip_prefix("@executable_path/").unwrap(); + let parent = binary_path.parent()?; + let resolved = parent.join(relative); + if let Ok(path) = resolved.canonicalize() { + return Some(path); + } + } else if install_name.starts_with("@rpath/") { + // @rpath; search through rpaths. + let relative = install_name.strip_prefix("@rpath/").unwrap(); + for rpath in rpaths { + // Resolve rpath itself if it contains tokens. + let resolved_rpath = if rpath.starts_with("@loader_path/") { + let rpath_relative = rpath.strip_prefix("@loader_path/").unwrap(); + binary_path.parent()?.join(rpath_relative) + } else if rpath.starts_with("@executable_path/") { + let rpath_relative = rpath.strip_prefix("@executable_path/").unwrap(); + binary_path.parent()?.join(rpath_relative) + } else { + PathBuf::from(rpath) + }; + + let candidate = resolved_rpath.join(relative); + if let Ok(path) = candidate.canonicalize() { + return Some(path); + } + } + } else if !install_name.starts_with('@') { + // Absolute or relative path. + let path = PathBuf::from(install_name); + if path.is_absolute() { + if let Ok(resolved) = path.canonicalize() { + return Some(resolved); + } + } + // Try relative to binary. + if let Some(parent) = binary_path.parent() { + let candidate = parent.join(&path); + if let Ok(resolved) = candidate.canonicalize() { + return Some(resolved); + } + } + // Try DYLD environment paths for bare library names. + if let Some(lib_name) = Path::new(install_name).file_name() { + if let Some(resolved) = search_dyld_paths(&lib_name.to_string_lossy()) { + return Some(resolved); + } + } + } + + None +} + +/// Information about a library and its dependents. +#[derive(Debug)] +struct LibraryInfo { + /// Binaries that depend on this library and their install names. + dependents: HashMap, +} + +/// Remove absolute rpaths from a binary. +/// +/// This prevents issues when wheels are installed in different locations, +/// as absolute rpaths would point to the original build location. +fn sanitize_rpaths(path: &Path) -> Result<(), DelocateError> { + let macho = macho::parse_macho(path)?; + + for rpath in &macho.rpaths { + // Remove rpaths that are absolute and don't use @ tokens. + if !rpath.starts_with('@') && Path::new(rpath).is_absolute() { + macho::delete_rpath(path, rpath)?; + } + } + + Ok(()) +} + +/// Get the minimum macOS version from platform tags. +/// +/// Returns the minimum version across all macOS platform tags, or `None` if there are no macOS tags. +fn get_macos_version(platform_tags: &[PlatformTag]) -> Option { + platform_tags + .iter() + .filter_map(|tag| match tag { + PlatformTag::Macos { major, minor, .. } => Some(MacOSVersion::new(*major, *minor)), + _ => None, + }) + .min() +} + +/// Check that a library's macOS version requirement is compatible with the wheel's platform tag. +fn check_macos_version_compatible( + lib_path: &Path, + wheel_version: MacOSVersion, +) -> Result<(), DelocateError> { + let macho = macho::parse_macho(lib_path)?; + + if let Some(lib_version) = macho.min_macos_version { + if lib_version > wheel_version { + return Err(DelocateError::IncompatibleMacOSVersion { + library: lib_path.to_path_buf(), + library_version: lib_version.to_string(), + wheel_version: wheel_version.to_string(), + }); + } + } + + Ok(()) +} + +/// Check that a library has all architectures required by its dependents. +fn check_dependency_archs(lib_path: &Path, required_archs: &[Arch]) -> Result<(), DelocateError> { + if required_archs.is_empty() { + return Ok(()); + } + + macho::check_archs(lib_path, required_archs) +} + +/// Find the maximum macOS version required by any of the given binaries. +fn find_max_macos_version<'a>(paths: impl Iterator) -> Option { + let mut max_version: Option = None; + + for path in paths { + if let Ok(macho) = macho::parse_macho(path) { + if let Some(version) = macho.min_macos_version { + max_version = + Some(max_version.map_or(version, |current| std::cmp::max(current, version))); + } + } + } + + max_version +} + +/// Update platform tags to reflect a new macOS version. +/// +/// For example, `macosx_10_9_x86_64` with version 11.0 becomes `macosx_11_0_x86_64`. +/// Non-macOS tags are preserved unchanged. +fn update_platform_tags_version( + platform_tags: &[PlatformTag], + version: MacOSVersion, +) -> Vec { + // Handle macOS 11+ where minor version is always 0 for tagging. + let (major, minor) = if version.major >= 11 { + (version.major, 0) + } else { + (version.major, version.minor) + }; + + platform_tags + .iter() + .map(|tag| match tag { + PlatformTag::Macos { binary_format, .. } => PlatformTag::Macos { + major, + minor, + binary_format: *binary_format, + }, + other => other.clone(), + }) + .collect() +} + +/// Find all Mach-O binaries in a directory. +fn find_binaries(dir: &Path) -> Result, DelocateError> { + let mut binaries = Vec::new(); + + for entry in WalkDir::new(dir) { + let entry = entry?; + if !entry.file_type().is_file() { + continue; + } + + let path = entry.path(); + + // Check extension. + let ext = path.extension().and_then(|ext| ext.to_str()); + if !matches!(ext, Some("so" | "dylib")) { + // Also check if it's a Mach-O without extension. + if ext.is_some() { + continue; + } + } + + if macho::is_macho_file(path)? { + binaries.push(path.to_path_buf()); + } + } + + Ok(binaries) +} + +/// Analyze dependencies for all binaries in a directory. +fn analyze_dependencies( + dir: &Path, + binaries: &[PathBuf], + exclude: &[String], +) -> Result, DelocateError> { + let mut libraries: HashMap = HashMap::new(); + let mut to_process: Vec<(PathBuf, PathBuf, String)> = Vec::new(); + + // Initial pass: collect direct dependencies. + for binary_path in binaries { + let macho = macho::parse_macho(binary_path)?; + + for dep in &macho.dependencies { + // Skip excluded libraries. + if exclude.iter().any(|pat| dep.contains(pat)) { + continue; + } + + // Skip system libraries. + if is_system_library(dep) { + continue; + } + + // Try to resolve the dependency. + if let Some(resolved) = resolve_dynamic_path(dep, binary_path, &macho.rpaths) { + // Skip if the resolved path is within our directory (already bundled). + if resolved.starts_with(dir) { + continue; + } + + to_process.push((resolved, binary_path.clone(), dep.clone())); + } + } + } + + // Process dependencies and their transitive dependencies. + let mut processed: HashSet = HashSet::new(); + + while let Some((lib_path, dependent_path, install_name)) = to_process.pop() { + // Add to libraries map. + libraries + .entry(lib_path.clone()) + .or_insert_with(|| LibraryInfo { + dependents: HashMap::new(), + }) + .dependents + .insert(dependent_path, install_name); + + // Process transitive dependencies. + if processed.insert(lib_path.clone()) { + if let Ok(macho) = macho::parse_macho(&lib_path) { + for dep in &macho.dependencies { + if exclude.iter().any(|pat| dep.contains(pat)) { + continue; + } + + if is_system_library(dep) { + continue; + } + + if let Some(resolved) = resolve_dynamic_path(dep, &lib_path, &macho.rpaths) { + if resolved.starts_with(dir) { + continue; + } + + to_process.push((resolved, lib_path.clone(), dep.clone())); + } + } + } + } + } + + Ok(libraries) +} + +/// List dependencies of a wheel. +pub fn list_wheel_dependencies( + wheel_path: &Path, +) -> Result)>, DelocateError> { + let temp_dir = tempfile::tempdir()?; + let wheel_dir = temp_dir.path(); + + wheel::unpack_wheel(wheel_path, wheel_dir)?; + + let binaries = find_binaries(wheel_dir)?; + let libraries = analyze_dependencies(wheel_dir, &binaries, &[])?; + + let mut deps: Vec<(String, Vec)> = libraries + .into_iter() + .map(|(path, info)| { + let dependents: Vec = info + .dependents + .keys() + .map(|dep_path| { + dep_path + .strip_prefix(wheel_dir) + .unwrap_or(dep_path) + .to_path_buf() + }) + .collect(); + (path.to_string_lossy().into_owned(), dependents) + }) + .collect(); + + deps.sort_by(|a, b| a.0.cmp(&b.0)); + Ok(deps) +} + +/// Delocate a wheel: copy external libraries and update install names. +pub fn delocate_wheel( + wheel_path: &Path, + dest_dir: &Path, + options: &DelocateOptions, +) -> Result { + debug!("Delocating wheel: {}", wheel_path.display()); + + let filename_str = wheel_path + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| DelocateError::InvalidWheelPath { + path: wheel_path.to_path_buf(), + })?; + let filename = WheelFilename::from_str(filename_str).map_err(|err| { + DelocateError::InvalidWheelFilename { + filename: filename_str.to_string(), + err, + } + })?; + + let temp_dir = tempfile::tempdir()?; + let wheel_dir = temp_dir.path(); + let platform_tags = filename.platform_tags(); + + // Determine the target macOS version: + // 1. Use explicitly set target_macos_version from options (includes MACOSX_DEPLOYMENT_TARGET). + // 2. Fall back to parsing from wheel's platform tag. + let wheel_platform_version = get_macos_version(platform_tags); + let target_version = options.target_macos_version.or(wheel_platform_version); + + // Unpack the wheel. + wheel::unpack_wheel(wheel_path, wheel_dir)?; + + // Find all binaries. + let binaries = find_binaries(wheel_dir)?; + + // Check required architectures. + if !options.require_archs.is_empty() { + for binary in &binaries { + macho::check_archs(binary, &options.require_archs)?; + } + } + + // Analyze dependencies. + let libraries = analyze_dependencies(wheel_dir, &binaries, &options.exclude)?; + + // Validate dependencies before copying. + for lib_path in libraries.keys() { + // Check architecture compatibility. + if !options.require_archs.is_empty() { + check_dependency_archs(lib_path, &options.require_archs)?; + } + + // Check macOS version compatibility against target version. + if options.check_version_compatibility { + if let Some(version) = target_version { + check_macos_version_compatible(lib_path, version)?; + } + } + } + + // Find the maximum macOS version required by all binaries. + let max_required_version = find_max_macos_version( + binaries + .iter() + .map(PathBuf::as_path) + .chain(libraries.keys().map(PathBuf::as_path)), + ); + + // Determine the final platform tags; update if binaries require higher version. + let final_platform_tags = match (&wheel_platform_version, &max_required_version) { + (Some(wheel_ver), Some(max_ver)) if max_ver > wheel_ver => { + update_platform_tags_version(platform_tags, *max_ver) + } + _ => platform_tags.to_vec(), + }; + + if libraries.is_empty() { + debug!("No external dependencies found"); + + // No external dependencies, but still sanitize rpaths if requested. + if options.sanitize_rpaths { + for binary in &binaries { + sanitize_rpaths(binary)?; + } + // Need to update RECORD after modifying binaries. + let dist_info = wheel::find_dist_info(wheel_dir)?; + wheel::update_record(wheel_dir, &dist_info)?; + + // Repack the wheel. + let output_filename = filename.with_platform_tags(&final_platform_tags); + let output_path = dest_dir.join(output_filename.to_string()); + wheel::pack_wheel(wheel_dir, &output_path)?; + return Ok(output_path); + } + + // No modifications needed, just copy the wheel. + let dest_path = dest_dir.join(wheel_path.file_name().unwrap()); + fs::copy(wheel_path, &dest_path)?; + return Ok(dest_path); + } + + debug!("Found {} external libraries to bundle", libraries.len()); + + // Find the package directory (first directory that's not .dist-info or .data) + let package_dir = find_package_dir(wheel_dir, &filename)?; + + // Create the library directory. + let lib_dir = package_dir.join(&options.lib_sdir); + fs::create_dir_all(&lib_dir)?; + + // Check for library name collisions. + let mut lib_names: HashMap> = HashMap::new(); + for path in libraries.keys() { + let name = path + .file_name() + .and_then(|file_name| file_name.to_str()) + .unwrap_or("unknown") + .to_string(); + lib_names.entry(name).or_default().push(path.clone()); + } + + for (name, paths) in &lib_names { + if paths.len() > 1 { + return Err(DelocateError::LibraryCollision { + name: name.clone(), + paths: paths.clone(), + }); + } + } + + // Copy libraries and update install names. + for (lib_path, info) in &libraries { + let lib_name = lib_path.file_name().unwrap(); + let dest_lib = lib_dir.join(lib_name); + + // Copy the library. + trace!("Copying {} to {}", lib_path.display(), dest_lib.display()); + fs::copy(lib_path, &dest_lib)?; + + // Sanitize rpaths in the copied library. + if options.sanitize_rpaths { + sanitize_rpaths(&dest_lib)?; + } + + // Set the install ID of the copied library. + let new_id = format!( + "@loader_path/{}/{}", + options.lib_sdir, + lib_name.to_string_lossy() + ); + macho::change_install_id(&dest_lib, &new_id)?; + + // Update install names in dependents. + for (dependent_path, old_install_name) in &info.dependents { + // Calculate the relative path from dependent to library. + let dependent_in_wheel = if dependent_path.starts_with(wheel_dir) { + dependent_path.clone() + } else { + // This is a transitive dependency that was copied. + lib_dir.join(dependent_path.file_name().unwrap()) + }; + + let dependent_parent = dependent_in_wheel.parent().unwrap(); + let relative_to_package = pathdiff::diff_paths(&lib_dir, dependent_parent) + .unwrap_or_else(|| PathBuf::from(&options.lib_sdir)); + + let new_install_name = format!( + "@loader_path/{}/{}", + relative_to_package.to_string_lossy(), + lib_name.to_string_lossy() + ); + + // Update the install name. + if dependent_in_wheel.exists() { + macho::change_install_name( + &dependent_in_wheel, + old_install_name, + &new_install_name, + )?; + } + } + } + + // Sanitize rpaths in original wheel binaries. + if options.sanitize_rpaths { + for binary in &binaries { + sanitize_rpaths(binary)?; + } + } + + // Update RECORD. + let dist_info = wheel::find_dist_info(wheel_dir)?; + wheel::update_record(wheel_dir, &dist_info)?; + + // Create output wheel with potentially updated platform tag. + let output_filename = filename.with_platform_tags(&final_platform_tags); + let output_path = dest_dir.join(output_filename.to_string()); + + wheel::pack_wheel(wheel_dir, &output_path)?; + + Ok(output_path) +} + +/// Find the main package directory in a wheel. +fn find_package_dir(wheel_dir: &Path, filename: &WheelFilename) -> Result { + // Look for a directory that matches the package name. + let dist_info_name = filename.name.as_dist_info_name(); + + for entry in fs::read_dir(wheel_dir)? { + let entry = entry?; + if !entry.file_type()?.is_dir() { + continue; + } + + let name = entry.file_name(); + let name_str = name.to_string_lossy(); + + // Skip `.dist-info` and `.data` directories. + if name_str.ends_with(".dist-info") || name_str.ends_with(".data") { + continue; + } + + // Check if it matches normalized name. + if name_str == *dist_info_name || name_str.replace('-', "_") == *dist_info_name { + return Ok(entry.path()); + } + } + + // If no matching directory, use the wheel directory itself. + Ok(wheel_dir.to_path_buf()) +} + +#[cfg(test)] +mod tests { + use super::*; + use uv_platform_tags::BinaryFormat; + + #[test] + fn test_is_system_library() { + assert!(is_system_library("/usr/lib/libSystem.B.dylib")); + assert!(is_system_library( + "/System/Library/Frameworks/CoreFoundation.framework/CoreFoundation" + )); + assert!(!is_system_library("/usr/local/lib/libfoo.dylib")); + assert!(!is_system_library("/opt/homebrew/lib/libbar.dylib")); + } + + #[test] + fn test_get_macos_version() { + // Standard format. + let tags = [PlatformTag::Macos { + major: 10, + minor: 9, + binary_format: BinaryFormat::X86_64, + }]; + let version = get_macos_version(&tags).unwrap(); + assert_eq!(version.major, 10); + assert_eq!(version.minor, 9); + + // macOS 11+. + let tags = [PlatformTag::Macos { + major: 11, + minor: 0, + binary_format: BinaryFormat::Arm64, + }]; + let version = get_macos_version(&tags).unwrap(); + assert_eq!(version.major, 11); + assert_eq!(version.minor, 0); + + // Universal binary. + let tags = [PlatformTag::Macos { + major: 10, + minor: 9, + binary_format: BinaryFormat::Universal2, + }]; + let version = get_macos_version(&tags).unwrap(); + assert_eq!(version.major, 10); + assert_eq!(version.minor, 9); + + // Multiple tags; returns minimum version. + let tags = [ + PlatformTag::Macos { + major: 11, + minor: 0, + binary_format: BinaryFormat::Arm64, + }, + PlatformTag::Macos { + major: 10, + minor: 9, + binary_format: BinaryFormat::X86_64, + }, + ]; + let version = get_macos_version(&tags).unwrap(); + assert_eq!(version.major, 10); + assert_eq!(version.minor, 9); + + // Not a macOS platform. + let tags = [PlatformTag::Linux { + arch: uv_platform_tags::Arch::X86_64, + }]; + assert!(get_macos_version(&tags).is_none()); + + let tags = [PlatformTag::WinAmd64]; + assert!(get_macos_version(&tags).is_none()); + } + + #[test] + fn test_parse_macos_version() { + let version = MacOSVersion::parse("10.9").unwrap(); + assert_eq!(version.major, 10); + assert_eq!(version.minor, 9); + + let version = MacOSVersion::parse("11.0").unwrap(); + assert_eq!(version.major, 11); + assert_eq!(version.minor, 0); + + // Patch is ignored. + let version = MacOSVersion::parse("14.2.1").unwrap(); + assert_eq!(version.major, 14); + assert_eq!(version.minor, 2); + + let version = MacOSVersion::parse("15").unwrap(); + assert_eq!(version.major, 15); + assert_eq!(version.minor, 0); + } + + #[test] + fn test_update_platform_tags_version() { + // Upgrade from 10.9 to 11.0. + let tags = [PlatformTag::Macos { + major: 10, + minor: 9, + binary_format: BinaryFormat::X86_64, + }]; + let updated = update_platform_tags_version(&tags, MacOSVersion::new(11, 0)); + assert_eq!( + updated, + vec![PlatformTag::Macos { + major: 11, + minor: 0, + binary_format: BinaryFormat::X86_64, + }] + ); + + // Upgrade from 10.9 to 10.15. + let updated = update_platform_tags_version(&tags, MacOSVersion::new(10, 15)); + assert_eq!( + updated, + vec![PlatformTag::Macos { + major: 10, + minor: 15, + binary_format: BinaryFormat::X86_64, + }] + ); + + // Universal2. + let tags = [PlatformTag::Macos { + major: 10, + minor: 9, + binary_format: BinaryFormat::Universal2, + }]; + let updated = update_platform_tags_version(&tags, MacOSVersion::new(11, 0)); + assert_eq!( + updated, + vec![PlatformTag::Macos { + major: 11, + minor: 0, + binary_format: BinaryFormat::Universal2, + }] + ); + + // macOS 11+ always has minor=0 for tagging. + let tags = [PlatformTag::Macos { + major: 11, + minor: 0, + binary_format: BinaryFormat::Arm64, + }]; + let updated = update_platform_tags_version(&tags, MacOSVersion::new(14, 2)); + assert_eq!( + updated, + vec![PlatformTag::Macos { + major: 14, + minor: 0, + binary_format: BinaryFormat::Arm64, + }] + ); + + // Non-macOS tag unchanged. + let tags = [PlatformTag::Linux { + arch: uv_platform_tags::Arch::X86_64, + }]; + let updated = update_platform_tags_version(&tags, MacOSVersion::new(11, 0)); + assert_eq!( + updated, + vec![PlatformTag::Linux { + arch: uv_platform_tags::Arch::X86_64 + }] + ); + } +} diff --git a/crates/uv-delocate/src/error.rs b/crates/uv-delocate/src/error.rs new file mode 100644 index 0000000000000..bb2d17c989b9b --- /dev/null +++ b/crates/uv-delocate/src/error.rs @@ -0,0 +1,72 @@ +use std::path::PathBuf; + +use thiserror::Error; + +use uv_distribution_filename::WheelFilenameError; +use uv_fs::Simplified; + +/// Errors that can occur during delocate operations. +#[derive(Debug, Error)] +pub enum DelocateError { + #[error(transparent)] + Io(#[from] std::io::Error), + + #[error(transparent)] + Zip(#[from] zip::result::ZipError), + + #[error(transparent)] + WalkDir(#[from] walkdir::Error), + + #[error(transparent)] + Extract(#[from] uv_extract::Error), + + #[error(transparent)] + InstallWheel(#[from] uv_install_wheel::Error), + + #[error("Failed to parse Mach-O binary: {0}")] + MachOParse(String), + + #[error("Dependency not found: {name} (required by {})", required_by.user_display())] + DependencyNotFound { name: String, required_by: PathBuf }, + + #[error("Missing required architecture {arch} in {}", path.user_display())] + MissingArchitecture { arch: String, path: PathBuf }, + + #[error("Library name collision: {name} exists at multiple paths: {paths:?}")] + LibraryCollision { name: String, paths: Vec }, + + #[error("Invalid wheel filename: {filename}")] + InvalidWheelFilename { + filename: String, + #[source] + err: WheelFilenameError, + }, + + #[error("Invalid wheel path: {}", path.user_display())] + InvalidWheelPath { path: PathBuf }, + + #[error("Missing `.dist-info` directory in wheel")] + MissingDistInfo, + + #[error("Path `{}` is not within wheel directory `{}`", path.user_display(), wheel_dir.user_display())] + PathNotInWheel { path: PathBuf, wheel_dir: PathBuf }, + + #[error("Unsupported Mach-O format: {0}")] + UnsupportedFormat(String), + + #[error( + "Library {} requires macOS {library_version}, but wheel declares {wheel_version}", + library.user_display() + )] + IncompatibleMacOSVersion { + library: PathBuf, + library_version: String, + wheel_version: String, + }, + + #[error("`codesign` failed for {}: {stderr}", path.user_display())] + CodesignFailed { path: PathBuf, stderr: String }, + + #[error("`install_name_tool` failed for {}: {stderr}", path.user_display())] + InstallNameToolFailed { path: PathBuf, stderr: String }, +} diff --git a/crates/uv-delocate/src/lib.rs b/crates/uv-delocate/src/lib.rs new file mode 100644 index 0000000000000..b79955ebc3b18 --- /dev/null +++ b/crates/uv-delocate/src/lib.rs @@ -0,0 +1,47 @@ +//! Mach-O delocate functionality for Python wheels. +//! +//! This crate provides functionality to: +//! +//! 1. Parse Mach-O binaries and extract dependency information. +//! 2. Copy external library dependencies into Python wheels. +//! 3. Update install names to use relative paths (`@loader_path`). +//! 4. Validate binary architectures. +//! +//! This library is derived from [`delocate`](https://github.com/matthew-brett/delocate) by Matthew +//! Brett and contributors, which is available under the following BSD-2-Clause license: +//! +//! ```text +//! Copyright (c) 2014-2025, Matthew Brett and the Delocate contributors. +//! All rights reserved. +//! +//! Redistribution and use in source and binary forms, with or without +//! modification, are permitted provided that the following conditions are met: +//! +//! 1. Redistributions of source code must retain the above copyright notice, this +//! list of conditions and the following disclaimer. +//! +//! 2. Redistributions in binary form must reproduce the above copyright notice, +//! this list of conditions and the following disclaimer in the documentation +//! and/or other materials provided with the distribution. +//! +//! THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +//! AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +//! IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +//! DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +//! FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +//! DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +//! SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +//! CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +//! OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +//! OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +//! ``` + +mod delocate; +mod error; +pub mod macho; +pub mod wheel; + +pub use delocate::{DelocateOptions, delocate_wheel, list_wheel_dependencies}; +pub use error::DelocateError; +pub use macho::{Arch, MachOFile}; +pub use uv_platform::MacOSVersion; diff --git a/crates/uv-delocate/src/macho.rs b/crates/uv-delocate/src/macho.rs new file mode 100644 index 0000000000000..4dc687c3eb5ae --- /dev/null +++ b/crates/uv-delocate/src/macho.rs @@ -0,0 +1,332 @@ +//! Mach-O binary parsing and modification. +//! +//! This module provides functionality to detect Mach-O binaries, parse their +//! dependencies and rpaths, detect architectures, and modify install names. + +use std::collections::HashSet; +use std::io::Read; +use std::path::Path; +use std::process::Command; + +use fs_err as fs; +use goblin::Hint; +use goblin::mach::load_command; +use goblin::mach::{Mach, MachO}; +use tracing::trace; + +use uv_platform::MacOSVersion; + +use crate::error::DelocateError; + +/// CPU architecture of a Mach-O binary. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum Arch { + X86_64, + Arm64, + I386, + Arm64_32, + PowerPC, + PowerPC64, + Unknown(u32), +} + +impl Arch { + fn from_cputype(cputype: u32) -> Self { + use goblin::mach::cputype::{ + CPU_TYPE_ARM64, CPU_TYPE_ARM64_32, CPU_TYPE_I386, CPU_TYPE_POWERPC, CPU_TYPE_POWERPC64, + CPU_TYPE_X86_64, + }; + match cputype { + CPU_TYPE_X86_64 => Self::X86_64, + CPU_TYPE_ARM64 => Self::Arm64, + CPU_TYPE_I386 => Self::I386, + CPU_TYPE_ARM64_32 => Self::Arm64_32, + CPU_TYPE_POWERPC => Self::PowerPC, + CPU_TYPE_POWERPC64 => Self::PowerPC64, + other => Self::Unknown(other), + } + } + + /// Returns the architecture name as used in wheel platform tags. + pub fn as_str(&self) -> &'static str { + match self { + Self::X86_64 => "x86_64", + Self::Arm64 => "arm64", + Self::I386 => "i386", + Self::Arm64_32 => "arm64_32", + Self::PowerPC => "ppc", + Self::PowerPC64 => "ppc64", + Self::Unknown(_) => "unknown", + } + } +} + +impl std::fmt::Display for Arch { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.as_str()) + } +} + +impl std::str::FromStr for Arch { + type Err = String; + + fn from_str(s: &str) -> Result { + match s.to_lowercase().as_str() { + "x86_64" => Ok(Self::X86_64), + "arm64" | "aarch64" => Ok(Self::Arm64), + "i386" | "i686" | "x86" => Ok(Self::I386), + "arm64_32" => Ok(Self::Arm64_32), + "ppc" | "powerpc" => Ok(Self::PowerPC), + "ppc64" | "powerpc64" => Ok(Self::PowerPC64), + _ => Err(format!("Unknown architecture: {s}")), + } + } +} + +/// Parsed Mach-O file information. +#[derive(Debug)] +pub struct MachOFile { + /// Architectures present in the binary. + pub archs: HashSet, + /// Dylib dependencies (`LC_LOAD_DYLIB`, `LC_LOAD_WEAK_DYLIB`, etc.). + pub dependencies: Vec, + /// Runtime search paths (`LC_RPATH`). + pub rpaths: Vec, + /// Install name of this library (`LC_ID_DYLIB`), if present. + pub install_name: Option, + /// Minimum macOS version required, per architecture. + pub min_macos_version: Option, +} + +/// Check if a file is a Mach-O binary by examining its magic bytes. +pub fn is_macho_file(path: &Path) -> Result { + let mut file = match fs::File::open(path) { + Ok(f) => f, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(false), + Err(e) => return Err(e.into()), + }; + + let mut bytes = [0u8; 16]; + if file.read_exact(&mut bytes).is_err() { + return Ok(false); + } + + Ok(matches!( + goblin::mach::peek_bytes(&bytes), + Ok(Hint::Mach(_) | Hint::MachFat(_)) + )) +} + +/// Parse a Mach-O file and extract dependency information. +pub fn parse_macho(path: &Path) -> Result { + let data = fs::read(path)?; + parse_macho_bytes(&data) +} + +/// Parse Mach-O data from bytes. +pub fn parse_macho_bytes(data: &[u8]) -> Result { + let mach = Mach::parse(data).map_err(|err| DelocateError::MachOParse(err.to_string()))?; + + match mach { + Mach::Binary(macho) => Ok(parse_single_macho(&macho)), + Mach::Fat(fat) => { + let mut archs = HashSet::new(); + let mut all_deps: HashSet = HashSet::new(); + let mut all_rpaths: HashSet = HashSet::new(); + let mut install_name: Option = None; + let mut min_macos_version: Option = None; + + for arch in fat.iter_arches().flatten() { + let slice_data = &data[arch.offset as usize..(arch.offset + arch.size) as usize]; + let macho = MachO::parse(slice_data, 0) + .map_err(|err| DelocateError::MachOParse(err.to_string()))?; + + let parsed = parse_single_macho(&macho); + archs.extend(parsed.archs); + all_deps.extend(parsed.dependencies); + all_rpaths.extend(parsed.rpaths); + install_name = install_name.or(parsed.install_name); + + // Take the maximum macOS version across all architectures. + if let Some(version) = parsed.min_macos_version { + min_macos_version = Some( + min_macos_version + .map_or(version, |current| std::cmp::max(current, version)), + ); + } + } + + Ok(MachOFile { + archs, + dependencies: all_deps.into_iter().collect(), + rpaths: all_rpaths.into_iter().collect(), + install_name, + min_macos_version, + }) + } + } +} + +fn parse_single_macho(macho: &MachO) -> MachOFile { + let mut min_macos_version: Option = None; + + for cmd in &macho.load_commands { + match cmd.command { + load_command::CommandVariant::BuildVersion(ref build_ver) => { + // LC_BUILD_VERSION is used in modern binaries; platform 1 = MACOS. + if build_ver.platform == 1 { + let version = MacOSVersion::from_packed(build_ver.minos); + min_macos_version = Some( + min_macos_version + .map_or(version, |current| std::cmp::max(current, version)), + ); + } + } + load_command::CommandVariant::VersionMinMacosx(ref ver) => { + // LC_VERSION_MIN_MACOSX is used in older binaries. + let version = MacOSVersion::from_packed(ver.version); + min_macos_version = Some( + min_macos_version.map_or(version, |current| std::cmp::max(current, version)), + ); + } + _ => {} + } + } + + MachOFile { + archs: HashSet::from([Arch::from_cputype(macho.header.cputype())]), + dependencies: macho.libs.iter().map(|s| (*s).to_string()).collect(), + rpaths: macho.rpaths.iter().map(|s| (*s).to_string()).collect(), + install_name: macho.name.map(ToString::to_string), + min_macos_version, + } +} + +/// Change an install name in a Mach-O binary file. +pub fn change_install_name( + path: &Path, + old_name: &str, + new_name: &str, +) -> Result<(), DelocateError> { + trace!( + "Changing install name in {}: {} -> {}", + path.display(), + old_name, + new_name + ); + + let output = Command::new("install_name_tool") + .args(["-change", old_name, new_name]) + .arg(path) + .output()?; + + if output.status.success() { + sign_adhoc(path) + } else { + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); + Err(DelocateError::InstallNameToolFailed { + path: path.to_path_buf(), + stderr, + }) + } +} + +/// Change the install ID (`LC_ID_DYLIB`) of a Mach-O library. +pub fn change_install_id(path: &Path, new_id: &str) -> Result<(), DelocateError> { + trace!("Changing install ID of {} to {}", path.display(), new_id); + + let output = Command::new("install_name_tool") + .args(["-id", new_id]) + .arg(path) + .output()?; + + if output.status.success() { + sign_adhoc(path) + } else { + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); + Err(DelocateError::InstallNameToolFailed { + path: path.to_path_buf(), + stderr, + }) + } +} + +/// Delete an rpath from a Mach-O binary. +pub fn delete_rpath(path: &Path, rpath: &str) -> Result<(), DelocateError> { + trace!("Deleting rpath {} from {}", rpath, path.display()); + + let output = Command::new("install_name_tool") + .args(["-delete_rpath", rpath]) + .arg(path) + .output()?; + + if output.status.success() { + sign_adhoc(path) + } else { + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); + Err(DelocateError::InstallNameToolFailed { + path: path.to_path_buf(), + stderr, + }) + } +} + +/// Apply ad-hoc code signing to a binary. +/// +/// This forcefully replaces any existing signature with an ad-hoc signature. +/// This is required on macOS (especially Apple Silicon) after modifying binaries, +/// as the modification invalidates the existing signature. +fn sign_adhoc(path: &Path) -> Result<(), DelocateError> { + trace!("Applying ad-hoc code signature to {}", path.display()); + + let output = Command::new("codesign") + .args(["--force", "--sign", "-"]) + .arg(path) + .output()?; + + if output.status.success() { + Ok(()) + } else { + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); + Err(DelocateError::CodesignFailed { + path: path.to_path_buf(), + stderr, + }) + } +} + +/// Check if a binary has all the required architectures. +pub fn check_archs(path: &Path, required: &[Arch]) -> Result<(), DelocateError> { + let macho = parse_macho(path)?; + + for arch in required { + if !macho.archs.contains(arch) { + return Err(DelocateError::MissingArchitecture { + arch: arch.to_string(), + path: path.to_path_buf(), + }); + } + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_arch_display() { + assert_eq!(Arch::X86_64.as_str(), "x86_64"); + assert_eq!(Arch::Arm64.as_str(), "arm64"); + } + + #[test] + fn test_arch_from_str() { + assert_eq!("x86_64".parse::().unwrap(), Arch::X86_64); + assert_eq!("arm64".parse::().unwrap(), Arch::Arm64); + assert_eq!("aarch64".parse::().unwrap(), Arch::Arm64); + assert_eq!("i386".parse::().unwrap(), Arch::I386); + assert!("unknown_arch".parse::().is_err()); + } +} diff --git a/crates/uv-delocate/src/wheel.rs b/crates/uv-delocate/src/wheel.rs new file mode 100644 index 0000000000000..d5ddb4d5605f6 --- /dev/null +++ b/crates/uv-delocate/src/wheel.rs @@ -0,0 +1,149 @@ +//! Python wheel file operations. +//! +//! Provides functionality for unpacking, modifying, and repacking wheel files, +//! including RECORD file updates. + +use std::io::{self, Read}; +use std::path::Path; + +use base64::Engine; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use fs_err as fs; +use fs_err::File; +use sha2::{Digest, Sha256}; +use walkdir::WalkDir; +use zip::ZipWriter; +use zip::write::FileOptions; + +use uv_install_wheel::{RecordEntry, write_record_file}; + +use crate::error::DelocateError; + +/// Unpack a wheel to a directory. +pub fn unpack_wheel(wheel_path: &Path, dest_dir: &Path) -> Result<(), DelocateError> { + let file = File::open(wheel_path)?; + uv_extract::unzip(file, dest_dir)?; + Ok(()) +} + +/// Repack a directory into a wheel file. +pub fn pack_wheel(source_dir: &Path, wheel_path: &Path) -> Result<(), DelocateError> { + let file = File::create(wheel_path)?; + let mut zip = ZipWriter::new(file); + + let options = FileOptions::<()>::default() + .compression_method(zip::CompressionMethod::Deflated) + .unix_permissions(0o644); + + let walkdir = WalkDir::new(source_dir); + let mut paths: Vec<_> = walkdir + .into_iter() + .filter_map(Result::ok) + .filter(|entry| entry.file_type().is_file()) + .collect(); + + // Sort for reproducibility. + paths.sort_by(|a, b| a.path().cmp(b.path())); + + for entry in paths { + let path = entry.path(); + let relative = + path.strip_prefix(source_dir) + .map_err(|_| DelocateError::PathNotInWheel { + path: path.to_path_buf(), + wheel_dir: source_dir.to_path_buf(), + })?; + + let relative_str = relative.to_string_lossy(); + + // Normalize permissions: 0o755 for executables, 0o644 for non-executable files. + #[cfg(unix)] + let options = { + use std::os::unix::fs::PermissionsExt; + let metadata = fs::metadata(path)?; + let is_executable = metadata.permissions().mode() & 0o111 != 0; + let permissions = if is_executable { 0o755 } else { 0o644 }; + options.unix_permissions(permissions) + }; + + zip.start_file(relative_str.as_ref(), options)?; + + let mut f = File::open(path)?; + io::copy(&mut f, &mut zip)?; + } + + zip.finish()?; + Ok(()) +} + +/// Compute the SHA256 hash of a file in the format used by RECORD files. +fn hash_file(path: &Path) -> Result<(String, u64), DelocateError> { + let mut file = File::open(path)?; + let mut hasher = Sha256::new(); + let mut buffer = [0u8; 8192]; + let mut size = 0u64; + + loop { + let n = file.read(&mut buffer)?; + if n == 0 { + break; + } + hasher.update(&buffer[..n]); + size += n as u64; + } + + let hash = hasher.finalize(); + let hash_str = format!("sha256={}", URL_SAFE_NO_PAD.encode(hash)); + + Ok((hash_str, size)) +} + +/// Update the RECORD file in a wheel directory. +pub fn update_record(wheel_dir: &Path, dist_info_dir: &str) -> Result<(), DelocateError> { + let record_path = wheel_dir.join(dist_info_dir).join("RECORD"); + + let mut records = Vec::new(); + + for entry in WalkDir::new(wheel_dir) { + let entry = entry?; + if !entry.file_type().is_file() { + continue; + } + + let path = entry.path(); + let relative = path + .strip_prefix(wheel_dir) + .map_err(|_| DelocateError::PathNotInWheel { + path: path.to_path_buf(), + wheel_dir: wheel_dir.to_path_buf(), + })?; + + let relative_str = relative.to_string_lossy().replace('\\', "/"); + + // RECORD file itself has no hash. + if relative_str == format!("{dist_info_dir}/RECORD") { + records.push(RecordEntry::unhashed(relative_str)); + } else { + let (hash, size) = hash_file(path)?; + records.push(RecordEntry::new(relative_str, hash, size)); + } + } + + write_record_file(&record_path, records)?; + + Ok(()) +} + +/// Find the .dist-info directory in an unpacked wheel. +pub fn find_dist_info(wheel_dir: &Path) -> Result { + for entry in fs::read_dir(wheel_dir)? { + let entry = entry?; + let name = entry.file_name(); + let name_str = name.to_string_lossy(); + if name_str.ends_with(".dist-info") && entry.file_type()?.is_dir() { + return Ok(name_str.into_owned()); + } + } + + Err(DelocateError::MissingDistInfo) +} diff --git a/crates/uv-delocate/tests/arch_checking.rs b/crates/uv-delocate/tests/arch_checking.rs new file mode 100644 index 0000000000000..952e65596437d --- /dev/null +++ b/crates/uv-delocate/tests/arch_checking.rs @@ -0,0 +1,46 @@ +//! Tests for architecture checking functionality. + +use std::path::PathBuf; + +use uv_delocate::{Arch, DelocateError}; + +fn test_data_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/data") +} + +#[test] +fn test_check_archs_single_present() { + let data_dir = test_data_dir(); + // x86_64 only library. + assert!(uv_delocate::macho::check_archs(&data_dir.join("liba.dylib"), &[Arch::X86_64]).is_ok()); +} + +#[test] +fn test_check_archs_single_missing() { + let data_dir = test_data_dir(); + // x86_64 library doesn't have arm64. + let result = uv_delocate::macho::check_archs(&data_dir.join("liba.dylib"), &[Arch::Arm64]); + assert!(matches!( + result, + Err(DelocateError::MissingArchitecture { .. }) + )); +} + +#[test] +fn test_check_archs_universal() { + let data_dir = test_data_dir(); + // Universal binary should have both. + assert!( + uv_delocate::macho::check_archs(&data_dir.join("liba_both.dylib"), &[Arch::X86_64]).is_ok() + ); + assert!( + uv_delocate::macho::check_archs(&data_dir.join("liba_both.dylib"), &[Arch::Arm64]).is_ok() + ); + assert!( + uv_delocate::macho::check_archs( + &data_dir.join("liba_both.dylib"), + &[Arch::X86_64, Arch::Arm64] + ) + .is_ok() + ); +} diff --git a/crates/uv-delocate/tests/code_signing.rs b/crates/uv-delocate/tests/code_signing.rs new file mode 100644 index 0000000000000..9ea1c24df371e --- /dev/null +++ b/crates/uv-delocate/tests/code_signing.rs @@ -0,0 +1,68 @@ +//! Tests for code signing after binary modification (macOS only). + +#![cfg(target_os = "macos")] + +use std::path::PathBuf; +use std::process::Command; + +use fs_err as fs; +use tempfile::TempDir; + +fn test_data_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/data") +} + +fn copy_dylib(name: &str, temp_dir: &TempDir) -> PathBuf { + let src = test_data_dir().join(name); + let dest = temp_dir.path().join(name); + fs::copy(&src, &dest).unwrap(); + dest +} + +#[test] +fn test_codesign_after_modification() { + let temp_dir = TempDir::new().unwrap(); + let dylib = copy_dylib("liba.dylib", &temp_dir); + + // Modify the install ID. + uv_delocate::macho::change_install_id(&dylib, "@loader_path/new_id.dylib").unwrap(); + + // Verify the binary is still valid (codesign should have been applied). + let output = Command::new("codesign") + .args(["--verify", &dylib.to_string_lossy()]) + .output() + .unwrap(); + + // Ad-hoc signed binaries should verify successfully. + assert!( + output.status.success(), + "Binary should be valid after modification: {:?}", + String::from_utf8_lossy(&output.stderr) + ); +} + +#[test] +fn test_codesign_after_rpath_deletion() { + let temp_dir = TempDir::new().unwrap(); + let dylib = copy_dylib("libextfunc_rpath.dylib", &temp_dir); + + // Parse and check rpaths. + let macho = uv_delocate::macho::parse_macho(&dylib).unwrap(); + + // Delete an rpath if there is one. + if !macho.rpaths.is_empty() { + let rpath = &macho.rpaths[0]; + uv_delocate::macho::delete_rpath(&dylib, rpath).unwrap(); + + // Verify the binary is still valid. + let output = Command::new("codesign") + .args(["--verify", &dylib.to_string_lossy()]) + .output() + .unwrap(); + + assert!( + output.status.success(), + "Binary should be valid after rpath deletion" + ); + } +} diff --git a/crates/uv-delocate/tests/data/fakepkg1-1.0-cp36-abi3-macosx_10_9_universal2.whl b/crates/uv-delocate/tests/data/fakepkg1-1.0-cp36-abi3-macosx_10_9_universal2.whl new file mode 100644 index 0000000000000..fae621f85edeb Binary files /dev/null and b/crates/uv-delocate/tests/data/fakepkg1-1.0-cp36-abi3-macosx_10_9_universal2.whl differ diff --git a/crates/uv-delocate/tests/data/fakepkg2-1.0-py3-none-any.whl b/crates/uv-delocate/tests/data/fakepkg2-1.0-py3-none-any.whl new file mode 100644 index 0000000000000..976048880c458 Binary files /dev/null and b/crates/uv-delocate/tests/data/fakepkg2-1.0-py3-none-any.whl differ diff --git a/crates/uv-delocate/tests/data/fakepkg_namespace-1.0-cp36-abi3-macosx_10_9_universal2.whl b/crates/uv-delocate/tests/data/fakepkg_namespace-1.0-cp36-abi3-macosx_10_9_universal2.whl new file mode 100644 index 0000000000000..fe4bad10c855a Binary files /dev/null and b/crates/uv-delocate/tests/data/fakepkg_namespace-1.0-cp36-abi3-macosx_10_9_universal2.whl differ diff --git a/crates/uv-delocate/tests/data/fakepkg_rpath-1.0-cp36-abi3-macosx_10_9_universal2.whl b/crates/uv-delocate/tests/data/fakepkg_rpath-1.0-cp36-abi3-macosx_10_9_universal2.whl new file mode 100644 index 0000000000000..8a3267b65852e Binary files /dev/null and b/crates/uv-delocate/tests/data/fakepkg_rpath-1.0-cp36-abi3-macosx_10_9_universal2.whl differ diff --git a/crates/uv-delocate/tests/data/fakepkg_toplevel-1.0-cp36-abi3-macosx_10_9_universal2.whl b/crates/uv-delocate/tests/data/fakepkg_toplevel-1.0-cp36-abi3-macosx_10_9_universal2.whl new file mode 100644 index 0000000000000..ac2a95edcb4e5 Binary files /dev/null and b/crates/uv-delocate/tests/data/fakepkg_toplevel-1.0-cp36-abi3-macosx_10_9_universal2.whl differ diff --git a/crates/uv-delocate/tests/data/liba.dylib b/crates/uv-delocate/tests/data/liba.dylib new file mode 100644 index 0000000000000..16d74fa54377d Binary files /dev/null and b/crates/uv-delocate/tests/data/liba.dylib differ diff --git a/crates/uv-delocate/tests/data/liba_12.dylib b/crates/uv-delocate/tests/data/liba_12.dylib new file mode 100755 index 0000000000000..d07bf701dd2dd Binary files /dev/null and b/crates/uv-delocate/tests/data/liba_12.dylib differ diff --git a/crates/uv-delocate/tests/data/liba_12_1.dylib b/crates/uv-delocate/tests/data/liba_12_1.dylib new file mode 100644 index 0000000000000..608596eb73f80 Binary files /dev/null and b/crates/uv-delocate/tests/data/liba_12_1.dylib differ diff --git a/crates/uv-delocate/tests/data/liba_both.dylib b/crates/uv-delocate/tests/data/liba_both.dylib new file mode 100755 index 0000000000000..54759ae93c0a6 Binary files /dev/null and b/crates/uv-delocate/tests/data/liba_both.dylib differ diff --git a/crates/uv-delocate/tests/data/libam1-arch.dylib b/crates/uv-delocate/tests/data/libam1-arch.dylib new file mode 100755 index 0000000000000..206019192c47f Binary files /dev/null and b/crates/uv-delocate/tests/data/libam1-arch.dylib differ diff --git a/crates/uv-delocate/tests/data/libam1.dylib b/crates/uv-delocate/tests/data/libam1.dylib new file mode 100755 index 0000000000000..d9011ddc394f5 Binary files /dev/null and b/crates/uv-delocate/tests/data/libam1.dylib differ diff --git a/crates/uv-delocate/tests/data/libam1_12.dylib b/crates/uv-delocate/tests/data/libam1_12.dylib new file mode 100755 index 0000000000000..cf849edadb923 Binary files /dev/null and b/crates/uv-delocate/tests/data/libam1_12.dylib differ diff --git a/crates/uv-delocate/tests/data/libb.dylib b/crates/uv-delocate/tests/data/libb.dylib new file mode 100644 index 0000000000000..8d80183446cdc Binary files /dev/null and b/crates/uv-delocate/tests/data/libb.dylib differ diff --git a/crates/uv-delocate/tests/data/libc.dylib b/crates/uv-delocate/tests/data/libc.dylib new file mode 100755 index 0000000000000..e1c076f165df3 Binary files /dev/null and b/crates/uv-delocate/tests/data/libc.dylib differ diff --git a/crates/uv-delocate/tests/data/libc_12.dylib b/crates/uv-delocate/tests/data/libc_12.dylib new file mode 100755 index 0000000000000..6eedbd7201d71 Binary files /dev/null and b/crates/uv-delocate/tests/data/libc_12.dylib differ diff --git a/crates/uv-delocate/tests/data/libextfunc.dylib b/crates/uv-delocate/tests/data/libextfunc.dylib new file mode 100755 index 0000000000000..4732f7f37ab47 Binary files /dev/null and b/crates/uv-delocate/tests/data/libextfunc.dylib differ diff --git a/crates/uv-delocate/tests/data/libextfunc2_rpath.dylib b/crates/uv-delocate/tests/data/libextfunc2_rpath.dylib new file mode 100644 index 0000000000000..7336c14da8165 Binary files /dev/null and b/crates/uv-delocate/tests/data/libextfunc2_rpath.dylib differ diff --git a/crates/uv-delocate/tests/data/libextfunc_rpath.dylib b/crates/uv-delocate/tests/data/libextfunc_rpath.dylib new file mode 100755 index 0000000000000..161bfaf12db64 Binary files /dev/null and b/crates/uv-delocate/tests/data/libextfunc_rpath.dylib differ diff --git a/crates/uv-delocate/tests/data/np-1.24.1_arm_random__sfc64.cpython-311-darwin.so b/crates/uv-delocate/tests/data/np-1.24.1_arm_random__sfc64.cpython-311-darwin.so new file mode 100755 index 0000000000000..8b4177a859a12 Binary files /dev/null and b/crates/uv-delocate/tests/data/np-1.24.1_arm_random__sfc64.cpython-311-darwin.so differ diff --git a/crates/uv-delocate/tests/data/np-1.24.1_x86_64_random__sfc64.cpython-311-darwin.so b/crates/uv-delocate/tests/data/np-1.24.1_x86_64_random__sfc64.cpython-311-darwin.so new file mode 100755 index 0000000000000..2ea0efa7de22f Binary files /dev/null and b/crates/uv-delocate/tests/data/np-1.24.1_x86_64_random__sfc64.cpython-311-darwin.so differ diff --git a/crates/uv-delocate/tests/data/test-lib b/crates/uv-delocate/tests/data/test-lib new file mode 100755 index 0000000000000..f7e245e1b21e3 Binary files /dev/null and b/crates/uv-delocate/tests/data/test-lib differ diff --git a/crates/uv-delocate/tests/delocate_integration.rs b/crates/uv-delocate/tests/delocate_integration.rs new file mode 100644 index 0000000000000..bf305349f6069 --- /dev/null +++ b/crates/uv-delocate/tests/delocate_integration.rs @@ -0,0 +1,60 @@ +//! Integration tests for wheel delocate functionality. + +use std::path::PathBuf; + +use tempfile::TempDir; + +use uv_delocate::{Arch, DelocateOptions}; + +fn test_data_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/data") +} + +#[test] +fn test_list_wheel_dependencies_pure_python() { + let data_dir = test_data_dir(); + let deps = + uv_delocate::list_wheel_dependencies(&data_dir.join("fakepkg2-1.0-py3-none-any.whl")) + .unwrap(); + + // Pure Python wheel should have no external dependencies. + assert!(deps.is_empty()); +} + +#[test] +fn test_delocate_pure_python_wheel() { + let data_dir = test_data_dir(); + let temp_dir = TempDir::new().unwrap(); + + let result = uv_delocate::delocate_wheel( + &data_dir.join("fakepkg2-1.0-py3-none-any.whl"), + temp_dir.path(), + &DelocateOptions::default(), + ) + .unwrap(); + + // Should just copy the wheel unchanged. + assert!(result.exists()); + assert_eq!(result.file_name().unwrap(), "fakepkg2-1.0-py3-none-any.whl"); +} + +#[test] +fn test_delocate_wheel_with_require_archs() { + let data_dir = test_data_dir(); + let temp_dir = TempDir::new().unwrap(); + + // This should work because the wheel has universal2 binaries. + let options = DelocateOptions { + require_archs: vec![Arch::X86_64, Arch::Arm64], + ..Default::default() + }; + + let result = uv_delocate::delocate_wheel( + &data_dir.join("fakepkg1-1.0-cp36-abi3-macosx_10_9_universal2.whl"), + temp_dir.path(), + &options, + ); + + // The wheel should be processed (it may or may not have external deps). + assert!(result.is_ok()); +} diff --git a/crates/uv-delocate/tests/install_name_modification.rs b/crates/uv-delocate/tests/install_name_modification.rs new file mode 100644 index 0000000000000..48d2226710dfa --- /dev/null +++ b/crates/uv-delocate/tests/install_name_modification.rs @@ -0,0 +1,143 @@ +//! Tests for install name modification (macOS only). + +#![cfg(target_os = "macos")] + +use std::path::PathBuf; + +use fs_err as fs; +use tempfile::TempDir; + +use uv_delocate::Arch; + +fn test_data_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/data") +} + +fn copy_dylib(name: &str, temp_dir: &TempDir) -> PathBuf { + let src = test_data_dir().join(name); + let dst = temp_dir.path().join(name); + fs::copy(&src, &dst).unwrap(); + dst +} + +#[test] +fn test_change_install_name() { + let temp_dir = TempDir::new().unwrap(); + let dylib = copy_dylib("libextfunc_rpath.dylib", &temp_dir); + + // This dylib depends on @rpath/libextfunc2_rpath.dylib. + // Change it to a shorter @loader_path path (which will fit). + uv_delocate::macho::change_install_name( + &dylib, + "@rpath/libextfunc2_rpath.dylib", + "@loader_path/ext2.dylib", + ) + .unwrap(); + + // Verify the change. + let macho = uv_delocate::macho::parse_macho(&dylib).unwrap(); + let dep_names: Vec<&str> = macho.dependencies.iter().map(String::as_str).collect(); + assert!(dep_names.contains(&"@loader_path/ext2.dylib")); + assert!(!dep_names.contains(&"@rpath/libextfunc2_rpath.dylib")); +} + +#[test] +fn test_change_install_name_longer_via_install_name_tool() { + let temp_dir = TempDir::new().unwrap(); + let dylib = copy_dylib("libb.dylib", &temp_dir); + + // libb depends on liba.dylib (short name). + // Change to a longer name - should succeed via install_name_tool fallback. + uv_delocate::macho::change_install_name( + &dylib, + "liba.dylib", + "@loader_path/long/path/liba.dylib", + ) + .unwrap(); + + // Verify the change. + let macho = uv_delocate::macho::parse_macho(&dylib).unwrap(); + let dep_names: Vec<&str> = macho.dependencies.iter().map(String::as_str).collect(); + assert!(dep_names.contains(&"@loader_path/long/path/liba.dylib")); +} + +#[test] +fn test_change_install_name_not_found() { + let temp_dir = TempDir::new().unwrap(); + let dylib = copy_dylib("liba.dylib", &temp_dir); + + // Get original dependencies. + let original = uv_delocate::macho::parse_macho(&dylib).unwrap(); + let original_deps = original.dependencies.clone(); + + // liba doesn't depend on "nonexistent.dylib". + // install_name_tool silently does nothing if the old name doesn't exist. + let result = uv_delocate::macho::change_install_name( + &dylib, + "nonexistent.dylib", + "@loader_path/foo.dylib", + ); + assert!(result.is_ok()); + + // Verify dependencies are unchanged. + let after = uv_delocate::macho::parse_macho(&dylib).unwrap(); + let after_deps = after.dependencies.clone(); + assert_eq!(original_deps, after_deps); +} + +#[test] +fn test_change_install_id() { + let temp_dir = TempDir::new().unwrap(); + let dylib = copy_dylib("libextfunc_rpath.dylib", &temp_dir); + + // Change install ID (original is @rpath/libextfunc_rpath.dylib). + // Use a shorter name that fits. + uv_delocate::macho::change_install_id(&dylib, "@loader_path/ext.dylib").unwrap(); + + // Verify the change. + let macho = uv_delocate::macho::parse_macho(&dylib).unwrap(); + assert!(macho.install_name.is_some()); + assert_eq!( + macho.install_name.as_ref().unwrap(), + "@loader_path/ext.dylib" + ); +} + +#[test] +fn test_change_install_id_longer_via_install_name_tool() { + let temp_dir = TempDir::new().unwrap(); + let dylib = copy_dylib("liba.dylib", &temp_dir); + + // Original ID is "liba.dylib" (short). + // Change to a longer name - should succeed via install_name_tool fallback. + uv_delocate::macho::change_install_id(&dylib, "@loader_path/long/path/liba.dylib").unwrap(); + + // Verify the change. + let macho = uv_delocate::macho::parse_macho(&dylib).unwrap(); + assert!(macho.install_name.is_some()); + assert_eq!( + macho.install_name.as_ref().unwrap(), + "@loader_path/long/path/liba.dylib" + ); +} + +#[test] +fn test_change_install_id_universal_binary() { + let temp_dir = TempDir::new().unwrap(); + let dylib = copy_dylib("libextfunc_rpath.dylib", &temp_dir); + + // Change install ID in universal binary - should update both slices. + // Original is @rpath/libextfunc_rpath.dylib. + uv_delocate::macho::change_install_id(&dylib, "@loader_path/ext.dylib").unwrap(); + + // Verify both architectures see the change. + let macho = uv_delocate::macho::parse_macho(&dylib).unwrap(); + assert!(macho.install_name.is_some()); + assert_eq!( + macho.install_name.as_ref().unwrap(), + "@loader_path/ext.dylib" + ); + // Should still have both architectures. + assert!(macho.archs.contains(&Arch::X86_64)); + assert!(macho.archs.contains(&Arch::Arm64)); +} diff --git a/crates/uv-delocate/tests/macho_parsing.rs b/crates/uv-delocate/tests/macho_parsing.rs new file mode 100644 index 0000000000000..e92922a5b0b1f --- /dev/null +++ b/crates/uv-delocate/tests/macho_parsing.rs @@ -0,0 +1,122 @@ +//! Tests for Mach-O parsing functionality. + +use std::collections::HashSet; +use std::path::PathBuf; + +use uv_delocate::Arch; + +fn test_data_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/data") +} + +#[test] +fn test_is_macho_file() { + let data_dir = test_data_dir(); + + // Should recognize dylibs. + assert!(uv_delocate::macho::is_macho_file(&data_dir.join("liba.dylib")).unwrap()); + assert!(uv_delocate::macho::is_macho_file(&data_dir.join("libb.dylib")).unwrap()); + assert!(uv_delocate::macho::is_macho_file(&data_dir.join("liba_both.dylib")).unwrap()); + + // Should recognize .so files. + assert!( + uv_delocate::macho::is_macho_file( + &data_dir.join("np-1.24.1_arm_random__sfc64.cpython-311-darwin.so") + ) + .unwrap() + ); + + // Non-existent files should return false. + assert!(!uv_delocate::macho::is_macho_file(&data_dir.join("nonexistent.dylib")).unwrap()); +} + +#[test] +fn test_parse_single_arch_x86_64() { + let data_dir = test_data_dir(); + let macho = uv_delocate::macho::parse_macho(&data_dir.join("liba.dylib")).unwrap(); + + // Check architecture. + assert!(macho.archs.contains(&Arch::X86_64)); + assert_eq!(macho.archs.len(), 1); + + // Check dependencies; should have system libs. + let dep_names: Vec<&str> = macho.dependencies.iter().map(String::as_str).collect(); + assert!(dep_names.contains(&"/usr/lib/libc++.1.dylib")); + assert!(dep_names.contains(&"/usr/lib/libSystem.B.dylib")); +} + +#[test] +fn test_parse_single_arch_arm64() { + let data_dir = test_data_dir(); + let macho = uv_delocate::macho::parse_macho(&data_dir.join("libam1.dylib")).unwrap(); + + // Check architecture. + assert!(macho.archs.contains(&Arch::Arm64)); + assert_eq!(macho.archs.len(), 1); +} + +#[test] +fn test_parse_universal_binary() { + let data_dir = test_data_dir(); + let macho = uv_delocate::macho::parse_macho(&data_dir.join("liba_both.dylib")).unwrap(); + + // Should have both architectures. + assert!(macho.archs.contains(&Arch::X86_64)); + assert!(macho.archs.contains(&Arch::Arm64)); + assert_eq!(macho.archs.len(), 2); +} + +#[test] +fn test_parse_with_dependencies() { + let data_dir = test_data_dir(); + + // libb depends on liba. + let macho = uv_delocate::macho::parse_macho(&data_dir.join("libb.dylib")).unwrap(); + let dep_names: Vec<&str> = macho.dependencies.iter().map(String::as_str).collect(); + assert!(dep_names.contains(&"liba.dylib")); + + // libc depends on liba and libb. + let macho = uv_delocate::macho::parse_macho(&data_dir.join("libc.dylib")).unwrap(); + let dep_names: Vec<&str> = macho.dependencies.iter().map(String::as_str).collect(); + assert!(dep_names.contains(&"liba.dylib")); + assert!(dep_names.contains(&"libb.dylib")); +} + +#[test] +fn test_parse_with_rpath() { + let data_dir = test_data_dir(); + let macho = uv_delocate::macho::parse_macho(&data_dir.join("libextfunc_rpath.dylib")).unwrap(); + + // Should have @rpath dependencies. + let dep_names: Vec<&str> = macho.dependencies.iter().map(String::as_str).collect(); + assert!(dep_names.contains(&"@rpath/libextfunc2_rpath.dylib")); + + // Should have rpaths. + assert!(!macho.rpaths.is_empty()); + let rpath_set: HashSet<&str> = macho + .rpaths + .iter() + .map(std::string::String::as_str) + .collect(); + assert!(rpath_set.contains("@loader_path/")); + assert!(rpath_set.contains("@executable_path/")); +} + +#[test] +fn test_parse_numpy_extension() { + let data_dir = test_data_dir(); + + // x86_64 numpy extension. + let macho = uv_delocate::macho::parse_macho( + &data_dir.join("np-1.24.1_x86_64_random__sfc64.cpython-311-darwin.so"), + ) + .unwrap(); + assert!(macho.archs.contains(&Arch::X86_64)); + + // arm64 numpy extension. + let macho = uv_delocate::macho::parse_macho( + &data_dir.join("np-1.24.1_arm_random__sfc64.cpython-311-darwin.so"), + ) + .unwrap(); + assert!(macho.archs.contains(&Arch::Arm64)); +} diff --git a/crates/uv-delocate/tests/macos_version.rs b/crates/uv-delocate/tests/macos_version.rs new file mode 100644 index 0000000000000..6bcaa8f5c1f06 --- /dev/null +++ b/crates/uv-delocate/tests/macos_version.rs @@ -0,0 +1,62 @@ +//! Tests for macOS version parsing functionality. + +use std::path::PathBuf; + +use uv_delocate::MacOSVersion; + +fn test_data_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/data") +} + +#[test] +fn test_parse_macos_version_from_binary() { + let data_dir = test_data_dir(); + + // Parse a modern ARM64 binary. + let macho = uv_delocate::macho::parse_macho( + &data_dir.join("np-1.24.1_arm_random__sfc64.cpython-311-darwin.so"), + ) + .unwrap(); + + // ARM64 binaries require macOS 11.0 or later. + if let Some(version) = macho.min_macos_version { + assert!(version.major >= 11, "ARM64 binary should require macOS 11+"); + } +} + +#[test] +fn test_parse_macos_version_from_x86_binary() { + let data_dir = test_data_dir(); + + // Parse an x86_64 binary. + let macho = uv_delocate::macho::parse_macho( + &data_dir.join("np-1.24.1_x86_64_random__sfc64.cpython-311-darwin.so"), + ) + .unwrap(); + + // x86_64 binaries can target older macOS versions. + if let Some(version) = macho.min_macos_version { + // Just verify we can read it. + assert!(version.major >= 10); + } +} + +#[test] +fn test_macos_version_ordering() { + let v10_9 = MacOSVersion::new(10, 9); + let v10_15 = MacOSVersion::new(10, 15); + let v11_0 = MacOSVersion::new(11, 0); + let v14_0 = MacOSVersion::new(14, 0); + + assert!(v10_9 < v10_15); + assert!(v10_15 < v11_0); + assert!(v11_0 < v14_0); + assert!(v10_9 < v14_0); +} + +#[test] +fn test_macos_version_display() { + assert_eq!(MacOSVersion::new(10, 9).to_string(), "10.9"); + assert_eq!(MacOSVersion::new(11, 0).to_string(), "11.0"); + assert_eq!(MacOSVersion::new(14, 2).to_string(), "14.2"); +} diff --git a/crates/uv-delocate/tests/rpath_handling.rs b/crates/uv-delocate/tests/rpath_handling.rs new file mode 100644 index 0000000000000..af352b35afdfc --- /dev/null +++ b/crates/uv-delocate/tests/rpath_handling.rs @@ -0,0 +1,52 @@ +//! Tests for rpath handling functionality. + +use std::path::PathBuf; + +#[cfg(target_os = "macos")] +use fs_err as fs; +#[cfg(target_os = "macos")] +use tempfile::TempDir; + +fn test_data_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/data") +} + +#[cfg(target_os = "macos")] +fn copy_dylib(name: &str, temp_dir: &TempDir) -> PathBuf { + let src = test_data_dir().join(name); + let dest = temp_dir.path().join(name); + fs::copy(&src, &dest).unwrap(); + dest +} + +#[test] +fn test_parse_rpath_from_binary() { + let data_dir = test_data_dir(); + + // This binary has rpaths. + let macho = uv_delocate::macho::parse_macho(&data_dir.join("libextfunc_rpath.dylib")).unwrap(); + + // Should have at least one rpath. + assert!(!macho.rpaths.is_empty(), "Binary should have rpaths"); +} + +#[test] +#[cfg(target_os = "macos")] +fn test_delete_rpath() { + let temp_dir = TempDir::new().unwrap(); + let dylib = copy_dylib("libextfunc_rpath.dylib", &temp_dir); + + // Parse the binary and get rpaths. + let macho = uv_delocate::macho::parse_macho(&dylib).unwrap(); + let original_count = macho.rpaths.len(); + + if original_count > 0 { + let rpath_to_delete = macho.rpaths[0].clone(); + uv_delocate::macho::delete_rpath(&dylib, &rpath_to_delete).unwrap(); + + // Verify the rpath was deleted. + let macho = uv_delocate::macho::parse_macho(&dylib).unwrap(); + assert_eq!(macho.rpaths.len(), original_count - 1); + assert!(!macho.rpaths.contains(&rpath_to_delete)); + } +} diff --git a/crates/uv-delocate/tests/wheel_operations.rs b/crates/uv-delocate/tests/wheel_operations.rs new file mode 100644 index 0000000000000..13a749fce4f16 --- /dev/null +++ b/crates/uv-delocate/tests/wheel_operations.rs @@ -0,0 +1,78 @@ +//! Tests for wheel packing and unpacking operations. + +use std::path::PathBuf; + +use fs_err as fs; +use tempfile::TempDir; + +fn test_data_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/data") +} + +#[test] +fn test_unpack_wheel() { + let data_dir = test_data_dir(); + let temp_dir = TempDir::new().unwrap(); + + uv_delocate::wheel::unpack_wheel( + &data_dir.join("fakepkg1-1.0-cp36-abi3-macosx_10_9_universal2.whl"), + temp_dir.path(), + ) + .unwrap(); + + // Check that dist-info exists. + assert!(temp_dir.path().join("fakepkg1-1.0.dist-info").exists()); + assert!( + temp_dir + .path() + .join("fakepkg1-1.0.dist-info/WHEEL") + .exists() + ); + assert!( + temp_dir + .path() + .join("fakepkg1-1.0.dist-info/RECORD") + .exists() + ); + + // Check that package files exist. + assert!(temp_dir.path().join("fakepkg1").exists()); +} + +#[test] +fn test_find_dist_info() { + let data_dir = test_data_dir(); + let temp_dir = TempDir::new().unwrap(); + + uv_delocate::wheel::unpack_wheel( + &data_dir.join("fakepkg1-1.0-cp36-abi3-macosx_10_9_universal2.whl"), + temp_dir.path(), + ) + .unwrap(); + + let dist_info = uv_delocate::wheel::find_dist_info(temp_dir.path()).unwrap(); + assert_eq!(dist_info, "fakepkg1-1.0.dist-info"); +} + +#[test] +fn test_unpack_repack_wheel() { + let data_dir = test_data_dir(); + let temp_dir = TempDir::new().unwrap(); + let unpack_dir = temp_dir.path().join("unpacked"); + let output_wheel = temp_dir.path().join("output.whl"); + + fs::create_dir(&unpack_dir).unwrap(); + + // Unpack. + uv_delocate::wheel::unpack_wheel(&data_dir.join("fakepkg2-1.0-py3-none-any.whl"), &unpack_dir) + .unwrap(); + + // Repack. + uv_delocate::wheel::pack_wheel(&unpack_dir, &output_wheel).unwrap(); + + // Verify the output is a valid zip. + assert!(output_wheel.exists()); + let file = fs::File::open(&output_wheel).unwrap(); + let archive = zip::ZipArchive::new(file).unwrap(); + assert!(!archive.is_empty()); +} diff --git a/crates/uv-distribution-filename/src/expanded_tags.rs b/crates/uv-distribution-filename/src/expanded_tags.rs index b04898089fad3..b9990982290cc 100644 --- a/crates/uv-distribution-filename/src/expanded_tags.rs +++ b/crates/uv-distribution-filename/src/expanded_tags.rs @@ -137,7 +137,7 @@ fn parse_expanded_tag(tag: &str) -> Result { .map(PlatformTag::from_str) .filter_map(Result::ok) .collect(), - repr: tag.into(), + repr: Some(tag.into()), }), }) } @@ -267,7 +267,9 @@ mod tests { arch: X86_64, }, ], - repr: "cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64", + repr: Some( + "cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64", + ), }, }, ], @@ -295,7 +297,9 @@ mod tests { platform_tag: [ Any, ], - repr: "py3-foo-any", + repr: Some( + "py3-foo-any", + ), }, }, ], @@ -329,7 +333,9 @@ mod tests { platform_tag: [ Any, ], - repr: "py2.py3-none-any", + repr: Some( + "py2.py3-none-any", + ), }, }, ], @@ -445,7 +451,9 @@ mod tests { arch: X86, }, ], - repr: "cp39.cp310-cp39.cp310-linux_x86_64.linux_i686", + repr: Some( + "cp39.cp310-cp39.cp310-linux_x86_64.linux_i686", + ), }, } "#); diff --git a/crates/uv-distribution-filename/src/snapshots/uv_distribution_filename__wheel__tests__ok_build_tag.snap b/crates/uv-distribution-filename/src/snapshots/uv_distribution_filename__wheel__tests__ok_build_tag.snap index ce7dcc62a6e23..10ad652bb6d25 100644 --- a/crates/uv-distribution-filename/src/snapshots/uv_distribution_filename__wheel__tests__ok_build_tag.snap +++ b/crates/uv-distribution-filename/src/snapshots/uv_distribution_filename__wheel__tests__ok_build_tag.snap @@ -28,7 +28,9 @@ Ok( platform_tag: [ Any, ], - repr: "202206090410-py3-none-any", + repr: Some( + "202206090410-py3-none-any", + ), }, }, }, diff --git a/crates/uv-distribution-filename/src/snapshots/uv_distribution_filename__wheel__tests__ok_multiple_tags.snap b/crates/uv-distribution-filename/src/snapshots/uv_distribution_filename__wheel__tests__ok_multiple_tags.snap index a474bd98eaa83..35b87e23cbbe6 100644 --- a/crates/uv-distribution-filename/src/snapshots/uv_distribution_filename__wheel__tests__ok_multiple_tags.snap +++ b/crates/uv-distribution-filename/src/snapshots/uv_distribution_filename__wheel__tests__ok_multiple_tags.snap @@ -38,7 +38,9 @@ Ok( arch: X86_64, }, ], - repr: "cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64", + repr: Some( + "cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64", + ), }, }, }, diff --git a/crates/uv-distribution-filename/src/wheel.rs b/crates/uv-distribution-filename/src/wheel.rs index c3ba40bdfd5d1..abcd983d16862 100644 --- a/crates/uv-distribution-filename/src/wheel.rs +++ b/crates/uv-distribution-filename/src/wheel.rs @@ -152,6 +152,21 @@ impl WheelFilename { self.tags.build_tag() } + /// Create a new [`WheelFilename`] with the given platform tags. + #[must_use] + pub fn with_platform_tags(&self, platform_tags: &[PlatformTag]) -> Self { + Self { + name: self.name.clone(), + version: self.version.clone(), + tags: WheelTag::new( + self.build_tag().cloned(), + self.python_tags(), + self.abi_tags(), + platform_tags, + ), + } + } + /// Parse a wheel filename from the stem (e.g., `foo-1.2.3-py3-none-any`). pub fn from_stem(stem: &str) -> Result { // The wheel stem should not contain the `.whl` extension. @@ -274,7 +289,7 @@ impl WheelFilename { .map(PlatformTag::from_str) .filter_map(Result::ok) .collect(), - repr: repr.into(), + repr: Some(repr.into()), }), } }; @@ -470,4 +485,21 @@ mod tests { ).unwrap(); insta::assert_snapshot!(filename.cache_key(), @"1.2.3.4.5.6.7.8.9.0.1.2.3.4.5.6.7.8.9.0.1.2.1.2-80bf8598e9647cf7"); } + + #[test] + fn with_platform_tags() { + use uv_platform_tags::BinaryFormat; + + let filename = + WheelFilename::from_str("foo-1.0-cp311-cp311-macosx_10_9_x86_64.whl").unwrap(); + let new_filename = filename.with_platform_tags(&[PlatformTag::Macos { + major: 11, + minor: 0, + binary_format: BinaryFormat::X86_64, + }]); + assert_eq!( + new_filename.to_string(), + "foo-1.0-cp311-cp311-macosx_11_0_x86_64.whl" + ); + } } diff --git a/crates/uv-distribution-filename/src/wheel_tag.rs b/crates/uv-distribution-filename/src/wheel_tag.rs index 7d93e42a43ea8..49775aad5820b 100644 --- a/crates/uv-distribution-filename/src/wheel_tag.rs +++ b/crates/uv-distribution-filename/src/wheel_tag.rs @@ -38,6 +38,39 @@ pub(crate) enum WheelTag { } impl WheelTag { + /// Create a new [`WheelTag`] from its components. + pub(crate) fn new( + build_tag: Option, + python_tags: &[LanguageTag], + abi_tags: &[AbiTag], + platform_tags: &[PlatformTag], + ) -> Self { + // Use the small variant if possible (no build tag and single tags). + if build_tag.is_none() + && python_tags.len() == 1 + && abi_tags.len() == 1 + && platform_tags.len() == 1 + { + return Self::Small { + small: WheelTagSmall { + python_tag: python_tags[0], + abi_tag: abi_tags[0], + platform_tag: platform_tags[0].clone(), + }, + }; + } + + Self::Large { + large: Box::new(WheelTagLarge { + build_tag, + python_tag: python_tags.iter().copied().collect(), + abi_tag: abi_tags.iter().copied().collect(), + platform_tag: platform_tags.iter().cloned().collect(), + repr: None, + }), + } + } + /// Return the Python tags. pub(crate) fn python_tags(&self) -> &[LanguageTag] { match self { @@ -139,11 +172,40 @@ pub(crate) struct WheelTagLarge { /// The string representation of the tag. /// /// Preserves any unsupported tags that were filtered out when parsing the wheel filename. - pub(crate) repr: SmallString, + /// If `None`, the representation is generated from the tag fields. + pub(crate) repr: Option, } impl Display for WheelTagLarge { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.repr) + if let Some(repr) = &self.repr { + return write!(f, "{repr}"); + } + + // Generate from fields. + let python_str = self + .python_tag + .iter() + .map(ToString::to_string) + .collect::>() + .join("."); + let abi_str = self + .abi_tag + .iter() + .map(ToString::to_string) + .collect::>() + .join("."); + let platform_str = self + .platform_tag + .iter() + .map(ToString::to_string) + .collect::>() + .join("."); + + if let Some(build_tag) = &self.build_tag { + write!(f, "{build_tag}-{python_str}-{abi_str}-{platform_str}") + } else { + write!(f, "{python_str}-{abi_str}-{platform_str}") + } } } diff --git a/crates/uv-distribution/Cargo.toml b/crates/uv-distribution/Cargo.toml index e6e76bd06290d..842d71040c1fa 100644 --- a/crates/uv-distribution/Cargo.toml +++ b/crates/uv-distribution/Cargo.toml @@ -19,6 +19,7 @@ workspace = true uv-auth = { workspace = true } uv-cache = { workspace = true } uv-cache-info = { workspace = true } +uv-cache-key = { workspace = true } uv-client = { workspace = true } uv-configuration = { workspace = true } uv-distribution-filename = { workspace = true } @@ -35,6 +36,7 @@ uv-pep508 = { workspace = true } uv-platform-tags = { workspace = true } uv-pypi-types = { workspace = true } uv-redacted = { workspace = true } +uv-static = { workspace = true } uv-types = { workspace = true } uv-workspace = { workspace = true } @@ -46,13 +48,15 @@ nanoid = { workspace = true } owo-colors = { workspace = true } reqwest = { workspace = true } reqwest-middleware = { workspace = true } +rkyv = { workspace = true } rmp-serde = { workspace = true } rustc-hash = { workspace = true } serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } tempfile = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } -tokio-util = { workspace = true, features = ["compat"] } +tokio-util = { workspace = true, features = ["compat", "io"] } toml = { workspace = true } tracing = { workspace = true } url = { workspace = true } diff --git a/crates/uv-distribution/src/distribution_database.rs b/crates/uv-distribution/src/distribution_database.rs index cef91e75da8aa..200e60c229c24 100644 --- a/crates/uv-distribution/src/distribution_database.rs +++ b/crates/uv-distribution/src/distribution_database.rs @@ -10,7 +10,7 @@ use tempfile::TempDir; use tokio::io::{AsyncRead, AsyncSeekExt, ReadBuf}; use tokio::sync::Semaphore; use tokio_util::compat::FuturesAsyncReadCompatExt; -use tracing::{Instrument, info_span, instrument, warn}; +use tracing::{Instrument, debug, info_span, instrument, warn}; use url::Url; use uv_cache::{ArchiveId, CacheBucket, CacheEntry, WheelCache}; @@ -20,8 +20,8 @@ use uv_client::{ }; use uv_distribution_filename::WheelFilename; use uv_distribution_types::{ - BuildInfo, BuildableSource, BuiltDist, Dist, File, HashPolicy, Hashed, IndexUrl, InstalledDist, - Name, SourceDist, ToUrlError, + BuildInfo, BuildableSource, BuiltDist, CompatibleDist, Dist, File, HashPolicy, Hashed, + IndexUrl, InstalledDist, Name, RegistryBuiltDist, SourceDist, ToUrlError, }; use uv_extract::hash::Hasher; use uv_fs::write_atomic; @@ -32,6 +32,7 @@ use uv_types::{BuildContext, BuildStack}; use crate::archive::Archive; use crate::metadata::{ArchiveMetadata, Metadata}; +use crate::remote::RemoteCacheResolver; use crate::source::SourceDistributionBuilder; use crate::{Error, LocalWheel, Reporter, RequiresDist}; @@ -50,6 +51,7 @@ use crate::{Error, LocalWheel, Reporter, RequiresDist}; pub struct DistributionDatabase<'a, Context: BuildContext> { build_context: &'a Context, builder: SourceDistributionBuilder<'a, Context>, + resolver: RemoteCacheResolver<'a, Context>, client: ManagedClient<'a>, reporter: Option>, } @@ -63,6 +65,7 @@ impl<'a, Context: BuildContext> DistributionDatabase<'a, Context> { Self { build_context, builder: SourceDistributionBuilder::new(build_context), + resolver: RemoteCacheResolver::new(build_context), client: ManagedClient::new(client, concurrent_downloads), reporter: None, } @@ -378,6 +381,23 @@ impl<'a, Context: BuildContext> DistributionDatabase<'a, Context> { dist: &SourceDist, tags: &Tags, hashes: HashPolicy<'_>, + ) -> Result { + // If the metadata is available in a remote cache, fetch it. + if let Ok(Some(wheel)) = self.get_remote_wheel(dist, tags, hashes).await { + return Ok(wheel); + } + + // Otherwise, build the wheel locally. + self.build_wheel_inner(dist, tags, hashes).await + } + + /// Convert a source distribution into a wheel, fetching it from the cache or building it if + /// necessary. + async fn build_wheel_inner( + &self, + dist: &SourceDist, + tags: &Tags, + hashes: HashPolicy<'_>, ) -> Result { let built_wheel = self .builder @@ -406,6 +426,16 @@ impl<'a, Context: BuildContext> DistributionDatabase<'a, Context> { }; } + // Upload the wheel to the remote cache. + self.resolver + .upload_to_cache( + &BuildableSource::Dist(dist), + &built_wheel.path, + &built_wheel.filename, + &self.client, + ) + .await?; + // Acquire the advisory lock. #[cfg(windows)] let _lock = { @@ -544,6 +574,9 @@ impl<'a, Context: BuildContext> DistributionDatabase<'a, Context> { source: &BuildableSource<'_>, hashes: HashPolicy<'_>, ) -> Result { + // Resolve the source distribution to a precise revision (i.e., a specific Git commit). + self.builder.resolve_revision(source, &self.client).await?; + // If the metadata was provided by the user directly, prefer it. if let Some(dist) = source.as_dist() { if let Some(metadata) = self @@ -551,14 +584,25 @@ impl<'a, Context: BuildContext> DistributionDatabase<'a, Context> { .dependency_metadata() .get(dist.name(), dist.version()) { - // If we skipped the build, we should still resolve any Git dependencies to precise - // commits. - self.builder.resolve_revision(source, &self.client).await?; - return Ok(ArchiveMetadata::from_metadata23(metadata.clone())); } } + // If the metadata is available in a remote cache, fetch it. + if let Ok(Some(metadata)) = self.get_remote_metadata(source, hashes).await { + return Ok(metadata); + } + + // Otherwise, retrieve the metadata from the source distribution. + self.build_wheel_metadata_inner(source, hashes).await + } + + /// Build the wheel metadata for a source distribution, or fetch it from the cache if possible. + async fn build_wheel_metadata_inner( + &self, + source: &BuildableSource<'_>, + hashes: HashPolicy<'_>, + ) -> Result { let metadata = self .builder .download_and_build_metadata(source, hashes, &self.client) @@ -568,6 +612,88 @@ impl<'a, Context: BuildContext> DistributionDatabase<'a, Context> { Ok(metadata) } + /// Fetch a wheel from a remote cache, if available. + async fn get_remote_wheel( + &self, + source: &SourceDist, + tags: &Tags, + hashes: HashPolicy<'_>, + ) -> Result, Error> { + let Some(index) = self + .resolver + .get_cached_distribution(&BuildableSource::Dist(source), Some(tags), &self.client) + .await? + else { + return Ok(None); + }; + for prioritized_dist in index.iter() { + let Some(compatible_dist) = prioritized_dist.get() else { + continue; + }; + match compatible_dist { + CompatibleDist::InstalledDist(..) => {} + CompatibleDist::SourceDist { sdist, .. } => { + debug!("Found cached remote source distribution for: {source}"); + let dist = SourceDist::Registry(sdist.clone()); + return self.build_wheel_inner(&dist, tags, hashes).await.map(Some); + } + CompatibleDist::CompatibleWheel { wheel, .. } + | CompatibleDist::IncompatibleWheel { wheel, .. } => { + debug!("Found cached remote built distribution for: {source}"); + let dist = BuiltDist::Registry(RegistryBuiltDist { + wheels: vec![wheel.clone()], + best_wheel_index: 0, + sdist: None, + }); + return self.get_wheel(&dist, hashes).await.map(Some); + } + } + } + Ok(None) + } + + /// Fetch the wheel metadata from a remote cache, if available. + async fn get_remote_metadata( + &self, + source: &BuildableSource<'_>, + hashes: HashPolicy<'_>, + ) -> Result, Error> { + let Some(index) = self + .resolver + .get_cached_distribution(source, None, &self.client) + .await? + else { + return Ok(None); + }; + for prioritized_dist in index.iter() { + let Some(compatible_dist) = prioritized_dist.get() else { + continue; + }; + match compatible_dist { + CompatibleDist::InstalledDist(..) => {} + CompatibleDist::SourceDist { sdist, .. } => { + debug!("Found cached remote source distribution for: {source}"); + let dist = SourceDist::Registry(sdist.clone()); + return self + .build_wheel_metadata_inner(&BuildableSource::Dist(&dist), hashes) + .await + .map(Some); + } + CompatibleDist::CompatibleWheel { wheel, .. } + | CompatibleDist::IncompatibleWheel { wheel, .. } => { + debug!("Found cached remote built distribution for: {source}"); + let dist = BuiltDist::Registry(RegistryBuiltDist { + wheels: vec![wheel.clone()], + best_wheel_index: 0, + sdist: None, + }); + return self.get_wheel_metadata(&dist, hashes).await.map(Some); + } + } + } + Ok(None) + } + /// Return the [`RequiresDist`] from a `pyproject.toml`, if it can be statically extracted. pub async fn requires_dist( &self, diff --git a/crates/uv-distribution/src/lib.rs b/crates/uv-distribution/src/lib.rs index 6ffb2d6d87682..68f66811799b5 100644 --- a/crates/uv-distribution/src/lib.rs +++ b/crates/uv-distribution/src/lib.rs @@ -16,5 +16,6 @@ mod download; mod error; mod index; mod metadata; +mod remote; mod reporter; mod source; diff --git a/crates/uv-distribution/src/remote.rs b/crates/uv-distribution/src/remote.rs new file mode 100644 index 0000000000000..c074a30ac8f87 --- /dev/null +++ b/crates/uv-distribution/src/remote.rs @@ -0,0 +1,703 @@ +use std::borrow::Cow; +use std::collections::BTreeMap; +use std::collections::btree_map::Entry; +use std::path::Path; +use std::sync::Arc; + +use reqwest::Body; +use rustc_hash::FxHashMap; +use serde::{Deserialize, Serialize}; +use tokio::sync::Mutex; +use tokio_util::io::ReaderStream; +use tracing::{debug, instrument, warn}; +use url::Url; + +use uv_auth::PyxTokenStore; +use uv_cache_key::RepositoryUrl; +use uv_client::{MetadataFormat, VersionFiles}; +use uv_configuration::BuildOptions; +use uv_distribution_filename::{DistFilename, SourceDistFilename, WheelFilename}; +use uv_distribution_types::{ + BuildableSource, File, HashComparison, HashPolicy, IncompatibleSource, IncompatibleWheel, + IndexFormat, IndexMetadata, IndexUrl, PrioritizedDist, RegistryBuiltWheel, RegistrySourceDist, + SourceDist, SourceDistCompatibility, SourceUrl, WheelCompatibility, +}; +use uv_git_types::{GitOid, GitUrl}; +use uv_normalize::PackageName; +use uv_pep440::Version; +use uv_pep508::VerbatimUrl; +use uv_platform_tags::{TagCompatibility, Tags}; +use uv_pypi_types::HashDigest; +use uv_static::EnvVars; +use uv_types::{BuildContext, HashStrategy}; + +use crate::Error; +use crate::distribution_database::ManagedClient; + +/// A resolver for remote Git-based indexes. +pub(crate) struct RemoteCacheResolver<'a, Context: BuildContext> { + build_context: &'a Context, + /// Cache for Git index entries. + index_cache: Arc>, + /// Cache for server-provided cache keys. + key_cache: Arc>, + store: Option, + workspace: Option, +} + +/// A cache for server-provided cache keys. +/// +/// Maps (repository, commit, subdirectory) to the server-computed cache key. +type CacheKeyCache = FxHashMap; + +/// Request body for fetching a cache key from the server. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize)] +struct CacheKeyRequest { + repository: String, + commit: String, + #[serde(skip_serializing_if = "Option::is_none")] + subdirectory: Option, +} + +/// Response from the cache key endpoint. +#[derive(Debug, Deserialize)] +struct CacheKeyResponse { + key: String, +} + +impl<'a, T: BuildContext> RemoteCacheResolver<'a, T> { + /// Initialize a [`RemoteCacheResolver`] from a [`BuildContext`]. + pub(crate) fn new(build_context: &'a T) -> Self { + Self { + build_context, + index_cache: Arc::default(), + key_cache: Arc::default(), + store: PyxTokenStore::from_settings().ok(), + workspace: std::env::var(EnvVars::PYX_CACHE_WORKSPACE).ok(), + } + } + + /// Fetch the cache key from the server for the given Git source. + async fn get_cache_key( + &self, + repository: &RepositoryUrl, + commit: GitOid, + subdirectory: Option<&Path>, + client: &ManagedClient<'a>, + ) -> Result, Error> { + let Some(store) = &self.store else { + return Ok(None); + }; + + // Build the request. + let request = CacheKeyRequest { + repository: repository.to_string(), + commit: commit.to_string(), + subdirectory: subdirectory.and_then(|p| p.to_str()).map(String::from), + }; + + // Check the local cache first. + { + let cache = self.key_cache.lock().await; + if let Some(key) = cache.get(&request) { + return Ok(Some(key.clone())); + } + } + + // Build the API URL with query parameters. + let Some(workspace) = &self.workspace else { + return Ok(None); + }; + let url = { + let mut url = store.api().clone(); + url.set_path(&format!("v1/cache/{workspace}/key")); + url.query_pairs_mut() + .append_pair("repository", &request.repository) + .append_pair("commit", &request.commit); + if let Some(ref subdir) = request.subdirectory { + url.query_pairs_mut().append_pair("subdirectory", subdir); + } + url + }; + debug!("Fetching cache key from: {url}"); + + // Build and send the request. + let response = match client + .unmanaged + .uncached_client(&url) + .get(Url::from(url.clone())) + .send() + .await + { + Ok(response) => response, + Err(err) => { + debug!("Failed to fetch cache key: {err}"); + return Ok(None); + } + }; + + if !response.status().is_success() { + debug!( + "Failed to fetch cache key: {} {}", + response.status(), + response.text().await.unwrap_or_default() + ); + return Ok(None); + } + + let response: CacheKeyResponse = response.json().await?; + + // Cache the key. + { + let mut cache = self.key_cache.lock().await; + cache.insert(request, response.key.clone()); + } + + Ok(Some(response.key)) + } + + /// Create a cache entry on the server and return the cache key. + /// + /// Unlike [`get_cache_key`], this creates the necessary server-side resources + /// (registry, view) that are required before uploading wheels. + async fn create_cache_entry( + &self, + repository: &RepositoryUrl, + commit: GitOid, + subdirectory: Option<&Path>, + client: &ManagedClient<'a>, + ) -> Result, Error> { + let Some(store) = &self.store else { + return Ok(None); + }; + + // Build the request. + let request = CacheKeyRequest { + repository: repository.to_string(), + commit: commit.to_string(), + subdirectory: subdirectory.and_then(|p| p.to_str()).map(String::from), + }; + + // Build the API URL. + let Some(workspace) = &self.workspace else { + return Ok(None); + }; + let url = { + let mut url = store.api().clone(); + url.set_path(&format!("v1/cache/{workspace}")); + url + }; + debug!("Creating cache entry at: {url}"); + + // Build and send the request. + let body = serde_json::to_vec(&request).expect("failed to serialize cache key request"); + let response = match client + .unmanaged + .uncached_client(&url) + .post(Url::from(url.clone())) + .header(reqwest::header::CONTENT_TYPE, "application/json") + .body(body) + .send() + .await + { + Ok(response) => response, + Err(err) => { + debug!("Failed to create cache entry: {err}"); + return Ok(None); + } + }; + + if !response.status().is_success() { + debug!( + "Failed to create cache entry: {} {}", + response.status(), + response.text().await.unwrap_or_default() + ); + return Ok(None); + } + + let response: CacheKeyResponse = response.json().await?; + + // Cache the key. + { + let mut cache = self.key_cache.lock().await; + cache.insert(request, response.key.clone()); + } + + Ok(Some(response.key)) + } + + /// Return the cached Git index for the given distribution, if any. + pub(crate) async fn get_cached_distribution( + &self, + source: &BuildableSource<'_>, + tags: Option<&Tags>, + client: &ManagedClient<'a>, + ) -> Result, Error> { + // Fetch the entries for the given distribution. + let entries = self.get_or_fetch_index(source, client).await?; + if entries.is_empty() { + return Ok(None); + } + + // Create the index. + let index = GitIndex::from_entries( + entries, + tags, + &HashStrategy::default(), + self.build_context.build_options(), + ); + Ok(Some(index)) + } + + /// Fetch the remote Git index for the given distribution. + async fn get_or_fetch_index( + &self, + source: &BuildableSource<'_>, + client: &ManagedClient<'a>, + ) -> Result, Error> { + #[derive(Debug)] + struct BuildableGitSource<'a> { + git: &'a GitUrl, + subdirectory: Option<&'a Path>, + name: Option<&'a PackageName>, + } + + let Some(workspace) = &self.workspace else { + return Ok(Vec::default()); + }; + + let source = match source { + BuildableSource::Dist(SourceDist::Git(dist)) => BuildableGitSource { + git: &dist.git, + subdirectory: dist.subdirectory.as_deref(), + name: Some(&dist.name), + }, + BuildableSource::Url(SourceUrl::Git(url)) => BuildableGitSource { + git: url.git, + subdirectory: url.subdirectory, + name: None, + }, + _ => { + return Ok(Vec::default()); + } + }; + + let Some(precise) = self.build_context.git().get_precise(source.git) else { + return Ok(Vec::default()); + }; + + // Fetch the cache key from the server. + let repository = RepositoryUrl::new(source.git.repository()); + let Some(cache_key) = self + .get_cache_key(&repository, precise, source.subdirectory, client) + .await? + else { + return Ok(Vec::default()); + }; + + // Build the index URL using the server-provided cache key. + let Some(store) = &self.store else { + return Ok(Vec::default()); + }; + let index = IndexUrl::from( + VerbatimUrl::parse_url(format!( + "{}/v1/cache/{workspace}/{cache_key}", + store.api().as_str().trim_end_matches('/'), + )) + .unwrap(), + ); + debug!("Using remote Git index URL: {}", index); + + // Determine the package name. + let name = if let Some(name) = source.name { + Cow::Borrowed(name) + } else { + // Fetch the list of packages from the Simple API. + let index_metadata = client + .managed(|client| client.fetch_simple_index(&index)) + .await?; + + // Ensure that the index contains exactly one package. + let mut packages = index_metadata.iter().cloned(); + let Some(name) = packages.next() else { + debug!("Remote Git index at `{index}` contains no packages"); + return Ok(Vec::default()); + }; + if packages.next().is_some() { + debug!("Remote Git index at `{index}` contains multiple packages"); + return Ok(Vec::default()); + } + Cow::Owned(name) + }; + + // Store the index entries in a cache, to avoid redundant fetches. + { + let cache = self.index_cache.lock().await; + if let Some(entries) = cache.get(&index) { + return Ok(entries.to_vec()); + } + } + + // Perform a remote fetch via the Simple API. + let metadata = IndexMetadata { + url: index.clone(), + format: IndexFormat::Simple, + }; + let archives = client + .manual(|client, semaphore| { + client.simple_detail( + name.as_ref(), + Some(metadata.as_ref()), + self.build_context.capabilities(), + semaphore, + ) + }) + .await?; + + // Collect the files from the remote index. + let mut entries = Vec::new(); + for (_, archive) in archives { + let MetadataFormat::Simple(archive) = archive else { + continue; + }; + for datum in archive.iter().rev() { + let files = rkyv::deserialize::(&datum.files) + .expect("archived version files always deserializes"); + for (filename, file) in files.all() { + if *filename.name() != *name { + warn!( + "Skipping file `{filename}` from remote Git index at `{index}` due to name mismatch (expected: `{name}`)" + ); + continue; + } + + entries.push(GitIndexEntry { + filename, + file, + index: index.clone(), + }); + } + } + } + + // Write to the cache. + { + let mut cache = self.index_cache.lock().await; + cache.insert(index.clone(), entries.clone()); + } + + Ok(entries) + } + + /// Upload a built wheel to the remote cache. + pub(crate) async fn upload_to_cache( + &self, + source: &BuildableSource<'_>, + wheel_path: &Path, + filename: &WheelFilename, + client: &ManagedClient<'a>, + ) -> Result<(), Error> { + #[derive(Debug)] + struct BuildableGitSource<'a> { + git: &'a GitUrl, + subdirectory: Option<&'a Path>, + } + + let Some(workspace) = &self.workspace else { + return Ok(()); + }; + + let Some(store) = &self.store else { + return Ok(()); + }; + + let source = match source { + BuildableSource::Dist(SourceDist::Git(dist)) => BuildableGitSource { + git: &dist.git, + subdirectory: dist.subdirectory.as_deref(), + }, + BuildableSource::Url(SourceUrl::Git(url)) => BuildableGitSource { + git: url.git, + subdirectory: url.subdirectory, + }, + _ => { + return Ok(()); + } + }; + + let Some(precise) = self.build_context.git().get_precise(source.git) else { + return Ok(()); + }; + + // Create the cache entry on the server (or get existing key). + let repository = RepositoryUrl::new(source.git.repository()); + let Some(cache_key) = self + .create_cache_entry(&repository, precise, source.subdirectory, client) + .await? + else { + return Ok(()); + }; + + // Build the upload URL using the server-provided cache key. + let url = { + let mut url = store.api().clone(); + url.set_path(&format!("v1/cache/{workspace}/{cache_key}")); + url + }; + debug!("Uploading wheel to remote cache: {url}"); + + // Get the file size for the Content-Length header. + let file_size = fs_err::tokio::metadata(wheel_path) + .await + .map_err(Error::CacheRead)? + .len(); + + // Open the wheel file. + let file = fs_err::tokio::File::open(wheel_path) + .await + .map_err(Error::CacheRead)?; + let stream = ReaderStream::new(file); + let body = Body::wrap_stream(stream); + + // Build the multipart form with streaming body. + let part = reqwest::multipart::Part::stream_with_length(body, file_size) + .file_name(filename.to_string()); + let form = reqwest::multipart::Form::new().part("content", part); + + // Build and send the request. + let response = match client + .unmanaged + .uncached_client(&url) + .post(Url::from(url.clone())) + .multipart(form) + .send() + .await + { + Ok(response) => response, + Err(err) => { + warn!("Failed to upload wheel to cache: {err}"); + return Ok(()); + } + }; + + if !response.status().is_success() { + warn!( + "Failed to upload wheel to cache: {} {}", + response.status(), + response.text().await.unwrap_or_default() + ); + } + + Ok(()) + } +} + +/// An entry in a remote Git index. +#[derive(Debug, Clone)] +struct GitIndexEntry { + filename: DistFilename, + file: File, + index: IndexUrl, +} + +/// A set of [`PrioritizedDist`] from a Git index. +/// +/// In practice, it's assumed that the [`GitIndex`] will only contain distributions for a single +/// package. +#[derive(Debug, Clone, Default)] +pub(crate) struct GitIndex(FxHashMap); + +impl GitIndex { + /// Collect all files from a Git index. + #[instrument(skip_all)] + fn from_entries( + entries: Vec, + tags: Option<&Tags>, + hasher: &HashStrategy, + build_options: &BuildOptions, + ) -> Self { + let mut index = FxHashMap::::default(); + for entry in entries { + let distributions = index.entry(entry.filename.name().clone()).or_default(); + distributions.add_file( + entry.file, + entry.filename, + tags, + hasher, + build_options, + entry.index, + ); + } + Self(index) + } + + /// Returns an [`Iterator`] over the distributions. + pub(crate) fn iter(&self) -> impl Iterator { + self.0 + .iter() + .flat_map(|(.., distributions)| distributions.0.iter().map(|(.., dist)| dist)) + } +} + +/// A set of [`PrioritizedDist`] from a Git index, indexed by [`Version`]. +#[derive(Debug, Clone, Default)] +pub(crate) struct GitIndexDistributions(BTreeMap); + +impl GitIndexDistributions { + /// Add the given [`File`] to the [`GitIndexDistributions`] for the given package. + fn add_file( + &mut self, + file: File, + filename: DistFilename, + tags: Option<&Tags>, + hasher: &HashStrategy, + build_options: &BuildOptions, + index: IndexUrl, + ) { + // TODO(charlie): Incorporate `Requires-Python`, yanked status, etc. + match filename { + DistFilename::WheelFilename(filename) => { + let version = filename.version.clone(); + + let compatibility = Self::wheel_compatibility( + &filename, + file.hashes.as_slice(), + tags, + hasher, + build_options, + ); + let dist = RegistryBuiltWheel { + filename, + file: Box::new(file), + index, + }; + match self.0.entry(version) { + Entry::Occupied(mut entry) => { + entry.get_mut().insert_built(dist, vec![], compatibility); + } + Entry::Vacant(entry) => { + entry.insert(PrioritizedDist::from_built(dist, vec![], compatibility)); + } + } + } + DistFilename::SourceDistFilename(filename) => { + let compatibility = Self::source_dist_compatibility( + &filename, + file.hashes.as_slice(), + hasher, + build_options, + ); + let dist = RegistrySourceDist { + name: filename.name.clone(), + version: filename.version.clone(), + ext: filename.extension, + file: Box::new(file), + index, + wheels: vec![], + }; + match self.0.entry(filename.version) { + Entry::Occupied(mut entry) => { + entry.get_mut().insert_source(dist, vec![], compatibility); + } + Entry::Vacant(entry) => { + entry.insert(PrioritizedDist::from_source(dist, vec![], compatibility)); + } + } + } + } + } + + fn source_dist_compatibility( + filename: &SourceDistFilename, + hashes: &[HashDigest], + hasher: &HashStrategy, + build_options: &BuildOptions, + ) -> SourceDistCompatibility { + // Check if source distributions are allowed for this package. + if build_options.no_build_package(&filename.name) { + return SourceDistCompatibility::Incompatible(IncompatibleSource::NoBuild); + } + + // Check if hashes line up. + let hash = if let HashPolicy::Validate(required) = + hasher.get_package(&filename.name, &filename.version) + { + if hashes.is_empty() { + HashComparison::Missing + } else if required.iter().any(|hash| hashes.contains(hash)) { + HashComparison::Matched + } else { + HashComparison::Mismatched + } + } else { + HashComparison::Matched + }; + + SourceDistCompatibility::Compatible(hash) + } + + fn wheel_compatibility( + filename: &WheelFilename, + hashes: &[HashDigest], + tags: Option<&Tags>, + hasher: &HashStrategy, + build_options: &BuildOptions, + ) -> WheelCompatibility { + // Check if binaries are allowed for this package. + if build_options.no_binary_package(&filename.name) { + return WheelCompatibility::Incompatible(IncompatibleWheel::NoBinary); + } + + // Determine a compatibility for the wheel based on tags. + let priority = match tags { + Some(tags) => match filename.compatibility(tags) { + TagCompatibility::Incompatible(tag) => { + return WheelCompatibility::Incompatible(IncompatibleWheel::Tag(tag)); + } + TagCompatibility::Compatible(priority) => Some(priority), + }, + None => None, + }; + + // Check if hashes line up. + let hash = if let HashPolicy::Validate(required) = + hasher.get_package(&filename.name, &filename.version) + { + if hashes.is_empty() { + HashComparison::Missing + } else if required.iter().any(|hash| hashes.contains(hash)) { + HashComparison::Matched + } else { + HashComparison::Mismatched + } + } else { + HashComparison::Matched + }; + + // Break ties with the build tag. + let build_tag = filename.build_tag().cloned(); + + WheelCompatibility::Compatible(hash, priority, build_tag) + } +} + +/// A map from [`IndexUrl`] to [`GitIndex`] entries found at the given URL. +#[derive(Default, Debug, Clone)] +struct GitIndexCache(FxHashMap>); + +impl GitIndexCache { + /// Get the entries for a given index URL. + fn get(&self, index: &IndexUrl) -> Option<&[GitIndexEntry]> { + self.0.get(index).map(Vec::as_slice) + } + + /// Insert the entries for a given index URL. + fn insert( + &mut self, + index: IndexUrl, + entries: Vec, + ) -> Option> { + self.0.insert(index, entries) + } +} diff --git a/crates/uv-install-wheel/src/lib.rs b/crates/uv-install-wheel/src/lib.rs index 97a1eb234e856..0b5840f04af7a 100644 --- a/crates/uv-install-wheel/src/lib.rs +++ b/crates/uv-install-wheel/src/lib.rs @@ -13,8 +13,9 @@ use uv_pypi_types::Scheme; pub use install::install_wheel; pub use linker::{LinkMode, Locks}; +pub use record::RecordEntry; pub use uninstall::{Uninstall, uninstall_egg, uninstall_legacy_editable, uninstall_wheel}; -pub use wheel::{LibKind, WheelFile, read_record_file}; +pub use wheel::{LibKind, WheelFile, read_record_file, write_record_file}; mod install; mod linker; diff --git a/crates/uv-install-wheel/src/record.rs b/crates/uv-install-wheel/src/record.rs index 404cee5310c57..6a677757b3d35 100644 --- a/crates/uv-install-wheel/src/record.rs +++ b/crates/uv-install-wheel/src/record.rs @@ -1,7 +1,8 @@ use serde::{Deserialize, Serialize}; -/// Line in a RECORD file -/// +/// Line in a RECORD file. +/// +/// See: /// /// ```csv /// tqdm/cli.py,sha256=x_c8nmc4Huc-lKEsAXj78ZiyqSJ9hJ71j7vltY67icw,10509 @@ -11,6 +12,25 @@ use serde::{Deserialize, Serialize}; pub struct RecordEntry { pub path: String, pub hash: Option, - #[allow(dead_code)] pub size: Option, } + +impl RecordEntry { + /// Create a new record entry with a hash and size. + pub fn new(path: String, hash: String, size: u64) -> Self { + Self { + path, + hash: Some(hash), + size: Some(size), + } + } + + /// Create a record entry for a file that should not be hashed (e.g., the RECORD file itself). + pub fn unhashed(path: String) -> Self { + Self { + path, + hash: None, + size: None, + } + } +} diff --git a/crates/uv-install-wheel/src/wheel.rs b/crates/uv-install-wheel/src/wheel.rs index 4037567e26773..210ac0a138fc0 100644 --- a/crates/uv-install-wheel/src/wheel.rs +++ b/crates/uv-install-wheel/src/wheel.rs @@ -791,8 +791,9 @@ pub(crate) fn get_relocatable_executable( }) } -/// Reads the record file -/// +/// Reads the record file. +/// +/// See: pub fn read_record_file(record: &mut impl Read) -> Result, Error> { csv::ReaderBuilder::new() .has_headers(false) @@ -810,6 +811,23 @@ pub fn read_record_file(record: &mut impl Read) -> Result, Erro .collect() } +/// Writes a record file. +/// +/// The records are sorted for reproducibility before writing. +/// +/// See: +pub fn write_record_file(path: &Path, mut records: Vec) -> Result<(), Error> { + records.sort(); + let mut writer = csv::WriterBuilder::new() + .has_headers(false) + .escape(b'"') + .from_path(path)?; + for record in records { + writer.serialize(record)?; + } + Ok(()) +} + /// Parse a file with email message format such as WHEEL and METADATA fn parse_email_message_file( file: impl Read, diff --git a/crates/uv-platform/src/lib.rs b/crates/uv-platform/src/lib.rs index d0e6fdd4bf35d..6628bf433d1d6 100644 --- a/crates/uv-platform/src/lib.rs +++ b/crates/uv-platform/src/lib.rs @@ -9,11 +9,13 @@ use tracing::trace; pub use crate::arch::{Arch, ArchVariant}; pub use crate::libc::{Libc, LibcDetectionError, LibcVersion}; +pub use crate::macos::MacOSVersion; pub use crate::os::Os; mod arch; mod cpuinfo; mod libc; +mod macos; mod os; #[derive(Error, Debug)] diff --git a/crates/uv-platform/src/macos.rs b/crates/uv-platform/src/macos.rs new file mode 100644 index 0000000000000..8280aec87c800 --- /dev/null +++ b/crates/uv-platform/src/macos.rs @@ -0,0 +1,87 @@ +//! macOS version parsing and representation. + +use std::fmt; + +/// macOS version requirement. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub struct MacOSVersion { + pub major: u16, + pub minor: u16, +} + +impl MacOSVersion { + pub const fn new(major: u16, minor: u16) -> Self { + Self { major, minor } + } + + /// Parse a macOS version string like "10.9" or "11.0" or "14.0". + pub fn parse(s: &str) -> Option { + let mut parts = s.split('.'); + let major: u16 = parts.next()?.parse().ok()?; + let minor: u16 = parts.next().and_then(|part| part.parse().ok()).unwrap_or(0); + Some(Self::new(major, minor)) + } + + /// Parse from a packed version (used in Mach-O `LC_BUILD_VERSION` and `LC_VERSION_MIN_MACOSX`). + /// + /// Format: `xxxx.yy.zz` where `x` is major, `y` is minor, `z` is patch (ignored). + #[allow(clippy::cast_possible_truncation)] + pub const fn from_packed(packed: u32) -> Self { + Self { + major: ((packed >> 16) & 0xFFFF) as u16, + minor: ((packed >> 8) & 0xFF) as u16, + } + } +} + +impl fmt::Display for MacOSVersion { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}.{}", self.major, self.minor) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse() { + assert_eq!(MacOSVersion::parse("10.9"), Some(MacOSVersion::new(10, 9))); + assert_eq!(MacOSVersion::parse("11.0"), Some(MacOSVersion::new(11, 0))); + assert_eq!(MacOSVersion::parse("14"), Some(MacOSVersion::new(14, 0))); + assert_eq!(MacOSVersion::parse(""), None); + assert_eq!(MacOSVersion::parse("abc"), None); + } + + #[test] + fn test_from_packed() { + // 10.9.0 = 0x000A0900 + assert_eq!( + MacOSVersion::from_packed(0x000A_0900), + MacOSVersion::new(10, 9) + ); + // 11.0.0 = 0x000B0000 + assert_eq!( + MacOSVersion::from_packed(0x000B_0000), + MacOSVersion::new(11, 0) + ); + // 14.0.0 = 0x000E0000 + assert_eq!( + MacOSVersion::from_packed(0x000E_0000), + MacOSVersion::new(14, 0) + ); + } + + #[test] + fn test_display() { + assert_eq!(MacOSVersion::new(10, 9).to_string(), "10.9"); + assert_eq!(MacOSVersion::new(14, 0).to_string(), "14.0"); + } + + #[test] + fn test_ord() { + assert!(MacOSVersion::new(10, 9) < MacOSVersion::new(10, 10)); + assert!(MacOSVersion::new(10, 15) < MacOSVersion::new(11, 0)); + assert!(MacOSVersion::new(11, 0) < MacOSVersion::new(14, 0)); + } +} diff --git a/crates/uv-static/src/env_vars.rs b/crates/uv-static/src/env_vars.rs index 69451f38696d0..161302091c9b7 100644 --- a/crates/uv-static/src/env_vars.rs +++ b/crates/uv-static/src/env_vars.rs @@ -716,6 +716,18 @@ impl EnvVars { #[attr_added_in("0.1.42")] pub const MACOSX_DEPLOYMENT_TARGET: &'static str = "MACOSX_DEPLOYMENT_TARGET"; + /// Search path for dynamic libraries on macOS (checked before system paths). + /// + /// Used during wheel delocating to find library dependencies. + #[attr_added_in("0.9.22")] + pub const DYLD_LIBRARY_PATH: &'static str = "DYLD_LIBRARY_PATH"; + + /// Fallback search path for dynamic libraries on macOS. + /// + /// Used during wheel delocating to find library dependencies. + #[attr_added_in("0.9.22")] + pub const DYLD_FALLBACK_LIBRARY_PATH: &'static str = "DYLD_FALLBACK_LIBRARY_PATH"; + /// Used with `--python-platform arm64-apple-ios` and related variants to set the /// deployment target (i.e., the minimum supported iOS version). /// @@ -1216,6 +1228,10 @@ impl EnvVars { #[attr_added_in("0.8.15")] pub const PYX_API_KEY: &'static str = "PYX_API_KEY"; + /// The pyx workspace in which to search for cached wheels. + #[attr_added_in("0.9.9")] + pub const PYX_CACHE_WORKSPACE: &'static str = "PYX_CACHE_WORKSPACE"; + /// The pyx API key, for backwards compatibility. #[attr_hidden] #[attr_added_in("0.8.15")] diff --git a/crates/uv/tests/it/pip_install.rs b/crates/uv/tests/it/pip_install.rs index 58f2f6274686e..5225e5b9c2b8a 100644 --- a/crates/uv/tests/it/pip_install.rs +++ b/crates/uv/tests/it/pip_install.rs @@ -3845,9 +3845,7 @@ fn install_git_source_respects_offline_mode() { ----- stderr ----- × Failed to download and build `uv-public-pypackage @ git+https://github.com/astral-test/uv-public-pypackage` - ├─▶ Git operation failed - ├─▶ failed to clone into: [CACHE_DIR]/git-v0/db/8dab139913c4b566 - ╰─▶ Remote Git fetches are not allowed because network connectivity is disabled (i.e., with `--offline`) + ╰─▶ Network connectivity is disabled, but the requested data wasn't found in the cache for: `https://api.github.com/repos/astral-test/uv-public-pypackage/commits/HEAD` " ); }