diff --git a/Cargo.lock b/Cargo.lock index ffd4b52749cb..cdc10889b924 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -518,9 +518,9 @@ checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" [[package]] name = "cpufeatures" -version = "0.2.10" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fbc60abd742b35f2492f808e1abbb83d45f72db402e14c55057edc9c7b1e9e4" +checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" dependencies = [ "libc", ] @@ -1045,9 +1045,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.14.2" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156" +checksum = "7dfda62a12f55daeae5015f81b0baea145391cb4520f86c248fc615d72640d12" [[package]] name = "heck" @@ -1183,7 +1183,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.10", + "socket2 0.4.9", "tokio", "tower-service", "tracing", @@ -1270,7 +1270,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8adf3ddd720272c6ea8bf59463c04e0f93d0bbf7c5439b691bca2987e0270897" dependencies = [ "equivalent", - "hashbrown 0.14.2", + "hashbrown 0.14.1", "serde", ] @@ -1350,7 +1350,6 @@ dependencies = [ "reflink-copy", "regex", "serde", - "serde_json", "sha2", "target-lexicon", "tempfile", @@ -2015,22 +2014,14 @@ version = "0.0.1" dependencies = [ "anyhow", "clap", - "colored", - "directories", "flate2", "fs-err", "gourgeist", "indoc 2.0.4", - "itertools", "pep508_rs", "platform-host", "platform-tags", - "puffin-client", - "puffin-installer", - "puffin-interpreter", - "puffin-package", - "puffin-resolver", - "puffin-workspace", + "puffin-traits", "pyproject-toml", "serde", "serde_json", @@ -2040,11 +2031,37 @@ dependencies = [ "tokio", "toml 0.8.2", "tracing", - "tracing-subscriber", "which", "zip", ] +[[package]] +name = "puffin-build-cli" +version = "0.0.1" +dependencies = [ + "anyhow", + "clap", + "colored", + "directories", + "fs-err", + "futures", + "gourgeist", + "itertools", + "pep508_rs", + "platform-host", + "platform-tags", + "puffin-build", + "puffin-client", + "puffin-dispatch", + "puffin-interpreter", + "puffin-package", + "tempfile", + "tokio", + "tracing", + "tracing-subscriber", + "which", +] + [[package]] name = "puffin-cli" version = "0.0.1" @@ -2073,6 +2090,7 @@ dependencies = [ "predicates", "pubgrub", "puffin-client", + "puffin-dispatch", "puffin-installer", "puffin-interpreter", "puffin-package", @@ -2103,10 +2121,32 @@ dependencies = [ "serde", "serde_json", "thiserror", + "tokio", "tracing", "url", ] +[[package]] +name = "puffin-dispatch" +version = "0.1.0" +dependencies = [ + "anyhow", + "gourgeist", + "itertools", + "log", + "pep508_rs", + "platform-host", + "platform-tags", + "puffin-build", + "puffin-client", + "puffin-installer", + "puffin-interpreter", + "puffin-package", + "puffin-resolver", + "puffin-traits", + "tempfile", +] + [[package]] name = "puffin-installer" version = "0.0.1" @@ -2139,7 +2179,6 @@ dependencies = [ "pep440_rs 0.3.12", "pep508_rs", "platform-host", - "puffin-package", "serde_json", "tokio", "tracing", @@ -2178,9 +2217,12 @@ dependencies = [ "clap", "colored", "distribution-filename", + "fs-err", "futures", "fxhash", + "gourgeist", "insta", + "install-wheel-rs", "itertools", "once_cell", "pep440_rs 0.3.12", @@ -2191,10 +2233,25 @@ dependencies = [ "pubgrub", "puffin-client", "puffin-package", + "puffin-traits", + "tempfile", "thiserror", "tokio", + "tokio-util", "tracing", + "url", "waitmap", + "which", + "zip", +] + +[[package]] +name = "puffin-traits" +version = "0.1.0" +dependencies = [ + "anyhow", + "gourgeist", + "pep508_rs", ] [[package]] @@ -2581,9 +2638,9 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustix" -version = "0.38.20" +version = "0.38.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67ce50cb2e16c2903e30d1cbccfd8387a74b9d4c938b6a4c5ec6cc7556f7a8a0" +checksum = "745ecfa778e66b2b63c88a61cb36e0eea109e803b0b86bf9879fbc77c70e86ed" dependencies = [ "bitflags 2.4.1", "errno", @@ -2800,9 +2857,9 @@ checksum = "b7c388c1b5e93756d0c740965c41e8822f866621d41acbdf6336a6a168f8840c" [[package]] name = "socket2" -version = "0.4.10" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" +checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" dependencies = [ "libc", "winapi", @@ -2810,9 +2867,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.5" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" +checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e" dependencies = [ "libc", "windows-sys 0.48.0", @@ -2938,9 +2995,9 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.12.12" +version = "0.12.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c39fd04924ca3a864207c66fc2cd7d22d7c016007f9ce846cbb9326331930a" +checksum = "9d0e916b1148c8e263850e1ebcbd046f333e0683c724876bb0da63ea4373dc8a" [[package]] name = "task-local-extensions" @@ -3037,18 +3094,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.50" +version = "1.0.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2" +checksum = "1177e8c6d7ede7afde3585fd2513e611227efd6481bd78d2e82ba1ce16557ed4" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.50" +version = "1.0.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" +checksum = "10712f02019e9288794769fba95cd6847df9874d49d871d062172f9dd41bc4cc" dependencies = [ "proc-macro2", "quote", @@ -3130,7 +3187,7 @@ dependencies = [ "mio", "num_cpus", "pin-project-lite", - "socket2 0.5.5", + "socket2 0.5.4", "tokio-macros", "windows-sys 0.48.0", ] @@ -3249,9 +3306,9 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.40" +version = "0.1.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +checksum = "ee2ef2af84856a50c1d430afce2fdded0a4ec7eda868db86409b4543df0797f9" dependencies = [ "log", "pin-project-lite", diff --git a/crates/install-wheel-rs/Cargo.toml b/crates/install-wheel-rs/Cargo.toml index 0325d401970d..22406cb11f65 100644 --- a/crates/install-wheel-rs/Cargo.toml +++ b/crates/install-wheel-rs/Cargo.toml @@ -38,7 +38,6 @@ rayon = { version = "1.8.0", optional = true } reflink-copy = { workspace = true } regex = { workspace = true } serde = { workspace = true, features = ["derive"] } -serde_json = { workspace = true } sha2 = { workspace = true } target-lexicon = { workspace = true } tempfile = { workspace = true } diff --git a/crates/install-wheel-rs/src/lib.rs b/crates/install-wheel-rs/src/lib.rs index 832ff8a58458..dfef4256b011 100644 --- a/crates/install-wheel-rs/src/lib.rs +++ b/crates/install-wheel-rs/src/lib.rs @@ -12,8 +12,8 @@ pub use record::RecordEntry; pub use script::Script; pub use uninstall::{uninstall_wheel, Uninstall}; pub use wheel::{ - get_script_launcher, install_wheel, parse_key_value_file, read_record_file, relative_to, - SHEBANG_PYTHON, + find_dist_info, get_script_launcher, install_wheel, parse_key_value_file, read_record_file, + relative_to, SHEBANG_PYTHON, }; mod install_location; diff --git a/crates/install-wheel-rs/src/wheel.rs b/crates/install-wheel-rs/src/wheel.rs index d5c4482ab191..3f41ed5cbc90 100644 --- a/crates/install-wheel-rs/src/wheel.rs +++ b/crates/install-wheel-rs/src/wheel.rs @@ -1026,7 +1026,7 @@ pub fn install_wheel( /// Either way, we just search the wheel for the name /// /// -fn find_dist_info( +pub fn find_dist_info( filename: &WheelFilename, archive: &mut ZipArchive, ) -> Result { diff --git a/crates/pep440-rs/src/version_specifier.rs b/crates/pep440-rs/src/version_specifier.rs index 298a89c17e4b..13fdf0395919 100644 --- a/crates/pep440-rs/src/version_specifier.rs +++ b/crates/pep440-rs/src/version_specifier.rs @@ -86,6 +86,12 @@ impl FromStr for VersionSpecifiers { } } +impl From for VersionSpecifiers { + fn from(specifier: VersionSpecifier) -> Self { + Self(vec![specifier]) + } +} + impl Display for VersionSpecifiers { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { for (idx, version_specifier) in self.0.iter().enumerate() { @@ -341,6 +347,14 @@ impl VersionSpecifier { Ok(Self { operator, version }) } + /// `==` + pub fn equals_version(version: Version) -> Self { + Self { + operator: Operator::Equal, + version, + } + } + /// Get the operator, e.g. `>=` in `>= 2.0.0` pub fn operator(&self) -> &Operator { &self.operator diff --git a/crates/pep508-rs/src/marker.rs b/crates/pep508-rs/src/marker.rs index 2e73a34cc4b9..25cf56db7cc7 100644 --- a/crates/pep508-rs/src/marker.rs +++ b/crates/pep508-rs/src/marker.rs @@ -312,6 +312,12 @@ impl FromStr for StringVersion { } } +impl Display for StringVersion { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + self.version.fmt(f) + } +} + #[cfg(feature = "serde")] impl Serialize for StringVersion { fn serialize(&self, serializer: S) -> Result diff --git a/crates/puffin-build-cli/.gitignore b/crates/puffin-build-cli/.gitignore new file mode 100644 index 000000000000..4d668e3d9167 --- /dev/null +++ b/crates/puffin-build-cli/.gitignore @@ -0,0 +1,2 @@ +downloads +wheels diff --git a/crates/puffin-build-cli/Cargo.toml b/crates/puffin-build-cli/Cargo.toml new file mode 100644 index 000000000000..5b67463933e3 --- /dev/null +++ b/crates/puffin-build-cli/Cargo.toml @@ -0,0 +1,35 @@ +[package] +name = "puffin-build-cli" +version = "0.0.1" +description = "Build wheels from source distributions" +edition = { workspace = true } +rust-version = { workspace = true } +homepage = { workspace = true } +documentation = { workspace = true } +repository = { workspace = true } +authors = { workspace = true } +license = { workspace = true } + +[dependencies] +gourgeist = { path = "../gourgeist" } +pep508_rs = { path = "../pep508-rs" } +platform-host = { path = "../platform-host" } +platform-tags = { path = "../platform-tags" } +puffin-build = { path = "../puffin-build" } +puffin-client = { path = "../puffin-client" } +puffin-dispatch = { path = "../puffin-dispatch" } +puffin-interpreter = { path = "../puffin-interpreter" } +puffin-package = { path = "../puffin-package" } + +anyhow = { workspace = true } +clap = { workspace = true, features = ["derive"] } +directories = { workspace = true } +fs-err = { workspace = true } +futures = { workspace = true } +itertools = { workspace = true } +tempfile = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } +which = { workspace = true } +colored = "2.0.4" diff --git a/crates/puffin-build/src/main.rs b/crates/puffin-build-cli/src/main.rs similarity index 78% rename from crates/puffin-build/src/main.rs rename to crates/puffin-build-cli/src/main.rs index 248de6a981a2..fe7a83e56fe8 100644 --- a/crates/puffin-build/src/main.rs +++ b/crates/puffin-build-cli/src/main.rs @@ -5,7 +5,11 @@ use clap::Parser; use colored::Colorize; use directories::ProjectDirs; use fs_err as fs; +use platform_host::Platform; use puffin_build::{Error, SourceDistributionBuilder}; +use puffin_client::RegistryClientBuilder; +use puffin_dispatch::PuffinDispatch; +use puffin_interpreter::PythonExecutable; use std::path::PathBuf; use std::process::ExitCode; use std::time::Instant; @@ -49,11 +53,19 @@ async fn run() -> anyhow::Result<()> { })?; let interpreter_info = gourgeist::get_interpreter_info(&base_python)?; - let builder = - SourceDistributionBuilder::setup(&args.sdist, &base_python, &interpreter_info, cache) - .await?; + let platform = Platform::current()?; + let python = PythonExecutable::from_env(platform, cache)?; + let puffin_dispatch = + PuffinDispatch::new(RegistryClientBuilder::default().build(), python, cache); + let builder = SourceDistributionBuilder::setup( + &args.sdist, + &base_python, + &interpreter_info, + &puffin_dispatch, + ) + .await?; let wheel = builder.build(&wheel_dir)?; - println!("Wheel built to {}", wheel.display()); + println!("Wheel built to {}", wheel_dir.join(wheel).display()); Ok(()) } diff --git a/crates/puffin-build/test.sh b/crates/puffin-build-cli/test.sh similarity index 100% rename from crates/puffin-build/test.sh rename to crates/puffin-build-cli/test.sh diff --git a/crates/puffin-build/Cargo.toml b/crates/puffin-build/Cargo.toml index 2648d9651057..f8ebae19d3fa 100644 --- a/crates/puffin-build/Cargo.toml +++ b/crates/puffin-build/Cargo.toml @@ -15,21 +15,13 @@ gourgeist = { path = "../gourgeist" } pep508_rs = { path = "../pep508-rs" } platform-host = { path = "../platform-host" } platform-tags = { path = "../platform-tags" } -puffin-client = { path = "../puffin-client" } -puffin-installer = { path = "../puffin-installer" } -puffin-interpreter = { path = "../puffin-interpreter" } -puffin-package = { path = "../puffin-package" } -puffin-resolver = { path = "../puffin-resolver" } -puffin-workspace = { path = "../puffin-workspace" } +puffin-traits = { path = "../puffin-traits" } anyhow = { workspace = true } clap = { workspace = true, features = ["derive"] } -colored = { workspace = true } -directories = { workspace = true } flate2 = { workspace = true } fs-err = { workspace = true } indoc = { workspace = true } -itertools = { workspace = true } pyproject-toml = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } @@ -39,6 +31,5 @@ thiserror = { workspace = true } tokio = { workspace = true } toml = { workspace = true } tracing = { workspace = true } -tracing-subscriber = { workspace = true } which = { workspace = true} zip = { workspace = true } diff --git a/crates/puffin-build/src/lib.rs b/crates/puffin-build/src/lib.rs index c2d070b90c71..b5d151c368a7 100644 --- a/crates/puffin-build/src/lib.rs +++ b/crates/puffin-build/src/lib.rs @@ -2,34 +2,27 @@ //! //! -use anyhow::Context; -use flate2::read::GzDecoder; -use fs_err as fs; -use fs_err::{DirEntry, File}; -use gourgeist::{InterpreterInfo, Venv}; -use indoc::formatdoc; -use itertools::{Either, Itertools}; -use pep508_rs::Requirement; -use platform_host::Platform; -use platform_tags::Tags; -use puffin_client::RegistryClientBuilder; -use puffin_installer::{CachedDistribution, Downloader, LocalIndex, RemoteDistribution, Unzipper}; -use puffin_interpreter::PythonExecutable; -use puffin_package::package_name::PackageName; -use puffin_resolver::WheelFinder; -use pyproject_toml::PyProjectToml; use std::io; use std::io::BufRead; -use std::ops::Deref; use std::path::{Path, PathBuf}; use std::process::{Command, Output}; use std::str::FromStr; + +use flate2::read::GzDecoder; +use fs_err as fs; +use fs_err::{DirEntry, File}; +use indoc::formatdoc; +use pyproject_toml::PyProjectToml; use tar::Archive; use tempfile::{tempdir, TempDir}; use thiserror::Error; use tracing::{debug, instrument}; use zip::ZipArchive; +use gourgeist::{InterpreterInfo, Venv}; +use pep508_rs::Requirement; +use puffin_traits::PuffinCtx; + #[derive(Error, Debug)] pub enum Error { #[error(transparent)] @@ -42,8 +35,8 @@ pub enum Error { InvalidSourceDistribution(String), #[error("Invalid pyproject.toml")] InvalidPyprojectToml(#[from] toml::de::Error), - #[error("Failed to install requirements")] - RequirementsInstall(#[source] anyhow::Error), + #[error("Failed to install requirements from {0}")] + RequirementsInstall(&'static str, #[source] anyhow::Error), #[error("Failed to create temporary virtual environment")] Gourgeist(#[from] gourgeist::Error), #[error("Failed to run {0}")] @@ -114,7 +107,7 @@ impl SourceDistributionBuilder { sdist: &Path, base_python: &Path, interpreter_info: &InterpreterInfo, - cache: Option<&Path>, + puffin_ctx: &impl PuffinCtx, ) -> Result { let temp_dir = tempdir()?; @@ -151,7 +144,7 @@ impl SourceDistributionBuilder { base_python, interpreter_info, pep517_backend, - cache, + puffin_ctx, ) .await? } else { @@ -173,9 +166,14 @@ impl SourceDistributionBuilder { Requirement::from_str("setuptools").unwrap(), Requirement::from_str("pip").unwrap(), ]; - resolve_and_install(venv.as_std_path(), &requirements, cache) + let resolved_requirements = puffin_ctx + .resolve(&requirements) + .await + .map_err(|err| Error::RequirementsInstall("setup.py build", err))?; + puffin_ctx + .install(&resolved_requirements, &venv) .await - .map_err(Error::RequirementsInstall)?; + .map_err(|err| Error::RequirementsInstall("setup.py build", err))?; venv }; @@ -209,8 +207,8 @@ impl SourceDistributionBuilder { r#"{} as backend import json - if get_requires_for_build_wheel := getattr(backend, "prepare_metadata_for_build_wheel", None): - print(get_requires_for_build_wheel("{}")) + if prepare_metadata_for_build_wheel := getattr(backend, "prepare_metadata_for_build_wheel", None): + print(prepare_metadata_for_build_wheel("{}")) else: print() "#, pep517_backend.backend_import(), escape_path_for_python(&metadata_directory) @@ -255,7 +253,7 @@ impl SourceDistributionBuilder { /// /// #[instrument(skip(self))] - pub fn build(&self, wheel_dir: &Path) -> Result { + pub fn build(&self, wheel_dir: &Path) -> Result { // The build scripts run with the extracted root as cwd, so they need the absolute path let wheel_dir = fs::canonicalize(wheel_dir)?; @@ -287,9 +285,9 @@ impl SourceDistributionBuilder { }; // TODO(konstin): Faster copy such as reflink? Or maybe don't really let the user pick the target dir let wheel = wheel_dir.join(dist_wheel.file_name()); - fs::copy(dist_wheel.path(), &wheel)?; + fs::copy(dist_wheel.path(), wheel)?; // TODO(konstin): Check wheel filename - Ok(wheel) + Ok(dist_wheel.file_name().to_string_lossy().to_string()) } } @@ -297,7 +295,7 @@ impl SourceDistributionBuilder { &self, wheel_dir: &Path, pep517_backend: &Pep517Backend, - ) -> Result { + ) -> Result { let metadata_directory = self .metadata_directory .as_deref() @@ -323,18 +321,17 @@ impl SourceDistributionBuilder { )); } let stdout = String::from_utf8_lossy(&output.stdout); - let wheel = stdout - .lines() - .last() - .map(|distribution_filename| wheel_dir.join(distribution_filename)); - let Some(wheel) = wheel.filter(|wheel| wheel.is_file()) else { + let distribution_filename = stdout.lines().last(); + let Some(distribution_filename) = + distribution_filename.filter(|wheel| wheel_dir.join(wheel).is_file()) + else { return Err(Error::from_command_output( "Build backend did not return the wheel filename through `build_wheel()`" .to_string(), &output, )); }; - Ok(wheel) + Ok(distribution_filename.to_string()) } } @@ -351,16 +348,17 @@ async fn create_pep517_build_environment( base_python: &Path, data: &InterpreterInfo, pep517_backend: &Pep517Backend, - cache: Option<&Path>, + puffin_ctx: &impl PuffinCtx, ) -> Result { let venv = gourgeist::create_venv(root.join(".venv"), base_python, data, true)?; - resolve_and_install( - venv.deref().as_std_path(), - &pep517_backend.requirements, - cache, - ) - .await - .map_err(Error::RequirementsInstall)?; + let resolved_requirements = puffin_ctx + .resolve(&pep517_backend.requirements) + .await + .map_err(|err| Error::RequirementsInstall("get_requires_for_build_wheel", err))?; + puffin_ctx + .install(&resolved_requirements, &venv) + .await + .map_err(|err| Error::RequirementsInstall("get_requires_for_build_wheel", err))?; debug!( "Calling `{}.get_requires_for_build_wheel()`", @@ -375,7 +373,7 @@ async fn create_pep517_build_environment( else: requires = [] print(json.dumps(requires)) - "#, pep517_backend.backend_import() + "#, pep517_backend.backend_import() }; let output = run_python_script(&venv.python_interpreter(), &script, source_tree)?; if !output.status.success() { @@ -420,63 +418,18 @@ async fn create_pep517_build_environment( .cloned() .chain(extra_requires) .collect(); - resolve_and_install(&*venv, &requirements, cache) + let resolved_requirements = puffin_ctx + .resolve(&requirements) .await - .map_err(Error::RequirementsInstall)?; + .map_err(|err| Error::RequirementsInstall("build-system.requires", err))?; + + puffin_ctx + .install(&resolved_requirements, &venv) + .await + .map_err(|err| Error::RequirementsInstall("build-system.requires", err))?; } Ok(venv) } - -#[instrument(skip_all)] -async fn resolve_and_install( - venv: impl AsRef, - requirements: &[Requirement], - cache: Option<&Path>, -) -> anyhow::Result<()> { - debug!("Installing {} build requirements", requirements.len()); - - let local_index = if let Some(cache) = cache { - LocalIndex::try_from_directory(cache)? - } else { - LocalIndex::default() - }; - let (cached, uncached): (Vec, Vec) = - requirements.iter().partition_map(|requirement| { - let package = PackageName::normalize(&requirement.name); - if let Some(distribution) = local_index - .get(&package) - .filter(|dist| requirement.is_satisfied_by(dist.version())) - { - Either::Left(distribution.clone()) - } else { - Either::Right(requirement.clone()) - } - }); - - let client = RegistryClientBuilder::default().cache(cache).build(); - - let platform = Platform::current()?; - let python = PythonExecutable::from_venv(platform, venv.as_ref(), cache)?; - let tags = Tags::from_env(python.platform(), python.simple_version())?; - let resolution = WheelFinder::new(&tags, &client).resolve(&uncached).await?; - let uncached = resolution - .into_files() - .map(RemoteDistribution::from_file) - .collect::>>()?; - let staging = tempdir()?; - let downloads = Downloader::new(&client, cache) - .download(&uncached, cache.unwrap_or(staging.path())) - .await?; - let unzips = Unzipper::default() - .download(downloads, cache.unwrap_or(staging.path())) - .await - .context("Failed to download and unpack wheels")?; - let wheels = unzips.into_iter().chain(cached).collect::>(); - puffin_installer::Installer::new(&python).install(&wheels)?; - - Ok(()) -} - /// Returns the directory with the `pyproject.toml`/`setup.py` #[instrument(skip_all, fields(path))] fn extract_archive(sdist: &Path, extracted: &PathBuf) -> Result { diff --git a/crates/puffin-cli/Cargo.toml b/crates/puffin-cli/Cargo.toml index 6a960907856a..e10b2569104a 100644 --- a/crates/puffin-cli/Cargo.toml +++ b/crates/puffin-cli/Cargo.toml @@ -16,6 +16,7 @@ platform-host = { path = "../platform-host" } platform-tags = { path = "../platform-tags" } pubgrub = { path = "../../vendor/pubgrub" } puffin-client = { path = "../puffin-client" } +puffin-dispatch = { path = "../puffin-dispatch" } puffin-installer = { path = "../puffin-installer" } puffin-interpreter = { path = "../puffin-interpreter" } puffin-package = { path = "../puffin-package" } diff --git a/crates/puffin-cli/src/commands/pip_compile.rs b/crates/puffin-cli/src/commands/pip_compile.rs index 73be1829ae46..283fc4934de2 100644 --- a/crates/puffin-cli/src/commands/pip_compile.rs +++ b/crates/puffin-cli/src/commands/pip_compile.rs @@ -7,15 +7,15 @@ use anyhow::Result; use colored::Colorize; use fs_err::File; use itertools::Itertools; -use pubgrub::report::Reporter; -use tracing::debug; - use pep508_rs::Requirement; use platform_host::Platform; use platform_tags::Tags; +use pubgrub::report::Reporter; use puffin_client::RegistryClientBuilder; +use puffin_dispatch::PuffinDispatch; use puffin_interpreter::PythonExecutable; use puffin_resolver::ResolutionMode; +use tracing::debug; use crate::commands::{elapsed, ExitStatus}; use crate::index_urls::IndexUrls; @@ -51,14 +51,16 @@ pub(crate) async fn pip_compile( // Detect the current Python interpreter. let platform = Platform::current()?; let python = PythonExecutable::from_env(platform, cache)?; + + // Determine the current environment markers. + let markers = python.markers().clone(); + debug!( - "Using Python interpreter: {}", + "Using Python {} at {}", + markers.python_version, python.executable().display() ); - // Determine the current environment markers. - let markers = python.markers(); - // Determine the compatible platform tags. let tags = Tags::from_env(python.platform(), python.simple_version())?; @@ -77,9 +79,19 @@ pub(crate) async fn pip_compile( builder.build() }; + let puffin_dispatch = + PuffinDispatch::new(RegistryClientBuilder::default().build(), python, cache); + // Resolve the dependencies. - let resolver = - puffin_resolver::Resolver::new(requirements, constraints, mode, markers, &tags, &client); + let resolver = puffin_resolver::Resolver::new( + requirements, + constraints, + mode, + &markers, + &tags, + &client, + &puffin_dispatch, + ); let resolution = match resolver.resolve().await { Err(puffin_resolver::ResolveError::PubGrub(pubgrub::error::PubGrubError::NoSolution( mut derivation_tree, diff --git a/crates/puffin-client/Cargo.toml b/crates/puffin-client/Cargo.toml index b7028cfafa8f..5cdc3e6e147d 100644 --- a/crates/puffin-client/Cargo.toml +++ b/crates/puffin-client/Cargo.toml @@ -6,6 +6,7 @@ edition = "2021" [dependencies] puffin-package = { path = "../puffin-package" } +futures = { workspace = true } http-cache-reqwest = { workspace = true } reqwest = { workspace = true } reqwest-middleware = { workspace = true } @@ -13,6 +14,6 @@ reqwest-retry = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } thiserror = { workspace = true } -url = { workspace = true } +tokio = { workspace = true } tracing = { workspace = true } -futures = { workspace = true } +url = { workspace = true } diff --git a/crates/puffin-dispatch/Cargo.toml b/crates/puffin-dispatch/Cargo.toml new file mode 100644 index 000000000000..ea70ae0f337a --- /dev/null +++ b/crates/puffin-dispatch/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "puffin-dispatch" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +homepage.workspace = true +documentation.workspace = true +repository.workspace = true +authors.workspace = true +license.workspace = true + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +gourgeist = { path = "../gourgeist" } +pep508_rs = { path = "../pep508-rs" } +platform-host = { path = "../platform-host" } +platform-tags = { path = "../platform-tags" } +puffin-build = { path = "../puffin-build" } +puffin-client = { path = "../puffin-client" } +puffin-installer = { path = "../puffin-installer" } +puffin-interpreter = { path = "../puffin-interpreter" } +puffin-package = { path = "../puffin-package" } +puffin-resolver = { path = "../puffin-resolver" } +puffin-traits = { path = "../puffin-traits" } + +anyhow = { workspace = true } +itertools = { workspace = true } +tempfile = { workspace = true } +log = "0.4.20" \ No newline at end of file diff --git a/crates/puffin-dispatch/src/lib.rs b/crates/puffin-dispatch/src/lib.rs new file mode 100644 index 000000000000..d05ae9b7fe5e --- /dev/null +++ b/crates/puffin-dispatch/src/lib.rs @@ -0,0 +1,200 @@ +#![allow(unstable_name_collisions)] // intersperse + +use std::future::Future; +use std::path::{Path, PathBuf}; +use std::pin::Pin; + +use anyhow::Context; +use itertools::{Either, Itertools}; +use tempfile::tempdir; + +use gourgeist::Venv; +use log::debug; +use pep508_rs::Requirement; +use platform_tags::Tags; +use puffin_build::SourceDistributionBuilder; +use puffin_client::RegistryClient; +use puffin_installer::{ + uninstall, CachedDistribution, Downloader, Installer, LocalIndex, RemoteDistribution, + SitePackages, Unzipper, +}; +use puffin_interpreter::PythonExecutable; +use puffin_package::package_name::PackageName; +use puffin_resolver::{ResolutionMode, Resolver, WheelFinder}; +use puffin_traits::PuffinCtx; + +pub struct PuffinDispatch { + client: RegistryClient, + python: PythonExecutable, + cache: Option, +} + +impl PuffinDispatch { + pub fn new(client: RegistryClient, python: PythonExecutable, cache: Option) -> Self + where + T: Into, + { + Self { + client, + python, + cache: cache.map(Into::into), + } + } +} + +impl PuffinCtx for PuffinDispatch { + fn cache(&self) -> Option<&Path> { + self.cache.as_deref() + } + + fn resolve<'a>( + &'a self, + requirements: &'a [Requirement], + ) -> Pin>> + 'a>> { + Box::pin(async { + let tags = Tags::from_env(self.python.platform(), self.python.simple_version())?; + let resolver = Resolver::new( + requirements.to_vec(), + Vec::default(), + ResolutionMode::Highest, + self.python.markers(), + &tags, + &self.client, + // TODO: nested builds are not supported yet + self, + ); + let resolution_graph = resolver.resolve().await.context( + "No solution found when resolving build dependencies for source distribution build", + )?; + Ok(resolution_graph.requirements()) + }) + } + + fn install<'a>( + &'a self, + requirements: &'a [Requirement], + venv: &'a Venv, + ) -> Pin> + 'a>> { + Box::pin(async move { + debug!( + "Install in {} requirements {}", + venv.as_str(), + requirements + .iter() + .map(std::string::ToString::to_string) + .intersperse(", ".to_string()) + .collect::() + ); + let python = PythonExecutable::from_venv_with_base(venv.as_std_path(), &self.python); + + let site_packages = SitePackages::try_from_executable(&python)?; + + // We have three buckets: + // * Not installed + // * Installed and a matching version + // * Install but an incorrect version, we need to remove it first + let mut to_remove = Vec::new(); + let mut to_install = Vec::new(); + for requirement in requirements { + if let Some((_name, installed)) = + site_packages.iter().find(|(name, _distribution)| { + name == &&PackageName::normalize(&requirement.name) + }) + { + if requirement.is_satisfied_by(installed.version()) { + // Nothing to do + } else { + to_remove.push(installed); + to_install.push(requirement.clone()); + } + } else { + to_install.push(requirement.clone()); + } + } + + if !to_remove.is_empty() { + debug!( + "Removing {:?}", + to_remove + .iter() + .map(|dist| dist.id()) + .intersperse(", ".to_string()) + .collect::() + ); + + for dist_info in to_remove { + uninstall(dist_info).await?; + } + } + + debug!( + "Installing {}", + to_install + .iter() + .map(std::string::ToString::to_string) + .intersperse(", ".to_string()) + .collect::() + ); + + let local_index = if let Some(cache) = &self.cache { + LocalIndex::try_from_directory(cache)? + } else { + LocalIndex::default() + }; + + let (cached, uncached): (Vec, Vec) = + to_install.iter().partition_map(|requirement| { + let package = PackageName::normalize(&requirement.name); + if let Some(distribution) = local_index + .get(&package) + .filter(|dist| requirement.is_satisfied_by(dist.version())) + { + Either::Left(distribution.clone()) + } else { + Either::Right(requirement.clone()) + } + }); + + let tags = Tags::from_env(python.platform(), python.simple_version())?; + let resolution = WheelFinder::new(&tags, &self.client) + .resolve(&uncached) + .await?; + + let uncached = resolution + .into_files() + .map(RemoteDistribution::from_file) + .collect::>>()?; + let staging = tempdir()?; + let downloads = Downloader::new(&self.client, self.cache.as_deref()) + .download(&uncached, self.cache.as_deref().unwrap_or(staging.path())) + .await?; + let unzips = Unzipper::default() + .download(downloads, self.cache.as_deref().unwrap_or(staging.path())) + .await + .context("Failed to download and unpack wheels")?; + let wheels = unzips.into_iter().chain(cached).collect::>(); + Installer::new(&python).install(&wheels)?; + Ok(()) + }) + } + + fn build_source_distribution<'a>( + &'a self, + sdist: &'a Path, + wheel_dir: &'a Path, + ) -> Pin> + 'a>> { + Box::pin(async move { + // TODO: Merge this with PythonExecutable + let interpreter_info = gourgeist::get_interpreter_info(self.python.executable())?; + + let builder = SourceDistributionBuilder::setup( + sdist, + self.python.executable(), + &interpreter_info, + self, + ) + .await?; + Ok(builder.build(wheel_dir)?) + }) + } +} diff --git a/crates/puffin-installer/src/unzipper.rs b/crates/puffin-installer/src/unzipper.rs index d44e12f0ffd1..746ed575b5c9 100644 --- a/crates/puffin-installer/src/unzipper.rs +++ b/crates/puffin-installer/src/unzipper.rs @@ -55,11 +55,19 @@ impl Unzipper { .await??; // Write the unzipped wheel to the target directory. - fs_err::tokio::rename( + let result = fs_err::tokio::rename( staging.path().join(remote.id()), wheel_cache.entry(&remote.id()), ) - .await?; + .await; + + if let Err(err) = result { + // If the renaming failed because another instance was faster, that's fine + // (`DirectoryNotEmpty` is not stable so we can't match on it) + if !wheel_cache.entry(&remote.id()).is_dir() { + return Err(err.into()); + } + } wheels.push(CachedDistribution::new( remote.name().clone(), diff --git a/crates/puffin-interpreter/Cargo.toml b/crates/puffin-interpreter/Cargo.toml index 279f45e96cb2..c9708a9714bb 100644 --- a/crates/puffin-interpreter/Cargo.toml +++ b/crates/puffin-interpreter/Cargo.toml @@ -13,7 +13,6 @@ license = { workspace = true } pep440_rs = { path = "../pep440-rs" } pep508_rs = { path = "../pep508-rs" } platform-host = { path = "../platform-host" } -puffin-package = { path = "../puffin-package" } anyhow = { workspace = true } cacache = { workspace = true } diff --git a/crates/puffin-interpreter/src/lib.rs b/crates/puffin-interpreter/src/lib.rs index 19fa890be338..9b5036c69cef 100644 --- a/crates/puffin-interpreter/src/lib.rs +++ b/crates/puffin-interpreter/src/lib.rs @@ -13,7 +13,7 @@ mod python_platform; mod virtual_env; /// A Python executable and its associated platform markers. -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct PythonExecutable { platform: PythonPlatform, venv: PathBuf, @@ -50,6 +50,17 @@ impl PythonExecutable { }) } + /// Create a [`PythonExecutable`] for a venv with a known base [`PythonExecutable`]. + pub fn from_venv_with_base(venv: &Path, base: &Self) -> Self { + let executable = base.platform.venv_python(venv); + + Self { + venv: venv.to_path_buf(), + executable, + ..base.clone() + } + } + /// Returns the path to the Python virtual environment. pub fn platform(&self) -> &Platform { &self.platform diff --git a/crates/puffin-package/Cargo.toml b/crates/puffin-package/Cargo.toml index 4b796bb5958e..a2c7db8e3faf 100644 --- a/crates/puffin-package/Cargo.toml +++ b/crates/puffin-package/Cargo.toml @@ -7,7 +7,6 @@ edition = "2021" pep440_rs = { path = "../pep440-rs", features = ["serde"] } pep508_rs = { path = "../pep508-rs", features = ["serde"] } -anyhow = { workspace = true } fs-err = { workspace = true } mailparse = { workspace = true } memchr = { workspace = true } @@ -20,6 +19,7 @@ tracing.workspace = true unscanny = { workspace = true } [dev-dependencies] +anyhow = { version = "1.0.75" } indoc = { version = "2.0.4" } insta = { version = "1.33.0" } serde_json = { version = "1.0.107" } diff --git a/crates/puffin-resolver/Cargo.toml b/crates/puffin-resolver/Cargo.toml index c9bbc936d9ed..809fb2271553 100644 --- a/crates/puffin-resolver/Cargo.toml +++ b/crates/puffin-resolver/Cargo.toml @@ -10,6 +10,8 @@ authors = { workspace = true } license = { workspace = true } [dependencies] +gourgeist = { path = "../gourgeist" } +install-wheel-rs = { path = "../install-wheel-rs" } pep440_rs = { path = "../pep440-rs" } pep508_rs = { path = "../pep508-rs" } platform-host = { path = "../platform-host" } @@ -17,21 +19,28 @@ platform-tags = { path = "../platform-tags" } pubgrub = { path = "../../vendor/pubgrub" } puffin-client = { path = "../puffin-client" } puffin-package = { path = "../puffin-package" } +puffin-traits = { path = "../puffin-traits" } distribution-filename = { path = "../distribution-filename" } anyhow = { workspace = true } bitflags = { workspace = true } clap = { workspace = true, features = ["derive"], optional = true } colored = { workspace = true } +fs-err = { workspace = true, features = ["tokio"] } futures = { workspace = true } fxhash = { workspace = true } itertools = { workspace = true } once_cell = { workspace = true } petgraph = { workspace = true } +tempfile = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } +tokio-util = { workspace = true, features = ["compat"] } tracing = { workspace = true } +url = { workspace = true } waitmap = { workspace = true } +which = { workspace = true } +zip = { workspace = true } [dev-dependencies] once_cell = { version = "1.18.0" } diff --git a/crates/puffin-resolver/src/error.rs b/crates/puffin-resolver/src/error.rs index 24572d6acbdb..3a6c8f701124 100644 --- a/crates/puffin-resolver/src/error.rs +++ b/crates/puffin-resolver/src/error.rs @@ -25,6 +25,14 @@ pub enum ResolveError { #[error(transparent)] PubGrub(#[from] pubgrub::error::PubGrubError>), + + #[error("Failed to build source distribution {filename}")] + SourceDistribution { + filename: String, + // TODO: Gives this a proper error type + #[source] + err: anyhow::Error, + }, } impl From> for ResolveError { diff --git a/crates/puffin-resolver/src/lib.rs b/crates/puffin-resolver/src/lib.rs index df1c19a465ab..54ff22c84f29 100644 --- a/crates/puffin-resolver/src/lib.rs +++ b/crates/puffin-resolver/src/lib.rs @@ -2,6 +2,7 @@ pub use error::ResolveError; pub use mode::ResolutionMode; pub use resolution::PinnedPackage; pub use resolver::Resolver; +pub use source_distribution::BuiltSourceDistributionCache; pub use wheel_finder::{Reporter, WheelFinder}; mod error; @@ -9,4 +10,5 @@ mod mode; mod pubgrub; mod resolution; mod resolver; +mod source_distribution; mod wheel_finder; diff --git a/crates/puffin-resolver/src/resolution.rs b/crates/puffin-resolver/src/resolution.rs index 1fb44745a76b..dfd94d8f0e06 100644 --- a/crates/puffin-resolver/src/resolution.rs +++ b/crates/puffin-resolver/src/resolution.rs @@ -1,13 +1,14 @@ -use std::hash::BuildHasherDefault; - use colored::Colorize; use fxhash::FxHashMap; use petgraph::visit::EdgeRef; +use std::hash::BuildHasherDefault; + use pubgrub::range::Range; use pubgrub::solver::{Kind, State}; use pubgrub::type_aliases::SelectedDependencies; -use pep440_rs::Version; +use pep440_rs::{Version, VersionSpecifier, VersionSpecifiers}; +use pep508_rs::{Requirement, VersionOrUrl}; use puffin_client::File; use puffin_package::package_name::PackageName; @@ -148,6 +149,27 @@ impl Graph { pub fn is_empty(&self) -> bool { self.0.node_count() == 0 } + + pub fn requirements(&self) -> Vec { + // Collect and sort all packages. + let mut nodes = self + .0 + .node_indices() + .map(|node| (node, &self.0[node])) + .collect::>(); + nodes.sort_unstable_by_key(|(_, package)| package.name()); + self.0 + .node_indices() + .map(|node| Requirement { + name: self.0[node].name.to_string(), + extras: None, + version_or_url: Some(VersionOrUrl::VersionSpecifier(VersionSpecifiers::from( + VersionSpecifier::equals_version(self.0[node].version.clone()), + ))), + marker: None, + }) + .collect() + } } /// Write the graph in the `{name}=={version}` format of requirements.txt that pip uses. diff --git a/crates/puffin-resolver/src/resolver.rs b/crates/puffin-resolver/src/resolver.rs index 1b1c5291fc35..002ec99c122f 100644 --- a/crates/puffin-resolver/src/resolver.rs +++ b/crates/puffin-resolver/src/resolver.rs @@ -2,12 +2,14 @@ use std::borrow::Borrow; use std::collections::hash_map::Entry; +use std::future::Future; +use std::path::{Path, PathBuf}; +use std::pin::Pin; use std::str::FromStr; use std::sync::Arc; use anyhow::Result; use futures::channel::mpsc::UnboundedReceiver; -use futures::future::Either; use futures::{pin_mut, FutureExt, StreamExt, TryFutureExt}; use fxhash::{FxHashMap, FxHashSet}; use pubgrub::error::PubGrubError; @@ -18,13 +20,14 @@ use tokio::select; use tracing::{debug, trace}; use waitmap::WaitMap; -use distribution_filename::WheelFilename; +use distribution_filename::{SourceDistributionFilename, WheelFilename}; use pep508_rs::{MarkerEnvironment, Requirement}; use platform_tags::Tags; use puffin_client::{File, RegistryClient, SimpleJson}; use puffin_package::dist_info_name::DistInfoName; use puffin_package::metadata::Metadata21; use puffin_package::package_name::PackageName; +use puffin_traits::PuffinCtx; use crate::error::ResolveError; use crate::mode::{CandidateSelector, ResolutionMode}; @@ -32,8 +35,10 @@ use crate::pubgrub::package::PubGrubPackage; use crate::pubgrub::version::{PubGrubVersion, MIN_VERSION}; use crate::pubgrub::{iter_requirements, version_range}; use crate::resolution::Graph; +use crate::source_distribution::{download_and_build_sdist, read_dist_info}; +use crate::BuiltSourceDistributionCache; -pub struct Resolver<'a> { +pub struct Resolver<'a, Ctx: PuffinCtx> { requirements: Vec, constraints: Vec, markers: &'a MarkerEnvironment, @@ -41,9 +46,10 @@ pub struct Resolver<'a> { client: &'a RegistryClient, selector: CandidateSelector, cache: Arc, + puffin_ctx: &'a Ctx, } -impl<'a> Resolver<'a> { +impl<'a, Ctx: PuffinCtx> Resolver<'a, Ctx> { /// Initialize a new resolver. pub fn new( requirements: Vec, @@ -52,6 +58,7 @@ impl<'a> Resolver<'a> { markers: &'a MarkerEnvironment, tags: &'a Tags, client: &'a RegistryClient, + puffin_ctx: &'a Ctx, ) -> Self { Self { selector: CandidateSelector::from_mode(mode, &requirements), @@ -61,6 +68,7 @@ impl<'a> Resolver<'a> { markers, tags, client, + puffin_ctx, } } @@ -266,27 +274,46 @@ impl<'a> Resolver<'a> { }; let simple_json = entry.value(); - // Select the latest compatible version. - let Some(file) = self + // Try to find a wheel. If there isn't any, to a find a source distribution. If there + // isn't any either, short circuit and fail the resolution. + // TODO: Group files by version, then check for each version first for compatible wheels + // and then for a compatible sdist. This is required to still select the most recent + // version. + let Some((file, request)) = self .selector .iter_candidates(package_name, &simple_json.files) - .find(|file| { - let Ok(name) = WheelFilename::from_str(file.filename.as_str()) else { - return false; - }; - - if !name.is_compatible(self.tags) { - return false; + .find_map(|file| { + let wheel_filename = WheelFilename::from_str(file.filename.as_str()).ok()?; + if !wheel_filename.is_compatible(self.tags) { + return None; } - if !range + if range .borrow() - .contains(&PubGrubVersion::from(name.version.clone())) + .contains(&PubGrubVersion::from(wheel_filename.version.clone())) { - return false; - }; - - true + Some((file, Request::WheelVersion(file.clone()))) + } else { + None + } + }) + .or_else(|| { + self.selector + .iter_candidates(package_name, &simple_json.files) + .find_map(|file| { + let sdist_filename = + SourceDistributionFilename::parse(&file.filename, package_name) + .ok()?; + + if range + .borrow() + .contains(&PubGrubVersion::from(sdist_filename.version.clone())) + { + Some((file, Request::SdistVersion((file.clone(), sdist_filename)))) + } else { + None + } + }) }) else { // Short circuit: we couldn't find _any_ compatible versions for a package. @@ -296,7 +323,7 @@ impl<'a> Resolver<'a> { // Emit a request to fetch the metadata for this version. if in_flight.insert(file.hashes.sha256.clone()) { - request_sink.unbounded_send(Request::Version(file.clone()))?; + request_sink.unbounded_send(request)?; } selection = index; @@ -321,7 +348,7 @@ impl<'a> Resolver<'a> { ); // Find a compatible version. - let Some(wheel) = self + let mut wheel = self .selector .iter_candidates(package_name, &simple_json.files) .find_map(|file| { @@ -345,29 +372,69 @@ impl<'a> Resolver<'a> { name: package_name.clone(), version: name.version.clone(), }) - }) - else { - // Short circuit: we couldn't find _any_ compatible versions for a package. - return Ok((package, None)); - }; + }); - debug!( - "Selecting: {}=={} ({})", - wheel.name, wheel.version, wheel.file.filename - ); + if wheel.is_none() { + if let Some((sdist_file, parsed_filename)) = simple_json + .files + .iter() + .rev() + .filter_map(|file| { + let Ok(parsed_filename) = + SourceDistributionFilename::parse(&file.filename, package_name) + else { + return None; + }; + + if !range + .borrow() + .contains(&PubGrubVersion::from(parsed_filename.version.clone())) + { + return None; + }; + + Some((file, parsed_filename)) + }) + .max_by(|left, right| left.1.version.cmp(&right.1.version)) + { + // Emit a request to fetch the metadata for this version. + if in_flight.insert(sdist_file.hashes.sha256.clone()) { + request_sink.unbounded_send(Request::SdistVersion(( + sdist_file.clone(), + parsed_filename.clone(), + )))?; + } + // TODO(konstin): That's not a wheel + wheel = Some(Wheel { + file: sdist_file.clone(), + name: package_name.clone(), + version: parsed_filename.version.clone(), + }); + } + } + + if let Some(wheel) = wheel { + debug!( + "Selecting: {}=={} ({})", + wheel.name, wheel.version, wheel.file.filename + ); - // We want to return a package pinned to a specific version; but we _also_ want to - // store the exact file that we selected to satisfy that version. - pins.entry(wheel.name) - .or_default() - .insert(wheel.version.clone(), wheel.file.clone()); + // We want to return a package pinned to a specific version; but we _also_ want to + // store the exact file that we selected to satisfy that version. + pins.entry(wheel.name) + .or_default() + .insert(wheel.version.clone(), wheel.file.clone()); - // Emit a request to fetch the metadata for this version. - if in_flight.insert(wheel.file.hashes.sha256.clone()) { - request_sink.unbounded_send(Request::Version(wheel.file.clone()))?; - } + // Emit a request to fetch the metadata for this version. + if in_flight.insert(wheel.file.hashes.sha256.clone()) { + request_sink.unbounded_send(Request::WheelVersion(wheel.file.clone()))?; + } - Ok((package, Some(PubGrubVersion::from(wheel.version)))) + Ok((package, Some(PubGrubVersion::from(wheel.version)))) + } else { + // Short circuit: we couldn't find _any_ compatible versions for a package. + Ok((package, None)) + } } }; } @@ -485,20 +552,7 @@ impl<'a> Resolver<'a> { /// Fetch the metadata for a stream of packages and versions. async fn fetch(&self, request_stream: UnboundedReceiver) -> Result<(), ResolveError> { let mut response_stream = request_stream - .map({ - |request: Request| match request { - Request::Package(package_name) => Either::Left( - self.client - .simple(package_name.clone()) - .map_ok(move |metadata| Response::Package(package_name, metadata)), - ), - Request::Version(file) => Either::Right( - self.client - .file(file.clone()) - .map_ok(move |metadata| Response::Version(file, metadata)), - ), - } - }) + .map(|request| self.process_request(request)) .buffer_unordered(32) .ready_chunks(32); @@ -515,12 +569,80 @@ impl<'a> Resolver<'a> { .versions .insert(file.hashes.sha256.clone(), metadata); } + Response::Sdist(file, metadata) => { + trace!("Received sdist build metadata for {}", file.filename); + self.cache + .versions + .insert(file.hashes.sha256.clone(), metadata); + } } } } Ok::<(), ResolveError>(()) } + + fn process_request( + &'a self, + request: Request, + ) -> Pin> + 'a>> { + match request { + Request::Package(package_name) => Box::pin( + self.client + .simple(package_name.clone()) + .map_ok(move |metadata| Response::Package(package_name, metadata)) + .map_err(ResolveError::Client), + ), + Request::WheelVersion(file) => Box::pin( + self.client + .file(file.clone()) + .map_ok(move |metadata| Response::Version(file, metadata)) + .map_err(ResolveError::Client), + ), + Request::SdistVersion((file, filename)) => Box::pin(async move { + let cached_wheel = self.find_cached_built_wheel(self.puffin_ctx.cache(), &filename); + let metadata21 = if let Some(cached_wheel) = cached_wheel { + read_dist_info(cached_wheel).await + } else { + download_and_build_sdist(&file, self.client, self.puffin_ctx, &filename).await + } + .map_err(|err| ResolveError::SourceDistribution { + filename: file.filename.clone(), + err, + })?; + + Ok(Response::Sdist(file, metadata21)) + }), + } + } + + fn find_cached_built_wheel( + &self, + cache: Option<&Path>, + filename: &SourceDistributionFilename, + ) -> Option { + let Some(cache) = cache else { + return None; + }; + let cache = BuiltSourceDistributionCache::new(cache); + let Ok(read_dir) = fs_err::read_dir(cache.version(&filename.name, &filename.version)) + else { + return None; + }; + + for entry in read_dir { + let Ok(entry) = entry else { continue }; + let Ok(wheel) = WheelFilename::from_str(entry.file_name().to_string_lossy().as_ref()) + else { + continue; + }; + + if wheel.is_compatible(self.tags) { + return Some(entry.path().clone()); + } + } + None + } } #[derive(Debug, Clone)] @@ -537,8 +659,10 @@ struct Wheel { enum Request { /// A request to fetch the metadata for a package. Package(PackageName), + /// A request to fetch and build the source distribution for a specific package version + SdistVersion((File, SourceDistributionFilename)), /// A request to fetch the metadata for a specific version of a package. - Version(File), + WheelVersion(File), } #[derive(Debug)] @@ -547,6 +671,8 @@ enum Response { Package(PackageName, SimpleJson), /// The returned metadata for a specific version of a package. Version(File, Metadata21), + /// The returned metadata for an sdist build. + Sdist(File, Metadata21), } struct SolverCache { diff --git a/crates/puffin-resolver/src/source_distribution.rs b/crates/puffin-resolver/src/source_distribution.rs new file mode 100644 index 000000000000..c6a02a986c40 --- /dev/null +++ b/crates/puffin-resolver/src/source_distribution.rs @@ -0,0 +1,90 @@ +use std::path::{Path, PathBuf}; +use std::str::FromStr; + +use fs_err::tokio as fs; +use tempfile::tempdir; +use tokio::task::spawn_blocking; +use tokio_util::compat::FuturesAsyncReadCompatExt; +use tracing::debug; +use url::Url; +use zip::ZipArchive; + +use distribution_filename::{SourceDistributionFilename, WheelFilename}; +use pep440_rs::Version; +use puffin_client::{File, RegistryClient}; +use puffin_package::metadata::Metadata21; +use puffin_package::package_name::PackageName; +use puffin_traits::PuffinCtx; + +const BUILT_WHEELS_CACHE: &str = "built-wheels-v0"; + +/// TODO: Find a better home for me? +/// +/// Stores wheels built from source distributions. We need to keep those separate from the regular +/// wheel cache since a wheel with the same name may be uploaded after we made our build and in that +/// case the hashes would clash. +pub struct BuiltSourceDistributionCache(PathBuf); + +impl BuiltSourceDistributionCache { + pub fn new(path: impl AsRef) -> Self { + Self(path.as_ref().join(BUILT_WHEELS_CACHE)) + } + + pub fn version(&self, name: &PackageName, version: &Version) -> PathBuf { + self.0.join(name.to_string()).join(version.to_string()) + } +} + +pub(crate) async fn download_and_build_sdist( + file: &File, + client: &RegistryClient, + puffin_ctx: &impl PuffinCtx, + sdist_filename: &SourceDistributionFilename, +) -> anyhow::Result { + debug!("Building {}", &file.filename); + let url = Url::parse(&file.url)?; + let reader = client.stream_external(&url).await?; + let mut reader = tokio::io::BufReader::new(reader.compat()); + let temp_dir = tempdir()?; + + let sdist_dir = temp_dir.path().join("sdist"); + tokio::fs::create_dir(&sdist_dir).await?; + let sdist_file = sdist_dir.join(&file.filename); + let mut writer = tokio::fs::File::create(&sdist_file).await?; + tokio::io::copy(&mut reader, &mut writer).await?; + + let wheel_dir = if let Some(cache) = &puffin_ctx.cache() { + BuiltSourceDistributionCache::new(cache) + .version(&sdist_filename.name, &sdist_filename.version) + } else { + temp_dir.path().join("wheels") + }; + + fs::create_dir_all(&wheel_dir).await?; + + let disk_filename = puffin_ctx + .build_source_distribution(&sdist_file, &wheel_dir) + .await?; + + let metadata21 = read_dist_info(wheel_dir.join(disk_filename)).await?; + + debug!("Finished Building {}", &file.filename); + Ok(metadata21) +} + +pub(crate) async fn read_dist_info(wheel: PathBuf) -> anyhow::Result { + let dist_info = spawn_blocking(move || -> anyhow::Result { + let mut archive = ZipArchive::new(std::fs::File::open(&wheel)?)?; + let dist_info_prefix = install_wheel_rs::find_dist_info( + &WheelFilename::from_str(wheel.file_name().unwrap().to_string_lossy().as_ref())?, + &mut archive, + )?; + let dist_info = std::io::read_to_string( + archive.by_name(&format!("{dist_info_prefix}.dist-info/METADATA"))?, + )?; + Ok(dist_info) + }) + .await + .unwrap()?; + Ok(Metadata21::parse(dist_info.as_bytes())?) +} diff --git a/crates/puffin-resolver/tests/resolver.rs b/crates/puffin-resolver/tests/resolver.rs index 50878d868f9e..c6776fad9d2c 100644 --- a/crates/puffin-resolver/tests/resolver.rs +++ b/crates/puffin-resolver/tests/resolver.rs @@ -29,6 +29,7 @@ async fn pylint() -> Result<()> { &MARKERS_311, &TAGS_311, &client, + None, ); let resolution = resolver.resolve().await?; @@ -52,6 +53,7 @@ async fn black() -> Result<()> { &MARKERS_311, &TAGS_311, &client, + None, ); let resolution = resolver.resolve().await?; @@ -75,6 +77,7 @@ async fn black_colorama() -> Result<()> { &MARKERS_311, &TAGS_311, &client, + None, ); let resolution = resolver.resolve().await?; @@ -98,6 +101,7 @@ async fn black_python_310() -> Result<()> { &MARKERS_310, &TAGS_310, &client, + None, ); let resolution = resolver.resolve().await?; @@ -123,6 +127,7 @@ async fn black_mypy_extensions() -> Result<()> { &MARKERS_311, &TAGS_311, &client, + None, ); let resolution = resolver.resolve().await?; @@ -148,6 +153,7 @@ async fn black_mypy_extensions_extra() -> Result<()> { &MARKERS_311, &TAGS_311, &client, + None, ); let resolution = resolver.resolve().await?; @@ -173,6 +179,7 @@ async fn black_flake8() -> Result<()> { &MARKERS_311, &TAGS_311, &client, + None, ); let resolution = resolver.resolve().await?; @@ -196,6 +203,7 @@ async fn black_lowest() -> Result<()> { &MARKERS_311, &TAGS_311, &client, + None, ); let resolution = resolver.resolve().await?; @@ -219,6 +227,7 @@ async fn black_lowest_direct() -> Result<()> { &MARKERS_311, &TAGS_311, &client, + None, ); let resolution = resolver.resolve().await?; diff --git a/crates/puffin-traits/Cargo.toml b/crates/puffin-traits/Cargo.toml new file mode 100644 index 000000000000..c5d52bad9368 --- /dev/null +++ b/crates/puffin-traits/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "puffin-traits" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +homepage.workspace = true +documentation.workspace = true +repository.workspace = true +authors.workspace = true +license.workspace = true + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +gourgeist = { path = "../gourgeist" } +pep508_rs = { path = "../pep508-rs" } + +anyhow = "1.0.75" \ No newline at end of file diff --git a/crates/puffin-traits/src/lib.rs b/crates/puffin-traits/src/lib.rs new file mode 100644 index 000000000000..c651fcbb5969 --- /dev/null +++ b/crates/puffin-traits/src/lib.rs @@ -0,0 +1,27 @@ +use gourgeist::Venv; +use pep508_rs::Requirement; +use std::future::Future; +use std::path::Path; +use std::pin::Pin; + +// TODO(konstin): Proper error types +pub trait PuffinCtx { + // TODO(konstin): Add a cache abstraction + fn cache(&self) -> Option<&Path>; + + fn resolve<'a>( + &'a self, + requirements: &'a [Requirement], + ) -> Pin>> + 'a>>; + fn install<'a>( + &'a self, + requirements: &'a [Requirement], + venv: &'a Venv, + ) -> Pin> + 'a>>; + /// Returns the filename of the built wheel + fn build_source_distribution<'a>( + &'a self, + sdist: &'a Path, + wheel_dir: &'a Path, + ) -> Pin> + 'a>>; +} diff --git a/scripts/pypi_top8k/.gitignore b/scripts/pypi_top8k/.gitignore new file mode 100644 index 000000000000..3f69456a678f --- /dev/null +++ b/scripts/pypi_top8k/.gitignore @@ -0,0 +1,3 @@ +* +!.gitignore +!download.sh \ No newline at end of file diff --git a/scripts/pypi_top8k/download.sh b/scripts/pypi_top8k/download.sh new file mode 100644 index 000000000000..89c904d6a286 --- /dev/null +++ b/scripts/pypi_top8k/download.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +set -e + +curl https://hugovk.github.io/top-pypi-packages/top-pypi-packages-30-days.min.json > top-pypi-packages-30-days.min.json diff --git a/vendor/pubgrub/src/internal/partial_solution.rs b/vendor/pubgrub/src/internal/partial_solution.rs index 1b683b9d84d8..6c445ed84279 100644 --- a/vendor/pubgrub/src/internal/partial_solution.rs +++ b/vendor/pubgrub/src/internal/partial_solution.rs @@ -141,7 +141,13 @@ impl PartialSolution { AssignmentsIntersection::Decision(_) => panic!("Already existing decision"), // Cannot be called if the versions is not contained in the terms intersection. AssignmentsIntersection::Derivations(term) => { - debug_assert!(term.contains(&version)) + debug_assert!( + term.contains(&version), + "{} {:?} {}", + package, + version, + term, + ) } }, }