diff --git a/Cargo.lock b/Cargo.lock index 59ce99529eccf..ef63b50056497 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1976,12 +1976,6 @@ dependencies = [ "pep508_rs", "platform-host", "platform-tags", - "puffin-client", - "puffin-installer", - "puffin-interpreter", - "puffin-package", - "puffin-resolver", - "puffin-workspace", "pyproject-toml", "serde", "serde_json", @@ -1996,6 +1990,35 @@ dependencies = [ "zip", ] +[[package]] +name = "puffin-build-cli" +version = "0.0.1" +dependencies = [ + "anyhow", + "clap", + "directories", + "fs-err", + "futures", + "gourgeist", + "itertools", + "owo-colors", + "pep508_rs", + "platform-host", + "platform-tags", + "puffin-build", + "puffin-client", + "puffin-installer", + "puffin-interpreter", + "puffin-package", + "puffin-resolver", + "puffin-workspace", + "tempfile", + "tokio", + "tracing", + "tracing-subscriber", + "which", +] + [[package]] name = "puffin-cli" version = "0.0.1" @@ -2022,6 +2045,7 @@ dependencies = [ "platform-tags", "predicates", "pubgrub", + "puffin-build-cli", "puffin-client", "puffin-installer", "puffin-interpreter", @@ -2051,6 +2075,7 @@ dependencies = [ "serde", "serde_json", "thiserror", + "tokio", "tracing", "url", ] @@ -2123,20 +2148,30 @@ version = "0.0.1" dependencies = [ "anyhow", "bitflags 2.4.1", + "fs-err", "futures", + "gourgeist", + "install-wheel-rs", "once_cell", "pep440_rs 0.3.12", "pep508_rs", "platform-host", "platform-tags", "pubgrub", + "puffin-build", "puffin-client", + "puffin-interpreter", "puffin-package", + "tempfile", "thiserror", "tokio", + "tokio-util", "tracing", + "url", "waitmap", "wheel-filename", + "which", + "zip", ] [[package]] diff --git a/crates/install-wheel-rs/src/lib.rs b/crates/install-wheel-rs/src/lib.rs index 0190bd84a63ea..a4bf0be5c6be1 100644 --- a/crates/install-wheel-rs/src/lib.rs +++ b/crates/install-wheel-rs/src/lib.rs @@ -2,18 +2,18 @@ use std::io; +use platform_host::{Arch, Os}; use platform_info::PlatformInfoError; use thiserror::Error; use zip::result::ZipError; pub use install_location::{normalize_name, InstallLocation, LockedDir}; -use platform_host::{Arch, Os}; pub use record::RecordEntry; pub use script::Script; pub use uninstall::{uninstall_wheel, Uninstall}; pub use wheel::{ - get_script_launcher, install_wheel, parse_key_value_file, read_record_file, relative_to, - SHEBANG_PYTHON, + find_dist_info, get_script_launcher, install_wheel, parse_key_value_file, read_record_file, + relative_to, SHEBANG_PYTHON, }; mod install_location; diff --git a/crates/install-wheel-rs/src/wheel.rs b/crates/install-wheel-rs/src/wheel.rs index 03e2dd3e78069..edf18e2383697 100644 --- a/crates/install-wheel-rs/src/wheel.rs +++ b/crates/install-wheel-rs/src/wheel.rs @@ -1026,7 +1026,7 @@ pub fn install_wheel( /// Either way, we just search the wheel for the name /// /// -fn find_dist_info( +pub fn find_dist_info( filename: &WheelFilename, archive: &mut ZipArchive, ) -> Result { diff --git a/crates/puffin-build-cli/.gitignore b/crates/puffin-build-cli/.gitignore new file mode 100644 index 0000000000000..4d668e3d91671 --- /dev/null +++ b/crates/puffin-build-cli/.gitignore @@ -0,0 +1,2 @@ +downloads +wheels diff --git a/crates/puffin-build-cli/Cargo.toml b/crates/puffin-build-cli/Cargo.toml new file mode 100644 index 0000000000000..7d11bc41e0b4e --- /dev/null +++ b/crates/puffin-build-cli/Cargo.toml @@ -0,0 +1,37 @@ +[package] +name = "puffin-build-cli" +version = "0.0.1" +description = "Build wheels from source distributions" +edition = { workspace = true } +rust-version = { workspace = true } +homepage = { workspace = true } +documentation = { workspace = true } +repository = { workspace = true } +authors = { workspace = true } +license = { workspace = true } + +[dependencies] +gourgeist = { path = "../gourgeist" } +pep508_rs = { path = "../pep508-rs" } +platform-host = { path = "../platform-host" } +platform-tags = { path = "../platform-tags" } +puffin-build = { path = "../puffin-build" } +puffin-client = { path = "../puffin-client" } +puffin-installer = { path = "../puffin-installer" } +puffin-interpreter = { path = "../puffin-interpreter" } +puffin-package = { path = "../puffin-package" } +puffin-resolver = { path = "../puffin-resolver" } +puffin-workspace = { path = "../puffin-workspace" } + +anyhow = { workspace = true } +clap = { workspace = true, features = ["derive"] } +directories = { workspace = true } +fs-err = { workspace = true } +futures = { workspace = true } +itertools = { workspace = true } +owo-colors = { workspace = true } +tempfile = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } +which = { workspace = true } diff --git a/crates/puffin-build-cli/src/lib.rs b/crates/puffin-build-cli/src/lib.rs new file mode 100644 index 0000000000000..4e1c4c771b81e --- /dev/null +++ b/crates/puffin-build-cli/src/lib.rs @@ -0,0 +1,3 @@ +mod sdist; + +pub use sdist::resolve_and_install; diff --git a/crates/puffin-build/src/main.rs b/crates/puffin-build-cli/src/main.rs similarity index 90% rename from crates/puffin-build/src/main.rs rename to crates/puffin-build-cli/src/main.rs index c18ffb4f9e334..d2d44d3ba02a8 100644 --- a/crates/puffin-build/src/main.rs +++ b/crates/puffin-build-cli/src/main.rs @@ -1,5 +1,6 @@ #![allow(clippy::print_stdout, clippy::print_stderr)] +use crate::sdist::resolve_and_install; use anyhow::Context; use clap::Parser; use directories::ProjectDirs; @@ -16,6 +17,8 @@ use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::util::SubscriberInitExt; use tracing_subscriber::{fmt, EnvFilter}; +pub mod sdist; + #[derive(Parser)] struct Args { /// Base python in a way that can be found with `which` @@ -49,9 +52,14 @@ async fn run() -> anyhow::Result<()> { })?; let interpreter_info = gourgeist::get_interpreter_info(&base_python)?; - let builder = - SourceDistributionBuilder::setup(&args.sdist, &base_python, &interpreter_info, cache) - .await?; + let builder = SourceDistributionBuilder::setup( + &args.sdist, + &base_python, + &interpreter_info, + cache, + resolve_and_install, + ) + .await?; let wheel = builder.build(&wheel_dir)?; println!("Wheel built to {}", wheel.display()); Ok(()) diff --git a/crates/puffin-build-cli/src/sdist.rs b/crates/puffin-build-cli/src/sdist.rs new file mode 100644 index 0000000000000..9469eaa08b600 --- /dev/null +++ b/crates/puffin-build-cli/src/sdist.rs @@ -0,0 +1,74 @@ +use anyhow::Context; +use futures::FutureExt; +use itertools::{Either, Itertools}; +use pep508_rs::Requirement; +use platform_host::Platform; +use platform_tags::Tags; +use puffin_client::PypiClientBuilder; +use puffin_installer::{CachedDistribution, Downloader, LocalIndex, RemoteDistribution, Unzipper}; +use puffin_interpreter::PythonExecutable; +use puffin_package::package_name::PackageName; +use puffin_resolver::WheelFinder; +use std::future::Future; +use std::path::Path; +use std::pin::Pin; +use tempfile::tempdir; +use tracing::{debug, instrument}; + +#[instrument(skip_all)] +pub async fn resolve_and_install_impl( + venv: &Path, + requirements: &[Requirement], + cache: Option<&Path>, +) -> anyhow::Result<()> { + debug!("Installing {} build requirements", requirements.len()); + + let local_index = if let Some(cache) = cache { + LocalIndex::from_directory(cache).await? + } else { + LocalIndex::default() + }; + let (cached, uncached): (Vec, Vec) = + requirements.iter().partition_map(|requirement| { + let package = PackageName::normalize(&requirement.name); + if let Some(distribution) = local_index + .get(&package) + .filter(|dist| requirement.is_satisfied_by(dist.version())) + { + Either::Left(distribution.clone()) + } else { + Either::Right(requirement.clone()) + } + }); + + let client = PypiClientBuilder::default().cache(cache).build(); + + let platform = Platform::current()?; + let python = PythonExecutable::from_venv(platform, venv, cache)?; + let tags = Tags::from_env(python.platform(), python.simple_version())?; + let resolution = WheelFinder::new(&tags, &client).resolve(&uncached).await?; + let uncached = resolution + .into_files() + .map(RemoteDistribution::from_file) + .collect::>>()?; + let staging = tempdir()?; + let downloads = Downloader::new(&client, cache) + .download(&uncached, cache.unwrap_or(staging.path())) + .await?; + let unzips = Unzipper::default() + .download(downloads, cache.unwrap_or(staging.path())) + .await + .context("Failed to download and unpack wheels")?; + let wheels = unzips.into_iter().chain(cached).collect::>(); + puffin_installer::Installer::new(&python).install(&wheels)?; + + Ok(()) +} + +pub fn resolve_and_install<'a>( + venv: &'a Path, + requirements: &'a [Requirement], + cache: Option<&'a Path>, +) -> Pin> + Send + 'a>> { + resolve_and_install_impl(venv, requirements, cache).boxed() +} diff --git a/crates/puffin-build/test.sh b/crates/puffin-build-cli/test.sh similarity index 100% rename from crates/puffin-build/test.sh rename to crates/puffin-build-cli/test.sh diff --git a/crates/puffin-build/Cargo.toml b/crates/puffin-build/Cargo.toml index 85f18cc134fc0..238599dcf23e9 100644 --- a/crates/puffin-build/Cargo.toml +++ b/crates/puffin-build/Cargo.toml @@ -15,12 +15,6 @@ gourgeist = { path = "../gourgeist" } pep508_rs = { path = "../pep508-rs" } platform-host = { path = "../platform-host" } platform-tags = { path = "../platform-tags" } -puffin-client = { path = "../puffin-client" } -puffin-installer = { path = "../puffin-installer" } -puffin-interpreter = { path = "../puffin-interpreter" } -puffin-package = { path = "../puffin-package" } -puffin-resolver = { path = "../puffin-resolver" } -puffin-workspace = { path = "../puffin-workspace" } anyhow = { workspace = true } clap = { workspace = true, features = ["derive"] } diff --git a/crates/puffin-build/src/lib.rs b/crates/puffin-build/src/lib.rs index fb50318a647c3..8b95f3b6c39d6 100644 --- a/crates/puffin-build/src/lib.rs +++ b/crates/puffin-build/src/lib.rs @@ -2,26 +2,19 @@ //! //! -use anyhow::Context; use flate2::read::GzDecoder; use fs_err as fs; use fs_err::{DirEntry, File}; use gourgeist::{InterpreterInfo, Venv}; use indoc::formatdoc; -use itertools::{Either, Itertools}; use pep508_rs::Requirement; -use platform_host::Platform; -use platform_tags::Tags; -use puffin_client::PypiClientBuilder; -use puffin_installer::{CachedDistribution, Downloader, LocalIndex, RemoteDistribution, Unzipper}; -use puffin_interpreter::PythonExecutable; -use puffin_package::package_name::PackageName; -use puffin_resolver::WheelFinder; use pyproject_toml::PyProjectToml; +use std::future::Future; use std::io; use std::io::BufRead; use std::ops::Deref; use std::path::{Path, PathBuf}; +use std::pin::Pin; use std::process::{Command, Output}; use std::str::FromStr; use tar::Archive; @@ -108,6 +101,13 @@ pub struct SourceDistributionBuilder { metadata_directory: Option, } +pub type ResolveAndInstall = + for<'a> fn( + &'a Path, + &'a [Requirement], + Option<&'a Path>, + ) -> Pin> + Send + 'a)>>; + impl SourceDistributionBuilder { /// Extract the source distribution and create a venv with the required packages pub async fn setup( @@ -115,6 +115,7 @@ impl SourceDistributionBuilder { base_python: &Path, interpreter_info: &InterpreterInfo, cache: Option<&Path>, + resolve_and_install: ResolveAndInstall, ) -> Result { let temp_dir = tempdir()?; @@ -152,6 +153,7 @@ impl SourceDistributionBuilder { interpreter_info, pep517_backend, cache, + resolve_and_install, ) .await? } else { @@ -352,6 +354,7 @@ async fn create_pep517_build_environment( data: &InterpreterInfo, pep517_backend: &Pep517Backend, cache: Option<&Path>, + resolve_and_install: ResolveAndInstall, ) -> Result { let venv = gourgeist::create_venv(root.join(".venv"), base_python, data, true)?; resolve_and_install( @@ -420,63 +423,13 @@ async fn create_pep517_build_environment( .cloned() .chain(extra_requires) .collect(); - resolve_and_install(&*venv, &requirements, cache) + resolve_and_install(venv.as_std_path(), &requirements, cache) .await .map_err(Error::RequirementsInstall)?; } Ok(venv) } -#[instrument(skip_all)] -async fn resolve_and_install( - venv: impl AsRef, - requirements: &[Requirement], - cache: Option<&Path>, -) -> anyhow::Result<()> { - debug!("Installing {} build requirements", requirements.len()); - - let local_index = if let Some(cache) = cache { - LocalIndex::from_directory(cache).await? - } else { - LocalIndex::default() - }; - let (cached, uncached): (Vec, Vec) = - requirements.iter().partition_map(|requirement| { - let package = PackageName::normalize(&requirement.name); - if let Some(distribution) = local_index - .get(&package) - .filter(|dist| requirement.is_satisfied_by(dist.version())) - { - Either::Left(distribution.clone()) - } else { - Either::Right(requirement.clone()) - } - }); - - let client = PypiClientBuilder::default().cache(cache).build(); - - let platform = Platform::current()?; - let python = PythonExecutable::from_venv(platform, venv.as_ref(), cache)?; - let tags = Tags::from_env(python.platform(), python.simple_version())?; - let resolution = WheelFinder::new(&tags, &client).resolve(&uncached).await?; - let uncached = resolution - .into_files() - .map(RemoteDistribution::from_file) - .collect::>>()?; - let staging = tempdir()?; - let downloads = Downloader::new(&client, cache) - .download(&uncached, cache.unwrap_or(staging.path())) - .await?; - let unzips = Unzipper::default() - .download(downloads, cache.unwrap_or(staging.path())) - .await - .context("Failed to download and unpack wheels")?; - let wheels = unzips.into_iter().chain(cached).collect::>(); - puffin_installer::Installer::new(&python).install(&wheels)?; - - Ok(()) -} - /// Returns the directory with the `pyproject.toml`/`setup.py` #[instrument(skip_all, fields(path))] fn extract_archive(sdist: &Path, extracted: &PathBuf) -> Result { diff --git a/crates/puffin-cli/Cargo.toml b/crates/puffin-cli/Cargo.toml index c5a2e7a7d6610..be7f7df6728a4 100644 --- a/crates/puffin-cli/Cargo.toml +++ b/crates/puffin-cli/Cargo.toml @@ -15,6 +15,7 @@ pep508_rs = { path = "../pep508-rs" } platform-host = { path = "../platform-host" } platform-tags = { path = "../platform-tags" } pubgrub = { path = "../../vendor/pubgrub" } +puffin-build-cli = { path = "../puffin-build-cli" } puffin-client = { path = "../puffin-client" } puffin-installer = { path = "../puffin-installer" } puffin-interpreter = { path = "../puffin-interpreter" } diff --git a/crates/puffin-cli/src/commands/pip_compile.rs b/crates/puffin-cli/src/commands/pip_compile.rs index 3839e1581535e..337e623790b4a 100644 --- a/crates/puffin-cli/src/commands/pip_compile.rs +++ b/crates/puffin-cli/src/commands/pip_compile.rs @@ -1,21 +1,21 @@ +use anyhow::Result; use fs_err::File; +use owo_colors::OwoColorize; +use pubgrub::report::Reporter; use std::fmt::Write; use std::io::{stdout, BufWriter}; use std::path::Path; - -use anyhow::Result; -use owo_colors::OwoColorize; -use pubgrub::report::Reporter; use tracing::debug; +use crate::commands::{elapsed, ExitStatus}; +use crate::printer::Printer; use platform_host::Platform; use platform_tags::Tags; +use puffin_build_cli::resolve_and_install; use puffin_client::PypiClientBuilder; use puffin_interpreter::PythonExecutable; use puffin_package::requirements_txt::RequirementsTxt; - -use crate::commands::{elapsed, ExitStatus}; -use crate::printer::Printer; +use puffin_resolver::SourceDistributionBuildContext; /// Resolve a set of requirements into a set of pinned versions. pub(crate) async fn pip_compile( @@ -55,8 +55,14 @@ pub(crate) async fn pip_compile( // Instantiate a client. let client = PypiClientBuilder::default().cache(cache).build(); + let build_context = SourceDistributionBuildContext { + resolve_and_install, + python: python.clone(), + }; + // Resolve the dependencies. - let resolver = puffin_resolver::Resolver::new(requirements, markers, &tags, &client); + let resolver = + puffin_resolver::Resolver::new(requirements, markers, &tags, &client, Some(build_context)); let resolution = match resolver.resolve().await { Err(puffin_resolver::ResolveError::PubGrub(pubgrub::error::PubGrubError::NoSolution( mut derivation_tree, diff --git a/crates/puffin-client/Cargo.toml b/crates/puffin-client/Cargo.toml index b7028cfafa8f8..5cdc3e6e147d4 100644 --- a/crates/puffin-client/Cargo.toml +++ b/crates/puffin-client/Cargo.toml @@ -6,6 +6,7 @@ edition = "2021" [dependencies] puffin-package = { path = "../puffin-package" } +futures = { workspace = true } http-cache-reqwest = { workspace = true } reqwest = { workspace = true } reqwest-middleware = { workspace = true } @@ -13,6 +14,6 @@ reqwest-retry = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } thiserror = { workspace = true } -url = { workspace = true } +tokio = { workspace = true } tracing = { workspace = true } -futures = { workspace = true } +url = { workspace = true } diff --git a/crates/puffin-interpreter/src/lib.rs b/crates/puffin-interpreter/src/lib.rs index 19fa890be3383..786b269f6feb4 100644 --- a/crates/puffin-interpreter/src/lib.rs +++ b/crates/puffin-interpreter/src/lib.rs @@ -13,7 +13,7 @@ mod python_platform; mod virtual_env; /// A Python executable and its associated platform markers. -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct PythonExecutable { platform: PythonPlatform, venv: PathBuf, diff --git a/crates/puffin-resolver/Cargo.toml b/crates/puffin-resolver/Cargo.toml index 23bb23228cab1..bb6c714d6dec6 100644 --- a/crates/puffin-resolver/Cargo.toml +++ b/crates/puffin-resolver/Cargo.toml @@ -10,23 +10,33 @@ authors = { workspace = true } license = { workspace = true } [dependencies] +gourgeist = { path = "../gourgeist" } +install-wheel-rs = { path = "../install-wheel-rs" } pep440_rs = { path = "../pep440-rs" } pep508_rs = { path = "../pep508-rs" } platform-host = { path = "../platform-host" } platform-tags = { path = "../platform-tags" } pubgrub = { path = "../../vendor/pubgrub" } +puffin-build = { path = "../puffin-build" } puffin-client = { path = "../puffin-client" } +puffin-interpreter = { path = "../puffin-interpreter" } puffin-package = { path = "../puffin-package" } wheel-filename = { path = "../wheel-filename" } anyhow = { workspace = true } bitflags = { workspace = true } +fs-err = { workspace = true } futures = { workspace = true } once_cell = { workspace = true } +tempfile = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } +tokio-util = { workspace = true, features = ["compat"] } tracing = { workspace = true } +url = { workspace = true } waitmap = { workspace = true } +which = { workspace = true } +zip = { workspace = true } [dev-dependencies] once_cell = { version = "1.18.0" } diff --git a/crates/puffin-resolver/src/error.rs b/crates/puffin-resolver/src/error.rs index a39e77aaeb072..a88c4466b4611 100644 --- a/crates/puffin-resolver/src/error.rs +++ b/crates/puffin-resolver/src/error.rs @@ -24,6 +24,14 @@ pub enum ResolveError { #[error(transparent)] PubGrub(#[from] pubgrub::error::PubGrubError), + + #[error("Failed to build source distribution {filename}")] + SourceDistribution { + filename: String, + // TODO: Gives this a proper error type + #[source] + err: anyhow::Error, + }, } impl From> for ResolveError { diff --git a/crates/puffin-resolver/src/lib.rs b/crates/puffin-resolver/src/lib.rs index 2bf55e9edfd5c..28da6aaacb51f 100644 --- a/crates/puffin-resolver/src/lib.rs +++ b/crates/puffin-resolver/src/lib.rs @@ -1,6 +1,6 @@ pub use error::ResolveError; pub use resolution::{PinnedPackage, Resolution}; -pub use resolver::Resolver; +pub use resolver::{Resolver, SourceDistributionBuildContext}; pub use wheel_finder::{Reporter, WheelFinder}; mod error; diff --git a/crates/puffin-resolver/src/resolver.rs b/crates/puffin-resolver/src/resolver.rs index 733b84b042ed6..c4350dc32981d 100644 --- a/crates/puffin-resolver/src/resolver.rs +++ b/crates/puffin-resolver/src/resolver.rs @@ -3,24 +3,33 @@ use std::borrow::Borrow; use std::collections::hash_map::Entry; use std::collections::{BTreeMap, HashMap, HashSet}; +use std::future::Future; +use std::pin::Pin; use std::str::FromStr; use std::sync::Arc; +use std::{fs, io}; -use anyhow::Result; +use anyhow::{format_err, Result}; use futures::channel::mpsc::UnboundedReceiver; -use futures::future::Either; use futures::{pin_mut, FutureExt, StreamExt, TryFutureExt}; use pubgrub::error::PubGrubError; use pubgrub::range::Range; use pubgrub::solver::{DependencyConstraints, Incompatibility, State}; use pubgrub::type_aliases::SelectedDependencies; +use tempfile::tempdir; use tokio::select; +use tokio::task::spawn_blocking; +use tokio_util::compat::FuturesAsyncReadCompatExt; use tracing::{debug, trace}; +use url::Url; use waitmap::WaitMap; +use zip::ZipArchive; use pep508_rs::{MarkerEnvironment, Requirement}; use platform_tags::Tags; +use puffin_build::ResolveAndInstall; use puffin_client::{File, PypiClient, SimpleJson}; +use puffin_interpreter::PythonExecutable; use puffin_package::dist_info_name::DistInfoName; use puffin_package::metadata::Metadata21; use puffin_package::package_name::PackageName; @@ -32,12 +41,21 @@ use crate::pubgrub::package::PubGrubPackage; use crate::pubgrub::version::{PubGrubVersion, MIN_VERSION}; use crate::resolution::{PinnedPackage, Resolution}; +/// Pass the information required to build source distributions around +pub struct SourceDistributionBuildContext { + /// Avoid circular crate dependencies by using a callback + pub resolve_and_install: ResolveAndInstall, + pub python: PythonExecutable, +} + pub struct Resolver<'a> { requirements: Vec, markers: &'a MarkerEnvironment, tags: &'a Tags, client: &'a PypiClient, cache: Arc, + /// sdist builder + source_distribution_build_context: Option, } impl<'a> Resolver<'a> { @@ -47,6 +65,7 @@ impl<'a> Resolver<'a> { markers: &'a MarkerEnvironment, tags: &'a Tags, client: &'a PypiClient, + resolve_and_install: Option, ) -> Self { Self { requirements, @@ -54,6 +73,7 @@ impl<'a> Resolver<'a> { tags, client, cache: Arc::new(SolverCache::default()), + source_distribution_build_context: resolve_and_install, } } @@ -311,7 +331,7 @@ impl<'a> Resolver<'a> { ); // Find a compatible version. - let wheel = simple_json.files.iter().rev().find_map(|file| { + let mut wheel = simple_json.files.iter().rev().find_map(|file| { let Ok(name) = WheelFilename::from_str(file.filename.as_str()) else { return None; }; @@ -338,6 +358,48 @@ impl<'a> Resolver<'a> { }) }); + if wheel.is_none() { + if let Some((sdist_file, version)) = simple_json + .files + .iter() + .rev() + .filter_map(|file| { + let filename = file.filename.as_str().strip_suffix(".tar.gz")?; + if filename.len() < package_name.to_string().len() + 1 { + return None; + } + + // TODO: Handle names like properly + let version = &filename[package_name.to_string().len() + "-".len()..]; + + filename.rsplit_once('-')?; + // TODO(konstin): Don't silently ignore errors + let version = pep440_rs::Version::from_str(version).ok()?; + + if !range + .borrow() + .contains(&PubGrubVersion::from(version.clone())) + { + return None; + }; + + Some((file, version)) + }) + .max_by(|left, right| left.1.cmp(&right.1)) + { + // Emit a request to fetch the metadata for this version. + if in_flight.insert(sdist_file.hashes.sha256.clone()) { + request_sink.unbounded_send(Request::Sdist(sdist_file.clone()))?; + } + // TODO(konstin): That's not a wheel + wheel = Some(Wheel { + file: sdist_file.clone(), + name: package_name.clone(), + version: version.clone(), + }); + } + } + if let Some(wheel) = wheel { debug!( "Selecting: {}=={} ({})", @@ -450,17 +512,35 @@ impl<'a> Resolver<'a> { async fn fetch(&self, request_stream: UnboundedReceiver) -> Result<(), ResolveError> { let mut response_stream = request_stream .map({ - |request: Request| match request { - Request::Package(package_name) => Either::Left( - self.client - .simple(package_name.clone()) - .map_ok(move |metadata| Response::Package(package_name, metadata)), - ), - Request::Version(file) => Either::Right( - self.client - .file(file.clone()) - .map_ok(move |metadata| Response::Version(file, metadata)), - ), + |request: Request| -> Pin>>> { + match request { + Request::Package(package_name) => Box::pin( + self.client + .simple(package_name.clone()) + .map_ok(move |metadata| Response::Package(package_name, metadata)).map_err(ResolveError::Client), + ), + Request::Version(file) => Box::pin( + self.client + .file(file.clone()) + .map_ok(move |metadata| Response::Version(file, metadata)).map_err(ResolveError::Client), + ), + Request::Sdist(file) => Box::pin(async { + if let Some(build_context )=&self.source_distribution_build_context { + let metadata21 = + download_and_build_sdist(&file, self.client, build_context).await.map_err(|err| ResolveError::SourceDistribution { + filename: file.filename.clone(), + err, + })?; + + Ok(Response::Sdist(file, metadata21)) + } else { + Err(ResolveError::SourceDistribution { + filename: file.filename, + err: format_err!("No build context registered"), + }) + } + }), + } } }) .buffer_unordered(32) @@ -479,6 +559,12 @@ impl<'a> Resolver<'a> { .versions .insert(file.hashes.sha256.clone(), metadata); } + Response::Sdist(file, metadata) => { + trace!("Received sdist build metadata for {}", file.filename); + self.cache + .versions + .insert(file.hashes.sha256.clone(), metadata); + } } } } @@ -487,6 +573,57 @@ impl<'a> Resolver<'a> { } } +async fn download_and_build_sdist( + file: &File, + client: &PypiClient, + build_context: &SourceDistributionBuildContext, +) -> Result { + debug!("Building {}", &file.filename); + let url = Url::parse(&file.url)?; + let reader = client.stream_external(&url).await?; + let mut reader = tokio::io::BufReader::new(reader.compat()); + let temp_dir = tempdir()?; + + let sdist_dir = temp_dir.path().join("sdist"); + tokio::fs::create_dir(&sdist_dir).await?; + let sdist_file = sdist_dir.join(&file.filename); + let mut writer = tokio::fs::File::create(&sdist_file).await?; + tokio::io::copy(&mut reader, &mut writer).await?; + + // TODO: Merge this with PythonExecutable + let interpreter_info = gourgeist::get_interpreter_info(build_context.python.executable()) + .expect("TODO: get interpreter info"); + + let wheels_dir = temp_dir.path().join("wheels"); + tokio::fs::create_dir(&wheels_dir).await?; + + let builder = puffin_build::SourceDistributionBuilder::setup( + &sdist_file, + build_context.python.executable(), + &interpreter_info, + None, + build_context.resolve_and_install, + ) + .await?; + let filename = builder.build(&wheels_dir)?; + let dist_info = spawn_blocking(move || -> Result { + let mut archive = ZipArchive::new(fs::File::open(&filename)?)?; + let dist_info_prefix = install_wheel_rs::find_dist_info( + &WheelFilename::from_str(filename.file_name().unwrap().to_string_lossy().as_ref())?, + &mut archive, + )?; + let dist_info = io::read_to_string( + archive.by_name(&format!("{dist_info_prefix}.dist-info/METADATA"))?, + )?; + Ok(dist_info) + }) + .await??; + let metadata21 = Metadata21::parse(dist_info.as_bytes())?; + + debug!("Finished {}", &file.filename); + Ok(metadata21) +} + #[derive(Debug, Clone)] struct Wheel { /// The underlying [`File`] for this wheel. @@ -501,6 +638,8 @@ struct Wheel { enum Request { /// A request to fetch the metadata for a package. Package(PackageName), + /// A request to fetch and build the source distribution for a specific package version + Sdist(File), /// A request to fetch the metadata for a specific version of a package. Version(File), } @@ -511,6 +650,8 @@ enum Response { Package(PackageName, SimpleJson), /// The returned metadata for a specific version of a package. Version(File, Metadata21), + /// The returned metadata for an sdist build. + Sdist(File, Metadata21), } struct SolverCache { diff --git a/crates/puffin-resolver/tests/resolver.rs b/crates/puffin-resolver/tests/resolver.rs index d15261f9bbb96..3e4bc512136fd 100644 --- a/crates/puffin-resolver/tests/resolver.rs +++ b/crates/puffin-resolver/tests/resolver.rs @@ -17,7 +17,7 @@ async fn pylint() -> Result<()> { let client = PypiClientBuilder::default().build(); let requirements = vec![Requirement::from_str("pylint==2.3.0").unwrap()]; - let resolver = Resolver::new(requirements, &MARKERS_311, &TAGS_311, &client); + let resolver = Resolver::new(requirements, &MARKERS_311, &TAGS_311, &client, None); let resolution = resolver.resolve().await?; assert_eq!( @@ -39,7 +39,7 @@ async fn black() -> Result<()> { let client = PypiClientBuilder::default().build(); let requirements = vec![Requirement::from_str("black<=23.9.1").unwrap()]; - let resolver = Resolver::new(requirements, &MARKERS_311, &TAGS_311, &client); + let resolver = Resolver::new(requirements, &MARKERS_311, &TAGS_311, &client, None); let resolution = resolver.resolve().await?; assert_eq!( @@ -63,7 +63,7 @@ async fn black_colorama() -> Result<()> { let client = PypiClientBuilder::default().build(); let requirements = vec![Requirement::from_str("black[colorama]<=23.9.1").unwrap()]; - let resolver = Resolver::new(requirements, &MARKERS_311, &TAGS_311, &client); + let resolver = Resolver::new(requirements, &MARKERS_311, &TAGS_311, &client, None); let resolution = resolver.resolve().await?; assert_eq!( @@ -88,7 +88,7 @@ async fn black_python_310() -> Result<()> { let client = PypiClientBuilder::default().build(); let requirements = vec![Requirement::from_str("black<=23.9.1").unwrap()]; - let resolver = Resolver::new(requirements, &MARKERS_310, &TAGS_310, &client); + let resolver = Resolver::new(requirements, &MARKERS_310, &TAGS_310, &client, None); let resolution = resolver.resolve().await?; assert_eq!( @@ -114,7 +114,7 @@ async fn htmldate() -> Result<()> { let client = PypiClientBuilder::default().build(); let requirements = vec![Requirement::from_str("htmldate<=1.5.0").unwrap()]; - let resolver = Resolver::new(requirements, &MARKERS_311, &TAGS_311, &client); + let resolver = Resolver::new(requirements, &MARKERS_311, &TAGS_311, &client, None); let resolution = resolver.resolve().await?; // Resolves to `htmldate==1.4.3` (rather than `htmldate==1.5.2`) because `htmldate==1.5.2` has diff --git a/requirements_mst.txt b/requirements_mst.txt new file mode 100644 index 0000000000000..156eb7d814175 --- /dev/null +++ b/requirements_mst.txt @@ -0,0 +1 @@ +meine_stadt_transparent==0.2.14 \ No newline at end of file diff --git a/requirements_sdist.txt b/requirements_sdist.txt new file mode 100644 index 0000000000000..5a1521c46637b --- /dev/null +++ b/requirements_sdist.txt @@ -0,0 +1 @@ +future \ No newline at end of file diff --git a/vendor/pubgrub/src/internal/partial_solution.rs b/vendor/pubgrub/src/internal/partial_solution.rs index 390fcac47144a..475e9bb5525c9 100644 --- a/vendor/pubgrub/src/internal/partial_solution.rs +++ b/vendor/pubgrub/src/internal/partial_solution.rs @@ -87,7 +87,13 @@ impl PartialSolution { AssignmentsIntersection::Decision(_) => panic!("Already existing decision"), // Cannot be called if the versions is not contained in the terms intersection. AssignmentsIntersection::Derivations(term) => { - debug_assert!(term.contains(&version)) + debug_assert!( + term.contains(&version), + "{} {:?} {}", + package, + version, + term, + ) } }, }