diff --git a/Cargo.lock b/Cargo.lock index 99b46140..0c71da97 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -125,6 +125,17 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9338790e78aa95a416786ec8389546c4b6a1dfc3dc36071ed9518a9413a542eb" +[[package]] +name = "async-recursion" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd55a5ba1179988837d24ab4c7cc8ed6efdeff578ede0416b4225a5fca35bd0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "async-stream" version = "0.3.5" @@ -2017,6 +2028,7 @@ name = "rattler_installs_packages" version = "0.1.0" dependencies = [ "async-once-cell", + "async-recursion", "async-trait", "async_http_range_reader", "async_zip", diff --git a/crates/rattler_installs_packages/Cargo.toml b/crates/rattler_installs_packages/Cargo.toml index 4ac40f02..b7a8662b 100644 --- a/crates/rattler_installs_packages/Cargo.toml +++ b/crates/rattler_installs_packages/Cargo.toml @@ -67,6 +67,7 @@ pyproject-toml = "0.8.0" async-once-cell = "0.5.3" configparser = "3.0.3" cacache = { version = "12.0.0", default-features = false, features = ["tokio-runtime", "mmap"] } +async-recursion = "1.0.5" [dev-dependencies] criterion = "0.5" diff --git a/crates/rattler_installs_packages/src/artifacts/sdist.rs b/crates/rattler_installs_packages/src/artifacts/sdist.rs index dd18c81f..88287271 100644 --- a/crates/rattler_installs_packages/src/artifacts/sdist.rs +++ b/crates/rattler_installs_packages/src/artifacts/sdist.rs @@ -260,7 +260,7 @@ mod tests { &env_markers, None, &resolve_options, - &package_db.1.path(), + package_db.1.path(), ); let result = wheel_builder.get_sdist_metadata(&sdist).await.unwrap(); @@ -283,7 +283,7 @@ mod tests { &env_markers, None, &resolve_options, - &package_db.1.path(), + package_db.1.path(), ); // Build the wheel @@ -308,7 +308,7 @@ mod tests { &env_markers, None, &resolve_options, - &package_db.1.path(), + package_db.1.path(), ); // Build the wheel diff --git a/crates/rattler_installs_packages/src/index/package_database.rs b/crates/rattler_installs_packages/src/index/package_database.rs index 1a365a96..21f4a656 100644 --- a/crates/rattler_installs_packages/src/index/package_database.rs +++ b/crates/rattler_installs_packages/src/index/package_database.rs @@ -9,6 +9,7 @@ use crate::{ types::WheelFilename, }; use async_http_range_reader::{AsyncHttpRangeReader, CheckSupportMethod}; +use async_recursion::async_recursion; use elsa::sync::FrozenMap; use futures::{pin_mut, stream, StreamExt}; use http::{header::CONTENT_TYPE, HeaderMap, HeaderValue, Method}; @@ -429,11 +430,27 @@ impl PackageDb { /// Opens the specified artifact info. Downloads the artifact data from the remote location if /// the information is not already cached. - pub async fn get_artifact( + #[async_recursion] + pub async fn get_wheel<'db, 'i>( &self, artifact_info: &ArtifactInfo, - ) -> miette::Result { - self.get_artifact_with_cache(artifact_info, CacheMode::Default) + builder: Option<&'async_recursion WheelBuilder<'db, 'i>>, + ) -> miette::Result { + // Try to build the wheel for this SDist if possible + if artifact_info.is::() { + if let Some(builder) = builder { + let sdist = self + .get_artifact_with_cache::(artifact_info, CacheMode::Default) + .await?; + + return builder.build_wheel(&sdist).await.into_diagnostic(); + } else { + miette::bail!("cannot build wheel without a wheel builder"); + } + } + + // Otherwise just retrieve the wheel + self.get_artifact_with_cache::(artifact_info, CacheMode::Default) .await } } diff --git a/crates/rattler_installs_packages/src/wheel_builder/build_environment.rs b/crates/rattler_installs_packages/src/wheel_builder/build_environment.rs index 93928bc8..24ca33d9 100644 --- a/crates/rattler_installs_packages/src/wheel_builder/build_environment.rs +++ b/crates/rattler_installs_packages/src/wheel_builder/build_environment.rs @@ -1,10 +1,10 @@ use crate::artifacts::wheel::UnpackWheelOptions; -use crate::artifacts::{SDist, Wheel}; -use crate::index::PackageDb; +use crate::artifacts::SDist; + use crate::python_env::{PythonLocation, VEnv, WheelTags}; use crate::resolve::{resolve, PinnedPackage, ResolveOptions}; use crate::types::Artifact; -use crate::wheel_builder::{build_requirements, WheelBuildError}; +use crate::wheel_builder::{build_requirements, WheelBuildError, WheelBuilder}; use pep508_rs::{MarkerEnvironment, Requirement}; use std::collections::{HashMap, HashSet}; use std::path::{Path, PathBuf}; @@ -73,9 +73,9 @@ impl<'db> BuildEnvironment<'db> { /// Install extra requirements into the venv, if any extra were found /// If the extra requirements are already installed, this will do nothing /// for that requirement. - pub(crate) async fn install_extra_requirements( + pub(crate) async fn install_extra_requirements<'i>( &self, - package_db: &'db PackageDb, + wheel_builder: &WheelBuilder<'db, 'i>, env_markers: &MarkerEnvironment, wheel_tags: Option<&WheelTags>, resolve_options: &ResolveOptions, @@ -98,7 +98,7 @@ impl<'db> BuildEnvironment<'db> { let favored_packages = HashMap::default(); let all_requirements = combined_requirements.to_vec(); let extra_resolved_wheels = resolve( - package_db, + wheel_builder.package_db, all_requirements.iter(), env_markers, wheel_tags, @@ -120,8 +120,9 @@ impl<'db> BuildEnvironment<'db> { package_info.version ); let artifact_info = package_info.artifacts.first().unwrap(); - let artifact = package_db - .get_artifact::(artifact_info) + let artifact = wheel_builder + .package_db + .get_wheel(artifact_info, Some(wheel_builder)) .await .expect("could not get artifact"); @@ -145,9 +146,9 @@ impl<'db> BuildEnvironment<'db> { } /// Setup the build environment so that we can build a wheel from an sdist - pub(crate) async fn setup( + pub(crate) async fn setup<'i>( sdist: &SDist, - package_db: &'db PackageDb, + wheel_builder: &WheelBuilder<'db, 'i>, env_markers: &MarkerEnvironment, wheel_tags: Option<&WheelTags>, resolve_options: &ResolveOptions, @@ -167,9 +168,16 @@ impl<'db> BuildEnvironment<'db> { }); // Find the build requirements let build_requirements = build_requirements(&build_system); + tracing::info!( + "build requirements: {:?}", + build_requirements + .iter() + .map(|r| r.to_string()) + .collect::>() + ); // Resolve the build environment let resolved_wheels = resolve( - package_db, + wheel_builder.package_db, build_requirements.iter(), env_markers, wheel_tags, @@ -183,10 +191,12 @@ impl<'db> BuildEnvironment<'db> { // Install into venv for package_info in resolved_wheels.iter() { let artifact_info = package_info.artifacts.first().unwrap(); - let artifact = package_db - .get_artifact::(artifact_info) + + let artifact = wheel_builder + .package_db + .get_wheel(artifact_info, Some(wheel_builder)) .await - .map_err(|_| WheelBuildError::CouldNotGetArtifact)?; + .map_err(WheelBuildError::CouldNotGetArtifact)?; venv.install_wheel( &artifact, diff --git a/crates/rattler_installs_packages/src/wheel_builder/mod.rs b/crates/rattler_installs_packages/src/wheel_builder/mod.rs index c9f21fee..77290f8b 100644 --- a/crates/rattler_installs_packages/src/wheel_builder/mod.rs +++ b/crates/rattler_installs_packages/src/wheel_builder/mod.rs @@ -55,31 +55,31 @@ pub struct WheelBuilder<'db, 'i> { #[allow(missing_docs)] #[derive(thiserror::Error, Debug)] pub enum WheelBuildError { - #[error("Could not build wheel: {0}")] + #[error("could not build wheel: {0}")] Error(String), - #[error("Could not install artifact in virtual environment")] + #[error("could not install artifact in virtual environment: {0}")] UnpackError(#[from] UnpackError), - #[error("Could not build wheel: {0}")] + #[error("could not build wheel: {0}")] IoError(#[from] std::io::Error), - #[error("Could not run command {0} to build wheel: {1}")] + #[error("could not run command {0} to build wheel: {1}")] CouldNotRunCommand(String, std::io::Error), - #[error("Could not resolve environment for wheel building")] + #[error("could not resolve environment for wheel building")] CouldNotResolveEnvironment(Vec), - #[error("Error parsing JSON from extra_requirements.json: {0}")] + #[error("error parsing JSON from extra_requirements.json: {0}")] JSONError(#[from] serde_json::Error), - #[error("Could not parse generated wheel metadata: {0}")] + #[error("could not parse generated wheel metadata: {0}")] WheelCoreMetadataError(#[from] WheelCoreMetaDataError), - #[error("Could not get artifact")] - CouldNotGetArtifact, + #[error("could not get artifact: {0}")] + CouldNotGetArtifact(miette::Report), - #[error("Could not get artifact: {0}")] + #[error("could not get artifact from cache: {0}")] CacheError(#[from] wheel_cache::WheelCacheError), #[error("error parsing artifact name: {0}")] @@ -123,24 +123,20 @@ impl<'db, 'i> WheelBuilder<'db, 'i> { package_db: &'db PackageDb, env_markers: &'i MarkerEnvironment, wheel_tags: Option<&'i WheelTags>, - _resolve_options: &'i ResolveOptions, + resolve_options: &'i ResolveOptions, wheel_cache_dir: &Path, ) -> Self { - // TODO: add this back later when we have a wheel cache // We are running into a chicken & egg problem if we want to build wheels for packages that // require their build system as sdist as well. For example, `hatchling` requires `hatchling` as // build system. Hypothetically we'd have to look through all the hatchling sdists to find the one // that doesn't depend on itself. // Instead, we use wheels to build wheels. - // let resolve_options = if resolve_options.sdist_resolution == SDistResolution::OnlySDists { - // ResolveOptions { - // sdist_resolution: SDistResolution::Only, - // } - // } else { - // resolve_options.clone() - // }; - let resolve_options = ResolveOptions { - sdist_resolution: SDistResolution::OnlyWheels, + let resolve_options = if resolve_options.sdist_resolution == SDistResolution::OnlySDists { + ResolveOptions { + sdist_resolution: SDistResolution::PreferWheels, + } + } else { + resolve_options.clone() }; Self { @@ -175,7 +171,7 @@ impl<'db, 'i> WheelBuilder<'db, 'i> { let build_environment = BuildEnvironment::setup( sdist, - self.package_db, + self, self.env_markers, self.wheel_tags, &self.resolve_options, @@ -187,7 +183,7 @@ impl<'db, 'i> WheelBuilder<'db, 'i> { // Install extra requirements if any build_environment .install_extra_requirements( - self.package_db, + self, self.env_markers, self.wheel_tags, &self.resolve_options, @@ -351,7 +347,7 @@ mod tests { &env_markers, None, &resolve_options, - &package_db.1.path(), + package_db.1.path(), ); // Build the wheel