feat: sdists can now build sdists if needed (#111)

crates/rattler_installs_packages/Cargo.toml

@@ -67,6 +67,7 @@ pyproject-toml = "0.8.0"
 async-once-cell = "0.5.3"
 configparser = "3.0.3"
 cacache = { version = "12.0.0", default-features = false, features = ["tokio-runtime", "mmap"] }
+async-recursion = "1.0.5"
 
 [dev-dependencies]
 criterion = "0.5"

crates/rattler_installs_packages/src/artifacts/sdist.rs

@@ -260,7 +260,7 @@ mod tests {
             &env_markers,
             None,
             &resolve_options,
-            &package_db.1.path(),
+            package_db.1.path(),
         );
 
         let result = wheel_builder.get_sdist_metadata(&sdist).await.unwrap();
@@ -283,7 +283,7 @@ mod tests {
             &env_markers,
             None,
             &resolve_options,
-            &package_db.1.path(),
+            package_db.1.path(),
         );
 
         // Build the wheel
@@ -308,7 +308,7 @@ mod tests {
             &env_markers,
             None,
             &resolve_options,
-            &package_db.1.path(),
+            package_db.1.path(),
         );
 
         // Build the wheel

crates/rattler_installs_packages/src/index/package_database.rs

@@ -9,6 +9,7 @@ use crate::{
     types::WheelFilename,
 };
 use async_http_range_reader::{AsyncHttpRangeReader, CheckSupportMethod};
+use async_recursion::async_recursion;
 use elsa::sync::FrozenMap;
 use futures::{pin_mut, stream, StreamExt};
 use http::{header::CONTENT_TYPE, HeaderMap, HeaderValue, Method};
@@ -429,11 +430,27 @@ impl PackageDb {
 
     /// Opens the specified artifact info. Downloads the artifact data from the remote location if
     /// the information is not already cached.
-    pub async fn get_artifact<A: Artifact>(
+    #[async_recursion]
+    pub async fn get_wheel<'db, 'i>(
         &self,
         artifact_info: &ArtifactInfo,
-    ) -> miette::Result<A> {
-        self.get_artifact_with_cache(artifact_info, CacheMode::Default)
+        builder: Option<&'async_recursion WheelBuilder<'db, 'i>>,
+    ) -> miette::Result<Wheel> {
+        // Try to build the wheel for this SDist if possible
+        if artifact_info.is::<SDist>() {
+            if let Some(builder) = builder {
+                let sdist = self
+                    .get_artifact_with_cache::<SDist>(artifact_info, CacheMode::Default)
+                    .await?;
+
+                return builder.build_wheel(&sdist).await.into_diagnostic();
+            } else {
+                miette::bail!("cannot build wheel without a wheel builder");
+            }
+        }
+
+        // Otherwise just retrieve the wheel
+        self.get_artifact_with_cache::<Wheel>(artifact_info, CacheMode::Default)
             .await
     }
 }

crates/rattler_installs_packages/src/wheel_builder/build_environment.rs

@@ -1,10 +1,10 @@
 use crate::artifacts::wheel::UnpackWheelOptions;
-use crate::artifacts::{SDist, Wheel};
-use crate::index::PackageDb;
+use crate::artifacts::SDist;
+
 use crate::python_env::{PythonLocation, VEnv, WheelTags};
 use crate::resolve::{resolve, PinnedPackage, ResolveOptions};
 use crate::types::Artifact;
-use crate::wheel_builder::{build_requirements, WheelBuildError};
+use crate::wheel_builder::{build_requirements, WheelBuildError, WheelBuilder};
 use pep508_rs::{MarkerEnvironment, Requirement};
 use std::collections::{HashMap, HashSet};
 use std::path::{Path, PathBuf};
@@ -73,9 +73,9 @@ impl<'db> BuildEnvironment<'db> {
     /// Install extra requirements into the venv, if any extra were found
     /// If the extra requirements are already installed, this will do nothing
     /// for that requirement.
-    pub(crate) async fn install_extra_requirements(
+    pub(crate) async fn install_extra_requirements<'i>(
         &self,
-        package_db: &'db PackageDb,
+        wheel_builder: &WheelBuilder<'db, 'i>,
         env_markers: &MarkerEnvironment,
         wheel_tags: Option<&WheelTags>,
         resolve_options: &ResolveOptions,
@@ -98,7 +98,7 @@ impl<'db> BuildEnvironment<'db> {
             let favored_packages = HashMap::default();
             let all_requirements = combined_requirements.to_vec();
             let extra_resolved_wheels = resolve(
-                package_db,
+                wheel_builder.package_db,
                 all_requirements.iter(),
                 env_markers,
                 wheel_tags,
@@ -120,8 +120,9 @@ impl<'db> BuildEnvironment<'db> {
                     package_info.version
                 );
                 let artifact_info = package_info.artifacts.first().unwrap();
-                let artifact = package_db
-                    .get_artifact::<Wheel>(artifact_info)
+                let artifact = wheel_builder
+                    .package_db
+                    .get_wheel(artifact_info, Some(wheel_builder))
                     .await
                     .expect("could not get artifact");
 
@@ -145,9 +146,9 @@ impl<'db> BuildEnvironment<'db> {
     }
 
     /// Setup the build environment so that we can build a wheel from an sdist
-    pub(crate) async fn setup(
+    pub(crate) async fn setup<'i>(
         sdist: &SDist,
-        package_db: &'db PackageDb,
+        wheel_builder: &WheelBuilder<'db, 'i>,
         env_markers: &MarkerEnvironment,
         wheel_tags: Option<&WheelTags>,
         resolve_options: &ResolveOptions,
@@ -167,9 +168,16 @@ impl<'db> BuildEnvironment<'db> {
                 });
         // Find the build requirements
         let build_requirements = build_requirements(&build_system);
+        tracing::info!(
+            "build requirements: {:?}",
+            build_requirements
+                .iter()
+                .map(|r| r.to_string())
+                .collect::<Vec<_>>()
+        );
         // Resolve the build environment
         let resolved_wheels = resolve(
-            package_db,
+            wheel_builder.package_db,
             build_requirements.iter(),
             env_markers,
             wheel_tags,
@@ -183,10 +191,12 @@ impl<'db> BuildEnvironment<'db> {
         // Install into venv
         for package_info in resolved_wheels.iter() {
             let artifact_info = package_info.artifacts.first().unwrap();
-            let artifact = package_db
-                .get_artifact::<Wheel>(artifact_info)
+
+            let artifact = wheel_builder
+                .package_db
+                .get_wheel(artifact_info, Some(wheel_builder))
                 .await
-                .map_err(|_| WheelBuildError::CouldNotGetArtifact)?;
+                .map_err(WheelBuildError::CouldNotGetArtifact)?;
 
             venv.install_wheel(
                 &artifact,

crates/rattler_installs_packages/src/wheel_builder/mod.rs

@@ -55,31 +55,31 @@ pub struct WheelBuilder<'db, 'i> {
 #[allow(missing_docs)]
 #[derive(thiserror::Error, Debug)]
 pub enum WheelBuildError {
-    #[error("Could not build wheel: {0}")]
+    #[error("could not build wheel: {0}")]
     Error(String),
 
-    #[error("Could not install artifact in virtual environment")]
+    #[error("could not install artifact in virtual environment: {0}")]
     UnpackError(#[from] UnpackError),
 
-    #[error("Could not build wheel: {0}")]
+    #[error("could not build wheel: {0}")]
     IoError(#[from] std::io::Error),
 
-    #[error("Could not run command {0} to build wheel: {1}")]
+    #[error("could not run command {0} to build wheel: {1}")]
     CouldNotRunCommand(String, std::io::Error),
 
-    #[error("Could not resolve environment for wheel building")]
+    #[error("could not resolve environment for wheel building")]
     CouldNotResolveEnvironment(Vec<Requirement>),
 
-    #[error("Error parsing JSON from extra_requirements.json: {0}")]
+    #[error("error parsing JSON from extra_requirements.json: {0}")]
     JSONError(#[from] serde_json::Error),
 
-    #[error("Could not parse generated wheel metadata: {0}")]
+    #[error("could not parse generated wheel metadata: {0}")]
     WheelCoreMetadataError(#[from] WheelCoreMetaDataError),
 
-    #[error("Could not get artifact")]
-    CouldNotGetArtifact,
+    #[error("could not get artifact: {0}")]
+    CouldNotGetArtifact(miette::Report),
 
-    #[error("Could not get artifact: {0}")]
+    #[error("could not get artifact from cache: {0}")]
     CacheError(#[from] wheel_cache::WheelCacheError),
 
     #[error("error parsing artifact name: {0}")]
@@ -123,24 +123,20 @@ impl<'db, 'i> WheelBuilder<'db, 'i> {
         package_db: &'db PackageDb,
         env_markers: &'i MarkerEnvironment,
         wheel_tags: Option<&'i WheelTags>,
-        _resolve_options: &'i ResolveOptions,
+        resolve_options: &'i ResolveOptions,
         wheel_cache_dir: &Path,
     ) -> Self {
-        // TODO: add this back later when we have a wheel cache
         // We are running into a chicken & egg problem if we want to build wheels for packages that
         // require their build system as sdist as well. For example, `hatchling` requires `hatchling` as
         // build system. Hypothetically we'd have to look through all the hatchling sdists to find the one
         // that doesn't depend on itself.
         // Instead, we use wheels to build wheels.
-        // let resolve_options = if resolve_options.sdist_resolution == SDistResolution::OnlySDists {
-        //     ResolveOptions {
-        //         sdist_resolution: SDistResolution::Only,
-        //     }
-        // } else {
-        //     resolve_options.clone()
-        // };
-        let resolve_options = ResolveOptions {
-            sdist_resolution: SDistResolution::OnlyWheels,
+        let resolve_options = if resolve_options.sdist_resolution == SDistResolution::OnlySDists {
+            ResolveOptions {
+                sdist_resolution: SDistResolution::PreferWheels,
+            }
+        } else {
+            resolve_options.clone()
         };
 
         Self {
@@ -175,7 +171,7 @@ impl<'db, 'i> WheelBuilder<'db, 'i> {
 
         let build_environment = BuildEnvironment::setup(
             sdist,
-            self.package_db,
+            self,
             self.env_markers,
             self.wheel_tags,
             &self.resolve_options,
@@ -187,7 +183,7 @@ impl<'db, 'i> WheelBuilder<'db, 'i> {
         // Install extra requirements if any
         build_environment
             .install_extra_requirements(
-                self.package_db,
+                self,
                 self.env_markers,
                 self.wheel_tags,
                 &self.resolve_options,
@@ -351,7 +347,7 @@ mod tests {
             &env_markers,
             None,
             &resolve_options,
-            &package_db.1.path(),
+            package_db.1.path(),
         );
 
         // Build the wheel