feat: always record direct_url.json

crates/rattler_installs_packages/src/artifacts/sdist.rs

@@ -920,7 +920,7 @@ mod tests {
             yanked: Yanked::default(),
         };
 
-        let whl = package_db
+        let (whl, _) = package_db
             .0
             .get_wheel(&artifact_info, Some(&wheel_builder))
             .await
@@ -983,7 +983,7 @@ mod tests {
             yanked: Yanked::default(),
         };
 
-        let whl = package_db
+        let (whl, _) = package_db
             .0
             .get_wheel(&artifact_info, Some(&wheel_builder))
             .await
@@ -1026,7 +1026,7 @@ mod tests {
             yanked: Yanked::default(),
         };
 
-        let whl = package_db
+        let (whl, _) = package_db
             .0
             .get_wheel(&artifact_info, Some(&wheel_builder))
             .await

crates/rattler_installs_packages/src/artifacts/wheel.rs

@@ -1450,9 +1450,9 @@ mod test {
         let direct_url = DirectUrlJson {
             url: Url::from_directory_path(&package_path).unwrap(),
             source: DirectUrlSource::Archive {
-                hashes: DirectUrlHashes {
+                hashes: Some(DirectUrlHashes {
                     sha256: "95a7e86f46de9b5da6ec9365e1e96d1644c67328".to_string(),
-                },
+                }),
             },
         };
         let wheel = venv

crates/rattler_installs_packages/src/index/direct_url/file.rs

@@ -2,9 +2,9 @@ use crate::artifacts::{SDist, STree, Wheel};
 use crate::index::package_database::DirectUrlArtifactResponse;
 use crate::resolve::PypiVersion;
 use crate::types::{
-    ArtifactFromBytes, ArtifactHashes, ArtifactInfo, ArtifactType, DistInfoMetadata,
-    NormalizedPackageName, PackageName, SDistFilename, SDistFormat, STreeFilename,
-    WheelCoreMetadata, Yanked,
+    ArtifactFromBytes, ArtifactHashes, ArtifactInfo, ArtifactType, DirectUrlJson, DirectUrlSource,
+    DistInfoMetadata, NormalizedPackageName, PackageName, SDistFilename, SDistFormat,
+    STreeFilename, WheelCoreMetadata, Yanked,
 };
 use crate::wheel_builder::{WheelBuildError, WheelBuilder};
 use indexmap::IndexMap;
@@ -179,10 +179,16 @@ pub(crate) async fn get_artifacts_and_metadata<P: Into<NormalizedPackageName>>(
     let mut result = IndexMap::default();
     result.insert(PypiVersion::Url(url.clone()), vec![artifact_info.clone()]);
 
+    let direct_url_json = DirectUrlJson {
+        url: url.clone(),
+        source: DirectUrlSource::Dir { editable: None },
+    };
+
     Ok(DirectUrlArtifactResponse {
         artifact_info,
         metadata: (metadata_bytes, metadata),
         artifact_versions: result,
         artifact,
+        direct_url_json,
     })
 }

crates/rattler_installs_packages/src/index/direct_url/git.rs

@@ -2,13 +2,14 @@ use crate::index::git_interop::{git_clone, GitSource, ParsedUrl};
 use crate::index::package_database::DirectUrlArtifactResponse;
 use crate::resolve::PypiVersion;
 use crate::types::{
-    ArtifactHashes, ArtifactInfo, ArtifactName, ArtifactType, DistInfoMetadata, HasArtifactName,
-    NormalizedPackageName, Yanked,
+    ArtifactHashes, ArtifactInfo, ArtifactName, ArtifactType, DirectUrlJson, DirectUrlSource,
+    DirectUrlVcs, DistInfoMetadata, HasArtifactName, NormalizedPackageName, Yanked,
 };
 use crate::wheel_builder::WheelBuilder;
 use indexmap::IndexMap;
 use miette::IntoDiagnostic;
 use rattler_digest::{compute_bytes_digest, Sha256};
+use std::str::FromStr;
 use std::sync::Arc;
 use url::Url;
 
@@ -27,7 +28,7 @@ pub(crate) async fn get_artifacts_and_metadata<P: Into<NormalizedPackageName>>(
         rev: parsed_url.revision,
     };
 
-    let mut location = git_clone(&git_source).into_diagnostic()?;
+    let (mut location, git_rev) = git_clone(&git_source).into_diagnostic()?;
 
     if let Some(subdirectory) = parsed_url.subdirectory {
         location.push(&subdirectory);
@@ -60,6 +61,15 @@ pub(crate) async fn get_artifacts_and_metadata<P: Into<NormalizedPackageName>>(
         reason: None,
     };
 
+    let direct_url_json = DirectUrlJson {
+        url: Url::from_str(parsed_url.url.as_str()).expect("URL should be parseable"),
+        source: DirectUrlSource::Vcs {
+            vcs: DirectUrlVcs::Git,
+            requested_revision: git_source.rev,
+            commit_id: git_rev.get_commit(),
+        },
+    };
+
     let project_hash = ArtifactHashes {
         sha256: Some(compute_bytes_digest::<Sha256>(url.as_str().as_bytes())),
     };
@@ -82,5 +92,6 @@ pub(crate) async fn get_artifacts_and_metadata<P: Into<NormalizedPackageName>>(
         metadata: (wheel_metadata.0, wheel_metadata.1),
         artifact_versions: result,
         artifact: ArtifactType::STree(artifact),
+        direct_url_json,
     })
 }

crates/rattler_installs_packages/src/index/direct_url/http.rs

@@ -3,8 +3,9 @@ use crate::index::http::Http;
 use crate::index::{parse_hash, CacheMode};
 use crate::resolve::PypiVersion;
 use crate::types::{
-    ArtifactFromBytes, ArtifactHashes, ArtifactInfo, ArtifactType, DistInfoMetadata,
-    NormalizedPackageName, PackageName, SDistFilename, SDistFormat, WheelCoreMetadata, Yanked,
+    ArtifactFromBytes, ArtifactHashes, ArtifactInfo, ArtifactType, DirectUrlHashes, DirectUrlJson,
+    DirectUrlSource, DistInfoMetadata, NormalizedPackageName, PackageName, SDistFilename,
+    SDistFormat, WheelCoreMetadata, Yanked,
 };
 use crate::utils::ReadAndSeek;
 use crate::wheel_builder::WheelBuilder;
@@ -52,7 +53,7 @@ pub(crate) async fn get_artifacts_and_metadata<P: Into<NormalizedPackageName>>(
         bytes.rewind().into_diagnostic()?;
         ArtifactHashes {
             sha256: Some(rattler_digest::compute_bytes_digest::<Sha256>(
-                bytes_for_hash,
+                bytes_for_hash.clone(),
             )),
         }
     };
@@ -61,6 +62,13 @@ pub(crate) async fn get_artifacts_and_metadata<P: Into<NormalizedPackageName>>(
         assert_eq!(hash, artifact_hash);
     };
 
+    let hash_str = format!(
+        "{:x}",
+        artifact_hash
+            .sha256
+            .expect("hash should be already calculated")
+    );
+
     let (metadata_bytes, metadata, artifact) = if str_name.ends_with(".whl") {
         let wheel = Wheel::from_url_and_bytes(url.path(), &normalized_package_name, bytes)?;
 
@@ -92,11 +100,20 @@ pub(crate) async fn get_artifacts_and_metadata<P: Into<NormalizedPackageName>>(
     let mut result = IndexMap::default();
     result.insert(PypiVersion::Url(url.clone()), vec![artifact_info.clone()]);
 
+    // let hash_string = String::from_ut(&bytes_for_hash);
+    let direct_url_json = DirectUrlJson {
+        url: url.clone(),
+        source: DirectUrlSource::Archive {
+            hashes: Some(DirectUrlHashes { sha256: hash_str }),
+        },
+    };
+
     Ok(crate::index::package_database::DirectUrlArtifactResponse {
         artifact_info,
         metadata: (metadata_bytes, metadata),
         artifact_versions: result,
         artifact,
+        direct_url_json,
     })
 }
 

crates/rattler_installs_packages/src/index/git_interop.rs

@@ -263,7 +263,7 @@ fn get_revision_sha(dest: &PathBuf, rev: Option<String>) -> Result<GitRev, Sourc
     let rev = if let Some(rev) = rev {
         rev
     } else {
-        return Ok(GitRev::Head);
+        "HEAD".to_owned()
     };
 
     let output = Command::new("git")
@@ -300,7 +300,7 @@ fn get_revision_sha(dest: &PathBuf, rev: Option<String>) -> Result<GitRev, Sourc
 }
 
 /// Fetch the git repository specified by the given source and place it in the cache directory.
-pub fn git_clone(source: &GitSource) -> Result<PathBuf, SourceError> {
+pub fn git_clone(source: &GitSource) -> Result<(PathBuf, GitRev), SourceError> {
     // test if git is available locally as we fetch the git from PATH,
     if !Command::new("git")
         .arg("--version")
@@ -436,5 +436,5 @@ pub fn git_clone(source: &GitSource) -> Result<PathBuf, SourceError> {
         }
     }
 
-    Ok(cache_path)
+    Ok((cache_path, git_rev))
 }

crates/rattler_installs_packages/src/index/package_database.rs

@@ -5,7 +5,10 @@ use crate::index::html::{parse_package_names_html, parse_project_info_html};
 use crate::index::http::{CacheMode, Http, HttpRequestError};
 use crate::index::package_sources::PackageSources;
 use crate::resolve::PypiVersion;
-use crate::types::{ArtifactInfo, ArtifactType, ProjectInfo, STreeFilename, WheelCoreMetadata};
+use crate::types::{
+    ArtifactInfo, ArtifactType, DirectUrlHashes, DirectUrlJson, DirectUrlSource, ProjectInfo,
+    STreeFilename, WheelCoreMetadata,
+};
 
 use crate::wheel_builder::{WheelBuildError, WheelBuilder, WheelCache};
 use crate::{
@@ -72,6 +75,7 @@ pub(crate) struct DirectUrlArtifactResponse {
     pub(crate) artifact_versions: VersionArtifacts,
     pub(crate) metadata: (Vec<u8>, WheelCoreMetadata),
     pub(crate) artifact: ArtifactType,
+    pub(crate) direct_url_json: DirectUrlJson,
 }
 
 impl PackageDb {
@@ -230,8 +234,8 @@ impl PackageDb {
         &self,
         artifact_info: &ArtifactInfo,
         builder: Option<&'async_recursion WheelBuilder>,
-    ) -> miette::Result<Wheel> {
-        // TODO: add support for this currently there are not cached, they will be repeatedly downloaded between runs
+    ) -> miette::Result<(Wheel, Option<DirectUrlJson>)> {
+        // TODO: add support for this currently there are not saved
         if artifact_info.is_direct_url {
             if let Some(builder) = builder {
                 let response = super::direct_url::fetch_artifact_and_metadata_by_direct_url(
@@ -242,15 +246,17 @@ impl PackageDb {
                 )
                 .await?;
 
-                match response.artifact {
-                    ArtifactType::Wheel(wheel) => return Ok(wheel),
+                let whl = match response.artifact {
+                    ArtifactType::Wheel(wheel) => wheel,
                     ArtifactType::SDist(sdist) => {
-                        return builder.build_wheel(&sdist).await.into_diagnostic()
+                        builder.build_wheel(&sdist).await.into_diagnostic()?
                     }
                     ArtifactType::STree(stree) => {
-                        return builder.build_wheel(&stree).await.into_diagnostic()
+                        builder.build_wheel(&stree).await.into_diagnostic()?
                     }
-                }
+                };
+
+                return Ok((whl, Some(response.direct_url_json)));
             } else {
                 miette::bail!("cannot build wheel without a wheel builder");
             }
@@ -263,15 +269,45 @@ impl PackageDb {
                     .get_cached_artifact::<SDist>(artifact_info, CacheMode::Default)
                     .await?;
 
-                return builder.build_wheel(&sdist).await.into_diagnostic();
+                let whl = builder
+                    .build_wheel(&sdist)
+                    .await
+                    .into_diagnostic()
+                    .expect("cannot build wheel");
+
+                let direct_url = if artifact_info.is_direct_url {
+                    let direct_url_hash = if let Some(hash) = artifact_info.hashes.clone() {
+                        if let Some(sha256) = hash.sha256 {
+                            let str = format!("{:x}", sha256);
+                            Some(DirectUrlHashes { sha256: str })
+                        } else {
+                            None
+                        }
+                    } else {
+                        None
+                    };
+                    Some(DirectUrlJson {
+                        url: artifact_info.url.clone(),
+                        source: DirectUrlSource::Archive {
+                            hashes: direct_url_hash,
+                        },
+                    })
+                } else {
+                    None
+                };
+
+                return Ok((whl, direct_url));
             } else {
                 miette::bail!("cannot build wheel without a wheel builder");
             }
         }
 
         // Otherwise just retrieve the wheel
-        self.get_cached_artifact::<Wheel>(artifact_info, CacheMode::Default)
+        let cached_whl = self
+            .get_cached_artifact::<Wheel>(artifact_info, CacheMode::Default)
             .await
+            .expect("cannot build wheel");
+        Ok((cached_whl, None))
     }
 
     /// Get artifact directly from file, vcs, or url

crates/rattler_installs_packages/src/types/artifact_name.rs

@@ -435,6 +435,10 @@ impl WheelFilename {
         let Some((distribution, rest)) =
             split_into_filename_rest(file_stem, normalized_package_name)
         else {
+            println!(
+                "FILE STEM IS {:?} NORM NAME {:?}",
+                file_stem, normalized_package_name
+            );
             return Err(ParseArtifactNameError::PackageNameNotFound(
                 normalized_package_name.clone(),
                 s.to_string(),

crates/rattler_installs_packages/src/types/direct_url_json.rs

@@ -23,7 +23,7 @@ pub enum DirectUrlSource {
     /// Information about the archive file.
     Archive {
         /// Hashes of the archive file.
-        hashes: DirectUrlHashes,
+        hashes: Option<DirectUrlHashes>,
     },
     /// Information about a source from a VCS directly
     #[serde(rename = "vcs_info")]

crates/rattler_installs_packages/src/wheel_builder/build_environment.rs

@@ -257,14 +257,19 @@ impl BuildEnvironment {
                     package_info.version
                 );
                 let artifact_info = package_info.artifacts.first().unwrap();
-                let artifact = wheel_builder
+                let (artifact, direct_url_json) = wheel_builder
                     .package_db
                     .get_wheel(artifact_info, Some(wheel_builder))
                     .await
                     .expect("could not get artifact");
 
-                self.venv
-                    .install_wheel(&artifact, &UnpackWheelOptions::default())?;
+                self.venv.install_wheel(
+                    &artifact,
+                    &UnpackWheelOptions {
+                        direct_url_json,
+                        ..Default::default()
+                    },
+                )?;
             }
         }
         Ok(())
@@ -410,7 +415,7 @@ impl BuildEnvironment {
         for package_info in resolved_wheels.iter() {
             let artifact_info = package_info.artifacts.first().unwrap();
 
-            let artifact = wheel_builder
+            let (artifact, _) = wheel_builder
                 .package_db
                 .get_wheel(artifact_info, Some(wheel_builder))
                 .await

crates/rip_bin/src/main.rs

@@ -301,11 +301,18 @@ async fn actual_main() -> miette::Result<()> {
                 console::style(pinned_package.version).italic()
             );
             let artifact_info = pinned_package.artifacts.first().unwrap();
-            let artifact = package_db
+            let (artifact, direct_url_json) = package_db
                 .get_wheel(artifact_info, Some(&wheel_builder))
                 .await?;
-            venv.install_wheel(&artifact, &UnpackWheelOptions::default())
-                .into_diagnostic()?;
+            println!("DIRECT URL JSON IS {:?}", direct_url_json);
+            venv.install_wheel(
+                &artifact,
+                &UnpackWheelOptions {
+                    direct_url_json,
+                    ..Default::default()
+                },
+            )
+            .into_diagnostic()?;
         }
     }