feat: download and cache repodata.json (#55)

* crates/rattler/resources/channels/empty/noarch/repodata.json: convert to Git LFS

* fix: initial version

* fix: tests

* fix: formatting

* fix: tests on linux

* feat: seperate into different crates

* fix: libc for unix

* fix(doc): typeo

* fix: doc link

* fix: documentation links to tokio

* fix: wrap filesystem operations in tokio spawn blocking

crates/rattler-bin/src/commands/create.rs

@@ -1,7 +1,3 @@
-use rattler::{
-    repo_data::fetch::{terminal_progress, MultiRequestRepoDataBuilder},
-    solver::SolverProblem,
-};
 use rattler_conda_types::{Channel, ChannelConfig, MatchSpec};
 
 #[derive(Debug, clap::Parser)]
@@ -17,7 +13,7 @@ pub async fn create(opt: Opt) -> anyhow::Result<()> {
     let channel_config = ChannelConfig::default();
 
     // Parse the match specs
-    let specs = opt
+    let _specs = opt
         .specs
         .iter()
         .map(|spec| MatchSpec::from_str(spec, &channel_config))
@@ -31,38 +27,38 @@ pub async fn create(opt: Opt) -> anyhow::Result<()> {
         .map_err(|e| anyhow::anyhow!("could not create cache directory: {}", e))?;
 
     // Get the channels to download
-    let channels = opt
+    let _channels = opt
         .channels
         .unwrap_or_else(|| vec![String::from("conda-forge")])
         .into_iter()
         .map(|channel_str| Channel::from_str(&channel_str, &channel_config))
         .collect::<Result<Vec<_>, _>>()?;
 
-    // Download all repo data from the channels and create an index
-    let repo_data_per_source = MultiRequestRepoDataBuilder::default()
-        .set_cache_dir(&cache_dir)
-        .set_listener(terminal_progress())
-        .set_fail_fast(false)
-        .add_channels(channels)
-        .request()
-        .await;
-
-    // Error out if fetching one of the sources resulted in an error.
-    let repo_data = repo_data_per_source
-        .into_iter()
-        .map(|(channel, _, result)| result.map(|data| (channel, data)))
-        .collect::<Result<Vec<_>, _>>()?;
-
-    let solver_problem = SolverProblem {
-        channels: repo_data
-            .iter()
-            .map(|(channel, repodata)| (channel.base_url().to_string(), repodata))
-            .collect(),
-        specs,
-    };
-
-    let result = solver_problem.solve()?;
-    println!("{:#?}", result);
+    // // Download all repo data from the channels and create an index
+    // let repo_data_per_source = MultiRequestRepoDataBuilder::default()
+    //     .set_cache_dir(&cache_dir)
+    //     .set_listener(terminal_progress())
+    //     .set_fail_fast(false)
+    //     .add_channels(channels)
+    //     .request()
+    //     .await;
+    //
+    // // Error out if fetching one of the sources resulted in an error.
+    // let repo_data = repo_data_per_source
+    //     .into_iter()
+    //     .map(|(channel, _, result)| result.map(|data| (channel, data)))
+    //     .collect::<Result<Vec<_>, _>>()?;
+    //
+    // let solver_problem = SolverProblem {
+    //     channels: repo_data
+    //         .iter()
+    //         .map(|(channel, repodata)| (channel.base_url().to_string(), repodata))
+    //         .collect(),
+    //     specs,
+    // };
+    //
+    // let result = solver_problem.solve()?;
+    // println!("{:#?}", result);
 
     Ok(())
 }

crates/rattler/Cargo.toml

@@ -13,15 +13,14 @@ rustls-tls = ['reqwest/rustls-tls']
 [dependencies]
 anyhow = "1.0.44"
 apple-codesign = "0.22.0"
-async-compression = { version = "0.3.12", features = ["gzip", "futures-bufread", "tokio", "bzip2"] }
+async-compression = { version = "0.3.12", features = ["gzip", "tokio", "bzip2", "zstd"] }
 bytes = "1.1.0"
+chrono = { version = "0.4.23", default-features = false, features = ["std", "serde", "alloc"] }
 digest = "0.10.6"
 dirs = "4.0.0"
-extendhash = "1.0.9"
 futures = "0.3.17"
 fxhash = "0.2.1"
 hex = "0.4.3"
-indicatif = { version = "0.17.1", features = ["improved_unicode"] }
 itertools = "0.10.3"
 libc = "0.2"
 libz-sys = { version = "1.1.0", default-features = false, features = ["static"] }
@@ -31,6 +30,7 @@ nom = "7.1.0"
 once_cell = "1.8.0"
 pin-project-lite = "0.2.9"
 rattler_conda_types = { version = "0.1.0", path = "../rattler_conda_types" }
+rattler_digest = { version = "0.1.0", path = "../rattler_digest" }
 rattler_package_streaming = { version = "0.1.0", path = "../rattler_package_streaming", features = ["reqwest", "tokio"] }
 regex = "1.5.4"
 reqwest = { version = "0.11.6", default-features = false, features = ["stream", "json", "gzip"] }
@@ -50,14 +50,9 @@ uuid = { version = "1.3.0", features = ["v4", "fast-rng"] }
 
 [dev-dependencies]
 assert_matches = "1.5.0"
-axum = "0.6.2"
-insta = { version = "1.16.0", features = ["yaml"] }
-proptest = "1.0.0"
 rand = "0.8.4"
 rstest = "0.16.0"
-tokio-test = "0.4.2"
-tower-http = { version = "0.3.5", features = ["fs", "compression-gzip"] }
-tracing-test = "0.2.4"
+tracing-test = { version = "0.2.4" }
 
 [build-dependencies]
 cc = "1"

crates/rattler/src/install/link.rs

@@ -1,7 +1,7 @@
-use crate::utils::{parse_sha256_from_hex, Sha256HashingWriter};
 use apple_codesign::{SigningSettings, UnifiedSigner};
 use rattler_conda_types::package::{FileMode, PathType, PathsEntry};
 use rattler_conda_types::Platform;
+use rattler_digest::{parse_digest_from_hex, HashingWriter};
 use std::fs::Permissions;
 use std::io::Write;
 use std::path::Path;
@@ -81,7 +81,7 @@ pub fn link_file(
         // Open the destination file
         let destination = std::fs::File::create(&destination_path)
             .map_err(LinkFileError::FailedToOpenDestinationFile)?;
-        let mut destination_writer = Sha256HashingWriter::new(destination);
+        let mut destination_writer = HashingWriter::<_, sha2::Sha256>::new(destination);
 
         // Replace the prefix placeholder in the file with the new placeholder
         copy_and_replace_placholders(
@@ -111,7 +111,7 @@ pub fn link_file(
                 let original_hash = path_json_entry
                     .sha256
                     .as_deref()
-                    .and_then(parse_sha256_from_hex);
+                    .and_then(parse_digest_from_hex::<sha2::Sha256>);
                 let content_changed = original_hash != Some(current_hash);
 
                 // If the binary changed it requires resigning.

crates/rattler/src/lib.rs

@@ -9,14 +9,13 @@
 //! interfacing with many other languages (WASM, Javascript, Python, C, etc) and is therefor a good
 //! candidate for a reimplementation.
 
+use std::path::PathBuf;
+
 pub mod install;
 pub mod package_cache;
-pub mod repo_data;
 pub mod solver;
 pub mod validation;
 
-pub(crate) mod utils;
-
 /// A helper function that returns a [`Channel`] instance that points to an empty channel on disk
 /// that is bundled with this repository.
 #[cfg(any(doctest, test))]
@@ -30,10 +29,14 @@ pub fn empty_channel() -> rattler_conda_types::Channel {
     .unwrap()
 }
 
-#[cfg(test)]
-use std::path::{Path, PathBuf};
-
 #[cfg(test)]
 pub(crate) fn get_test_data_dir() -> PathBuf {
-    Path::new(env!("CARGO_MANIFEST_DIR")).join("../../test-data")
+    std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../test-data")
+}
+
+/// Returns the default cache directory used by rattler.
+pub fn default_cache_dir() -> anyhow::Result<PathBuf> {
+    Ok(dirs::cache_dir()
+        .ok_or_else(|| anyhow::anyhow!("could not determine cache directory for current platform"))?
+        .join("rattler/cache"))
 }