diff --git a/crates/rattler-bin/Cargo.toml b/crates/rattler-bin/Cargo.toml index 5708a940a..c74a9c0d1 100644 --- a/crates/rattler-bin/Cargo.toml +++ b/crates/rattler-bin/Cargo.toml @@ -29,7 +29,7 @@ indicatif = { workspace = true } once_cell = { workspace = true } rattler = { path="../rattler", version = "0.27.8", default-features = false, features = ["indicatif"] } rattler_conda_types = { path="../rattler_conda_types", version = "0.27.4", default-features = false } -rattler_networking = { path="../rattler_networking", version = "0.21.3", default-features = false } +rattler_networking = { path="../rattler_networking", version = "0.21.3", default-features = false, features = ["google-cloud-auth"] } rattler_repodata_gateway = { path="../rattler_repodata_gateway", version = "0.21.10", default-features = false, features = ["gateway"] } rattler_solve = { path="../rattler_solve", version = "1.0.5", default-features = false, features = ["resolvo", "libsolv_c"] } rattler_virtual_packages = { path="../rattler_virtual_packages", version = "1.1.1", default-features = false } diff --git a/crates/rattler-bin/src/commands/create.rs b/crates/rattler-bin/src/commands/create.rs index 7c840a77a..1511ab191 100644 --- a/crates/rattler-bin/src/commands/create.rs +++ b/crates/rattler-bin/src/commands/create.rs @@ -140,6 +140,8 @@ pub async fn create(opt: Opt) -> anyhow::Result<()> { .with_arc(Arc::new(AuthenticationMiddleware::new( authentication_storage, ))) + .with(rattler_networking::OciMiddleware) + .with(rattler_networking::GCSMiddleware) .build(); // Get the package names from the matchspecs so we can only load the package records that we need. diff --git a/crates/rattler_cache/src/package_cache/mod.rs b/crates/rattler_cache/src/package_cache/mod.rs index f1b2ad3c9..b9b3547ab 100644 --- a/crates/rattler_cache/src/package_cache/mod.rs +++ b/crates/rattler_cache/src/package_cache/mod.rs @@ -372,7 +372,10 @@ where let read_revision = write_lock.read_revision()?; if read_revision != cache_revision { - tracing::warn!("cache revisions dont match '{}", lock_file_path.display()); + tracing::debug!( + "cache revisions dont match '{}', retrying to acquire lock file.", + lock_file_path.display() + ); // The cache has been modified since we last checked. We need to re-validate. continue; } @@ -386,8 +389,6 @@ where .await .map_err(|e| PackageCacheError::FetchError(Arc::new(e)))?; - tracing::warn!("fetched '{}", lock_file_path.display()); - validated_revision = Some(new_revision); } } diff --git a/crates/rattler_repodata_gateway/src/gateway/mod.rs b/crates/rattler_repodata_gateway/src/gateway/mod.rs index 3f7340963..cf0456068 100644 --- a/crates/rattler_repodata_gateway/src/gateway/mod.rs +++ b/crates/rattler_repodata_gateway/src/gateway/mod.rs @@ -31,6 +31,7 @@ use reqwest_middleware::ClientWithMiddleware; use subdir::{Subdir, SubdirData}; use tokio::sync::broadcast; use tracing::instrument; +use url::Url; use crate::{fetch::FetchRepoDataError, gateway::error::SubdirNotFoundError, Reporter}; @@ -285,32 +286,32 @@ impl GatewayInner { "unsupported file based url".to_string(), )); } - } else if url.scheme() == "http" || url.scheme() == "https" { - if url.host_str() == Some("fast.prefiks.dev") - || url.host_str() == Some("fast.prefix.dev") - { - sharded_subdir::ShardedSubdir::new( - channel.clone(), - platform.to_string(), - self.client.clone(), - self.cache.clone(), - self.concurrent_requests_semaphore.clone(), - reporter.as_deref(), - ) - .await - .map(SubdirData::from_client) - } else { - remote_subdir::RemoteSubdirClient::new( - channel.clone(), - platform, - self.client.clone(), - self.cache.clone(), - self.channel_config.get(channel).clone(), - reporter, - ) - .await - .map(SubdirData::from_client) - } + } else if supports_sharded_repodata(&url) { + sharded_subdir::ShardedSubdir::new( + channel.clone(), + platform.to_string(), + self.client.clone(), + self.cache.clone(), + self.concurrent_requests_semaphore.clone(), + reporter.as_deref(), + ) + .await + .map(SubdirData::from_client) + } else if url.scheme() == "http" + || url.scheme() == "https" + || url.scheme() == "gcs" + || url.scheme() == "oci" + { + remote_subdir::RemoteSubdirClient::new( + channel.clone(), + platform, + self.client.clone(), + self.cache.clone(), + self.channel_config.get(channel).clone(), + reporter, + ) + .await + .map(SubdirData::from_client) } else { return Err(GatewayError::UnsupportedUrl(format!( "'{}' is not a supported scheme", @@ -350,9 +351,13 @@ enum PendingOrFetched { Fetched(T), } +fn supports_sharded_repodata(url: &Url) -> bool { + (url.scheme() == "http" || url.scheme() == "https") + && (url.host_str() == Some("fast.prefiks.dev") || url.host_str() == Some("fast.prefix.dev")) +} + #[cfg(test)] mod test { - use assert_matches::assert_matches; use std::{ path::{Path, PathBuf}, str::FromStr, @@ -360,12 +365,13 @@ mod test { time::Instant, }; + use assert_matches::assert_matches; use dashmap::DashSet; - use rattler_cache::default_cache_dir; - use rattler_cache::package_cache::PackageCache; + use rattler_cache::{default_cache_dir, package_cache::PackageCache}; use rattler_conda_types::{ - Channel, ChannelConfig, MatchSpec, PackageName, ParseStrictness::Lenient, - ParseStrictness::Strict, Platform, RepoDataRecord, + Channel, ChannelConfig, MatchSpec, PackageName, + ParseStrictness::{Lenient, Strict}, + Platform, RepoDataRecord, }; use rstest::rstest; use url::Url; @@ -490,7 +496,8 @@ mod test { assert_eq!(non_openssl_total_records, non_openssl_direct_records); } - // Make sure that the direct url version of openssl is used instead of the one from the normal channel. + // Make sure that the direct url version of openssl is used instead of the one + // from the normal channel. #[tokio::test] async fn test_select_forced_url_instead_of_deps() { let gateway = Gateway::builder() diff --git a/crates/rattler_virtual_packages/src/lib.rs b/crates/rattler_virtual_packages/src/lib.rs index 825407c59..7afd72fe9 100644 --- a/crates/rattler_virtual_packages/src/lib.rs +++ b/crates/rattler_virtual_packages/src/lib.rs @@ -147,7 +147,7 @@ pub enum VirtualPackage { /// Available on `Unix` based platforms Unix, - /// Available when running on `Linux`` + /// Available when running on `Linux` Linux(Linux), /// Available when running on `OSX`