feat: adds the ability to use the server age while requesting available artifacts

crates/rattler_installs_packages/src/index/mod.rs

@@ -9,7 +9,7 @@ mod http;
 mod package_database;
 mod package_sources;
 
-pub use package_database::{ArtifactRequest, PackageDb};
+pub use package_database::{ArtifactRequest, CheckAvailablePackages, PackageDb};
 pub use package_sources::{PackageSources, PackageSourcesBuilder};
 
 pub use self::http::CacheMode;

crates/rattler_installs_packages/src/index/package_database.rs

@@ -55,6 +55,9 @@ pub struct PackageDb {
 
     /// Reference to the cache directory for all caches
     cache_dir: PathBuf,
+
+    /// Option to that determines if we always want to check if there are new available artifacts
+    check_available_artifacts: CheckAvailablePackages,
 }
 
 /// Type of request to get from the `available_artifacts` function.
@@ -72,6 +75,17 @@ pub enum ArtifactRequest {
     },
 }
 
+/// Specifies if we always want to check if there are new available artifacts
+/// for a package or if we use the time that the server says the request is fresh
+#[derive(Default, Eq, PartialEq, Copy, Clone)]
+pub enum CheckAvailablePackages {
+    /// Always check if there are new available artifacts
+    #[default]
+    Always,
+    /// Trust the time that the server says the request is fresh
+    UseServerTime,
+}
+
 pub(crate) struct DirectUrlArtifactResponse {
     pub(crate) artifact_info: Arc<ArtifactInfo>,
     pub(crate) artifact_versions: VersionArtifacts,
@@ -86,6 +100,7 @@ impl PackageDb {
         package_sources: PackageSources,
         client: ClientWithMiddleware,
         cache_dir: &Path,
+        check_available_artifacts: CheckAvailablePackages,
     ) -> miette::Result<Self> {
         let http = Http::new(
             client,
@@ -102,6 +117,7 @@ impl PackageDb {
             artifacts: Default::default(),
             local_wheel_cache,
             cache_dir: cache_dir.to_owned(),
+            check_available_artifacts,
         })
     }
 
@@ -134,7 +150,7 @@ impl PackageDb {
                     .map(|url| url.join(&format!("{}/", p.as_str())).expect("invalid url"))
                     .collect_vec();
                 let request_iter = stream::iter(urls)
-                    .map(|url| fetch_simple_api(&http, url))
+                    .map(|url| fetch_simple_api(&http, url, self.check_available_artifacts))
                     .buffer_unordered(10)
                     .filter_map(|result| async { result.transpose() });
 
@@ -722,9 +738,17 @@ impl PackageDb {
     }
 }
 
-async fn fetch_simple_api(http: &Http, url: Url) -> miette::Result<Option<ProjectInfo>> {
+async fn fetch_simple_api(
+    http: &Http,
+    url: Url,
+    check_available_artifacts: CheckAvailablePackages,
+) -> miette::Result<Option<ProjectInfo>> {
     let mut headers = HeaderMap::new();
-    headers.insert(CACHE_CONTROL, HeaderValue::from_static("max-age=0"));
+    // If we always want to check if there are new available artifacts, we'll set the cache control
+    // to max-age=0, so that we always get a non-cached server response.
+    if CheckAvailablePackages::Always == check_available_artifacts {
+        headers.insert(CACHE_CONTROL, HeaderValue::from_static("max-age=0"));
+    }
 
     let response = match http
         .request(url.to_owned(), Method::GET, headers, CacheMode::Default)
@@ -853,6 +877,7 @@ mod test {
             url.into(),
             ClientWithMiddleware::from(Client::new()),
             cache_dir.path(),
+            CheckAvailablePackages::default(),
         )
         .unwrap();
 
@@ -910,6 +935,7 @@ mod test {
             sources,
             ClientWithMiddleware::from(Client::new()),
             cache_dir.path(),
+            Default::default(),
         )
         .unwrap();
 

crates/rattler_installs_packages/src/utils/test.rs

@@ -19,7 +19,7 @@ pub fn get_package_db() -> (Arc<PackageDb>, TempDir) {
     let sources = PackageSourcesBuilder::new(url).build().unwrap();
 
     (
-        Arc::new(PackageDb::new(sources, client, tempdir.path()).unwrap()),
+        Arc::new(PackageDb::new(sources, client, tempdir.path(), Default::default()).unwrap()),
         tempdir,
     )
 }

crates/rattler_installs_packages/src/wheel_builder/mod.rs

@@ -396,7 +396,7 @@ mod tests {
         let sources = PackageSourcesBuilder::new(url).build().unwrap();
 
         (
-            Arc::new(PackageDb::new(sources, client, tempdir.path()).unwrap()),
+            Arc::new(PackageDb::new(sources, client, tempdir.path(), Default::default()).unwrap()),
             tempdir,
         )
     }

crates/rip_bin/src/main.rs

@@ -8,7 +8,7 @@ use miette::Context;
 use tracing_subscriber::filter::Directive;
 use tracing_subscriber::{fmt, layer::SubscriberExt, util::SubscriberInitExt, EnvFilter};
 
-use rattler_installs_packages::index::PackageSourcesBuilder;
+use rattler_installs_packages::index::{CheckAvailablePackages, PackageSourcesBuilder};
 
 use rattler_installs_packages::normalize_index_url;
 use reqwest::Client;
@@ -33,6 +33,12 @@ struct Cli {
     /// to a repository compliant with PEP 503 (the simple repository API).
     #[clap(default_value = "https://pypi.org/simple/", long, global = true)]
     index_url: Url,
+
+    /// Species whether to always check for new package versions
+    /// or if we have a server response for a package, use the
+    /// age provided by the server to determine should send a request
+    #[clap(long, global = true)]
+    use_server_timeout: bool,
 }
 
 #[derive(Subcommand)]
@@ -66,15 +72,26 @@ async fn actual_main() -> miette::Result<()> {
     let index_url = normalize_index_url(args.index_url.clone());
     let sources = PackageSourcesBuilder::new(index_url).build()?;
 
+    let check_available_packages = if args.use_server_timeout {
+        CheckAvailablePackages::UseServerTime
+    } else {
+        CheckAvailablePackages::Always
+    };
+
     let client = ClientWithMiddleware::from(Client::new());
     let package_db = Arc::new(
-        rattler_installs_packages::index::PackageDb::new(sources, client, &cache_dir)
-            .wrap_err_with(|| {
-                format!(
-                    "failed to construct package database for index {}",
-                    args.index_url
-                )
-            })?,
+        rattler_installs_packages::index::PackageDb::new(
+            sources,
+            client,
+            &cache_dir,
+            check_available_packages,
+        )
+        .wrap_err_with(|| {
+            format!(
+                "failed to construct package database for index {}",
+                args.index_url
+            )
+        })?,
     );
 
     match args.command {