From 604c942c709a5b95f149ea5d7930d8105851a4fc Mon Sep 17 00:00:00 2001 From: Tim de Jager Date: Tue, 26 Sep 2023 11:21:29 +0200 Subject: [PATCH 1/4] feat: pep658 support --- .../src/package_database.rs | 27 +++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/crates/rattler_installs_packages/src/package_database.rs b/crates/rattler_installs_packages/src/package_database.rs index d5521947..6ba4cdc7 100644 --- a/crates/rattler_installs_packages/src/package_database.rs +++ b/crates/rattler_installs_packages/src/package_database.rs @@ -156,11 +156,34 @@ impl PackageDb { // We have exhausted all options to read the metadata from the cache. We'll have to hit the // network to get to the information. - // TODO: PEP 658 support - // Get the information from the first artifact. We assume the metadata is consistent across // all matching artifacts if let Some(artifact_info) = matching_artifacts.next() { + + // Retrieve the metadata instead of the entire wheel + // If the dist-info is available separately, we can use that instead + if artifact_info.dist_info_metadata.available { + + // Turn into PEP658 compliant URL + let mut url = artifact_info.url.clone(); + url.set_path(&url.path().replace(".whl", ".whl.metadata")); + + let mut bytes = Vec::new(); + self.http + .request( + url, + Method::GET, + HeaderMap::default(), + CacheMode::Default, + ).await? + .into_body() + .read_to_end(&mut bytes).await.into_diagnostic()?; + + let metadata = A::parse_metadata(&bytes)?; + self.put_metadata_in_cache(artifact_info, &bytes)?; + return Ok((artifact_info, metadata)); + } + let body = self .http .request( From 4cc03e5e22a42630104c6ff0d450708764d05f22 Mon Sep 17 00:00:00 2001 From: Tim de Jager Date: Tue, 26 Sep 2023 11:37:06 +0200 Subject: [PATCH 2/4] feat: added test for pep658 support --- .../src/package_database.rs | 75 ++++++++++++++----- 1 file changed, 56 insertions(+), 19 deletions(-) diff --git a/crates/rattler_installs_packages/src/package_database.rs b/crates/rattler_installs_packages/src/package_database.rs index 6ba4cdc7..686b3de8 100644 --- a/crates/rattler_installs_packages/src/package_database.rs +++ b/crates/rattler_installs_packages/src/package_database.rs @@ -163,25 +163,7 @@ impl PackageDb { // Retrieve the metadata instead of the entire wheel // If the dist-info is available separately, we can use that instead if artifact_info.dist_info_metadata.available { - - // Turn into PEP658 compliant URL - let mut url = artifact_info.url.clone(); - url.set_path(&url.path().replace(".whl", ".whl.metadata")); - - let mut bytes = Vec::new(); - self.http - .request( - url, - Method::GET, - HeaderMap::default(), - CacheMode::Default, - ).await? - .into_body() - .read_to_end(&mut bytes).await.into_diagnostic()?; - - let metadata = A::parse_metadata(&bytes)?; - self.put_metadata_in_cache(artifact_info, &bytes)?; - return Ok((artifact_info, metadata)); + return self.get_pep658_metadata::(artifact_info).await; } let body = self @@ -220,6 +202,30 @@ impl PackageDb { ); } + /// Retrieve the PEP658 metadata for the given artifact. + /// This assumes that the metadata is available in the repository + /// This can be checked with the ArtifactInfo + async fn get_pep658_metadata<'a, A: MetadataArtifact>(&self, artifact_info: &'a ArtifactInfo) -> miette::Result<(&'a ArtifactInfo, A::Metadata)> { + // Turn into PEP658 compliant URL + let mut url = artifact_info.url.clone(); + url.set_path(&url.path().replace(".whl", ".whl.metadata")); + + let mut bytes = Vec::new(); + self.http + .request( + url, + Method::GET, + HeaderMap::default(), + CacheMode::Default, + ).await? + .into_body() + .read_to_end(&mut bytes).await.into_diagnostic()?; + + let metadata = A::parse_metadata(&bytes)?; + self.put_metadata_in_cache(artifact_info, &bytes)?; + return Ok((artifact_info, metadata)); + } + /// Get all package names in the index. pub async fn get_package_names(&self) -> miette::Result> { let index_url = self.index_urls.first(); @@ -364,6 +370,37 @@ mod test { .await .unwrap(); } + + + #[tokio::test] + async fn test_pep658() { + let cache_dir = TempDir::new().unwrap(); + let package_db = PackageDb::new( + Client::new(), + &[Url::parse("https://pypi.org/simple/").unwrap()], + cache_dir.path().into(), + ) + .unwrap(); + + // Get all the artifacts + let artifacts = package_db + .available_artifacts("numpy".parse::().unwrap()) + .await + .unwrap(); + + // Get the artifact with dist-info attribute + let artifact_info = artifacts + .iter() + .flat_map(|(_, artifacts)| artifacts.iter()) + // This signifies that a PEP658 metadata file is available + .find(|a| a.dist_info_metadata.available) + .unwrap(); + + let (_artifact, _metadata) = package_db + .get_pep658_metadata::(&artifact_info) + .await + .unwrap(); + } } #[derive(Debug, Diagnostic)] From 027be70d5b754a13f8c1d159e2ebeaff7214537d Mon Sep 17 00:00:00 2001 From: Tim de Jager Date: Tue, 26 Sep 2023 11:40:09 +0200 Subject: [PATCH 3/4] fix: formatting --- .../rattler_installs_packages/benches/html.rs | 4 +-- crates/rattler_installs_packages/src/html.rs | 5 +--- .../src/package_database.rs | 25 +++++++++---------- 3 files changed, 14 insertions(+), 20 deletions(-) diff --git a/crates/rattler_installs_packages/benches/html.rs b/crates/rattler_installs_packages/benches/html.rs index a8d3eb8b..fdca1120 100644 --- a/crates/rattler_installs_packages/benches/html.rs +++ b/crates/rattler_installs_packages/benches/html.rs @@ -1,7 +1,5 @@ use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; -use rattler_installs_packages::html::{ - parse_package_names_html, parse_project_info_html, -}; +use rattler_installs_packages::html::{parse_package_names_html, parse_project_info_html}; use std::str::FromStr; use url::Url; diff --git a/crates/rattler_installs_packages/src/html.rs b/crates/rattler_installs_packages/src/html.rs index 6d05acd3..0ae51a56 100644 --- a/crates/rattler_installs_packages/src/html.rs +++ b/crates/rattler_installs_packages/src/html.rs @@ -11,7 +11,6 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. - use std::{borrow::Borrow, default::Default}; use crate::{ArtifactHashes, ArtifactName}; @@ -42,9 +41,7 @@ pub fn into_artifact_info(base: &Url, tag: &HTMLTag) -> Option { // Join with base let url = base.join(href.as_ref()).ok()?; let filename = url.path_segments().and_then(|mut s| s.next_back()); - let filename: ArtifactName = filename - .map(|s| s.parse())? - .ok()?; + let filename: ArtifactName = filename.map(|s| s.parse())?.ok()?; // We found a valid link let hash = url.fragment().and_then(parse_hash); diff --git a/crates/rattler_installs_packages/src/package_database.rs b/crates/rattler_installs_packages/src/package_database.rs index 686b3de8..5a5e44af 100644 --- a/crates/rattler_installs_packages/src/package_database.rs +++ b/crates/rattler_installs_packages/src/package_database.rs @@ -19,7 +19,7 @@ use pep440::Version; use reqwest::{header::CACHE_CONTROL, Client, StatusCode}; use std::borrow::Borrow; use std::fmt::Display; -use std::io::{Read}; +use std::io::Read; use std::path::PathBuf; use url::Url; @@ -159,7 +159,6 @@ impl PackageDb { // Get the information from the first artifact. We assume the metadata is consistent across // all matching artifacts if let Some(artifact_info) = matching_artifacts.next() { - // Retrieve the metadata instead of the entire wheel // If the dist-info is available separately, we can use that instead if artifact_info.dist_info_metadata.available { @@ -205,21 +204,22 @@ impl PackageDb { /// Retrieve the PEP658 metadata for the given artifact. /// This assumes that the metadata is available in the repository /// This can be checked with the ArtifactInfo - async fn get_pep658_metadata<'a, A: MetadataArtifact>(&self, artifact_info: &'a ArtifactInfo) -> miette::Result<(&'a ArtifactInfo, A::Metadata)> { + async fn get_pep658_metadata<'a, A: MetadataArtifact>( + &self, + artifact_info: &'a ArtifactInfo, + ) -> miette::Result<(&'a ArtifactInfo, A::Metadata)> { // Turn into PEP658 compliant URL let mut url = artifact_info.url.clone(); url.set_path(&url.path().replace(".whl", ".whl.metadata")); let mut bytes = Vec::new(); self.http - .request( - url, - Method::GET, - HeaderMap::default(), - CacheMode::Default, - ).await? + .request(url, Method::GET, HeaderMap::default(), CacheMode::Default) + .await? .into_body() - .read_to_end(&mut bytes).await.into_diagnostic()?; + .read_to_end(&mut bytes) + .await + .into_diagnostic()?; let metadata = A::parse_metadata(&bytes)?; self.put_metadata_in_cache(artifact_info, &bytes)?; @@ -340,8 +340,8 @@ async fn fetch_simple_api(http: &Http, url: Url) -> miette::Result Date: Tue, 26 Sep 2023 13:35:45 +0200 Subject: [PATCH 4/4] fix: clippy --- crates/rattler_installs_packages/src/package_database.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/rattler_installs_packages/src/package_database.rs b/crates/rattler_installs_packages/src/package_database.rs index 5a5e44af..c923fee8 100644 --- a/crates/rattler_installs_packages/src/package_database.rs +++ b/crates/rattler_installs_packages/src/package_database.rs @@ -223,7 +223,7 @@ impl PackageDb { let metadata = A::parse_metadata(&bytes)?; self.put_metadata_in_cache(artifact_info, &bytes)?; - return Ok((artifact_info, metadata)); + Ok((artifact_info, metadata)) } /// Get all package names in the index.