From fae7afab469a384178fa989accfb1f00c55e34b6 Mon Sep 17 00:00:00 2001 From: Bas Zalmstra Date: Tue, 26 Sep 2023 10:34:17 +0200 Subject: [PATCH 1/8] fix: clean up docs a little bit --- .../rattler_installs_packages/benches/html.rs | 4 +--- crates/rattler_installs_packages/src/html.rs | 5 +---- crates/rattler_installs_packages/src/lib.rs | 17 +++++++++++------ .../src/package_database.rs | 4 ++-- crates/rip/src/main.rs | 2 +- crates/rip/src/pypi_provider.rs | 4 ++-- rust-toolchain | 1 + 7 files changed, 19 insertions(+), 18 deletions(-) create mode 100644 rust-toolchain diff --git a/crates/rattler_installs_packages/benches/html.rs b/crates/rattler_installs_packages/benches/html.rs index a8d3eb8b..fdca1120 100644 --- a/crates/rattler_installs_packages/benches/html.rs +++ b/crates/rattler_installs_packages/benches/html.rs @@ -1,7 +1,5 @@ use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; -use rattler_installs_packages::html::{ - parse_package_names_html, parse_project_info_html, -}; +use rattler_installs_packages::html::{parse_package_names_html, parse_project_info_html}; use std::str::FromStr; use url::Url; diff --git a/crates/rattler_installs_packages/src/html.rs b/crates/rattler_installs_packages/src/html.rs index 6d05acd3..0ae51a56 100644 --- a/crates/rattler_installs_packages/src/html.rs +++ b/crates/rattler_installs_packages/src/html.rs @@ -11,7 +11,6 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. - use std::{borrow::Borrow, default::Default}; use crate::{ArtifactHashes, ArtifactName}; @@ -42,9 +41,7 @@ pub fn into_artifact_info(base: &Url, tag: &HTMLTag) -> Option { // Join with base let url = base.join(href.as_ref()).ok()?; let filename = url.path_segments().and_then(|mut s| s.next_back()); - let filename: ArtifactName = filename - .map(|s| s.parse())? - .ok()?; + let filename: ArtifactName = filename.map(|s| s.parse())?.ok()?; // We found a valid link let hash = url.fragment().and_then(parse_hash); diff --git a/crates/rattler_installs_packages/src/lib.rs b/crates/rattler_installs_packages/src/lib.rs index c7ad6d24..dcbaa8e1 100644 --- a/crates/rattler_installs_packages/src/lib.rs +++ b/crates/rattler_installs_packages/src/lib.rs @@ -1,6 +1,6 @@ mod artifact; -pub mod artifact_name; -pub mod core_metadata; +mod artifact_name; +mod core_metadata; mod extra; mod file_store; pub mod html; @@ -9,7 +9,7 @@ mod package_database; mod package_name; mod project_info; mod reqparse; -pub mod requirement; +mod requirement; mod rfc822ish; mod seek_slice; mod specifier; @@ -19,10 +19,15 @@ pub use file_store::{CacheKey, FileStore}; pub use package_database::PackageDb; pub use artifact::{Artifact, MetadataArtifact, Wheel}; -pub use artifact_name::ArtifactName; +pub use artifact_name::{ + ArtifactName, BuildTag, InnerAsArtifactName, ParseArtifactNameError, SDistFormat, SDistName, + WheelName, +}; pub use extra::Extra; pub use package_name::{NormalizedPackageName, PackageName, ParsePackageNameError}; pub use pep440::Version; -pub use project_info::{ArtifactHashes, ArtifactInfo, DistInfoMetadata, Meta, Yanked}; -pub use requirement::PackageRequirement; +pub use project_info::{ArtifactHashes, ArtifactInfo, DistInfoMetadata, Meta, ProjectInfo, Yanked}; +pub use requirement::{ + marker, PackageRequirement, ParseExtra, PythonRequirement, Requirement, UserRequirement, +}; pub use specifier::{CompareOp, Specifier, Specifiers}; diff --git a/crates/rattler_installs_packages/src/package_database.rs b/crates/rattler_installs_packages/src/package_database.rs index d5521947..dbfa2967 100644 --- a/crates/rattler_installs_packages/src/package_database.rs +++ b/crates/rattler_installs_packages/src/package_database.rs @@ -19,7 +19,7 @@ use pep440::Version; use reqwest::{header::CACHE_CONTROL, Client, StatusCode}; use std::borrow::Borrow; use std::fmt::Display; -use std::io::{Read}; +use std::io::Read; use std::path::PathBuf; use url::Url; @@ -311,8 +311,8 @@ async fn fetch_simple_api(http: &Http, url: Url) -> miette::Result Date: Tue, 26 Sep 2023 10:48:08 +0200 Subject: [PATCH 2/8] fix: index to seperate crate and ci issues --- .github/workflows/rust-compile.yml | 3 -- Cargo.lock | 20 ++++++++++++ crates/index/Cargo.toml | 32 +++++++++++++++++++ .../src/bin/index.rs => index/src/main.rs} | 6 ++-- crates/rattler_installs_packages/Cargo.toml | 9 +++--- crates/rip/Cargo.toml | 19 ++++------- 6 files changed, 66 insertions(+), 23 deletions(-) create mode 100644 crates/index/Cargo.toml rename crates/{rip/src/bin/index.rs => index/src/main.rs} (99%) diff --git a/.github/workflows/rust-compile.yml b/.github/workflows/rust-compile.yml index 5e46cdf8..1ad11f08 100644 --- a/.github/workflows/rust-compile.yml +++ b/.github/workflows/rust-compile.yml @@ -14,7 +14,6 @@ env: RUST_BACKTRACE: 1 RUSTFLAGS: "-D warnings" CARGO_TERM_COLOR: always - DEFAULT_FEATURES: jobs: check-rustdoc-links: @@ -105,7 +104,6 @@ jobs: run: > cargo build --all-targets - --features ${{ env.DEFAULT_FEATURES }} ${{ steps.build-options.outputs.CARGO_BUILD_OPTIONS}} - name: Disable testing the tools crate if cross compiling @@ -119,7 +117,6 @@ jobs: run: > cargo test --workspace - --features ${{ env.DEFAULT_FEATURES }} ${{ steps.build-options.outputs.CARGO_BUILD_OPTIONS}} ${{ steps.test-options.outputs.CARGO_TEST_OPTIONS}} -- diff --git a/Cargo.lock b/Cargo.lock index 429f23da..b963bdcc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1104,6 +1104,26 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "index" +version = "0.1.0" +dependencies = [ + "clap 4.4.4", + "dirs", + "indexmap 2.0.0", + "indicatif", + "miette", + "rand", + "rattler_installs_packages", + "rip", + "rusqlite", + "serde_json", + "tokio", + "tracing", + "tracing-subscriber", + "url", +] + [[package]] name = "indexmap" version = "1.9.3" diff --git a/crates/index/Cargo.toml b/crates/index/Cargo.toml new file mode 100644 index 00000000..1385743a --- /dev/null +++ b/crates/index/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "index" +authors = ["Tim de Jager "] +version.workspace = true +categories.workspace = true +homepage.workspace = true +repository.workspace = true +license.workspace = true +edition.workspace = true +readme.workspace = true +rust-version.workspace = true + +[dependencies] +clap = { version = "4.3.23", features = ["derive"] } +dirs = "5.0.1" +indexmap = "2.0.0" +indicatif = "0.17.6" +miette = { version = "5.10.0", features = ["fancy"] } +rand = "0.8.4" +rattler_installs_packages = { path = "../rattler_installs_packages" } +rip = { path = "../rip" } +rusqlite = { version = "0.29.0", features = ["bundled"] } +serde_json = "1.0.107" +tokio = { version = "1.29.1", features = ["rt", "macros", "rt-multi-thread"] } +tracing = "0.1.37" +tracing-subscriber = { version = "0.3.17", features = ["env-filter"] } +url = "2.4.0" + + +[package.metadata.release] +# Dont publish the binary +release = false diff --git a/crates/rip/src/bin/index.rs b/crates/index/src/main.rs similarity index 99% rename from crates/rip/src/bin/index.rs rename to crates/index/src/main.rs index de252713..ba630f68 100644 --- a/crates/rip/src/bin/index.rs +++ b/crates/index/src/main.rs @@ -51,7 +51,7 @@ pub async fn index(index_url: Url) -> Result<(), miette::Error> { &[normalize_index_url(index_url)], cache_dir.clone(), ) - .into_diagnostic()?; + .into_diagnostic()?; let mut names = package_db.get_package_names().await?; names.shuffle(&mut rand::thread_rng()); @@ -65,7 +65,7 @@ pub async fn index(index_url: Url) -> Result<(), miette::Error> { "CREATE UNIQUE INDEX IF NOT EXISTS idx_name_version ON `metadata` (`name`, `version`)", (), ) - .into_diagnostic()?; + .into_diagnostic()?; let mut insert_stmt = conn.prepare("INSERT INTO metadata (name, version, requires_dist, requires_python, extras) VALUES (?, ?, ?, ?, ?)").into_diagnostic()?; let mut request = conn @@ -180,7 +180,7 @@ pub fn query_extras() -> Result<(), miette::Error> { let requires_dist = serde_json::from_str::>( requirement.into_diagnostic()?.as_str(), ) - .into_diagnostic()?; + .into_diagnostic()?; total += requires_dist.len(); for req in requires_dist { if !req.extras.is_empty() { diff --git a/crates/rattler_installs_packages/Cargo.toml b/crates/rattler_installs_packages/Cargo.toml index f779fd81..35da225d 100644 --- a/crates/rattler_installs_packages/Cargo.toml +++ b/crates/rattler_installs_packages/Cargo.toml @@ -38,20 +38,19 @@ serde_with = "3.0.0" smallvec = { version = "1.11.0", features = ["const_generics", "const_new"] } tempfile = "3.6.0" thiserror = "1.0.43" +tl = "0.7.7" tokio = { version = "1.29.1" } tokio-util = { version = "0.7.8", features = ["compat"] } tracing = { version = "0.1.37", default-features = false, features = ["attributes"] } url = { version = "2.4.0", features = ["serde"] } zip = "0.6.6" -tl = "0.7.7" - [dev-dependencies] -tokio = { version = "1.29.1", features = ["rt", "macros"] } -miette = { version = "5.9.0", features = ["fancy"] } +criterion = "0.3" insta = { version = "1.30.0", features = ["ron"] } +miette = { version = "5.9.0", features = ["fancy"] } once_cell = "1.18.0" -criterion = "0.3" +tokio = { version = "1.29.1", features = ["rt", "macros"] } [[bench]] name = "html" diff --git a/crates/rip/Cargo.toml b/crates/rip/Cargo.toml index 7882108a..5386129c 100644 --- a/crates/rip/Cargo.toml +++ b/crates/rip/Cargo.toml @@ -11,11 +11,6 @@ license.workspace = true readme.workspace = true default-run = "rip" -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[[bin]] -name = "index" - [dependencies] clap = { version = "4.3.23", features = ["derive"] } console = { version = "0.15.7", features = ["windows-console-colors"] } @@ -24,19 +19,19 @@ indexmap = "2.0.0" indicatif = "0.17.6" itertools = "0.11.0" miette = { version = "5.10.0", features = ["fancy"] } +rand = "0.8.4" rattler_installs_packages = { path = "../rattler_installs_packages" } -resolvo = "0.1.0" reqwest = "0.11.19" +resolvo = "0.1.0" +rusqlite = { version = "0.29.0", features = ["bundled"] } +serde = "1.0.188" +serde_json = "1.0.107" +serde_with = "3.0.0" tabwriter = { version = "1.2.1", features = ["ansi_formatting"] } tokio = { version = "1.29.1", features = ["rt", "macros", "rt-multi-thread"] } tracing = "0.1.37" -tracing-subscriber = { version = "0.3.17", features = ["env-filter"]} +tracing-subscriber = { version = "0.3.17", features = ["env-filter"] } url = "2.4.0" -rand = "0.8.4" -serde = "1.0.188" -serde_with = "3.0.0" -serde_json = "1.0.107" -rusqlite = { version = "0.29.0", features = ["bundled"] } [package.metadata.release] # Dont publish the binary From 4cfd77483869573d8d2319cd069fc8849a0b5dde Mon Sep 17 00:00:00 2001 From: Bas Zalmstra Date: Tue, 26 Sep 2023 10:52:55 +0200 Subject: [PATCH 3/8] bump: upgrade all dependencies --- Cargo.lock | 16 ++++----- crates/index/Cargo.toml | 10 +++--- crates/rattler_installs_packages/Cargo.toml | 38 ++++++++++----------- crates/rip/Cargo.toml | 16 ++++----- 4 files changed, 40 insertions(+), 40 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b963bdcc..863b51d4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -353,9 +353,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.4" +version = "4.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1d7b8d5ec32af0fadc644bf1fd509a688c2103b185644bb1e29d164e0703136" +checksum = "824956d0dca8334758a5b7f7e50518d66ea319330cbceedcf76905c2f6ab30e3" dependencies = [ "clap_builder", "clap_derive", @@ -363,9 +363,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.4" +version = "4.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5179bb514e4d7c2051749d8fcefa2ed6d06a9f4e6d69faf3805f5d80b8cf8d56" +checksum = "122ec64120a49b4563ccaedcbea7818d069ed8e9aa6d829b82d8a4128936b2ab" dependencies = [ "anstream", "anstyle", @@ -1108,7 +1108,7 @@ dependencies = [ name = "index" version = "0.1.0" dependencies = [ - "clap 4.4.4", + "clap 4.4.5", "dirs", "indexmap 2.0.0", "indicatif", @@ -1749,9 +1749,9 @@ dependencies = [ [[package]] name = "rattler_digest" -version = "0.6.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "747c978f52c6a6509a6b06b60486436057f9aeb3c11ed47fd0b5c1e07bfcf783" +checksum = "881d6040b07c00070386bc7d98f21b71474a0b7847fa5b32a7c5a0a01d028f8f" dependencies = [ "blake2", "digest", @@ -1951,7 +1951,7 @@ dependencies = [ name = "rip" version = "0.1.0" dependencies = [ - "clap 4.4.4", + "clap 4.4.5", "console", "dirs", "indexmap 2.0.0", diff --git a/crates/index/Cargo.toml b/crates/index/Cargo.toml index 1385743a..359c578a 100644 --- a/crates/index/Cargo.toml +++ b/crates/index/Cargo.toml @@ -11,20 +11,20 @@ readme.workspace = true rust-version.workspace = true [dependencies] -clap = { version = "4.3.23", features = ["derive"] } +clap = { version = "4.4.5", features = ["derive"] } dirs = "5.0.1" indexmap = "2.0.0" -indicatif = "0.17.6" +indicatif = "0.17.7" miette = { version = "5.10.0", features = ["fancy"] } -rand = "0.8.4" +rand = "0.8.5" rattler_installs_packages = { path = "../rattler_installs_packages" } rip = { path = "../rip" } rusqlite = { version = "0.29.0", features = ["bundled"] } serde_json = "1.0.107" -tokio = { version = "1.29.1", features = ["rt", "macros", "rt-multi-thread"] } +tokio = { version = "1.32.0", features = ["rt", "macros", "rt-multi-thread"] } tracing = "0.1.37" tracing-subscriber = { version = "0.3.17", features = ["env-filter"] } -url = "2.4.0" +url = "2.4.1" [package.metadata.release] diff --git a/crates/rattler_installs_packages/Cargo.toml b/crates/rattler_installs_packages/Cargo.toml index 35da225d..b39c6331 100644 --- a/crates/rattler_installs_packages/Cargo.toml +++ b/crates/rattler_installs_packages/Cargo.toml @@ -12,45 +12,45 @@ readme.workspace = true rust-version.workspace = true [dependencies] -async-trait = "0.1.71" -bytes = "1.4.0" +async-trait = "0.1.73" +bytes = "1.5.0" ciborium = "0.2.1" data-encoding = "2.4.0" -elsa = "1.8.1" +elsa = "1.9.0" fs4 = "0.6.6" futures = "0.3.28" http = "0.2.9" http-cache-semantics = { version = "1.0.1", default-features = false, features = ["with_serde", "reqwest"] } indexmap = "2.0.0" -miette = "5.9.0" +miette = "5.10.0" mime = "0.3.17" once_cell = "1.18.0" parking_lot = "0.12.1" peg = "0.8.1" pep440 = "0.2.0" -pin-project-lite = "0.2.10" -rattler_digest = { version = "0.6.0", features = ["serde"] } -regex = "1.9.1" -reqwest = { version = "0.11.18", features = ["json", "stream"] } +pin-project-lite = "0.2.13" +rattler_digest = { version = "0.9.0", features = ["serde"] } +regex = "1.9.5" +reqwest = { version = "0.11.20", features = ["json", "stream"] } serde = "1.0.188" -serde_json = "1.0.100" -serde_with = "3.0.0" -smallvec = { version = "1.11.0", features = ["const_generics", "const_new"] } -tempfile = "3.6.0" -thiserror = "1.0.43" +serde_json = "1.0.107" +serde_with = "3.3.0" +smallvec = { version = "1.11.1", features = ["const_generics", "const_new"] } +tempfile = "3.8.0" +thiserror = "1.0.48" tl = "0.7.7" -tokio = { version = "1.29.1" } -tokio-util = { version = "0.7.8", features = ["compat"] } +tokio = { version = "1.32.0" } +tokio-util = { version = "0.7.9", features = ["compat"] } tracing = { version = "0.1.37", default-features = false, features = ["attributes"] } -url = { version = "2.4.0", features = ["serde"] } +url = { version = "2.4.1", features = ["serde"] } zip = "0.6.6" [dev-dependencies] criterion = "0.3" -insta = { version = "1.30.0", features = ["ron"] } -miette = { version = "5.9.0", features = ["fancy"] } +insta = { version = "1.32.0", features = ["ron"] } +miette = { version = "5.10.0", features = ["fancy"] } once_cell = "1.18.0" -tokio = { version = "1.29.1", features = ["rt", "macros"] } +tokio = { version = "1.32.0", features = ["rt", "macros"] } [[bench]] name = "html" diff --git a/crates/rip/Cargo.toml b/crates/rip/Cargo.toml index 5386129c..fca16c5d 100644 --- a/crates/rip/Cargo.toml +++ b/crates/rip/Cargo.toml @@ -12,26 +12,26 @@ readme.workspace = true default-run = "rip" [dependencies] -clap = { version = "4.3.23", features = ["derive"] } +clap = { version = "4.4.5", features = ["derive"] } console = { version = "0.15.7", features = ["windows-console-colors"] } dirs = "5.0.1" indexmap = "2.0.0" -indicatif = "0.17.6" +indicatif = "0.17.7" itertools = "0.11.0" miette = { version = "5.10.0", features = ["fancy"] } -rand = "0.8.4" +rand = "0.8.5" rattler_installs_packages = { path = "../rattler_installs_packages" } -reqwest = "0.11.19" +reqwest = "0.11.20" resolvo = "0.1.0" rusqlite = { version = "0.29.0", features = ["bundled"] } serde = "1.0.188" serde_json = "1.0.107" -serde_with = "3.0.0" -tabwriter = { version = "1.2.1", features = ["ansi_formatting"] } -tokio = { version = "1.29.1", features = ["rt", "macros", "rt-multi-thread"] } +serde_with = "3.3.0" +tabwriter = { version = "1.3.0", features = ["ansi_formatting"] } +tokio = { version = "1.32.0", features = ["rt", "macros", "rt-multi-thread"] } tracing = "0.1.37" tracing-subscriber = { version = "0.3.17", features = ["env-filter"] } -url = "2.4.0" +url = "2.4.1" [package.metadata.release] # Dont publish the binary From b260aef45aae5916918d94a4335b5f86d2a1ff10 Mon Sep 17 00:00:00 2001 From: Bas Zalmstra Date: Tue, 26 Sep 2023 10:56:34 +0200 Subject: [PATCH 4/8] fix: formatting --- crates/index/src/main.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/index/src/main.rs b/crates/index/src/main.rs index ba630f68..de252713 100644 --- a/crates/index/src/main.rs +++ b/crates/index/src/main.rs @@ -51,7 +51,7 @@ pub async fn index(index_url: Url) -> Result<(), miette::Error> { &[normalize_index_url(index_url)], cache_dir.clone(), ) - .into_diagnostic()?; + .into_diagnostic()?; let mut names = package_db.get_package_names().await?; names.shuffle(&mut rand::thread_rng()); @@ -65,7 +65,7 @@ pub async fn index(index_url: Url) -> Result<(), miette::Error> { "CREATE UNIQUE INDEX IF NOT EXISTS idx_name_version ON `metadata` (`name`, `version`)", (), ) - .into_diagnostic()?; + .into_diagnostic()?; let mut insert_stmt = conn.prepare("INSERT INTO metadata (name, version, requires_dist, requires_python, extras) VALUES (?, ?, ?, ?, ?)").into_diagnostic()?; let mut request = conn @@ -180,7 +180,7 @@ pub fn query_extras() -> Result<(), miette::Error> { let requires_dist = serde_json::from_str::>( requirement.into_diagnostic()?.as_str(), ) - .into_diagnostic()?; + .into_diagnostic()?; total += requires_dist.len(); for req in requires_dist { if !req.extras.is_empty() { From 6ee3bec1e7d0c3cb6df494b40312f1ef4c47b81a Mon Sep 17 00:00:00 2001 From: Bas Zalmstra Date: Tue, 26 Sep 2023 10:57:09 +0200 Subject: [PATCH 5/8] fix: doc hyperlink --- crates/index/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/index/src/main.rs b/crates/index/src/main.rs index de252713..265c9774 100644 --- a/crates/index/src/main.rs +++ b/crates/index/src/main.rs @@ -20,7 +20,7 @@ use rip::writer::{global_multi_progress, IndicatifWriter}; #[derive(Parser)] #[command(author, version, about, long_about = None)] struct Args { - /// Base URL of the Python Package Index (default https://pypi.org/simple). This should point + /// Base URL of the Python Package Index (default ). This should point /// to a repository compliant with PEP 503 (the simple repository API). #[clap(default_value = "https://pypi.org/simple/", long)] index_url: Url, From 2a9705ad06d44e4a140116a967c91e7588c820be Mon Sep 17 00:00:00 2001 From: Bas Zalmstra Date: Tue, 26 Sep 2023 11:04:22 +0200 Subject: [PATCH 6/8] feat: use rusttls feature --- crates/rattler_installs_packages/Cargo.toml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/rattler_installs_packages/Cargo.toml b/crates/rattler_installs_packages/Cargo.toml index b39c6331..bb54abfe 100644 --- a/crates/rattler_installs_packages/Cargo.toml +++ b/crates/rattler_installs_packages/Cargo.toml @@ -11,6 +11,11 @@ license.workspace = true readme.workspace = true rust-version.workspace = true +[features] +default = ["native-tls"] +native-tls = ['reqwest/native-tls'] +rustls-tls = ['reqwest/rustls-tls'] + [dependencies] async-trait = "0.1.73" bytes = "1.5.0" @@ -31,7 +36,7 @@ pep440 = "0.2.0" pin-project-lite = "0.2.13" rattler_digest = { version = "0.9.0", features = ["serde"] } regex = "1.9.5" -reqwest = { version = "0.11.20", features = ["json", "stream"] } +reqwest = { version = "0.11.20", default-features = false, features = ["json", "stream"] } serde = "1.0.188" serde_json = "1.0.107" serde_with = "3.3.0" From 3aa565523ae7178626f6387c0028cc7bb0a717fa Mon Sep 17 00:00:00 2001 From: Bas Zalmstra Date: Tue, 26 Sep 2023 14:01:08 +0200 Subject: [PATCH 7/8] fix: rustls-tls feature --- Cargo.lock | 103 ++++++++++++++++++++++++++++++++++++++++ crates/index/Cargo.toml | 9 +++- crates/rip/Cargo.toml | 9 +++- 3 files changed, 117 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 863b51d4..7a8847ff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1052,6 +1052,20 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d78e1e73ec14cf7375674f74d7dde185c8206fd9dea6fb6295e8a98098aaa97" +dependencies = [ + "futures-util", + "http", + "hyper", + "rustls", + "tokio", + "tokio-rustls", +] + [[package]] name = "hyper-tls" version = "0.5.0" @@ -1910,6 +1924,7 @@ dependencies = [ "http", "http-body", "hyper", + "hyper-rustls", "hyper-tls", "ipnet", "js-sys", @@ -1919,11 +1934,14 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", + "rustls", + "rustls-pemfile", "serde", "serde_json", "serde_urlencoded", "tokio", "tokio-native-tls", + "tokio-rustls", "tokio-util", "tower-service", "url", @@ -1931,6 +1949,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", + "webpki-roots", "winreg", ] @@ -1947,6 +1966,21 @@ dependencies = [ "tracing", ] +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin", + "untrusted", + "web-sys", + "winapi", +] + [[package]] name = "rip" version = "0.1.0" @@ -2017,6 +2051,37 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "rustls" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8" +dependencies = [ + "log", + "ring", + "rustls-webpki", + "sct", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" +dependencies = [ + "base64 0.21.4", +] + +[[package]] +name = "rustls-webpki" +version = "0.101.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c7d5dece342910d9ba34d259310cae3e0154b873b35408b787b59bce53d34fe" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "ryu" version = "1.0.15" @@ -2047,6 +2112,16 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "sct" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "security-framework" version = "2.9.2" @@ -2230,6 +2305,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + [[package]] name = "stable_deref_trait" version = "1.2.0" @@ -2472,6 +2553,16 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.9" @@ -2600,6 +2691,12 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + [[package]] name = "url" version = "2.4.1" @@ -2750,6 +2847,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-roots" +version = "0.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14247bb57be4f377dfb94c72830b8ce8fc6beac03cf4bf7b9732eadd414123fc" + [[package]] name = "winapi" version = "0.3.9" diff --git a/crates/index/Cargo.toml b/crates/index/Cargo.toml index 359c578a..3eab6855 100644 --- a/crates/index/Cargo.toml +++ b/crates/index/Cargo.toml @@ -10,6 +10,11 @@ edition.workspace = true readme.workspace = true rust-version.workspace = true +[features] +default = ["native-tls"] +native-tls = ['rattler_installs_packages/native-tls', 'rip/native-tls'] +rustls-tls = ['rattler_installs_packages/rustls-tls', 'rip/rustls-tls'] + [dependencies] clap = { version = "4.4.5", features = ["derive"] } dirs = "5.0.1" @@ -17,8 +22,8 @@ indexmap = "2.0.0" indicatif = "0.17.7" miette = { version = "5.10.0", features = ["fancy"] } rand = "0.8.5" -rattler_installs_packages = { path = "../rattler_installs_packages" } -rip = { path = "../rip" } +rattler_installs_packages = { path = "../rattler_installs_packages", default-features = false } +rip = { path = "../rip", default-features = false } rusqlite = { version = "0.29.0", features = ["bundled"] } serde_json = "1.0.107" tokio = { version = "1.32.0", features = ["rt", "macros", "rt-multi-thread"] } diff --git a/crates/rip/Cargo.toml b/crates/rip/Cargo.toml index fca16c5d..f76e2614 100644 --- a/crates/rip/Cargo.toml +++ b/crates/rip/Cargo.toml @@ -11,6 +11,11 @@ license.workspace = true readme.workspace = true default-run = "rip" +[features] +default = ["native-tls"] +native-tls = ['reqwest/native-tls', 'rattler_installs_packages/native-tls'] +rustls-tls = ['reqwest/rustls-tls', 'rattler_installs_packages/rustls-tls'] + [dependencies] clap = { version = "4.4.5", features = ["derive"] } console = { version = "0.15.7", features = ["windows-console-colors"] } @@ -20,8 +25,8 @@ indicatif = "0.17.7" itertools = "0.11.0" miette = { version = "5.10.0", features = ["fancy"] } rand = "0.8.5" -rattler_installs_packages = { path = "../rattler_installs_packages" } -reqwest = "0.11.20" +rattler_installs_packages = { path = "../rattler_installs_packages", default-features = false } +reqwest = { version = "0.11.20", default-features = false } resolvo = "0.1.0" rusqlite = { version = "0.29.0", features = ["bundled"] } serde = "1.0.188" From 8faecd106cf3df5d74c505ca702ae854f58f16ab Mon Sep 17 00:00:00 2001 From: Tim de Jager Date: Tue, 26 Sep 2023 14:29:31 +0200 Subject: [PATCH 8/8] Feat/pep658 (#15) Adds support for [PEP658](https://peps.python.org/pep-0658/) metadata files and downloads these instead of entire wheel if possible. Added small test that tests these with numpy which definitely supports them in the latest versions. --- .../src/package_database.rs | 63 ++++++++++++++++++- 1 file changed, 61 insertions(+), 2 deletions(-) diff --git a/crates/rattler_installs_packages/src/package_database.rs b/crates/rattler_installs_packages/src/package_database.rs index dbfa2967..c923fee8 100644 --- a/crates/rattler_installs_packages/src/package_database.rs +++ b/crates/rattler_installs_packages/src/package_database.rs @@ -156,11 +156,15 @@ impl PackageDb { // We have exhausted all options to read the metadata from the cache. We'll have to hit the // network to get to the information. - // TODO: PEP 658 support - // Get the information from the first artifact. We assume the metadata is consistent across // all matching artifacts if let Some(artifact_info) = matching_artifacts.next() { + // Retrieve the metadata instead of the entire wheel + // If the dist-info is available separately, we can use that instead + if artifact_info.dist_info_metadata.available { + return self.get_pep658_metadata::(artifact_info).await; + } + let body = self .http .request( @@ -197,6 +201,31 @@ impl PackageDb { ); } + /// Retrieve the PEP658 metadata for the given artifact. + /// This assumes that the metadata is available in the repository + /// This can be checked with the ArtifactInfo + async fn get_pep658_metadata<'a, A: MetadataArtifact>( + &self, + artifact_info: &'a ArtifactInfo, + ) -> miette::Result<(&'a ArtifactInfo, A::Metadata)> { + // Turn into PEP658 compliant URL + let mut url = artifact_info.url.clone(); + url.set_path(&url.path().replace(".whl", ".whl.metadata")); + + let mut bytes = Vec::new(); + self.http + .request(url, Method::GET, HeaderMap::default(), CacheMode::Default) + .await? + .into_body() + .read_to_end(&mut bytes) + .await + .into_diagnostic()?; + + let metadata = A::parse_metadata(&bytes)?; + self.put_metadata_in_cache(artifact_info, &bytes)?; + Ok((artifact_info, metadata)) + } + /// Get all package names in the index. pub async fn get_package_names(&self) -> miette::Result> { let index_url = self.index_urls.first(); @@ -341,6 +370,36 @@ mod test { .await .unwrap(); } + + #[tokio::test] + async fn test_pep658() { + let cache_dir = TempDir::new().unwrap(); + let package_db = PackageDb::new( + Client::new(), + &[Url::parse("https://pypi.org/simple/").unwrap()], + cache_dir.path().into(), + ) + .unwrap(); + + // Get all the artifacts + let artifacts = package_db + .available_artifacts("numpy".parse::().unwrap()) + .await + .unwrap(); + + // Get the artifact with dist-info attribute + let artifact_info = artifacts + .iter() + .flat_map(|(_, artifacts)| artifacts.iter()) + // This signifies that a PEP658 metadata file is available + .find(|a| a.dist_info_metadata.available) + .unwrap(); + + let (_artifact, _metadata) = package_db + .get_pep658_metadata::(&artifact_info) + .await + .unwrap(); + } } #[derive(Debug, Diagnostic)]