wip: indexing program to query pypi stuff

.gitignore

@@ -1,2 +1,3 @@
 /target
 .idea/
+*.sqlite3

crates/rattler_installs_packages/src/html.rs

@@ -236,8 +236,7 @@ where
 #[tracing::instrument(level = "debug", skip(body))]
 pub fn parse_package_names_html(body: &str) -> miette::Result<Vec<String>> {
     let dom = tl::parse(body, tl::ParserOptions::default()).into_diagnostic()?;
-    let names = dom
-        .query_selector("a");
+    let names = dom.query_selector("a");
 
     if let Some(names) = names {
         let names = names
@@ -381,8 +380,7 @@ mod test {
    </html>
         "#;
 
-        let names =
-            parse_package_names_html(&html).unwrap();
+        let names = parse_package_names_html(&html).unwrap();
         insta::assert_ron_snapshot!(names, @r###"
         [
           "0",

crates/rattler_installs_packages/src/http.rs

@@ -273,7 +273,6 @@ fn fill_cache<R: Read>(
         &CacheData {
             policy: policy.clone(),
             url: url.clone(),
-
         },
         &mut cache_writer,
     )

crates/rattler_installs_packages/src/lib.rs

@@ -20,6 +20,7 @@ pub use package_database::PackageDb;
 
 pub use artifact::{Artifact, MetadataArtifact, Wheel};
 pub use artifact_name::ArtifactName;
+pub use extra::Extra;
 pub use package_name::{NormalizedPackageName, PackageName, ParsePackageNameError};
 pub use pep440::Version;
 pub use project_info::{ArtifactHashes, ArtifactInfo, DistInfoMetadata, Meta, Yanked};

crates/rattler_installs_packages/src/package_database.rs

@@ -1,10 +1,18 @@
 use crate::artifact::MetadataArtifact;
 use crate::html::parse_project_info_html;
 use crate::http::HttpRequestError;
-use crate::{artifact::Artifact, artifact_name::InnerAsArtifactName, http::{CacheMode, Http}, package_name::PackageName, project_info::{ArtifactInfo, ProjectInfo}, FileStore, html};
+use crate::{
+    artifact::Artifact,
+    artifact_name::InnerAsArtifactName,
+    html,
+    http::{CacheMode, Http},
+    package_name::PackageName,
+    project_info::{ArtifactInfo, ProjectInfo},
+    FileStore,
+};
 use elsa::FrozenMap;
 use futures::{pin_mut, stream, StreamExt};
-use http::header::{CONTENT_TYPE, IF_UNMODIFIED_SINCE};
+use http::header::{CONTENT_TYPE};
 use http::{HeaderMap, HeaderValue, Method};
 use indexmap::IndexMap;
 use miette::{self, Diagnostic, IntoDiagnostic};
@@ -193,12 +201,15 @@ impl PackageDb {
     pub async fn get_package_names(&self) -> miette::Result<Vec<String>> {
         let index_url = self.index_urls.first();
         if let Some(url) = index_url {
-            let response = self.http.request(
-                url.clone(),
-                Method::GET,
-                HeaderMap::default(),
-                CacheMode::Default
-            ).await?;
+            let response = self
+                .http
+                .request(
+                    url.clone(),
+                    Method::GET,
+                    HeaderMap::default(),
+                    CacheMode::Default,
+                )
+                .await?;
 
             let mut bytes = response.into_body().force_local().await.into_diagnostic()?;
             let mut source = String::new();
@@ -207,7 +218,6 @@ impl PackageDb {
         } else {
             Ok(vec![])
         }
-
     }
 
     /// Opens the specified artifact info. Depending on the specified `cache_mode`, downloads the
@@ -328,7 +338,6 @@ mod test {
             .await
             .unwrap();
 
-        dbg!(metadata);
     }
 }
 

crates/rip/Cargo.toml

@@ -7,8 +7,8 @@ default-run = "rip"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [[bin]]
-name = "extras"
-path = "src/extras.rs"
+name = "index"
+
 
 
 [dependencies]
@@ -27,3 +27,8 @@ tokio = { version = "1.29.1", features = ["rt", "macros", "rt-multi-thread"] }
 tracing = "0.1.37"
 tracing-subscriber = { version = "0.3.17", features = ["env-filter"]}
 url = "2.4.0"
+rand = "0.8.4"
+serde = "1.0.188"
+serde_with = "3.0.0"
+serde_json = "1.0.107"
+rusqlite = { version = "0.29.0", features = ["bundled"] }