Daedalus Rewrite + Code Cleanup (#16)

* [wip] rewrite daedalus, vanilla, fabric, and quilt

* finish forge + neo

* fix docker

* fix neoforge 1.21+

* update concurrency limit

* finish

* remove mac garb

.env

@@ -1,9 +1,15 @@
-RUST_LOG=info
+RUST_LOG=warn,daedalus_client=trace
 
-BASE_URL=https://modrinth-cdn-staging.nyc3.digitaloceanspaces.com
+BASE_URL=http://localhost:9000/meta
+
+CONCURRENCY_LIMIT=10
 
 S3_ACCESS_TOKEN=none
 S3_SECRET=none
-S3_URL=none
-S3_REGION=none
-S3_BUCKET_NAME=none
+S3_URL=http://localhost:9000
+S3_REGION=path-style
+S3_BUCKET_NAME=meta
+
+CLOUDFLARE_INTEGRATION=false
+CLOUDFLARE_TOKEN=none
+CLOUDFLARE_ZONE_ID=none

.github/workflows/run.yml

@@ -0,0 +1,49 @@
+name: Run Meta
+
+on:
+  schedule:
+    - cron: '*/5 * * * *'
+
+jobs:
+  run-docker:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+
+      - name: Pull Docker image from GHCR
+        run: docker pull ghcr.io/modrinth/daedalus:latest
+
+      - name: Run Docker container
+        env:
+          BASE_URL: ${{ secrets.BASE_URL }}
+          S3_ACCESS_TOKEN: ${{ secrets.S3_ACCESS_TOKEN }}
+          S3_SECRET: ${{ secrets.S3_SECRET }}
+          S3_URL: ${{ secrets.S3_URL }}
+          S3_REGION: ${{ secrets.S3_REGION }}
+          S3_BUCKET_NAME: ${{ secrets.S3_BUCKET_NAME }}
+          CLOUDFLARE_INTEGRATION: ${{ secrets.CLOUDFLARE_INTEGRATION }}
+          CLOUDFLARE_TOKEN: ${{ secrets.CLOUDFLARE_TOKEN }}
+          CLOUDFLARE_ZONE_ID: ${{ secrets.CLOUDFLARE_ZONE_ID }}
+        run: |
+          docker run -d \
+            --name daedalus \
+            -e BASE_URL=$BASE_URL \
+            -e S3_ACCESS_TOKEN=$S3_ACCESS_TOKEN \
+            -e S3_SECRET=$S3_SECRET \
+            -e S3_URL=$S3_URL \
+            -e S3_REGION=$S3_REGION \
+            -e S3_BUCKET_NAME=$S3_BUCKET_NAME \
+            -e CLOUDFLARE_INTEGRATION=$CLOUDFLARE_INTEGRATION \
+            -e CLOUDFLARE_TOKEN=$CLOUDFLARE_TOKEN \
+            -e CLOUDFLARE_ZONE_ID=$CLOUDFLARE_ZONE_ID \
+            ghcr.io/modrinth/daedalus:latest

.gitignore

@@ -1,6 +1,7 @@
 ### Intellij ###
 # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
 # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+caches/
 
 # User-specific stuff
 .idea/**/workspace.xml

Dockerfile

@@ -1,4 +1,4 @@
-FROM rust:1.68.2 as build
+FROM rust:1.79.0 as build
 ENV PKG_CONFIG_ALLOW_CROSS=1
 
 WORKDIR /usr/src/daedalus

daedalus/Cargo.toml

@@ -1,8 +1,8 @@
 [package]
 name = "daedalus"
-version = "0.1.27"
-authors = ["Jai A <[email protected]>"]
-edition = "2018"
+version = "0.2.0"
+authors = ["Jai A <[email protected]>"]
+edition = "2021"
 license = "MIT"
 description = "Utilities for querying and parsing Minecraft metadata"
 repository = "https://github.com/modrinth/daedalus/"
@@ -14,12 +14,8 @@ readme = "README.md"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-reqwest = { version = "0.11", features = ["json"] }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 chrono = { version = "0.4", features = ["serde"] }
 bytes = "1"
 thiserror = "1.0"
-tokio = { version = "1", features = ["full"] }
-sha1 = { version = "0.6.1", features = ["std"]}
-bincode = {version = "2.0.0-rc.2", features = ["serde"], optional = true}

daedalus/src/lib.rs

@@ -12,30 +12,6 @@ pub mod modded;
 #[derive(thiserror::Error, Debug)]
 /// An error type representing possible errors when fetching metadata
 pub enum Error {
-    #[error("Failed to validate file checksum at url {url} with hash {hash} after {tries} tries")]
-    /// A checksum was failed to validate for a file
-    ChecksumFailure {
-        /// The checksum's hash
-        hash: String,
-        /// The URL of the file attempted to be downloaded
-        url: String,
-        /// The amount of tries that the file was downloaded until failure
-        tries: u32,
-    },
-    /// There was an error while deserializing metadata
-    #[error("Error while deserializing JSON")]
-    SerdeError(#[from] serde_json::Error),
-    /// There was a network error when fetching an object
-    #[error("Unable to fetch {item}")]
-    FetchError {
-        /// The internal reqwest error
-        inner: reqwest::Error,
-        /// The item that was failed to be fetched
-        item: String,
-    },
-    /// There was an error when managing async tasks
-    #[error("Error while managing asynchronous tasks")]
-    TaskError(#[from] tokio::task::JoinError),
     /// Error while parsing input
     #[error("{0}")]
     ParseError(String),
@@ -124,100 +100,3 @@ pub fn get_path_from_artifact(artifact: &str) -> Result<String, Error> {
         ))
     }
 }
-
-/// Downloads a file from specified mirrors
-pub async fn download_file_mirrors(
-    base: &str,
-    mirrors: &[&str],
-    sha1: Option<&str>,
-) -> Result<bytes::Bytes, Error> {
-    if mirrors.is_empty() {
-        return Err(Error::ParseError("No mirrors provided!".to_string()));
-    }
-
-    for (index, mirror) in mirrors.iter().enumerate() {
-        let result = download_file(&format!("{}{}", mirror, base), sha1).await;
-
-        if result.is_ok() || (result.is_err() && index == (mirrors.len() - 1)) {
-            return result;
-        }
-    }
-
-    unreachable!()
-}
-
-/// Downloads a file with retry and checksum functionality
-pub async fn download_file(
-    url: &str,
-    sha1: Option<&str>,
-) -> Result<bytes::Bytes, Error> {
-    let mut headers = reqwest::header::HeaderMap::new();
-    if let Ok(header) = reqwest::header::HeaderValue::from_str(&format!(
-        "modrinth/daedalus/{} ([email protected])",
-        env!("CARGO_PKG_VERSION")
-    )) {
-        headers.insert(reqwest::header::USER_AGENT, header);
-    }
-    let client = reqwest::Client::builder()
-        .tcp_keepalive(Some(std::time::Duration::from_secs(10)))
-        .timeout(std::time::Duration::from_secs(15))
-        .default_headers(headers)
-        .build()
-        .map_err(|err| Error::FetchError {
-            inner: err,
-            item: url.to_string(),
-        })?;
-
-    for attempt in 1..=4 {
-        let result = client.get(url).send().await;
-
-        match result {
-            Ok(x) => {
-                let bytes = x.bytes().await;
-
-                if let Ok(bytes) = bytes {
-                    if let Some(sha1) = sha1 {
-                        if &*get_hash(bytes.clone()).await? != sha1 {
-                            if attempt <= 3 {
-                                continue;
-                            } else {
-                                return Err(Error::ChecksumFailure {
-                                    hash: sha1.to_string(),
-                                    url: url.to_string(),
-                                    tries: attempt,
-                                });
-                            }
-                        }
-                    }
-
-                    return Ok(bytes);
-                } else if attempt <= 3 {
-                    continue;
-                } else if let Err(err) = bytes {
-                    return Err(Error::FetchError {
-                        inner: err,
-                        item: url.to_string(),
-                    });
-                }
-            }
-            Err(_) if attempt <= 3 => continue,
-            Err(err) => {
-                return Err(Error::FetchError {
-                    inner: err,
-                    item: url.to_string(),
-                })
-            }
-        }
-    }
-
-    unreachable!()
-}
-
-/// Computes a checksum of the input bytes
-pub async fn get_hash(bytes: bytes::Bytes) -> Result<String, Error> {
-    let hash =
-        tokio::task::spawn_blocking(|| sha1::Sha1::from(bytes).hexdigest())
-            .await?;
-
-    Ok(hash)
-}