Check for repository existence

trailofbits · Nov 28, 2023 · 8be99dc · 8be99dc
1 parent 16e6ba8
commit 8be99dc
Show file tree

Hide file tree

Showing 10 changed files with 96 additions and 37 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -12,6 +12,7 @@ anyhow = { version = "1.0", features = ["backtrace"] }
 cargo_metadata = "0.18"
 clap = { version = "4.4", features = ["cargo", "derive", "wrap_help"] }
 crates-index = { version = "2.3", features = ["git-https"] }
+curl = "0.4"
 env_logger = "0.10"
 home = "0.5"
 log = "0.4"

diff --git a/README.md b/README.md
@@ -4,11 +4,13 @@
 
 `cargo-unmaintained` is similar to [`cargo-audit`]. However, `cargo-unmaintained` finds unmaintained packages automatically using heuristics, rather than rely on users to manually submit them to the [RustSec Advisory Database].
 
-`cargo-unmaintained` defines an unmaintained package X as one that satisfies either 1 or 2 below:
+`cargo-unmaintained` defines an unmaintained package X as one that satisfies one of 1 through 3 below:
 
-1. X's repository is archived (see [Notes] below).
+1. X's repository does not exist.
 
-2. Both a and b below.
+2. X's repository is archived (see [Notes] below).
+
+3. Both a and b below.
 
    a. X depends on a version of a package Y that is incompatible with the Y's latest version.
 
@@ -24,7 +26,11 @@ As of 2023-11-15, the RustSec Advisory Database contains 87 active advisories fo
 
 - The purpose of condition 2(b) is to give package maintainers a chance to update their packages. That is, an incompatible upgrade to one of X's dependencies could require time-consuming changes to X. Without this check, `cargo-unmaintained` would produce many false positives.
 
-- Of the 35 packages in the RustSec Advisory Database _not_ identified by `cargo-unmaintained`, 6 do not build, 9 are unarchived leaves, and 2 were updated within the past 365 days. The remaining 18 were not identified for other reasons.
+- Of the 34 packages in the RustSec Advisory Database _not_ identified by `cargo-unmaintained`:
+  - 6 do not build.
+  - 8 are existent, unarchived leaves.
+  - 2 were updated within the past 365 days.
+  - 17 were not identified for other reasons.
 
 ## Output
 

diff --git a/src/curl.rs b/src/curl.rs
@@ -0,0 +1,14 @@
+use anyhow::Result;
+use curl::easy::Easy;
+
+pub(crate) fn existence(url: &str) -> Result<Option<bool>> {
+    let mut handle = Easy::new();
+    handle.url(url)?;
+    handle.transfer().perform()?;
+    let response_code = handle.response_code()?;
+    match response_code {
+        200 => Ok(Some(true)),
+        404 => Ok(Some(false)),
+        _ => Ok(None),
+    }
+}
diff --git a/src/github.rs b/src/github.rs
@@ -1,9 +1,11 @@
+use super::RepoStatus;
 use anyhow::{anyhow, bail, Context, Result};
 use once_cell::sync::Lazy;
 use regex::Regex;
 use std::{
     cell::RefCell,
     collections::HashMap,
+    convert::Infallible,
     fs::read_to_string,
     rc::Rc,
     time::{Duration, SystemTime},
@@ -38,14 +40,18 @@ pub(crate) fn load_token(path: &str) -> Result<()> {
     })
 }
 
-pub(crate) fn archival_status(url: &str) -> Result<Option<(&str, bool)>> {
+pub(crate) fn archival_status(url: &str) -> Result<Option<RepoStatus<Infallible>>> {
     let (url, owner_slash_repo, owner, repo) = match_github_url(url)?;
 
     let Some(repository) = repository(owner_slash_repo, owner, repo)? else {
-        return Ok(None);
+        return Ok(Some(RepoStatus::Nonexistent));
     };
 
-    Ok(Some((url, repository.archived.unwrap_or_default())))
+    if repository.archived.unwrap_or_default() {
+        Ok(Some(RepoStatus::Archived(url)))
+    } else {
+        Ok(None)
+    }
 }
 
 pub(crate) fn timestamp(url: &str) -> Result<Option<(&str, SystemTime)>> {
@@ -112,20 +118,27 @@ fn repository(
                 .clone()),
             Err(error) => {
                 repository_cache.insert(owner_slash_repo.to_owned(), None);
-                Err(error)
+                if let octocrab::Error::GitHub { source, .. } = &error {
+                    if source.message == "Not Found" {
+                        Ok(None)
+                    } else {
+                        Err(error.into())
+                    }
+                } else {
+                    Err(error.into())
+                }
             }
         }
     })
 }
 
 #[cfg_attr(dylint_lib = "general", allow(non_local_effect_before_error_return))]
-fn repository_uncached(owner: &str, repo: &str) -> Result<octocrab::models::Repository> {
+fn repository_uncached(owner: &str, repo: &str) -> octocrab::Result<octocrab::models::Repository> {
     RT.block_on(async {
         let octocrab = octocrab::instance();
 
         octocrab.repos(owner, repo).get().await
     })
-    .map_err(Into::into)
 }
 
 fn match_github_url(url: &str) -> Result<(&str, &str, &str, &str)> {

diff --git a/src/main.rs b/src/main.rs
@@ -14,6 +14,7 @@ use regex::Regex;
 use std::{
     cell::RefCell,
     collections::{HashMap, HashSet},
+    convert::Infallible,
     env::{args, var},
     ffi::OsStr,
     fs::{read_to_string, remove_dir_all, File},
@@ -27,6 +28,7 @@ use termcolor::{Color, ColorChoice, ColorSpec, StandardStream, WriteColor};
 use toml::{Table, Value};
 use walkdir::WalkDir;
 
+mod curl;
 mod github;
 mod opts;
 mod verbose;
@@ -146,6 +148,17 @@ impl<'a, T> RepoStatus<'a, T> {
     }
 }
 
+impl<'a> RepoStatus<'a, Infallible> {
+    fn lift<T>(&self) -> RepoStatus<'a, T> {
+        match self {
+            Self::Uncloneable => RepoStatus::Uncloneable,
+            Self::Nonexistent => RepoStatus::Nonexistent,
+            Self::Success(_, _) => unreachable!(),
+            Self::Archived(url) => RepoStatus::Archived(url),
+        }
+    }
+}
+
 /// Multiples of `max_age` that cause the color to go completely from yellow to red.
 const SATURATION_MULTIPLIER: u64 = 3;
 
@@ -356,11 +369,20 @@ fn unmaintained() -> Result<bool> {
     );
 
     for pkg in packages {
-        if var("GITHUB_TOKEN_PATH").is_ok() {
-            if let Some((url, true)) = archival_status(pkg)? {
+        if let Some(url) = &pkg.repository {
+            if var("GITHUB_TOKEN_PATH").is_ok() && url.starts_with("https://github.com/") {
+                if let Some(status) = archival_status(&pkg.name, url)? {
+                    unnmaintained_pkgs.push(UnmaintainedPkg {
+                        pkg,
+                        repo_age: status.lift(),
+                        outdated_deps: Vec::new(),
+                    });
+                    continue;
+                }
+            } else if let Some(false) = existence(&pkg.name, url)? {
                 unnmaintained_pkgs.push(UnmaintainedPkg {
                     pkg,
-                    repo_age: RepoStatus::Archived(url),
+                    repo_age: RepoStatus::Nonexistent,
                     outdated_deps: Vec::new(),
                 });
                 continue;
@@ -495,25 +517,25 @@ fn build_metadata_latest_version_map(metadata: &Metadata) -> HashMap<String, Ver
     map
 }
 
-fn archival_status(pkg: &Package) -> Result<Option<(&str, bool)>> {
-    let Some(url) = &pkg.repository else {
-        return Ok(None);
-    };
-
-    if !url.starts_with("https://github.com/") {
-        return Ok(None);
-    }
-
+fn archival_status<'a>(name: &str, url: &'a str) -> Result<Option<RepoStatus<'a, Infallible>>> {
     verbose::wrap!(
         || github::archival_status(url).or_else(|error| {
-            warn!(
-                "failed to determine `{}` archival status: {}",
-                pkg.name, error
-            );
+            warn!("failed to determine `{}` archival status: {}", name, error);
             Ok(None)
         }),
         "archival status of `{}` using GitHub API",
-        pkg.name
+        name
+    )
+}
+
+fn existence(name: &str, url: &str) -> Result<Option<bool>> {
+    verbose::wrap!(
+        || curl::existence(url).or_else(|error| {
+            warn!("failed to determine `{}` existence: {}", name, error);
+            Ok(None)
+        }),
+        "existence of `{}` using HTTP request",
+        name
     )
 }
 

diff --git a/tests/cases/hyperfine.with_token.stdout b/tests/cases/hyperfine.with_token.stdout
@@ -1,3 +1,4 @@
+fuchsia-cprng (no repository)
 rdrand (https://github.com/nagisa/rust_rdrand updated [..] days ago)
     rand_core (requirement: ^0.3, version used: 0.3.1, latest: [..])
 atty (https://github.com/softprops/atty updated [..] days ago)

diff --git a/tests/cases/hyperfine.without_token.stdout b/tests/cases/hyperfine.without_token.stdout
@@ -1,3 +1,4 @@
+fuchsia-cprng (no repository)
 rdrand (https://github.com/nagisa/rust_rdrand updated [..] days ago)
     rand_core (requirement: ^0.3, version used: 0.3.1, latest: [..])
 atty (https://github.com/softprops/atty updated [..] days ago)

diff --git a/tests/rustsec_comparison.with_token.stdout b/tests/rustsec_comparison.with_token.stdout
@@ -24,7 +24,7 @@ Error: found no packages matching `stream-cipher`
 ```
 safe-nd...found
 ffi_utils...found
-fake_clock...leaf
+fake_clock...found
 safe_bindgen...found
 quic-p2p...found
 routing...found
@@ -125,8 +125,7 @@ not found - error (6)
     safe-api             https://rustsec.org/advisories/RUSTSEC-2021-0024.html
     miscreant            https://rustsec.org/advisories/RUSTSEC-2021-0062.html
     cargo-download       https://rustsec.org/advisories/RUSTSEC-2021-0133.html
-not found - leaf (9)
-    fake_clock           https://rustsec.org/advisories/RUSTSEC-2020-0065.html
+not found - leaf (8)
     aesni                https://rustsec.org/advisories/RUSTSEC-2021-0059.html
     aes-soft             https://rustsec.org/advisories/RUSTSEC-2021-0060.html
     aes-ctr              https://rustsec.org/advisories/RUSTSEC-2021-0061.html
@@ -157,14 +156,15 @@ not found - other (18)
     xsalsa20poly1305     https://rustsec.org/advisories/RUSTSEC-2023-0037.html
     users                https://rustsec.org/advisories/RUSTSEC-2023-0040.html
     fehler               https://rustsec.org/advisories/RUSTSEC-2023-0067.html
-found (52)
+found (53)
     lz4-compress         https://rustsec.org/advisories/RUSTSEC-2017-0007.html
     tempdir              https://rustsec.org/advisories/RUSTSEC-2018-0017.html
     boxfnonce            https://rustsec.org/advisories/RUSTSEC-2019-0040.html
     bigint               https://rustsec.org/advisories/RUSTSEC-2020-0025.html
     failure              https://rustsec.org/advisories/RUSTSEC-2020-0036.html
     safe-nd              https://rustsec.org/advisories/RUSTSEC-2020-0063.html
     ffi_utils            https://rustsec.org/advisories/RUSTSEC-2020-0064.html
+    fake_clock           https://rustsec.org/advisories/RUSTSEC-2020-0065.html
     safe_bindgen         https://rustsec.org/advisories/RUSTSEC-2020-0066.html
     quic-p2p             https://rustsec.org/advisories/RUSTSEC-2020-0067.html
     routing              https://rustsec.org/advisories/RUSTSEC-2020-0076.html

diff --git a/tests/rustsec_comparison.without_token.stdout b/tests/rustsec_comparison.without_token.stdout
@@ -24,7 +24,7 @@ Error: found no packages matching `stream-cipher`
 ```
 safe-nd...found
 ffi_utils...found
-fake_clock...leaf
+fake_clock...found
 safe_bindgen...found
 quic-p2p...found
 routing...found
@@ -125,9 +125,8 @@ not found - error (6)
     safe-api             https://rustsec.org/advisories/RUSTSEC-2021-0024.html
     miscreant            https://rustsec.org/advisories/RUSTSEC-2021-0062.html
     cargo-download       https://rustsec.org/advisories/RUSTSEC-2021-0133.html
-not found - leaf (10)
+not found - leaf (9)
     boxfnonce            https://rustsec.org/advisories/RUSTSEC-2019-0040.html
-    fake_clock           https://rustsec.org/advisories/RUSTSEC-2020-0065.html
     aesni                https://rustsec.org/advisories/RUSTSEC-2021-0059.html
     aes-soft             https://rustsec.org/advisories/RUSTSEC-2021-0060.html
     aes-ctr              https://rustsec.org/advisories/RUSTSEC-2021-0061.html
@@ -166,12 +165,13 @@ not found - other (24)
     xsalsa20poly1305     https://rustsec.org/advisories/RUSTSEC-2023-0037.html
     users                https://rustsec.org/advisories/RUSTSEC-2023-0040.html
     fehler               https://rustsec.org/advisories/RUSTSEC-2023-0067.html
-found (43)
+found (44)
     lz4-compress         https://rustsec.org/advisories/RUSTSEC-2017-0007.html
     tempdir              https://rustsec.org/advisories/RUSTSEC-2018-0017.html
     bigint               https://rustsec.org/advisories/RUSTSEC-2020-0025.html
     safe-nd              https://rustsec.org/advisories/RUSTSEC-2020-0063.html
     ffi_utils            https://rustsec.org/advisories/RUSTSEC-2020-0064.html
+    fake_clock           https://rustsec.org/advisories/RUSTSEC-2020-0065.html
     safe_bindgen         https://rustsec.org/advisories/RUSTSEC-2020-0066.html
     quic-p2p             https://rustsec.org/advisories/RUSTSEC-2020-0067.html
     routing              https://rustsec.org/advisories/RUSTSEC-2020-0076.html