Skip to content

Commit

Permalink
Change to purely use remote sparse index
Browse files Browse the repository at this point in the history
  • Loading branch information
Jake-Shadle committed Aug 23, 2023
1 parent 012d3e9 commit 145bb50
Show file tree
Hide file tree
Showing 10 changed files with 203 additions and 1,156 deletions.
1,172 changes: 85 additions & 1,087 deletions Cargo.lock

Large diffs are not rendered by default.

23 changes: 14 additions & 9 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ resolver = "2"
[workspace.package]
license = "MIT OR Apache-2.0"
edition = "2021"
rust-version = "1.66.0" # MSRV
rust-version = "1.66.0" # MSRV
include = [
"build.rs",
"src/**/*",
Expand All @@ -13,7 +13,7 @@ include = [
"LICENSE*",
"README.md",
"benches/**/*",
"examples/**/*"
"examples/**/*",
]

[package]
Expand All @@ -35,12 +35,12 @@ rustdoc-args = ["--cfg", "docsrs"]

[package.metadata.release]
pre-release-replacements = [
{file="README.md", search="Current release: [a-z0-9\\.-]+", replace="Current release: {{version}}"} ,
{file="CHANGELOG.md", search="Unreleased", replace="{{version}}", min=1},
{file="CHANGELOG.md", search="\\.\\.\\.HEAD", replace="...{{tag_name}}", exactly=1},
{file="CHANGELOG.md", search="ReleaseDate", replace="{{date}}", min=1},
{file="CHANGELOG.md", search="<!-- next-header -->", replace="<!-- next-header -->\n## [Unreleased] - ReleaseDate\n", exactly=1},
{file="CHANGELOG.md", search="<!-- next-url -->", replace="<!-- next-url -->\n[Unreleased]: https://github.com/crate-ci/cargo-release/compare/{{tag_name}}...HEAD", exactly=1},
{ file = "README.md", search = "Current release: [a-z0-9\\.-]+", replace = "Current release: {{version}}" },
{ file = "CHANGELOG.md", search = "Unreleased", replace = "{{version}}", min = 1 },
{ file = "CHANGELOG.md", search = "\\.\\.\\.HEAD", replace = "...{{tag_name}}", exactly = 1 },
{ file = "CHANGELOG.md", search = "ReleaseDate", replace = "{{date}}", min = 1 },
{ file = "CHANGELOG.md", search = "<!-- next-header -->", replace = "<!-- next-header -->\n## [Unreleased] - ReleaseDate\n", exactly = 1 },
{ file = "CHANGELOG.md", search = "<!-- next-url -->", replace = "<!-- next-url -->\n[Unreleased]: https://github.com/crate-ci/cargo-release/compare/{{tag_name}}...HEAD", exactly = 1 },
]

[features]
Expand All @@ -59,7 +59,7 @@ vendored-openssl = ["git2/vendored-openssl"]

[dependencies]
cargo_metadata = "0.15"
tame-index = { version = "0.4", features = ["git", "sparse"] }
tame-index = "0.5"
git2 = { version = "0.17.2", default-features = false }
toml_edit = "0.19.10"
toml = "0.7.4"
Expand Down Expand Up @@ -88,6 +88,11 @@ anyhow = "1.0.71"
concolor-control = { version = "0.0.7", features = ["auto"] }
git-conventional = "0.12.3"

[dependencies.reqwest]
version = "0.11"
default-features = false
features = ["blocking", "rustls-tls", "gzip"]

[dev-dependencies]
assert_fs = "1.0"
predicates = "3.0"
Expand Down
1 change: 1 addition & 0 deletions src/error.rs
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ process_error_from!(std::io::Error);
process_error_from!(semver::Error);
process_error_from!(ignore::Error);
process_error_from!(tame_index::Error);
process_error_from!(reqwest::Error);
process_error_from!(cargo_metadata::Error);
process_error_from!(toml::ser::Error);
process_error_from!(toml_edit::ser::Error);
Expand Down
21 changes: 8 additions & 13 deletions src/ops/cargo.rs
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ pub fn publish(
}

pub fn wait_for_publish(
index: &mut tame_index::index::ComboIndex,
index: &mut crate::steps::index::CratesIoIndex,
name: &str,
version: &str,
timeout: std::time::Duration,
Expand All @@ -122,11 +122,6 @@ pub fn wait_for_publish(
let sleep_time = std::time::Duration::from_secs(1);
let mut logged = false;
loop {
if let tame_index::index::ComboIndex::Git(gi) = index {
if let Err(e) = gi.fetch() {
log::debug!("crate index update failed with {}", e);
}
}
if is_published(index, name, version) {
break;
} else if timeout < now.elapsed() {
Expand All @@ -147,13 +142,13 @@ pub fn wait_for_publish(
Ok(())
}

pub fn is_published(index: &tame_index::index::ComboIndex, name: &str, version: &str) -> bool {
match index.krate(name.try_into().expect("crate name is invalid"), true) {
Ok(Some(crate_data)) => crate_data
.versions
.into_iter()
.any(|iv| iv.version == version),
Ok(None) => false,
pub fn is_published(
index: &mut crate::steps::index::CratesIoIndex,
name: &str,
version: &str,
) -> bool {
match index.has_krate_version(name, version) {
Ok(has_krate_version) => has_krate_version.unwrap_or(false),
Err(err) => {
// For both http and git indices, this _might_ be an error that goes away in
// a future call, but at least printing out something should give the user
Expand Down
4 changes: 2 additions & 2 deletions src/steps/hook.rs
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ pub struct HookStep {
impl HookStep {
pub fn run(&self) -> Result<(), CliError> {
git::git_version()?;
let index = crate::steps::index::open_crates_io_index()?;
let mut index = crate::steps::index::CratesIoIndex::open()?;

if self.dry_run {
let _ =
Expand Down Expand Up @@ -87,7 +87,7 @@ impl HookStep {
{
let version = &pkg.initial_version;
if !crate::ops::cargo::is_published(
&index,
&mut index,
crate_name,
&version.full_version_string,
) {
Expand Down
93 changes: 72 additions & 21 deletions src/steps/index.rs
Original file line number Diff line number Diff line change
@@ -1,24 +1,75 @@
pub(crate) fn open_crates_io_index() -> Result<tame_index::index::ComboIndex, crate::error::CliError>
{
let index = tame_index::index::ComboIndexCache::new(tame_index::IndexLocation::new(
tame_index::IndexUrl::crates_io(None, None, None)?,
))?;

let index = match index {
tame_index::index::ComboIndexCache::Git(gi) => {
tame_index::index::RemoteGitIndex::new(gi)?.into()
}
tame_index::index::ComboIndexCache::Sparse(si) => {
tame_index::index::RemoteSparseIndex::new(
si,
tame_index::external::reqwest::blocking::Client::builder()
.http2_prior_knowledge()
.build()
.map_err(tame_index::Error::from)?,
)
.into()
pub struct CratesIoIndex {
index: tame_index::SparseIndex,
client: reqwest::blocking::Client,
etags: Vec<(String, String)>,
}

impl CratesIoIndex {
#[inline]
pub(crate) fn open() -> Result<Self, crate::error::CliError> {
let index = tame_index::SparseIndex::new(tame_index::IndexLocation::new(
tame_index::IndexUrl::CratesIoSparse,
))?;
let client = reqwest::blocking::ClientBuilder::new()
.http2_prior_knowledge()
.build()?;

Ok(Self {
index,
client,
etags: Vec::new(),
})
}

/// Determines if the specified crate exists in the crates.io index
pub(crate) fn has_krate(&mut self, name: &str) -> Result<bool, crate::error::CliError> {
Ok(self.has_krate_version(name, "")?.unwrap_or(false))
}

/// Determines if the specified crate version exists in the crates.io index
pub(crate) fn has_krate_version(
&mut self,
name: &str,
version: &str,
) -> Result<Option<bool>, crate::error::CliError> {
let etag = self
.etags
.iter()
.find_map(|(krate, etag)| (krate == name).then_some(etag.as_str()))
.unwrap_or("");

let krate_name = name.try_into()?;
let req = self.index.make_remote_request(krate_name, Some(etag))?;
let res = self.client.execute(req.try_into()?)?;

// Grab the etag if it exists for future requests
if let Some(etag) = res.headers().get(reqwest::header::ETAG) {
if let Ok(etag) = etag.to_str() {
if let Some(i) = self.etags.iter().position(|(krate, _)| krate == name) {
self.etags[i].1 = etag.to_owned();
} else {
self.etags.push((name.to_owned(), etag.to_owned()));
}
}
}
};

Ok(index)
let mut builder = tame_index::external::http::Response::builder()
.status(res.status())
.version(res.version());

builder
.headers_mut()
.unwrap()
.extend(res.headers().iter().map(|(k, v)| (k.clone(), v.clone())));

let body = res.bytes()?;
let response = builder
.body(body.to_vec())
.map_err(|e| tame_index::Error::from(tame_index::error::HttpError::from(e)))?;

let index_krate = self
.index
.parse_remote_response(krate_name, response, false)?;
Ok(index_krate.map(|ik| ik.versions.iter().any(|iv| iv.version == version)))
}
}
10 changes: 3 additions & 7 deletions src/steps/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ pub mod changes;
pub mod commit;
pub mod config;
pub mod hook;
mod index;
pub mod index;
pub mod owner;
pub mod plan;
pub mod publish;
Expand Down Expand Up @@ -214,7 +214,7 @@ pub fn verify_monotonically_increasing(

pub fn verify_rate_limit(
pkgs: &[plan::PackageRelease],
index: &tame_index::index::ComboIndex,
index: &mut index::CratesIoIndex,
dry_run: bool,
level: log::Level,
) -> Result<bool, crate::error::CliError> {
Expand All @@ -228,11 +228,7 @@ pub fn verify_rate_limit(
for pkg in pkgs {
if pkg.config.registry().is_none() && pkg.config.publish() {
let crate_name = pkg.meta.name.as_str();
if index
.krate(crate_name.try_into()?, true)
.map(|ik| ik.is_some())
.unwrap_or(false)
{
if index.has_krate(crate_name)? {
existing += 1;
} else {
new += 1;
Expand Down
14 changes: 9 additions & 5 deletions src/steps/publish.rs
Original file line number Diff line number Diff line change
Expand Up @@ -80,13 +80,16 @@ impl PublishStep {

let mut pkgs = plan::plan(pkgs)?;

let mut index = crate::steps::index::open_crates_io_index()?;
let mut index = crate::steps::index::CratesIoIndex::open()?;
for pkg in pkgs.values_mut() {
if pkg.config.registry().is_none() && pkg.config.release() {
let crate_name = pkg.meta.name.as_str();
let version = pkg.planned_version.as_ref().unwrap_or(&pkg.initial_version);
if crate::ops::cargo::is_published(&index, crate_name, &version.full_version_string)
{
if crate::ops::cargo::is_published(
&mut index,
crate_name,
&version.full_version_string,
) {
let _ = crate::ops::shell::warn(format!(
"disabled due to previous publish ({}), skipping {}",
version.full_version_string, crate_name
Expand Down Expand Up @@ -131,7 +134,8 @@ impl PublishStep {
)?;

failed |= !super::verify_metadata(&selected_pkgs, dry_run, log::Level::Error)?;
failed |= !super::verify_rate_limit(&selected_pkgs, &index, dry_run, log::Level::Error)?;
failed |=
!super::verify_rate_limit(&selected_pkgs, &mut index, dry_run, log::Level::Error)?;

// STEP 1: Release Confirmation
super::confirm("Publish", &selected_pkgs, self.no_confirm, dry_run)?;
Expand All @@ -156,7 +160,7 @@ impl PublishStep {
pub fn publish(
ws_meta: &cargo_metadata::Metadata,
pkgs: &[plan::PackageRelease],
index: &mut tame_index::index::ComboIndex,
index: &mut crate::steps::index::CratesIoIndex,
dry_run: bool,
) -> Result<(), CliError> {
for pkg in pkgs {
Expand Down
17 changes: 7 additions & 10 deletions src/steps/release.rs
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ pub struct ReleaseStep {
impl ReleaseStep {
pub fn run(&self) -> Result<(), CliError> {
git::git_version()?;
let mut index = crate::steps::index::open_crates_io_index()?;
let mut index = crate::steps::index::CratesIoIndex::open()?;

if self.dry_run {
let _ =
Expand All @@ -73,11 +73,7 @@ impl ReleaseStep {
pkg.bump(level_or_version, self.metadata.as_deref())?;
}
}
if index
.krate(pkg.meta.name.as_str().try_into()?, true)
.map(|ik| ik.is_some())
.unwrap_or(false)
{
if index.has_krate(&pkg.meta.name)? {
// Already published, skip it. Use `cargo release owner` for one-time updates
pkg.ensure_owners = false;
}
Expand Down Expand Up @@ -105,7 +101,7 @@ impl ReleaseStep {
&& !explicitly_excluded
{
let version = &pkg.initial_version;
if !cargo::is_published(&index, crate_name, &version.full_version_string) {
if !cargo::is_published(&mut index, crate_name, &version.full_version_string) {
log::debug!(
"enabled {}, v{} is unpublished",
crate_name,
Expand Down Expand Up @@ -159,7 +155,7 @@ impl ReleaseStep {
if pkg.config.publish() && pkg.config.registry().is_none() {
let version = pkg.planned_version.as_ref().unwrap_or(&pkg.initial_version);
let crate_name = pkg.meta.name.as_str();
if !cargo::is_published(&index, crate_name, &version.full_version_string) {
if !cargo::is_published(&mut index, crate_name, &version.full_version_string) {
let _ = crate::ops::shell::warn(format!(
"disabled by user, skipping {} v{} despite being unpublished",
crate_name, version.full_version_string,
Expand Down Expand Up @@ -202,7 +198,7 @@ impl ReleaseStep {
if pkg.config.registry().is_none() {
let version = pkg.planned_version.as_ref().unwrap_or(&pkg.initial_version);
let crate_name = pkg.meta.name.as_str();
if cargo::is_published(&index, crate_name, &version.full_version_string) {
if cargo::is_published(&mut index, crate_name, &version.full_version_string) {
let _ = crate::ops::shell::error(format!(
"{} {} is already published",
crate_name, version.full_version_string
Expand Down Expand Up @@ -235,7 +231,8 @@ impl ReleaseStep {
)?;

failed |= !super::verify_metadata(&selected_pkgs, dry_run, log::Level::Error)?;
failed |= !super::verify_rate_limit(&selected_pkgs, &index, dry_run, log::Level::Error)?;
failed |=
!super::verify_rate_limit(&selected_pkgs, &mut index, dry_run, log::Level::Error)?;

// STEP 1: Release Confirmation
super::confirm("Release", &selected_pkgs, self.no_confirm, dry_run)?;
Expand Down
4 changes: 2 additions & 2 deletions src/steps/replace.rs
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ pub struct ReplaceStep {
impl ReplaceStep {
pub fn run(&self) -> Result<(), CliError> {
git::git_version()?;
let index = crate::steps::index::open_crates_io_index()?;
let mut index = crate::steps::index::CratesIoIndex::open()?;

if self.dry_run {
let _ =
Expand Down Expand Up @@ -83,7 +83,7 @@ impl ReplaceStep {
{
let version = &pkg.initial_version;
if !crate::ops::cargo::is_published(
&index,
&mut index,
crate_name,
&version.full_version_string,
) {
Expand Down

0 comments on commit 145bb50

Please sign in to comment.