Skip to content

Commit

Permalink
Add feature flag for multithreaded zstd
Browse files Browse the repository at this point in the history
This commit adds an optional feature flag which
enable zstd compression on all threads of the CPU.
  • Loading branch information
famfo authored and dralley committed Oct 16, 2024
1 parent 405e883 commit 755b9c8
Show file tree
Hide file tree
Showing 3 changed files with 18 additions and 4 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Changed

- The `Header::parse_header` function gained a speed up related to parsing of the binary headers.
- Added `zstdmt` feature which sets zstd compression to use all available cores.

## 0.15.0

Expand Down
2 changes: 2 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -70,3 +70,5 @@ signature-meta = []

# Segregate tests that require podman to be installed
test-with-podman = ["signature-pgp"]

zstdmt = ["zstd/zstdmt"]
19 changes: 15 additions & 4 deletions src/rpm/compressor.rs
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ impl std::str::FromStr for CompressionType {
pub enum Compressor {
None(Vec<u8>),
Gzip(flate2::write::GzEncoder<Vec<u8>>),
/// If the `zstdmt` feature flag is enabled, compression will use all available cores to
/// compress the file.
Zstd(zstd::stream::Encoder<'static, Vec<u8>>),
Xz(xz2::write::XzEncoder<Vec<u8>>),
Bzip2(bzip2::write::BzEncoder<Vec<u8>>),
Expand All @@ -43,10 +45,19 @@ impl TryFrom<CompressionWithLevel> for Compressor {
CompressionWithLevel::Gzip(level) => Ok(Compressor::Gzip(
flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::new(level)),
)),
CompressionWithLevel::Zstd(level) => Ok(Compressor::Zstd(zstd::stream::Encoder::new(
Vec::new(),
level,
)?)),
CompressionWithLevel::Zstd(level) => {
#[cfg_attr(not(feature = "zstdmt"), allow(unused_mut))]
let mut stream = zstd::stream::Encoder::new(Vec::new(), level)?;

#[cfg(feature = "zstdmt")]
{
let threads = std::thread::available_parallelism()?;
// If someone has more than 2^32 threads, I'm impressed
stream.multithread(threads.get() as u32)?;
}

Ok(Compressor::Zstd(stream))
}
CompressionWithLevel::Xz(level) => Ok(Compressor::Xz(xz2::write::XzEncoder::new(
Vec::new(),
level,
Expand Down

0 comments on commit 755b9c8

Please sign in to comment.