From e8fff7a172b432fc8e83d3ece59fdf84b54eaeba Mon Sep 17 00:00:00 2001 From: Ilia Pozdnyakov Date: Sun, 13 Oct 2024 13:34:45 +0500 Subject: [PATCH] optimize and parallelize IO and hashing --- Cargo.lock | 98 +++++++++++++++++++++++++++++++++----------- Cargo.toml | 3 +- src/bin/iso2god.rs | 64 +++++++++++++++++------------ src/god/hash_list.rs | 46 +++++++++------------ src/god/mod.rs | 71 ++++++++++++++++---------------- 5 files changed, 170 insertions(+), 112 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7890c2b..76d57ee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -158,9 +158,9 @@ checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" [[package]] name = "cc" -version = "1.1.28" +version = "1.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e80e3b6a3ab07840e1cae9b0666a63970dc28e8ed5ffbcdacbfc760c281bfc1" +checksum = "b16803a61b81d9eabb7eae2588776c4c1e584b738ede45fdbb4c972cec1e9945" dependencies = [ "shlex", ] @@ -185,9 +185,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.19" +version = "4.5.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7be5744db7978a28d9df86a214130d106a89ce49644cbc4e3f0c22c3fba30615" +checksum = "b97f376d85a664d5837dbae44bf546e6477a679ff6610010f17276f686d867e8" dependencies = [ "clap_builder", "clap_derive", @@ -195,9 +195,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.19" +version = "4.5.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5fbc17d3ef8278f55b282b2a2e75ae6f6c7d4bb70ed3d0382375104bfafdb4b" +checksum = "19bc80abd44e4bed93ca373a0704ccbd1b710dc5749406201bb018272808dc54" dependencies = [ "anstream", "anstyle", @@ -254,6 +254,31 @@ dependencies = [ "libc", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" + [[package]] name = "crypto-common" version = "0.1.6" @@ -274,6 +299,12 @@ dependencies = [ "crypto-common", ] +[[package]] +name = "either" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" + [[package]] name = "encoding_rs" version = "0.8.34" @@ -618,13 +649,14 @@ checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" [[package]] name = "iso2god" -version = "1.6.0" +version = "1.7.0" dependencies = [ "anyhow", "bitflags", "byteorder", "clap", "num_enum", + "rayon", "reqwest", "serde", "serde-aux", @@ -640,9 +672,9 @@ checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "js-sys" -version = "0.3.70" +version = "0.3.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" +checksum = "6a88f1bda2bd75b0452a14784937d796722fdebfe50df998aeb3f0b7603019a9" dependencies = [ "wasm-bindgen", ] @@ -855,6 +887,26 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "reqwest" version = "0.12.8" @@ -1370,9 +1422,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" +checksum = "128d1e363af62632b8eb57219c8fd7877144af57558fb2ef0368d0087bddeb2e" dependencies = [ "cfg-if", "once_cell", @@ -1381,9 +1433,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" +checksum = "cb6dd4d3ca0ddffd1dd1c9c04f94b868c37ff5fac97c30b97cff2d74fce3a358" dependencies = [ "bumpalo", "log", @@ -1396,9 +1448,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.43" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61e9300f63a621e96ed275155c108eb6f843b6a26d053f122ab69724559dc8ed" +checksum = "cc7ec4f8827a71586374db3e87abdb5a2bb3a15afed140221307c3ec06b1f63b" dependencies = [ "cfg-if", "js-sys", @@ -1408,9 +1460,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" +checksum = "e79384be7f8f5a9dd5d7167216f022090cf1f9ec128e6e6a482a2cb5c5422c56" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -1418,9 +1470,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" +checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" dependencies = [ "proc-macro2", "quote", @@ -1431,15 +1483,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" +checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" [[package]] name = "web-sys" -version = "0.3.70" +version = "0.3.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26fdeaafd9bd129f65e7c031593c24d62186301e0c72c8978fa1678be7d532c0" +checksum = "f6488b90108c040df0fe62fa815cbdee25124641df01814dd7282749234c6112" dependencies = [ "js-sys", "wasm-bindgen", diff --git a/Cargo.toml b/Cargo.toml index 0f728d2..52fe50a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "iso2god" -version = "1.6.0" +version = "1.7.0" description = "A tool to convert between Xbox 360 ISO and Games On Demand file formats" repository = "https://github.com/iliazeus/iso2god-rs" edition = "2021" @@ -13,6 +13,7 @@ bitflags = "2.6.0" byteorder = "1.5.0" clap = { version = "4.5.19", features = ["derive"] } num_enum = "0.7.3" +rayon = "1.10.0" sha1 = "0.10.6" [dev-dependencies] diff --git a/src/bin/iso2god.rs b/src/bin/iso2god.rs index b6fab59..407804e 100644 --- a/src/bin/iso2god.rs +++ b/src/bin/iso2god.rs @@ -1,13 +1,16 @@ -use std::io::{BufReader, BufWriter, Read, Seek, Write}; +use std::io::{BufReader, Seek, SeekFrom, Write}; use std::fs; use std::fs::File; use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicUsize, Ordering}; use anyhow::{Context, Error}; use clap::{arg, command, Parser}; +use rayon::prelude::*; + use iso2god::executable::TitleInfo; use iso2god::god::ContentType; use iso2god::{game_list, god, iso}; @@ -37,6 +40,10 @@ struct Cli { /// Trim off unused space from the ISO image #[arg(long)] trim: bool, + + /// Number of worker threads to use + #[arg(long, short = 'j')] + num_threads: Option, } fn main() -> Result<(), Error> { @@ -47,10 +54,13 @@ fn main() -> Result<(), Error> { eprintln!("the --offline flag is deprecated: the tool now has a built-in title database, so it is always offline"); } + rayon::ThreadPoolBuilder::new() + .num_threads(args.num_threads.unwrap_or(0)) + .build_global()?; + println!("extracting ISO metadata"); - let source_iso_file = open_file_for_buffered_reading(&args.source_iso) - .context("error opening source ISO file")?; + let source_iso_file = File::open(&args.source_iso).context("error opening source ISO file")?; let source_iso_file_meta = fs::metadata(&args.source_iso).context("error reading source ISO file metadata")?; @@ -96,23 +106,31 @@ fn main() -> Result<(), Error> { ensure_empty_dir(&file_layout.data_dir_path()).context("error clearing data directory")?; - let mut source_iso = source_iso - .get_root() - .context("error reading source iso")? - .take(data_size); + println!("writing part files: 0/{part_count}"); - println!("writing part files"); + let progress = AtomicUsize::new(0); - for part_index in 0..part_count { - println!("writing part {:2} of {:2}", part_index, part_count); + (0..part_count).into_par_iter().try_for_each(|part_index| { + let mut iso_data_volume = File::open(&args.source_iso)?; + iso_data_volume.seek(SeekFrom::Start(source_iso.volume_descriptor.root_offset))?; let part_file = file_layout.part_file_path(part_index); - let mut part_file = - open_file_for_buffered_writing(&part_file).context("error creating part file")?; + let part_file = File::options() + .write(true) + .create(true) + .truncate(true) + .open(&part_file) + .context("error creating part file")?; - god::write_part(&mut source_iso, &mut part_file).context("error writing part file")?; - } + god::write_part(iso_data_volume, part_index, part_file) + .context("error writing part file")?; + + let cur = 1 + progress.fetch_add(1, Ordering::Relaxed); + println!("writing part files: {cur:2}/{part_count}"); + + Ok::<_, anyhow::Error>(()) + })?; println!("calculating MHT hash chain"); @@ -156,7 +174,11 @@ fn main() -> Result<(), Error> { let con_header = con_header.finalize(); - let mut con_header_file = open_file_for_buffered_writing(&file_layout.con_header_file_path()) + let mut con_header_file = File::options() + .write(true) + .create(true) + .truncate(true) + .open(file_layout.con_header_file_path()) .context("cannot open con header file")?; con_header_file @@ -192,15 +214,3 @@ fn write_part_mht( mht.write(&mut part_file)?; Ok(()) } - -fn open_file_for_buffered_writing(path: &Path) -> Result { - let file = File::options().create(true).write(true).open(path)?; - let file = BufWriter::with_capacity(8 * 1024 * 1024, file); - Ok(file) -} - -fn open_file_for_buffered_reading(path: &Path) -> Result { - let file = File::options().read(true).open(path)?; - let file = BufReader::with_capacity(8 * 1024 * 1024, file); - Ok(file) -} diff --git a/src/god/hash_list.rs b/src/god/hash_list.rs index 73aa7fe..e2edcb5 100644 --- a/src/god/hash_list.rs +++ b/src/god/hash_list.rs @@ -5,37 +5,37 @@ use sha1::{Digest, Sha1}; use anyhow::Error; pub struct HashList { - buffer: Vec, + buffer: [u8; 4096], + len: usize, } impl HashList { + pub fn bytes(&self) -> &[u8; 4096] { + &self.buffer + } + pub fn new() -> HashList { HashList { - buffer: Vec::with_capacity(4096), + buffer: [0u8; 4096], + len: 0, } } - pub fn read(reader: &mut R) -> Result { - let mut reader = reader.by_ref().take(4096); - let mut buffer = Vec::::with_capacity(4096); - - let mut block_buffer = Vec::::with_capacity(20); - - loop { - reader.by_ref().take(20).read_to_end(&mut block_buffer)?; + pub fn read(mut reader: R) -> Result { + let mut buffer = [0u8; 4096]; + reader.read_exact(&mut buffer)?; - if block_buffer.is_empty() || block_buffer.iter().all(|x| *x == 0) { - break; - } + let len = buffer + .chunks(20) + .position(|c| *c == [0u8; 20]) + .unwrap_or(buffer.len()); - buffer.append(&mut block_buffer); - } - - Ok(HashList { buffer }) + Ok(HashList { buffer, len }) } pub fn add_hash(&mut self, hash: &[u8; 20]) { - self.buffer.extend_from_slice(hash); + self.buffer[self.len..self.len + 20].copy_from_slice(hash); + self.len += 20; } pub fn add_block_hash(&mut self, block: &[u8]) { @@ -43,17 +43,11 @@ impl HashList { } pub fn digest(&self) -> [u8; 20] { - Sha1::digest(self.to_bytes()).into() + Sha1::digest(&self.buffer).into() } pub fn write(&self, writer: &mut W) -> Result<(), Error> { - writer.write_all(&self.to_bytes())?; + writer.write_all(&self.buffer)?; Ok(()) } - - pub fn to_bytes(&self) -> Vec { - let mut buf = self.buffer.clone(); - buf.resize(4096, 0); - buf - } } diff --git a/src/god/mod.rs b/src/god/mod.rs index 6ae9456..a1046a5 100644 --- a/src/god/mod.rs +++ b/src/god/mod.rs @@ -17,59 +17,60 @@ pub use hash_list::*; pub const BLOCKS_PER_PART: u64 = 0xa1c4; pub const BLOCKS_PER_SUBPART: u64 = 0xcc; pub const BLOCK_SIZE: usize = 0x1000; -pub const FREE_SECTOR: u32 = 0x24; pub const SUBPARTS_PER_PART: u32 = 0xcb; +pub const SUBPART_SIZE: usize = BLOCK_SIZE * BLOCKS_PER_SUBPART as usize; -pub fn write_part(src: &mut R, dest: &mut W) -> Result<(), Error> { - let mut block_buffer = Vec::::with_capacity(BLOCK_SIZE); - let mut eof = false; +pub fn write_part( + mut data_volume: R, + part_index: u64, + mut part_file: W, +) -> Result<(), Error> { + data_volume.seek(SeekFrom::Start( + part_index * BLOCKS_PER_PART * BLOCK_SIZE as u64, + ))?; let mut master_hash_list = HashList::new(); - let master_hash_list_position = dest.stream_position()?; - master_hash_list.write(dest)?; + let master_hash_list_position = part_file.stream_position()?; + master_hash_list.write(&mut part_file)?; + + let mut subpart_buf = Vec::with_capacity(SUBPART_SIZE); for _subpart_index in 0..SUBPARTS_PER_PART { - if eof { + data_volume + .by_ref() + .take(SUBPART_SIZE as u64) + .read_to_end(&mut subpart_buf)?; + + if subpart_buf.len() == 0 { break; } let mut sub_hash_list = HashList::new(); - let sub_hash_list_position = dest.stream_position()?; - sub_hash_list.write(dest)?; - - for _block_index in 0..BLOCKS_PER_SUBPART { - src.by_ref() - .take(BLOCK_SIZE as u64) - .read_to_end(&mut block_buffer)?; - - if block_buffer.is_empty() { - eof = true; - break; - } - - sub_hash_list.add_block_hash(&block_buffer); - dest.write_all(&block_buffer)?; - block_buffer.clear(); + for block in subpart_buf.chunks(BLOCK_SIZE) { + sub_hash_list.add_block_hash(block); } - let next_position = dest.stream_position()?; + sub_hash_list.write(&mut part_file)?; + master_hash_list.add_block_hash(sub_hash_list.bytes()); - dest.seek(SeekFrom::Start(sub_hash_list_position))?; - sub_hash_list.write(dest)?; + // using io::copy here to benefit from potential reflink optimizations + // https://doc.rust-lang.org/std/io/fn.copy.html#platform-specific-behavior + data_volume.seek_relative(0 - subpart_buf.len() as i64)?; + std::io::copy( + &mut data_volume.by_ref().take(SUBPART_SIZE as u64), + &mut part_file, + )?; - master_hash_list.add_block_hash(&sub_hash_list.to_bytes()); - - dest.seek(SeekFrom::Start(next_position))?; + if subpart_buf.len() < SUBPART_SIZE { + break; + } + subpart_buf.clear(); } - let next_position = dest.stream_position()?; - - dest.seek(SeekFrom::Start(master_hash_list_position))?; - master_hash_list.write(dest)?; - - dest.seek(SeekFrom::Start(next_position))?; + part_file.seek(SeekFrom::Start(master_hash_list_position))?; + master_hash_list.write(&mut part_file)?; Ok(()) }