Improve memory usage when packing a model

[VivekPanyam]

The previous zip file library we used during packing required complete files to be available before they could be stored. This required us to load large (possibly multi GB) files into memory.

This is no longer required. The following two places within the packing code can be refactored to read, compute sha256, and store files in a streaming/incremental fashion:

carton/source/carton/src/format/v1/save.rs

Lines 259 to 279 in 33cc183

	// Load the data and compute the sha256
	let mut hasher = Sha256::new();
	let data = tokio::fs::read(entry.path()).await.unwrap();
	hasher.update(&data);
	let sha256 = format!("{:x}", hasher.finalize());
	manifest_contents.insert(relative_path.clone(), Some(sha256));
	// Add the entry to the zip file
	writer = tokio::task::spawn_blocking(move || {
	writer
	.start_file(
	relative_path,
	zip::write::FileOptions::default()
	.compression_method(zip::CompressionMethod::Zstd),
	)
	.unwrap();
	writer.write_all(&data).unwrap();
	writer
	})
	.await
	.unwrap();

carton/source/carton/src/format/v1/save.rs

Lines 354 to 390 in 33cc183

	// Load the data and compute the sha256
	let mut hasher = Sha256::new();
	let data = tokio::fs::read(entry.path()).await.unwrap();
	log::trace!("Done reading file {}", &relative_path);
	let (data, sha256) = tokio::task::spawn_blocking(move || {
	hasher.update(&data);
	(data, format!("{:x}", hasher.finalize()))
	})
	.await
	.unwrap();
	log::trace!("Computed sha256 of {}", &relative_path);
	// Only store the file in the zip if (1) we don't have any linked files or (2) the linked files don't include this sha256
	if linked_files
	.as_ref()
	.map_or(true, |v| !v.urls.contains_key(&sha256))
	{
	// Add the entry to the zip file
	let relative_path = relative_path.clone();
	writer = tokio::task::spawn_blocking(move || {
	writer
	.start_file(
	relative_path,
	zip::write::FileOptions::default()
	.compression_method(zip::CompressionMethod::Zstd)
	.large_file(data.len() >= 4 * 1024 * 1024 * 1024),
	)
	.unwrap();
	writer.write_all(&data).unwrap();
	writer
	})
	.await
	.unwrap();
	}