Skip to content

Commit

Permalink
compress.gzip: add compress flags
Browse files Browse the repository at this point in the history
  • Loading branch information
kbkpbot committed Jan 4, 2025
1 parent cb490a7 commit 172827c
Show file tree
Hide file tree
Showing 2 changed files with 77 additions and 2 deletions.
59 changes: 57 additions & 2 deletions vlib/compress/gzip/gzip.v
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,50 @@ module gzip
import compress as compr
import hash.crc32

// CompressFlags
pub enum CompressFlags {
// 0: Huffman only
// 1: Huffman+LZ (fastest/crap compression)
// 4095: Huffman+LZ (slowest/best compression)
huffman_only = 0
default_max_probes = 128
max_probes_mask = 0xfff

// If set, the compressor outputs a zlib header before the deflate data, and the Adler-32 of the source data at the end. Otherwise, you'll get raw deflate data.
// write_zlib_header = 0x01000
// Always compute the adler-32 of the input data (even when not writing zlib headers).
// compute_adler32 = 0x02000
// Set to use faster greedy parsing, instead of more efficient lazy parsing.
greedy_parsing_flag = 0x04000
// Enable to decrease the compressor's initialization time to the minimum, but the output may vary from run to run given the same input (depending on the contents of memory).
nondeterministic_parsing_flag = 0x08000
// Only look for RLE matches (matches with a distance of 1)
rle_matches = 0x10000
// Discards matches <= 5 chars if enabled.
filter_matches = 0x20000
// Disable usage of optimized Huffman tables.
force_all_static_blocks = 0x40000
// Only use raw (uncompressed) deflate blocks.
force_all_raw_blocks = 0x80000
// The low 12 bits are reserved to control the max # of hash probes per dictionary lookup.
}

// CompressParams set flags for compression:
// `flags`:
// 0: Huffman only;
// 1: Huffman+LZ (fastest/crap compression);
// 128: default_max_probes;
// 4095: Huffman+LZ (slowest/best compression)
@[params]
pub struct CompressParams {
pub:
flags int = int(CompressFlags.default_max_probes)
}

// compresses an array of bytes using gzip and returns the compressed bytes in a new array
// Example: compressed := gzip.compress(b)!
pub fn compress(data []u8) ![]u8 {
compressed := compr.compress(data, 0)!
pub fn compress(data []u8, params CompressParams) ![]u8 {
compressed := compr.compress(data, params.flags)!
// header
mut result := [
u8(0x1f), // magic numbers (1F 8B)
Expand Down Expand Up @@ -40,12 +80,27 @@ pub fn compress(data []u8) ![]u8 {
return result
}

// DecompressFlags
pub enum DecompressFlags {
// If set, the input has a valid zlib header and ends with an adler32 checksum (it's a valid zlib stream). Otherwise, the input is a raw deflate stream.
parse_zlib_header = 1
// If set, there are more input bytes available beyond the end of the supplied input buffer. If clear, the input buffer contains all remaining input.
has_more_input = 2
// If set, the output buffer is large enough to hold the entire decompressed stream. If clear, the output buffer is at least the size of the dictionary (typically 32KB).
using_non_wrapping_output_buf = 4
// Force adler-32 checksum computation of the decompressed bytes.
compute_adler32 = 8
}

// DecompressParams set flags for decompression:
// `flags` has no use now
@[params]
pub struct DecompressParams {
pub:
verify_header_checksum bool = true
verify_length bool = true
verify_checksum bool = true
flags int
}

pub const reserved_bits = 0b1110_0000
Expand Down
20 changes: 20 additions & 0 deletions vlib/compress/gzip/read_gz_files_test.v
Original file line number Diff line number Diff line change
Expand Up @@ -39,3 +39,23 @@ fn test_reading_gzip_files_compressed_with_different_options() {
assert content9 == content5
assert content5 == content1
}

fn test_compress_with_deferent_level() {
_, content9 := read_and_decode_file(s('readme_level_9.gz'))!

// compression: Huffman only=0
compress_0 := gzip.compress(content9, flags: int(gzip.CompressFlags.huffman_only))!

Check failure on line 47 in vlib/compress/gzip/read_gz_files_test.v

View workflow job for this annotation

GitHub Actions / gcc

cannot use `string` as `[]u8` in argument 1 to `compress.gzip.compress`

Check failure on line 47 in vlib/compress/gzip/read_gz_files_test.v

View workflow job for this annotation

GitHub Actions / clang (macos-14)

cannot use `string` as `[]u8` in argument 1 to `compress.gzip.compress`

Check failure on line 47 in vlib/compress/gzip/read_gz_files_test.v

View workflow job for this annotation

GitHub Actions / tcc

cannot use `string` as `[]u8` in argument 1 to `compress.gzip.compress`

Check failure on line 47 in vlib/compress/gzip/read_gz_files_test.v

View workflow job for this annotation

GitHub Actions / msvc

cannot use `string` as `[]u8` in argument 1 to `compress.gzip.compress`

Check failure on line 47 in vlib/compress/gzip/read_gz_files_test.v

View workflow job for this annotation

GitHub Actions / ubuntu-docker-musl

cannot use `string` as `[]u8` in argument 1 to `compress.gzip.compress`

Check failure on line 47 in vlib/compress/gzip/read_gz_files_test.v

View workflow job for this annotation

GitHub Actions / gcc

cannot use `string` as `[]u8` in argument 1 to `compress.gzip.compress`

Check failure on line 47 in vlib/compress/gzip/read_gz_files_test.v

View workflow job for this annotation

GitHub Actions / clang

cannot use `string` as `[]u8` in argument 1 to `compress.gzip.compress`

Check failure on line 47 in vlib/compress/gzip/read_gz_files_test.v

View workflow job for this annotation

GitHub Actions / tcc

cannot use `string` as `[]u8` in argument 1 to `compress.gzip.compress`
decompress_0 := gzip.decompress(compress_0)!

// compression: default_max_probes=128
compress_128 := gzip.compress(content9)!

Check failure on line 51 in vlib/compress/gzip/read_gz_files_test.v

View workflow job for this annotation

GitHub Actions / gcc

cannot use `string` as `[]u8` in argument 1 to `compress.gzip.compress`

Check failure on line 51 in vlib/compress/gzip/read_gz_files_test.v

View workflow job for this annotation

GitHub Actions / clang (macos-14)

cannot use `string` as `[]u8` in argument 1 to `compress.gzip.compress`

Check failure on line 51 in vlib/compress/gzip/read_gz_files_test.v

View workflow job for this annotation

GitHub Actions / tcc

cannot use `string` as `[]u8` in argument 1 to `compress.gzip.compress`

Check failure on line 51 in vlib/compress/gzip/read_gz_files_test.v

View workflow job for this annotation

GitHub Actions / msvc

cannot use `string` as `[]u8` in argument 1 to `compress.gzip.compress`

Check failure on line 51 in vlib/compress/gzip/read_gz_files_test.v

View workflow job for this annotation

GitHub Actions / ubuntu-docker-musl

cannot use `string` as `[]u8` in argument 1 to `compress.gzip.compress`

Check failure on line 51 in vlib/compress/gzip/read_gz_files_test.v

View workflow job for this annotation

GitHub Actions / gcc

cannot use `string` as `[]u8` in argument 1 to `compress.gzip.compress`

Check failure on line 51 in vlib/compress/gzip/read_gz_files_test.v

View workflow job for this annotation

GitHub Actions / clang

cannot use `string` as `[]u8` in argument 1 to `compress.gzip.compress`

Check failure on line 51 in vlib/compress/gzip/read_gz_files_test.v

View workflow job for this annotation

GitHub Actions / tcc

cannot use `string` as `[]u8` in argument 1 to `compress.gzip.compress`
decompress_128 := gzip.decompress(compress_128)!

// compression: Huffman+LZ=4095
compress_4095 := gzip.compress(content9, flags: int(gzip.CompressFlags.max_probes_mask))!

Check failure on line 55 in vlib/compress/gzip/read_gz_files_test.v

View workflow job for this annotation

GitHub Actions / gcc

cannot use `string` as `[]u8` in argument 1 to `compress.gzip.compress`

Check failure on line 55 in vlib/compress/gzip/read_gz_files_test.v

View workflow job for this annotation

GitHub Actions / clang (macos-14)

cannot use `string` as `[]u8` in argument 1 to `compress.gzip.compress`

Check failure on line 55 in vlib/compress/gzip/read_gz_files_test.v

View workflow job for this annotation

GitHub Actions / tcc

cannot use `string` as `[]u8` in argument 1 to `compress.gzip.compress`

Check failure on line 55 in vlib/compress/gzip/read_gz_files_test.v

View workflow job for this annotation

GitHub Actions / msvc

cannot use `string` as `[]u8` in argument 1 to `compress.gzip.compress`

Check failure on line 55 in vlib/compress/gzip/read_gz_files_test.v

View workflow job for this annotation

GitHub Actions / ubuntu-docker-musl

cannot use `string` as `[]u8` in argument 1 to `compress.gzip.compress`

Check failure on line 55 in vlib/compress/gzip/read_gz_files_test.v

View workflow job for this annotation

GitHub Actions / gcc

cannot use `string` as `[]u8` in argument 1 to `compress.gzip.compress`

Check failure on line 55 in vlib/compress/gzip/read_gz_files_test.v

View workflow job for this annotation

GitHub Actions / clang

cannot use `string` as `[]u8` in argument 1 to `compress.gzip.compress`

Check failure on line 55 in vlib/compress/gzip/read_gz_files_test.v

View workflow job for this annotation

GitHub Actions / tcc

cannot use `string` as `[]u8` in argument 1 to `compress.gzip.compress`
decompress_4095 := gzip.decompress(compress_4095)!

assert content9 == decompress_0

Check failure on line 58 in vlib/compress/gzip/read_gz_files_test.v

View workflow job for this annotation

GitHub Actions / gcc

infix expr: cannot use `[]u8` (right expression) as `string`

Check failure on line 58 in vlib/compress/gzip/read_gz_files_test.v

View workflow job for this annotation

GitHub Actions / clang (macos-14)

infix expr: cannot use `[]u8` (right expression) as `string`

Check failure on line 58 in vlib/compress/gzip/read_gz_files_test.v

View workflow job for this annotation

GitHub Actions / tcc

infix expr: cannot use `[]u8` (right expression) as `string`

Check failure on line 58 in vlib/compress/gzip/read_gz_files_test.v

View workflow job for this annotation

GitHub Actions / msvc

infix expr: cannot use `[]u8` (right expression) as `string`

Check failure on line 58 in vlib/compress/gzip/read_gz_files_test.v

View workflow job for this annotation

GitHub Actions / ubuntu-docker-musl

infix expr: cannot use `[]u8` (right expression) as `string`

Check failure on line 58 in vlib/compress/gzip/read_gz_files_test.v

View workflow job for this annotation

GitHub Actions / gcc

infix expr: cannot use `[]u8` (right expression) as `string`

Check failure on line 58 in vlib/compress/gzip/read_gz_files_test.v

View workflow job for this annotation

GitHub Actions / clang

infix expr: cannot use `[]u8` (right expression) as `string`

Check failure on line 58 in vlib/compress/gzip/read_gz_files_test.v

View workflow job for this annotation

GitHub Actions / tcc

infix expr: cannot use `[]u8` (right expression) as `string`
assert content9 == decompress_128

Check failure on line 59 in vlib/compress/gzip/read_gz_files_test.v

View workflow job for this annotation

GitHub Actions / gcc

infix expr: cannot use `[]u8` (right expression) as `string`

Check failure on line 59 in vlib/compress/gzip/read_gz_files_test.v

View workflow job for this annotation

GitHub Actions / clang (macos-14)

infix expr: cannot use `[]u8` (right expression) as `string`

Check failure on line 59 in vlib/compress/gzip/read_gz_files_test.v

View workflow job for this annotation

GitHub Actions / tcc

infix expr: cannot use `[]u8` (right expression) as `string`

Check failure on line 59 in vlib/compress/gzip/read_gz_files_test.v

View workflow job for this annotation

GitHub Actions / msvc

infix expr: cannot use `[]u8` (right expression) as `string`

Check failure on line 59 in vlib/compress/gzip/read_gz_files_test.v

View workflow job for this annotation

GitHub Actions / ubuntu-docker-musl

infix expr: cannot use `[]u8` (right expression) as `string`

Check failure on line 59 in vlib/compress/gzip/read_gz_files_test.v

View workflow job for this annotation

GitHub Actions / gcc

infix expr: cannot use `[]u8` (right expression) as `string`

Check failure on line 59 in vlib/compress/gzip/read_gz_files_test.v

View workflow job for this annotation

GitHub Actions / clang

infix expr: cannot use `[]u8` (right expression) as `string`

Check failure on line 59 in vlib/compress/gzip/read_gz_files_test.v

View workflow job for this annotation

GitHub Actions / tcc

infix expr: cannot use `[]u8` (right expression) as `string`
assert content9 == decompress_4095

Check failure on line 60 in vlib/compress/gzip/read_gz_files_test.v

View workflow job for this annotation

GitHub Actions / gcc

infix expr: cannot use `[]u8` (right expression) as `string`

Check failure on line 60 in vlib/compress/gzip/read_gz_files_test.v

View workflow job for this annotation

GitHub Actions / clang (macos-14)

infix expr: cannot use `[]u8` (right expression) as `string`

Check failure on line 60 in vlib/compress/gzip/read_gz_files_test.v

View workflow job for this annotation

GitHub Actions / tcc

infix expr: cannot use `[]u8` (right expression) as `string`

Check failure on line 60 in vlib/compress/gzip/read_gz_files_test.v

View workflow job for this annotation

GitHub Actions / msvc

infix expr: cannot use `[]u8` (right expression) as `string`

Check failure on line 60 in vlib/compress/gzip/read_gz_files_test.v

View workflow job for this annotation

GitHub Actions / ubuntu-docker-musl

infix expr: cannot use `[]u8` (right expression) as `string`

Check failure on line 60 in vlib/compress/gzip/read_gz_files_test.v

View workflow job for this annotation

GitHub Actions / gcc

infix expr: cannot use `[]u8` (right expression) as `string`

Check failure on line 60 in vlib/compress/gzip/read_gz_files_test.v

View workflow job for this annotation

GitHub Actions / clang

infix expr: cannot use `[]u8` (right expression) as `string`

Check failure on line 60 in vlib/compress/gzip/read_gz_files_test.v

View workflow job for this annotation

GitHub Actions / tcc

infix expr: cannot use `[]u8` (right expression) as `string`
}

0 comments on commit 172827c

Please sign in to comment.