From 3797eeeccb8bb15ecb6c023c7d865a099ff5de08 Mon Sep 17 00:00:00 2001 From: Ryan Oldenburg Date: Tue, 9 Apr 2024 20:43:50 -0500 Subject: [PATCH 1/9] new in-memory createZipArchive --- src/zippy/ziparchives.nim | 175 +++++++++++++++++- ...st_tarballs.nim => test_tarballs_read.nim} | 0 ...archives.nim => test_ziparchives_read.nim} | 2 +- tests/test_ziparchives_write.nim | 23 +++ 4 files changed, 194 insertions(+), 6 deletions(-) rename tests/{test_tarballs.nim => test_tarballs_read.nim} (100%) rename tests/{test_ziparchives.nim => test_ziparchives_read.nim} (97%) create mode 100644 tests/test_ziparchives_write.nim diff --git a/src/zippy/ziparchives.nim b/src/zippy/ziparchives.nim index 91fd8d9..061df31 100644 --- a/src/zippy/ziparchives.nim +++ b/src/zippy/ziparchives.nim @@ -1,14 +1,18 @@ import common, crc, internal, std/memfiles, std/os, std/strutils, std/tables, - std/times, std/unicode, ziparchives_v1, zippy + std/times, std/unicode, ziparchives_v1, zippy, std/sequtils export common, ziparchives_v1 const + fileHeaderLen = 30 fileHeaderSignature = 0x04034b50.uint32 + dataDescriptorSignature = 0x08074b50.uint32 centralDirectoryFileHeaderSignature = 0x02014b50.uint32 endOfCentralDirectorySignature = 0x06054b50.uint32 zip64EndOfCentralDirectorySignature = 0x06064b50.uint32 - zip64EndOfCentralDirectoryLocatorSignature = 0x07064b50 + zip64EndOfCentralDirectoryLocatorSignature = 0x07064b50.uint32 + zip64ExtraFieldId = 1.uint16 + zip64FileHeaderExtraLen = 28 type ZipArchiveRecordKind = enum @@ -51,7 +55,7 @@ proc extractFile*( var pos = record.fileHeaderOffset - if pos + 30 > reader.memFile.size: + if pos + fileHeaderLen > reader.memFile.size: failArchiveEOF() if read32(src, pos) != fileHeaderSignature: @@ -69,7 +73,7 @@ proc extractFile*( fileNameLen = read16(src, pos + 26).int extraFieldLen = read16(src, pos + 28).int - pos += 30 + fileNameLen + extraFieldLen + pos += fileHeaderLen + fileNameLen + extraFieldLen if pos + record.compressedSize > reader.memFile.size: failArchiveEOF() @@ -328,7 +332,7 @@ proc openZipArchive*( extraFieldsOffset += 4 - if fieldId != 1: + if fieldId != zip64ExtraFieldId: extraFieldsOffset += fieldLen else: # These are the zip64 sizes @@ -449,3 +453,164 @@ proc extractAll*( raise e finally: reader.close() + +proc createZipArchive*( + entries: sink OrderedTable[string, string] +): string {.raises: [ZippyError].} = + + proc add16(dst: var string, v: int16 | uint16) = + dst.setLen(dst.len + 2) + var tmp = v + copyMem(dst[^2].addr, tmp.addr, 2) + + proc add32(dst: var string, v: int32 | uint32) = + dst.setLen(dst.len + 4) + var tmp = v + copyMem(dst[^4].addr, tmp.addr, 4) + + proc add64(dst: var string, v: int | int64 | uint | uint64) = + dst.setLen(dst.len + 8) + var tmp = v + copyMem(dst[^8].addr, tmp.addr, 8) + + proc msdos(time: Time): (uint16, uint16) = + let + dt = time.local() + seconds = (dt.second div 2).uint16 + minutes = dt.minute.uint16 + hours = dt.hour.uint16 + days = dt.monthday.uint16 + months = dt.month.uint16 + years = (max(0, dt.year - 1980)).uint16 + + var time = seconds + time = (minutes shl 5) or time + time = (hours shl 11) or time + + var date = days + date = (months shl 5) or date + date = (years shl 9) or date + + (time, date) + + let (lastModifiedTime, lastModifiedDate) = msdos(getTime()) + + type ArchiveEntry = object + fileHeaderOffset: int + uncompressedLen: int + compressedLen: int + compressionMethod: uint16 + uncompressedCrc32: uint32 + + var records: seq[(string, ArchiveEntry)] + for path in toSeq(entries.keys): # The entries table is modified so use toSeq + if path == "": + raise newException(ZippyError, "Invalid empty path") + if path.len > uint16.high.int: + raise newException(ZippyError, "Zip archive entry path len > uint16.high") + + var contents: string + discard entries.pop(path, contents) + + var + compressed: string + compressionMethod: uint16 + if contents == "": + discard + else: + compressed = compress(contents, BestSpeed, dfDeflate) + compressionMethod = 8 + + let uncompressedCrc32 = crc32(contents) + + records.add((path, ArchiveEntry( + fileHeaderOffset: result.len, + uncompressedLen: contents.len, + compressedLen: compressed.len, + compressionMethod: compressionMethod, + uncompressedCrc32: uncompressedCrc32 + ))) + + result.add32(fileHeaderSignature) + result.add16(45) # Min version to extract + result.add16((1.uint16 shl 3) or (1.uint16 shl 11)) # General purpose flags + result.add16(compressionMethod) + result.add16(lastModifiedTime) + result.add16(lastModifiedDate) + result.add32(0) # CRC-32 of uncompressed data (in trailing data descriptor) + result.add32(uint32.high) # Compressed size (or 0xffffffff for ZIP64) + result.add32(uint32.high) # Uncompressed size (or 0xffffffff for ZIP64) + result.add16(cast[uint16](path.len)) # File name length + result.add16(0) # Extra field length + + result.add(path) + + # result.add(compressed) + result.setLen(result.len + compressed.len) + copyMem( + result[result.len - compressed.len].addr, + compressed.cstring, + compressed.len + ) + + result.add32(dataDescriptorSignature) + result.add32(uncompressedCrc32) + result.add64(compressed.len) + result.add64(contents.len) + + let centralDirectoryStart = result.len + + for i in 0 ..< records.len: + let entry = records[i][1] + result.add32(centralDirectoryFileHeaderSignature) + result.add16(20) # Version made by + result.add16(20) # Min version to extract + result.add16(1.uint16 shl 11) # General purpose flags + result.add16(entry.compressionMethod) + result.add16(lastModifiedTime) + result.add16(lastModifiedDate) + result.add32(entry.uncompressedCrc32) + result.add32(uint32.high) # Compressed size (or 0xffffffff for ZIP64) + result.add32(uint32.high) # Uncompressed size (or 0xffffffff for ZIP64) + result.add16(cast[uint16](records[i][0].len)) # File name length + result.add16(28) # Extra field length + result.add16(0) # File comment length + result.add16(0) # Disk number where file starts + result.add16(0) # Internal file attributes + result.add32(0) # External file attributes + result.add32(uint32.high) # Relative offset of local file header (or 0xffffffff for ZIP64) + + result.add(records[i][0]) + + result.add16(zip64ExtraFieldId) + result.add16(24) + result.add64(entry.uncompressedLen) + result.add64(entry.compressedLen) + result.add64(entry.fileHeaderOffset) + + let centralDirectoryEnd = result.len + + result.add32(zip64EndOfCentralDirectorySignature) + result.add64(44) + result.add16(45) + result.add16(45) + result.add32(0) + result.add32(0) + result.add64(records.len) + result.add64(records.len) + result.add64(centralDirectoryEnd - centralDirectoryStart) + result.add64(centralDirectoryStart) + + result.add32(zip64EndOfCentralDirectoryLocatorSignature) + result.add32(0) + result.add64(centralDirectoryEnd) + result.add32(1) + + result.add32(endOfCentralDirectorySignature) + result.add32(0) # Number of this disk + result.add32(0) # Disk where central directory starts + result.add32(uint32.high) # Number of central directory records on this disk (or 0xffff for ZIP64) + result.add32(uint32.high) # Total number of central directory records (or 0xffff for ZIP64) + result.add32(uint32.high) # Size of central directory (bytes) (or 0xffffffff for ZIP64) + result.add32(uint32.high) # Offset of start of central directory, relative to start of archive (or 0xffffffff for ZIP64) + result.add16(0) diff --git a/tests/test_tarballs.nim b/tests/test_tarballs_read.nim similarity index 100% rename from tests/test_tarballs.nim rename to tests/test_tarballs_read.nim diff --git a/tests/test_ziparchives.nim b/tests/test_ziparchives_read.nim similarity index 97% rename from tests/test_ziparchives.nim rename to tests/test_ziparchives_read.nim index 0ca3fb0..e5bb15d 100644 --- a/tests/test_ziparchives.nim +++ b/tests/test_ziparchives_read.nim @@ -1,4 +1,4 @@ -import std/os, zippy/ziparchives, tables +import std/os, zippy/ziparchives, std/tables const testDir = getTempDir() # "tmp/zip" diff --git a/tests/test_ziparchives_write.nim b/tests/test_ziparchives_write.nim new file mode 100644 index 0000000..875f992 --- /dev/null +++ b/tests/test_ziparchives_write.nim @@ -0,0 +1,23 @@ +import std/os, zippy/ziparchives, std/tables + +var entries: OrderedTable[string, string] +entries["a/b/tmp.txt"] = "a text file's contents here" + +let archive = createZipArchive(entries) +echo archive.len +writeFile("tmp.zip", archive) + +let reader = openZipArchive("tmp.zip") + +try: + # Iterate over the paths in the zip archive. + for path in reader.walkFiles: + echo path + + # # Extract a file from the archive. + let contents = reader.extractFile("a/b/tmp.txt") + echo contents.len +finally: + # Remember to close the reader when done. + reader.close() + From d6e316c0bec98dbec9a42701fcff199466e78be1 Mon Sep 17 00:00:00 2001 From: Ryan Oldenburg Date: Tue, 9 Apr 2024 21:02:29 -0500 Subject: [PATCH 2/9] f --- src/zippy/ziparchives.nim | 13 +++++----- tests/test_ziparchives_write.nim | 42 +++++++++++++++++++------------- 2 files changed, 32 insertions(+), 23 deletions(-) diff --git a/src/zippy/ziparchives.nim b/src/zippy/ziparchives.nim index 061df31..da6500c 100644 --- a/src/zippy/ziparchives.nim +++ b/src/zippy/ziparchives.nim @@ -546,12 +546,13 @@ proc createZipArchive*( result.add(path) # result.add(compressed) - result.setLen(result.len + compressed.len) - copyMem( - result[result.len - compressed.len].addr, - compressed.cstring, - compressed.len - ) + if compressed != "": + result.setLen(result.len + compressed.len) + copyMem( + result[result.len - compressed.len].addr, + compressed.cstring, + compressed.len + ) result.add32(dataDescriptorSignature) result.add32(uncompressedCrc32) diff --git a/tests/test_ziparchives_write.nim b/tests/test_ziparchives_write.nim index 875f992..a10ca4a 100644 --- a/tests/test_ziparchives_write.nim +++ b/tests/test_ziparchives_write.nim @@ -1,23 +1,31 @@ -import std/os, zippy/ziparchives, std/tables +import std/os, zippy/ziparchives, std/tables, std/strutils -var entries: OrderedTable[string, string] -entries["a/b/tmp.txt"] = "a text file's contents here" +# var entries: OrderedTable[string, string] +# entries["a/b/tmp.txt"] = "a text file's contents here" -let archive = createZipArchive(entries) -echo archive.len -writeFile("tmp.zip", archive) +# let archive = createZipArchive(entries) +# echo archive.len +# writeFile("tmp.zip", archive) -let reader = openZipArchive("tmp.zip") +# let reader = openZipArchive("tmp.zip") -try: - # Iterate over the paths in the zip archive. - for path in reader.walkFiles: - echo path +# try: +# # Iterate over the paths in the zip archive. +# for path in reader.walkFiles: +# echo path - # # Extract a file from the archive. - let contents = reader.extractFile("a/b/tmp.txt") - echo contents.len -finally: - # Remember to close the reader when done. - reader.close() +# # # Extract a file from the archive. +# let contents = reader.extractFile("a/b/tmp.txt") +# echo contents.len +# finally: +# # Remember to close the reader when done. +# reader.close() +# var entries: OrderedTable[string, string] +# for path in walkDirRec(getCurrentDir(), relative = true, skipSpecial = true): +# if path.startsWith(".git"): +# continue +# if path.endsWith(".exe"): +# continue +# entries[path] = readFile(path) +# writeFile("tmp.zip", createZipArchive(entries)) From 09ebb4089676fe029dd09e6a83dcf5ac7126ad0e Mon Sep 17 00:00:00 2001 From: Ryan Oldenburg Date: Tue, 9 Apr 2024 22:32:48 -0500 Subject: [PATCH 3/9] mac --- src/zippy/ziparchives.nim | 30 ++++++++++++++---------------- tests/test_ziparchives_write.nim | 17 ++++++++++++++--- 2 files changed, 28 insertions(+), 19 deletions(-) diff --git a/src/zippy/ziparchives.nim b/src/zippy/ziparchives.nim index da6500c..1add653 100644 --- a/src/zippy/ziparchives.nim +++ b/src/zippy/ziparchives.nim @@ -6,13 +6,11 @@ export common, ziparchives_v1 const fileHeaderLen = 30 fileHeaderSignature = 0x04034b50.uint32 - dataDescriptorSignature = 0x08074b50.uint32 centralDirectoryFileHeaderSignature = 0x02014b50.uint32 endOfCentralDirectorySignature = 0x06054b50.uint32 zip64EndOfCentralDirectorySignature = 0x06064b50.uint32 zip64EndOfCentralDirectoryLocatorSignature = 0x07064b50.uint32 zip64ExtraFieldId = 1.uint16 - zip64FileHeaderExtraLen = 28 type ZipArchiveRecordKind = enum @@ -533,18 +531,23 @@ proc createZipArchive*( result.add32(fileHeaderSignature) result.add16(45) # Min version to extract - result.add16((1.uint16 shl 3) or (1.uint16 shl 11)) # General purpose flags + result.add16(1.uint16 shl 11) # General purpose flags result.add16(compressionMethod) result.add16(lastModifiedTime) result.add16(lastModifiedDate) - result.add32(0) # CRC-32 of uncompressed data (in trailing data descriptor) + result.add32(uncompressedCrc32) # CRC-32 of uncompressed data result.add32(uint32.high) # Compressed size (or 0xffffffff for ZIP64) result.add32(uint32.high) # Uncompressed size (or 0xffffffff for ZIP64) result.add16(cast[uint16](path.len)) # File name length - result.add16(0) # Extra field length + result.add16(20) # Extra field length result.add(path) + result.add16(zip64ExtraFieldId) + result.add16(16) + result.add64(contents.len) + result.add64(compressed.len) + # result.add(compressed) if compressed != "": result.setLen(result.len + compressed.len) @@ -554,18 +557,13 @@ proc createZipArchive*( compressed.len ) - result.add32(dataDescriptorSignature) - result.add32(uncompressedCrc32) - result.add64(compressed.len) - result.add64(contents.len) - let centralDirectoryStart = result.len for i in 0 ..< records.len: let entry = records[i][1] result.add32(centralDirectoryFileHeaderSignature) - result.add16(20) # Version made by - result.add16(20) # Min version to extract + result.add16(45) # Version made by + result.add16(45) # Min version to extract result.add16(1.uint16 shl 11) # General purpose flags result.add16(entry.compressionMethod) result.add16(lastModifiedTime) @@ -608,10 +606,10 @@ proc createZipArchive*( result.add32(1) result.add32(endOfCentralDirectorySignature) - result.add32(0) # Number of this disk - result.add32(0) # Disk where central directory starts - result.add32(uint32.high) # Number of central directory records on this disk (or 0xffff for ZIP64) - result.add32(uint32.high) # Total number of central directory records (or 0xffff for ZIP64) + result.add16(0) # Number of this disk + result.add16(0) # Disk where central directory starts + result.add16(uint16.high) # Number of central directory records on this disk (or 0xffff for ZIP64) + result.add16(uint16.high) # Total number of central directory records (or 0xffff for ZIP64) result.add32(uint32.high) # Size of central directory (bytes) (or 0xffffffff for ZIP64) result.add32(uint32.high) # Offset of start of central directory, relative to start of archive (or 0xffffffff for ZIP64) result.add16(0) diff --git a/tests/test_ziparchives_write.nim b/tests/test_ziparchives_write.nim index a10ca4a..3fc7e7f 100644 --- a/tests/test_ziparchives_write.nim +++ b/tests/test_ziparchives_write.nim @@ -1,7 +1,7 @@ import std/os, zippy/ziparchives, std/tables, std/strutils # var entries: OrderedTable[string, string] -# entries["a/b/tmp.txt"] = "a text file's contents here" +# entries["README.txt"] = "Hello, World!" # let archive = createZipArchive(entries) # echo archive.len @@ -15,8 +15,8 @@ import std/os, zippy/ziparchives, std/tables, std/strutils # echo path # # # Extract a file from the archive. -# let contents = reader.extractFile("a/b/tmp.txt") -# echo contents.len +# # let contents = reader.extractFile("tmp.txt") +# # echo contents.len # finally: # # Remember to close the reader when done. # reader.close() @@ -29,3 +29,14 @@ import std/os, zippy/ziparchives, std/tables, std/strutils # continue # entries[path] = readFile(path) # writeFile("tmp.zip", createZipArchive(entries)) + +# let reader = openZipArchive("tmp.zip") + +# try: +# # Iterate over the paths in the zip archive. +# for path in reader.walkFiles: +# echo path + +# finally: +# # Remember to close the reader when done. +# reader.close() From d3b5ff20c2780003c2813d858217065faf8dccc2 Mon Sep 17 00:00:00 2001 From: Ryan Oldenburg Date: Tue, 9 Apr 2024 22:36:04 -0500 Subject: [PATCH 4/9] f --- tests/test_ziparchives_write.nim | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_ziparchives_write.nim b/tests/test_ziparchives_write.nim index 3fc7e7f..dbb56e3 100644 --- a/tests/test_ziparchives_write.nim +++ b/tests/test_ziparchives_write.nim @@ -1,10 +1,10 @@ import std/os, zippy/ziparchives, std/tables, std/strutils -# var entries: OrderedTable[string, string] -# entries["README.txt"] = "Hello, World!" +var entries: OrderedTable[string, string] +entries["README.txt"] = "Hello, World!" -# let archive = createZipArchive(entries) -# echo archive.len +let archive = createZipArchive(entries) +echo archive.len # writeFile("tmp.zip", archive) # let reader = openZipArchive("tmp.zip") From 3690080a04b7550c493a020358558769b1dea9f4 Mon Sep 17 00:00:00 2001 From: Ryan Oldenburg Date: Tue, 9 Apr 2024 22:42:01 -0500 Subject: [PATCH 5/9] f --- src/zippy/ziparchives.nim | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/zippy/ziparchives.nim b/src/zippy/ziparchives.nim index 1add653..9814774 100644 --- a/src/zippy/ziparchives.nim +++ b/src/zippy/ziparchives.nim @@ -501,14 +501,16 @@ proc createZipArchive*( uncompressedCrc32: uint32 var records: seq[(string, ArchiveEntry)] - for path in toSeq(entries.keys): # The entries table is modified so use toSeq - if path == "": - raise newException(ZippyError, "Invalid empty path") - if path.len > uint16.high.int: - raise newException(ZippyError, "Zip archive entry path len > uint16.high") + for fileName in toSeq(entries.keys): # The entries table is modified so use toSeq + if fileName == "": + raise newException(ZippyError, "Invalid empty file name") + if fileName[0] == '/': + raise newException(ZippyError, "File paths must be relative") + if fileName.len > uint16.high.int: + raise newException(ZippyError, "File name len > uint16.high") var contents: string - discard entries.pop(path, contents) + discard entries.pop(fileName, contents) var compressed: string @@ -521,7 +523,7 @@ proc createZipArchive*( let uncompressedCrc32 = crc32(contents) - records.add((path, ArchiveEntry( + records.add((fileName, ArchiveEntry( fileHeaderOffset: result.len, uncompressedLen: contents.len, compressedLen: compressed.len, @@ -538,10 +540,10 @@ proc createZipArchive*( result.add32(uncompressedCrc32) # CRC-32 of uncompressed data result.add32(uint32.high) # Compressed size (or 0xffffffff for ZIP64) result.add32(uint32.high) # Uncompressed size (or 0xffffffff for ZIP64) - result.add16(cast[uint16](path.len)) # File name length + result.add16(cast[uint16](fileName.len)) # File name length result.add16(20) # Extra field length - result.add(path) + result.add(fileName) result.add16(zip64ExtraFieldId) result.add16(16) From b1794d7177de72cc0dfc0d5c7ae9dba0a62b777a Mon Sep 17 00:00:00 2001 From: Ryan Oldenburg Date: Tue, 9 Apr 2024 22:45:55 -0500 Subject: [PATCH 6/9] f --- src/zippy/ziparchives.nim | 321 ++++++++++++++++--------------- tests/test_ziparchives_write.nim | 81 ++++---- 2 files changed, 202 insertions(+), 200 deletions(-) diff --git a/src/zippy/ziparchives.nim b/src/zippy/ziparchives.nim index 9814774..724c5f1 100644 --- a/src/zippy/ziparchives.nim +++ b/src/zippy/ziparchives.nim @@ -452,166 +452,167 @@ proc extractAll*( finally: reader.close() -proc createZipArchive*( - entries: sink OrderedTable[string, string] -): string {.raises: [ZippyError].} = - - proc add16(dst: var string, v: int16 | uint16) = - dst.setLen(dst.len + 2) - var tmp = v - copyMem(dst[^2].addr, tmp.addr, 2) - - proc add32(dst: var string, v: int32 | uint32) = - dst.setLen(dst.len + 4) - var tmp = v - copyMem(dst[^4].addr, tmp.addr, 4) - - proc add64(dst: var string, v: int | int64 | uint | uint64) = - dst.setLen(dst.len + 8) - var tmp = v - copyMem(dst[^8].addr, tmp.addr, 8) - - proc msdos(time: Time): (uint16, uint16) = - let - dt = time.local() - seconds = (dt.second div 2).uint16 - minutes = dt.minute.uint16 - hours = dt.hour.uint16 - days = dt.monthday.uint16 - months = dt.month.uint16 - years = (max(0, dt.year - 1980)).uint16 - - var time = seconds - time = (minutes shl 5) or time - time = (hours shl 11) or time - - var date = days - date = (months shl 5) or date - date = (years shl 9) or date - - (time, date) - - let (lastModifiedTime, lastModifiedDate) = msdos(getTime()) - - type ArchiveEntry = object - fileHeaderOffset: int - uncompressedLen: int - compressedLen: int - compressionMethod: uint16 - uncompressedCrc32: uint32 - - var records: seq[(string, ArchiveEntry)] - for fileName in toSeq(entries.keys): # The entries table is modified so use toSeq - if fileName == "": - raise newException(ZippyError, "Invalid empty file name") - if fileName[0] == '/': - raise newException(ZippyError, "File paths must be relative") - if fileName.len > uint16.high.int: - raise newException(ZippyError, "File name len > uint16.high") +when (NimMajor, NimMinor, NimPatch) >= (1, 6, 0): + proc createZipArchive*( + entries: sink OrderedTable[string, string] + ): string {.raises: [ZippyError].} = + + proc add16(dst: var string, v: int16 | uint16) = + dst.setLen(dst.len + 2) + var tmp = v + copyMem(dst[^2].addr, tmp.addr, 2) + + proc add32(dst: var string, v: int32 | uint32) = + dst.setLen(dst.len + 4) + var tmp = v + copyMem(dst[^4].addr, tmp.addr, 4) + + proc add64(dst: var string, v: int | int64 | uint | uint64) = + dst.setLen(dst.len + 8) + var tmp = v + copyMem(dst[^8].addr, tmp.addr, 8) + + proc msdos(time: Time): (uint16, uint16) = + let + dt = time.local() + seconds = (dt.second div 2).uint16 + minutes = dt.minute.uint16 + hours = dt.hour.uint16 + days = dt.monthday.uint16 + months = dt.month.uint16 + years = (max(0, dt.year - 1980)).uint16 + + var time = seconds + time = (minutes shl 5) or time + time = (hours shl 11) or time + + var date = days + date = (months shl 5) or date + date = (years shl 9) or date + + (time, date) + + let (lastModifiedTime, lastModifiedDate) = msdos(getTime()) + + type ArchiveEntry = object + fileHeaderOffset: int + uncompressedLen: int + compressedLen: int + compressionMethod: uint16 + uncompressedCrc32: uint32 - var contents: string - discard entries.pop(fileName, contents) + var records: seq[(string, ArchiveEntry)] + for fileName in toSeq(entries.keys): # The entries table is modified so use toSeq + if fileName == "": + raise newException(ZippyError, "Invalid empty file name") + if fileName[0] == '/': + raise newException(ZippyError, "File paths must be relative") + if fileName.len > uint16.high.int: + raise newException(ZippyError, "File name len > uint16.high") - var - compressed: string - compressionMethod: uint16 - if contents == "": - discard - else: - compressed = compress(contents, BestSpeed, dfDeflate) - compressionMethod = 8 - - let uncompressedCrc32 = crc32(contents) - - records.add((fileName, ArchiveEntry( - fileHeaderOffset: result.len, - uncompressedLen: contents.len, - compressedLen: compressed.len, - compressionMethod: compressionMethod, - uncompressedCrc32: uncompressedCrc32 - ))) - - result.add32(fileHeaderSignature) - result.add16(45) # Min version to extract - result.add16(1.uint16 shl 11) # General purpose flags - result.add16(compressionMethod) - result.add16(lastModifiedTime) - result.add16(lastModifiedDate) - result.add32(uncompressedCrc32) # CRC-32 of uncompressed data - result.add32(uint32.high) # Compressed size (or 0xffffffff for ZIP64) - result.add32(uint32.high) # Uncompressed size (or 0xffffffff for ZIP64) - result.add16(cast[uint16](fileName.len)) # File name length - result.add16(20) # Extra field length - - result.add(fileName) - - result.add16(zip64ExtraFieldId) - result.add16(16) - result.add64(contents.len) - result.add64(compressed.len) - - # result.add(compressed) - if compressed != "": - result.setLen(result.len + compressed.len) - copyMem( - result[result.len - compressed.len].addr, - compressed.cstring, - compressed.len - ) + var contents: string + discard entries.pop(fileName, contents) - let centralDirectoryStart = result.len - - for i in 0 ..< records.len: - let entry = records[i][1] - result.add32(centralDirectoryFileHeaderSignature) - result.add16(45) # Version made by - result.add16(45) # Min version to extract - result.add16(1.uint16 shl 11) # General purpose flags - result.add16(entry.compressionMethod) - result.add16(lastModifiedTime) - result.add16(lastModifiedDate) - result.add32(entry.uncompressedCrc32) - result.add32(uint32.high) # Compressed size (or 0xffffffff for ZIP64) - result.add32(uint32.high) # Uncompressed size (or 0xffffffff for ZIP64) - result.add16(cast[uint16](records[i][0].len)) # File name length - result.add16(28) # Extra field length - result.add16(0) # File comment length - result.add16(0) # Disk number where file starts - result.add16(0) # Internal file attributes - result.add32(0) # External file attributes - result.add32(uint32.high) # Relative offset of local file header (or 0xffffffff for ZIP64) - - result.add(records[i][0]) - - result.add16(zip64ExtraFieldId) - result.add16(24) - result.add64(entry.uncompressedLen) - result.add64(entry.compressedLen) - result.add64(entry.fileHeaderOffset) - - let centralDirectoryEnd = result.len - - result.add32(zip64EndOfCentralDirectorySignature) - result.add64(44) - result.add16(45) - result.add16(45) - result.add32(0) - result.add32(0) - result.add64(records.len) - result.add64(records.len) - result.add64(centralDirectoryEnd - centralDirectoryStart) - result.add64(centralDirectoryStart) - - result.add32(zip64EndOfCentralDirectoryLocatorSignature) - result.add32(0) - result.add64(centralDirectoryEnd) - result.add32(1) - - result.add32(endOfCentralDirectorySignature) - result.add16(0) # Number of this disk - result.add16(0) # Disk where central directory starts - result.add16(uint16.high) # Number of central directory records on this disk (or 0xffff for ZIP64) - result.add16(uint16.high) # Total number of central directory records (or 0xffff for ZIP64) - result.add32(uint32.high) # Size of central directory (bytes) (or 0xffffffff for ZIP64) - result.add32(uint32.high) # Offset of start of central directory, relative to start of archive (or 0xffffffff for ZIP64) - result.add16(0) + var + compressed: string + compressionMethod: uint16 + if contents == "": + discard + else: + compressed = compress(contents, BestSpeed, dfDeflate) + compressionMethod = 8 + + let uncompressedCrc32 = crc32(contents) + + records.add((fileName, ArchiveEntry( + fileHeaderOffset: result.len, + uncompressedLen: contents.len, + compressedLen: compressed.len, + compressionMethod: compressionMethod, + uncompressedCrc32: uncompressedCrc32 + ))) + + result.add32(fileHeaderSignature) + result.add16(45) # Min version to extract + result.add16(1.uint16 shl 11) # General purpose flags + result.add16(compressionMethod) + result.add16(lastModifiedTime) + result.add16(lastModifiedDate) + result.add32(uncompressedCrc32) # CRC-32 of uncompressed data + result.add32(uint32.high) # Compressed size (or 0xffffffff for ZIP64) + result.add32(uint32.high) # Uncompressed size (or 0xffffffff for ZIP64) + result.add16(cast[uint16](fileName.len)) # File name length + result.add16(20) # Extra field length + + result.add(fileName) + + result.add16(zip64ExtraFieldId) + result.add16(16) + result.add64(contents.len) + result.add64(compressed.len) + + # result.add(compressed) + if compressed != "": + result.setLen(result.len + compressed.len) + copyMem( + result[result.len - compressed.len].addr, + compressed.cstring, + compressed.len + ) + + let centralDirectoryStart = result.len + + for i in 0 ..< records.len: + let entry = records[i][1] + result.add32(centralDirectoryFileHeaderSignature) + result.add16(45) # Version made by + result.add16(45) # Min version to extract + result.add16(1.uint16 shl 11) # General purpose flags + result.add16(entry.compressionMethod) + result.add16(lastModifiedTime) + result.add16(lastModifiedDate) + result.add32(entry.uncompressedCrc32) + result.add32(uint32.high) # Compressed size (or 0xffffffff for ZIP64) + result.add32(uint32.high) # Uncompressed size (or 0xffffffff for ZIP64) + result.add16(cast[uint16](records[i][0].len)) # File name length + result.add16(28) # Extra field length + result.add16(0) # File comment length + result.add16(0) # Disk number where file starts + result.add16(0) # Internal file attributes + result.add32(0) # External file attributes + result.add32(uint32.high) # Relative offset of local file header (or 0xffffffff for ZIP64) + + result.add(records[i][0]) + + result.add16(zip64ExtraFieldId) + result.add16(24) + result.add64(entry.uncompressedLen) + result.add64(entry.compressedLen) + result.add64(entry.fileHeaderOffset) + + let centralDirectoryEnd = result.len + + result.add32(zip64EndOfCentralDirectorySignature) + result.add64(44) + result.add16(45) + result.add16(45) + result.add32(0) + result.add32(0) + result.add64(records.len) + result.add64(records.len) + result.add64(centralDirectoryEnd - centralDirectoryStart) + result.add64(centralDirectoryStart) + + result.add32(zip64EndOfCentralDirectoryLocatorSignature) + result.add32(0) + result.add64(centralDirectoryEnd) + result.add32(1) + + result.add32(endOfCentralDirectorySignature) + result.add16(0) # Number of this disk + result.add16(0) # Disk where central directory starts + result.add16(uint16.high) # Number of central directory records on this disk (or 0xffff for ZIP64) + result.add16(uint16.high) # Total number of central directory records (or 0xffff for ZIP64) + result.add32(uint32.high) # Size of central directory (bytes) (or 0xffffffff for ZIP64) + result.add32(uint32.high) # Offset of start of central directory, relative to start of archive (or 0xffffffff for ZIP64) + result.add16(0) diff --git a/tests/test_ziparchives_write.nim b/tests/test_ziparchives_write.nim index dbb56e3..a8f7844 100644 --- a/tests/test_ziparchives_write.nim +++ b/tests/test_ziparchives_write.nim @@ -1,42 +1,43 @@ import std/os, zippy/ziparchives, std/tables, std/strutils -var entries: OrderedTable[string, string] -entries["README.txt"] = "Hello, World!" - -let archive = createZipArchive(entries) -echo archive.len -# writeFile("tmp.zip", archive) - -# let reader = openZipArchive("tmp.zip") - -# try: -# # Iterate over the paths in the zip archive. -# for path in reader.walkFiles: -# echo path - -# # # Extract a file from the archive. -# # let contents = reader.extractFile("tmp.txt") -# # echo contents.len -# finally: -# # Remember to close the reader when done. -# reader.close() - -# var entries: OrderedTable[string, string] -# for path in walkDirRec(getCurrentDir(), relative = true, skipSpecial = true): -# if path.startsWith(".git"): -# continue -# if path.endsWith(".exe"): -# continue -# entries[path] = readFile(path) -# writeFile("tmp.zip", createZipArchive(entries)) - -# let reader = openZipArchive("tmp.zip") - -# try: -# # Iterate over the paths in the zip archive. -# for path in reader.walkFiles: -# echo path - -# finally: -# # Remember to close the reader when done. -# reader.close() +when (NimMajor, NimMinor, NimPatch) >= (1, 6, 0): + var entries: OrderedTable[string, string] + entries["README.txt"] = "Hello, World!" + + let archive = createZipArchive(entries) + echo archive.len + # writeFile("tmp.zip", archive) + + # let reader = openZipArchive("tmp.zip") + + # try: + # # Iterate over the paths in the zip archive. + # for path in reader.walkFiles: + # echo path + + # # # Extract a file from the archive. + # # let contents = reader.extractFile("tmp.txt") + # # echo contents.len + # finally: + # # Remember to close the reader when done. + # reader.close() + + # var entries: OrderedTable[string, string] + # for path in walkDirRec(getCurrentDir(), relative = true, skipSpecial = true): + # if path.startsWith(".git"): + # continue + # if path.endsWith(".exe"): + # continue + # entries[path] = readFile(path) + # writeFile("tmp.zip", createZipArchive(entries)) + + # let reader = openZipArchive("tmp.zip") + + # try: + # # Iterate over the paths in the zip archive. + # for path in reader.walkFiles: + # echo path + + # finally: + # # Remember to close the reader when done. + # reader.close() From 0dfebb6c755cc248e84a46c0df548a5ef8520e6d Mon Sep 17 00:00:00 2001 From: Ryan Oldenburg Date: Tue, 9 Apr 2024 22:47:27 -0500 Subject: [PATCH 7/9] f --- tests/all.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/all.nim b/tests/all.nim index 9f95c4b..b228298 100644 --- a/tests/all.nim +++ b/tests/all.nim @@ -1 +1 @@ -import test, test_known_bad, test_levels, test_tarballs, test_ziparchives +import test, test_known_bad, test_levels, test_tarballs_read, test_ziparchives_read, test_archives_write From 1b214b7a794d3a071f5bd6fdc12869e923e447c4 Mon Sep 17 00:00:00 2001 From: Ryan Oldenburg Date: Tue, 9 Apr 2024 22:48:40 -0500 Subject: [PATCH 8/9] f --- tests/all.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/all.nim b/tests/all.nim index b228298..5346be2 100644 --- a/tests/all.nim +++ b/tests/all.nim @@ -1 +1 @@ -import test, test_known_bad, test_levels, test_tarballs_read, test_ziparchives_read, test_archives_write +import test, test_known_bad, test_levels, test_tarballs_read, test_ziparchives_read, test_ziparchives_write From 6de6512a2dd4c57b55b0e0d784e567b1c1387f6f Mon Sep 17 00:00:00 2001 From: Ryan Oldenburg Date: Wed, 10 Apr 2024 00:04:35 -0500 Subject: [PATCH 9/9] f --- src/zippy/ziparchives.nim | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/src/zippy/ziparchives.nim b/src/zippy/ziparchives.nim index 724c5f1..7299fe3 100644 --- a/src/zippy/ziparchives.nim +++ b/src/zippy/ziparchives.nim @@ -510,23 +510,25 @@ when (NimMajor, NimMinor, NimPatch) >= (1, 6, 0): if fileName.len > uint16.high.int: raise newException(ZippyError, "File name len > uint16.high") - var contents: string - discard entries.pop(fileName, contents) - var + uncompressedLen: int + uncompressedCrc32: uint32 compressed: string compressionMethod: uint16 - if contents == "": - discard - else: - compressed = compress(contents, BestSpeed, dfDeflate) - compressionMethod = 8 - - let uncompressedCrc32 = crc32(contents) + block: # Free `contents` after this block + var contents: string + discard entries.pop(fileName, contents) + uncompressedLen = contents.len + uncompressedCrc32 = crc32(contents) + if contents == "": + discard + else: + compressed = compress(contents, BestSpeed, dfDeflate) + compressionMethod = 8 records.add((fileName, ArchiveEntry( fileHeaderOffset: result.len, - uncompressedLen: contents.len, + uncompressedLen: uncompressedLen, compressedLen: compressed.len, compressionMethod: compressionMethod, uncompressedCrc32: uncompressedCrc32 @@ -548,7 +550,7 @@ when (NimMajor, NimMinor, NimPatch) >= (1, 6, 0): result.add16(zip64ExtraFieldId) result.add16(16) - result.add64(contents.len) + result.add64(uncompressedLen) result.add64(compressed.len) # result.add(compressed)