From df58c7a328f042a2a02aa4c850b5929774a15cb7 Mon Sep 17 00:00:00 2001 From: Danixu Date: Sun, 7 Jul 2024 23:36:39 +0200 Subject: [PATCH 1/2] Fixed a bug when the last block of the file doesn't matches the blocksize --- lib/spdlog | 2 +- src/ziso.cpp | 47 +++++++++++++++++++++++++++++++++-------------- 2 files changed, 34 insertions(+), 15 deletions(-) diff --git a/lib/spdlog b/lib/spdlog index a2b4262..eeb22c1 160000 --- a/lib/spdlog +++ b/lib/spdlog @@ -1 +1 @@ -Subproject commit a2b4262090fd3f005c2315dcb5be2f0f1774a005 +Subproject commit eeb22c13bb65f342434c991641b80ecf4821e9b9 diff --git a/src/ziso.cpp b/src/ziso.cpp index 14fea17..f6124da 100644 --- a/src/ziso.cpp +++ b/src/ziso.cpp @@ -385,12 +385,12 @@ int main(int argc, char **argv) progress_compress((uint64_t)inFile.tellg() + readBufferPos, inputSize, (uint64_t)blockStartPosition - headerSize); } - spdlog::trace("Aligning the last block from: {}...", outFile.tellp()); + spdlog::trace("Aligning the last block from: {}...", (uint64_t)outFile.tellp()); // Align the file and set the eof position block file_align(outFile, fileHeader.indexShift); uint64_t blockEndPosition = outFile.tellp(); blocks[blocksNumber - 1] = (blockEndPosition >> fileHeader.indexShift); - spdlog::trace("Aligned block position: {}...", outFile.tellp()); + spdlog::trace("Aligned block position: {}...", (uint64_t)outFile.tellp()); // The HDL_dump bug trims the data at the end of the file if doesn't fit into a 2048 multiple. // This fix will pad the output file to the nearest 2048 bytes multiple. @@ -422,7 +422,7 @@ int main(int argc, char **argv) inFile.read(reinterpret_cast(&fileHeader), sizeof(fileHeader)); // Calculate the blocks number - blocksNumber = ceil(fileHeader.uncompressedSize / fileHeader.blockSize) + 1; + blocksNumber = ceil((float)fileHeader.uncompressedSize / fileHeader.blockSize) + 1; // Reserve and read the blocks index blocks.resize(blocksNumber, 0); @@ -478,19 +478,35 @@ int main(int argc, char **argv) uint64_t currentBlockSize = blockEndPosition - blockStartPosition; spdlog::trace( - "Block Start Position: {} - Block End Position: {} - Block Size: {} - Uncompressed: {}", + "Current Block: {} - Block Start Position: {} - Block End Position: {} - Block Size: {} - Uncompressed: {}", + currentBlock + 1, blockStartPosition, blockEndPosition, currentBlockSize, uncompressed); + // The current block size cannot exceed 2 x blockSize. + if (currentBlockSize > (fileHeader.blockSize * 2)) + { + // Looks like the header is corrupted + spdlog::error("The input file header is corrupt. Corrupted index block."); + return_code = 1; + goto exit; + } + uint32_t leftInReadBuffer = readBufferSize - readBufferPos; // If the current reader position is the end of the buffer, fill the buffer with new data if (currentBlockSize > leftInReadBuffer) { + // At the first block, the input file must be synced to the start point or can be missaligned + if (currentBlock == 0) + { + inFile.seekg(blockStartPosition); + } spdlog::trace("The reader buffer is empty... reading more data."); // Get the data left in file uint64_t leftInFile = inputSize - inFile.tellg(); + spdlog::trace("There are {} bytes left in the file.", leftInFile); // To Read uint64_t toRead = readBufferSize - leftInReadBuffer; @@ -498,22 +514,16 @@ int main(int argc, char **argv) { toRead = leftInFile; } + spdlog::trace("{} bytes will be read.", toRead); // Move the data buffer to the start point std::memmove(readBuffer.data(), readBuffer.data() + readBufferPos, leftInReadBuffer); + spdlog::trace("Current file position: {}", (uint64_t)inFile.tellp()); inFile.read(readBuffer.data() + leftInReadBuffer, toRead); + spdlog::trace("New file position: {}", (uint64_t)inFile.tellp()); readBufferPos = 0; } - // The current block size cannot exceed 2 x blockSize. - if (currentBlockSize > (fileHeader.blockSize * 2)) - { - // Looks like the header is corrupted - spdlog::error("The input file header is corrupt. Corrupted index block."); - return_code = 1; - goto exit; - } - int decompressedBytes = decompress_block( readBuffer.data() + readBufferPos, currentBlockSize, @@ -521,6 +531,15 @@ int main(int argc, char **argv) options.blockSize, uncompressed); + if (currentBlock == (blocksNumber - 2)) + { + // The LZ4 compressor seems not to be taking into account of the input data size to limit the decompressed data + // Or at least is not reporting the correct decompressed size. + // This is a fix to avoid this problem calculating the correct last block size instead to use the LZ4 compressor reported size. + decompressedBytes = fileHeader.uncompressedSize - ((uint64_t)outFile.tellp() + writeBufferPos); + spdlog::trace("Fixed the last block size to: {}", decompressedBytes); + } + spdlog::trace("Decompressed data bytes: {}", decompressedBytes); if (decompressedBytes > 0) @@ -548,7 +567,7 @@ int main(int argc, char **argv) if ((uint64_t)outFile.tellp() != fileHeader.uncompressedSize) { - spdlog::error("The output filesize doesn't matches the header filesize. {} vs {}", fileHeader.uncompressedSize, outFile.tellp()); + spdlog::error("The output filesize doesn't matches the header filesize. {} vs {}", fileHeader.uncompressedSize, (uint64_t)outFile.tellp()); return_code = 1; } } From 49657e62b8055727ab68dbbc7fc8d60acd2a2897 Mon Sep 17 00:00:00 2001 From: Danixu Date: Sun, 7 Jul 2024 23:52:05 +0200 Subject: [PATCH 2/2] Udated the version --- include/ziso.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/ziso.h b/include/ziso.h index cf4c7db..1adedf2 100644 --- a/include/ziso.h +++ b/include/ziso.h @@ -1,6 +1,6 @@ #define TITLE "ziso - ZSO compressor/decompressor" #define COPYR "Created by Daniel Carrasco (2023)" -#define VERSI "0.5.2" +#define VERSI "0.6.1" #include "banner.h" #include