diff --git a/components/core/src/clp/TraceableException.hpp b/components/core/src/clp/TraceableException.hpp index cd8e33f4b..f60273f93 100644 --- a/components/core/src/clp/TraceableException.hpp +++ b/components/core/src/clp/TraceableException.hpp @@ -39,6 +39,7 @@ class TraceableException : public std::exception { // Macros // Define a version of __FILE__ that's relative to the source directory #ifdef SOURCE_PATH_SIZE + // NOLINTNEXTLINE #define __FILENAME__ ((__FILE__) + SOURCE_PATH_SIZE) #else // We don't know the source path size, so just default to __FILE__ diff --git a/components/core/src/clp/streaming_compression/Compressor.hpp b/components/core/src/clp/streaming_compression/Compressor.hpp index 165696091..ee7846aee 100644 --- a/components/core/src/clp/streaming_compression/Compressor.hpp +++ b/components/core/src/clp/streaming_compression/Compressor.hpp @@ -1,14 +1,20 @@ #ifndef CLP_STREAMING_COMPRESSION_COMPRESSOR_HPP #define CLP_STREAMING_COMPRESSION_COMPRESSOR_HPP -#include -#include +#include +#include + +#include "../ErrorCode.hpp" +#include "../FileWriter.hpp" #include "../TraceableException.hpp" #include "../WriterInterface.hpp" #include "Constants.hpp" namespace clp::streaming_compression { +/** + * Generic compressor interface. + */ class Compressor : public WriterInterface { public: // Types @@ -19,7 +25,7 @@ class Compressor : public WriterInterface { : TraceableException(error_code, filename, line_number) {} // Methods - char const* what() const noexcept override { + [[nodiscard]] auto what() const noexcept -> char const* override { return "streaming_compression::Compressor operation failed"; } }; @@ -30,9 +36,12 @@ class Compressor : public WriterInterface { // Destructor virtual ~Compressor() = default; - // Explicitly disable copy and move constructor/assignment + // Explicitly disable copy constructor/assignment and enable the move version Compressor(Compressor const&) = delete; - Compressor& operator=(Compressor const&) = delete; + auto operator=(Compressor const&) -> Compressor& = delete; + + Compressor(Compressor&&) noexcept = default; + auto operator=(Compressor&&) -> Compressor& = default; // Methods implementing the WriterInterface /** @@ -40,24 +49,36 @@ class Compressor : public WriterInterface { * @param pos * @return ErrorCode_Unsupported */ - ErrorCode try_seek_from_begin(size_t pos) override { return ErrorCode_Unsupported; } + [[nodiscard]] auto try_seek_from_begin([[maybe_unused]] size_t pos) -> ErrorCode override { + return ErrorCode_Unsupported; + } /** * Unsupported operation * @param pos * @return ErrorCode_Unsupported */ - ErrorCode try_seek_from_current(off_t offset) override { return ErrorCode_Unsupported; } + [[nodiscard]] auto try_seek_from_current([[maybe_unused]] off_t offset) -> ErrorCode override { + return ErrorCode_Unsupported; + } // Methods /** * Closes the compression stream */ - virtual void close() = 0; + virtual auto close() -> void = 0; + + /** + * Initializes the compression stream with the given compression level + * @param file_writer + * @param compression_level + */ + virtual auto open(FileWriter& file_writer, [[maybe_unused]] int compression_level = 0) -> void + = 0; -protected: +private: // Variables - CompressorType m_type; + CompressorType m_type{}; }; } // namespace clp::streaming_compression diff --git a/components/core/src/clp/streaming_compression/Constants.hpp b/components/core/src/clp/streaming_compression/Constants.hpp index 4649c2e98..6c6a78bfc 100644 --- a/components/core/src/clp/streaming_compression/Constants.hpp +++ b/components/core/src/clp/streaming_compression/Constants.hpp @@ -1,7 +1,6 @@ #ifndef CLP_STREAMING_COMPRESSION_CONSTANTS_HPP #define CLP_STREAMING_COMPRESSION_CONSTANTS_HPP -#include #include namespace clp::streaming_compression { diff --git a/components/core/src/clp/streaming_compression/passthrough/Compressor.cpp b/components/core/src/clp/streaming_compression/passthrough/Compressor.cpp index 750ab48c1..112bf7d39 100644 --- a/components/core/src/clp/streaming_compression/passthrough/Compressor.cpp +++ b/components/core/src/clp/streaming_compression/passthrough/Compressor.cpp @@ -1,9 +1,12 @@ #include "Compressor.hpp" -#include "../../Defs.h" +#include + +#include "../../ErrorCode.hpp" +#include "../../TraceableException.hpp" namespace clp::streaming_compression::passthrough { -void Compressor::write(char const* data, size_t const data_length) { +auto Compressor::write(char const* data, size_t const data_length) -> void { if (nullptr == m_compressed_stream_file_writer) { throw OperationFailed(ErrorCode_NotInit, __FILENAME__, __LINE__); } @@ -19,7 +22,7 @@ void Compressor::write(char const* data, size_t const data_length) { m_compressed_stream_file_writer->write(data, data_length); } -void Compressor::flush() { +auto Compressor::flush() -> void { if (nullptr == m_compressed_stream_file_writer) { throw OperationFailed(ErrorCode_NotInit, __FILENAME__, __LINE__); } @@ -27,7 +30,7 @@ void Compressor::flush() { m_compressed_stream_file_writer->flush(); } -ErrorCode Compressor::try_get_pos(size_t& pos) const { +auto Compressor::try_get_pos(size_t& pos) const -> ErrorCode { if (nullptr == m_compressed_stream_file_writer) { return ErrorCode_NotInit; } @@ -35,11 +38,11 @@ ErrorCode Compressor::try_get_pos(size_t& pos) const { return m_compressed_stream_file_writer->try_get_pos(pos); } -void Compressor::close() { +auto Compressor::close() -> void { m_compressed_stream_file_writer = nullptr; } -void Compressor::open(FileWriter& file_writer) { +auto Compressor::open(FileWriter& file_writer, [[maybe_unused]] int compression_level) -> void { m_compressed_stream_file_writer = &file_writer; } } // namespace clp::streaming_compression::passthrough diff --git a/components/core/src/clp/streaming_compression/passthrough/Compressor.hpp b/components/core/src/clp/streaming_compression/passthrough/Compressor.hpp index b3735bd1e..a24024666 100644 --- a/components/core/src/clp/streaming_compression/passthrough/Compressor.hpp +++ b/components/core/src/clp/streaming_compression/passthrough/Compressor.hpp @@ -1,9 +1,13 @@ #ifndef CLP_STREAMING_COMPRESSION_PASSTHROUGH_COMPRESSOR_HPP #define CLP_STREAMING_COMPRESSION_PASSTHROUGH_COMPRESSOR_HPP +#include + +#include "../../ErrorCode.hpp" #include "../../FileWriter.hpp" #include "../../TraceableException.hpp" #include "../Compressor.hpp" +#include "../Constants.hpp" namespace clp::streaming_compression::passthrough { /** @@ -19,19 +23,23 @@ class Compressor : public ::clp::streaming_compression::Compressor { : TraceableException(error_code, filename, line_number) {} // Methods - char const* what() const noexcept override { + [[nodiscard]] auto what() const noexcept -> char const* override { return "streaming_compression::passthrough::Compressor operation failed"; } }; // Constructors - Compressor() - : ::clp::streaming_compression::Compressor(CompressorType::Passthrough), - m_compressed_stream_file_writer(nullptr) {} + Compressor() : ::clp::streaming_compression::Compressor{CompressorType::Passthrough} {} + + // Destructor + ~Compressor() override = default; - // Explicitly disable copy and move constructor/assignment + // Explicitly disable copy constructor/assignment and enable the move version Compressor(Compressor const&) = delete; - Compressor& operator=(Compressor const&) = delete; + auto operator=(Compressor const&) -> Compressor& = delete; + + Compressor(Compressor&&) noexcept = default; + auto operator=(Compressor&&) -> Compressor& = default; // Methods implementing the WriterInterface /** @@ -39,35 +47,37 @@ class Compressor : public ::clp::streaming_compression::Compressor { * @param data * @param data_length */ - void write(char const* data, size_t data_length) override; + auto write(char const* data, size_t data_length) -> void override; + /** * Flushes any buffered data */ - void flush() override; + auto flush() -> void override; + /** * Tries to get the current position of the write head * @param pos Position of the write head * @return ErrorCode_NotInit if the compressor is not open * @return Same as FileWriter::try_get_pos */ - ErrorCode try_get_pos(size_t& pos) const override; + [[nodiscard]] auto try_get_pos(size_t& pos) const -> ErrorCode override; // Methods implementing the Compressor interface /** * Closes the compressor */ - void close() override; + auto close() -> void override; - // Methods /** - * Initializes the compressor + * Initializes the compression stream * @param file_writer + * @param compression_level */ - void open(FileWriter& file_writer); + auto open(FileWriter& file_writer, [[maybe_unused]] int compression_level = 0) -> void override; private: // Variables - FileWriter* m_compressed_stream_file_writer; + FileWriter* m_compressed_stream_file_writer{nullptr}; }; } // namespace clp::streaming_compression::passthrough diff --git a/components/core/src/clp/streaming_compression/zstd/Compressor.cpp b/components/core/src/clp/streaming_compression/zstd/Compressor.cpp index ebbf9b574..8adc47aae 100644 --- a/components/core/src/clp/streaming_compression/zstd/Compressor.cpp +++ b/components/core/src/clp/streaming_compression/zstd/Compressor.cpp @@ -1,14 +1,25 @@ #include "Compressor.hpp" -#include "../../Defs.h" -#include "../../spdlog_with_specializations.hpp" +#include +#include + +#include +#include + +#include "../../ErrorCode.hpp" +#include "../../FileWriter.hpp" +#include "../../TraceableException.hpp" +#include "../Compressor.hpp" +#include "../Constants.hpp" namespace clp::streaming_compression::zstd { Compressor::Compressor() - : ::clp::streaming_compression::Compressor(CompressorType::ZSTD), - m_compression_stream_contains_data(false), - m_compressed_stream_file_writer(nullptr) { - m_compression_stream = ZSTD_createCStream(); + : ::clp::streaming_compression::Compressor{CompressorType::ZSTD}, + m_compressed_stream_block{}, + m_compressed_stream_file_writer{nullptr}, + m_compression_stream_contains_data{false}, + m_compression_stream{ZSTD_createCStream()}, + m_uncompressed_stream_pos{0} { if (nullptr == m_compression_stream) { SPDLOG_ERROR("streaming_compression::zstd::Compressor: ZSTD_createCStream() error"); throw OperationFailed(ErrorCode_Failure, __FILENAME__, __LINE__); @@ -19,20 +30,20 @@ Compressor::~Compressor() { ZSTD_freeCStream(m_compression_stream); } -void Compressor::open(FileWriter& file_writer, int const compression_level) { +auto Compressor::open(FileWriter& file_writer, int const compression_level) -> void { if (nullptr != m_compressed_stream_file_writer) { throw OperationFailed(ErrorCode_NotReady, __FILENAME__, __LINE__); } // Setup compressed stream parameters - size_t compressed_stream_block_size = ZSTD_CStreamOutSize(); - m_compressed_stream_block_buffer = std::make_unique(compressed_stream_block_size); - m_compressed_stream_block.dst = m_compressed_stream_block_buffer.get(); + auto const compressed_stream_block_size{ZSTD_CStreamOutSize()}; + m_compressed_stream_block_buffer.reserve(compressed_stream_block_size); + m_compressed_stream_block.dst = m_compressed_stream_block_buffer.data(); m_compressed_stream_block.size = compressed_stream_block_size; // Setup compression stream - auto init_result = ZSTD_initCStream(m_compression_stream, compression_level); - if (ZSTD_isError(init_result)) { + auto const init_result{ZSTD_initCStream(m_compression_stream, compression_level)}; + if (zstd_is_error(init_result)) { SPDLOG_ERROR( "streaming_compression::zstd::Compressor: ZSTD_initCStream() error: {}", ZSTD_getErrorName(init_result) @@ -45,7 +56,7 @@ void Compressor::open(FileWriter& file_writer, int const compression_level) { m_uncompressed_stream_pos = 0; } -void Compressor::close() { +auto Compressor::close() -> void { if (nullptr == m_compressed_stream_file_writer) { throw OperationFailed(ErrorCode_NotInit, __FILENAME__, __LINE__); } @@ -54,7 +65,7 @@ void Compressor::close() { m_compressed_stream_file_writer = nullptr; } -void Compressor::write(char const* data, size_t data_length) { +auto Compressor::write(char const* data, size_t data_length) -> void { if (nullptr == m_compressed_stream_file_writer) { throw OperationFailed(ErrorCode_NotInit, __FILENAME__, __LINE__); } @@ -70,23 +81,23 @@ void Compressor::write(char const* data, size_t data_length) { ZSTD_inBuffer uncompressed_stream_block = {data, data_length, 0}; while (uncompressed_stream_block.pos < uncompressed_stream_block.size) { m_compressed_stream_block.pos = 0; - auto error = ZSTD_compressStream( + auto const compress_result{ZSTD_compressStream( m_compression_stream, &m_compressed_stream_block, &uncompressed_stream_block - ); - if (ZSTD_isError(error)) { + )}; + if (zstd_is_error(compress_result)) { SPDLOG_ERROR( "streaming_compression::zstd::Compressor: ZSTD_compressStream() error: {}", - ZSTD_getErrorName(error) + ZSTD_getErrorName(compress_result) ); throw OperationFailed(ErrorCode_Failure, __FILENAME__, __LINE__); } - if (m_compressed_stream_block.pos) { + if (m_compressed_stream_block.pos > 0) { // Write to disk only if there is data in the compressed stream // block buffer m_compressed_stream_file_writer->write( - reinterpret_cast(m_compressed_stream_block.dst), + static_cast(m_compressed_stream_block.dst), m_compressed_stream_block.pos ); } @@ -96,14 +107,14 @@ void Compressor::write(char const* data, size_t data_length) { m_uncompressed_stream_pos += data_length; } -void Compressor::flush() { +auto Compressor::flush() -> void { if (false == m_compression_stream_contains_data) { return; } m_compressed_stream_block.pos = 0; - auto end_stream_result = ZSTD_endStream(m_compression_stream, &m_compressed_stream_block); - if (end_stream_result) { + auto const end_stream_result{ZSTD_endStream(m_compression_stream, &m_compressed_stream_block)}; + if (end_stream_result > 0) { // Note: Output buffer is large enough that it is guaranteed to have enough room to be // able to flush the entire buffer, so this can only be an error SPDLOG_ERROR( @@ -113,14 +124,14 @@ void Compressor::flush() { throw OperationFailed(ErrorCode_Failure, __FILENAME__, __LINE__); } m_compressed_stream_file_writer->write( - reinterpret_cast(m_compressed_stream_block.dst), + static_cast(m_compressed_stream_block.dst), m_compressed_stream_block.pos ); m_compression_stream_contains_data = false; } -ErrorCode Compressor::try_get_pos(size_t& pos) const { +auto Compressor::try_get_pos(size_t& pos) const -> ErrorCode { if (nullptr == m_compressed_stream_file_writer) { return ErrorCode_NotInit; } @@ -129,30 +140,35 @@ ErrorCode Compressor::try_get_pos(size_t& pos) const { return ErrorCode_Success; } -void Compressor::flush_without_ending_frame() { +auto Compressor::flush_without_ending_frame() -> void { if (false == m_compression_stream_contains_data) { return; } while (true) { m_compressed_stream_block.pos = 0; - auto result = ZSTD_flushStream(m_compression_stream, &m_compressed_stream_block); - if (ZSTD_isError(result)) { + auto flush_result{ZSTD_flushStream(m_compression_stream, &m_compressed_stream_block)}; + if (zstd_is_error(flush_result)) { SPDLOG_ERROR( "streaming_compression::zstd::Compressor: ZSTD_compressStream2() error: {}", - ZSTD_getErrorName(result) + ZSTD_getErrorName(flush_result) ); throw OperationFailed(ErrorCode_Failure, __FILENAME__, __LINE__); } - if (m_compressed_stream_block.pos) { + if (m_compressed_stream_block.pos > 0) { m_compressed_stream_file_writer->write( - reinterpret_cast(m_compressed_stream_block.dst), + static_cast(m_compressed_stream_block.dst), m_compressed_stream_block.pos ); } - if (0 == result) { + if (0 == flush_result) { break; } } } + +auto Compressor::zstd_is_error(size_t size_t_function_result) -> bool { + return ZSTD_isError(size_t_function_result) != 0 + && ZSTD_error_no_error != ZSTD_getErrorCode(size_t_function_result); +} } // namespace clp::streaming_compression::zstd diff --git a/components/core/src/clp/streaming_compression/zstd/Compressor.hpp b/components/core/src/clp/streaming_compression/zstd/Compressor.hpp index 75971dfa8..de41709bc 100644 --- a/components/core/src/clp/streaming_compression/zstd/Compressor.hpp +++ b/components/core/src/clp/streaming_compression/zstd/Compressor.hpp @@ -1,12 +1,12 @@ #ifndef CLP_STREAMING_COMPRESSION_ZSTD_COMPRESSOR_HPP #define CLP_STREAMING_COMPRESSION_ZSTD_COMPRESSOR_HPP -#include -#include +#include +#include #include -#include +#include "../../ErrorCode.hpp" #include "../../FileWriter.hpp" #include "../../TraceableException.hpp" #include "../Compressor.hpp" @@ -23,7 +23,7 @@ class Compressor : public ::clp::streaming_compression::Compressor { : TraceableException(error_code, filename, line_number) {} // Methods - char const* what() const noexcept override { + [[nodiscard]] auto what() const noexcept -> char const* override { return "streaming_compression::zstd::Compressor operation failed"; } }; @@ -32,11 +32,14 @@ class Compressor : public ::clp::streaming_compression::Compressor { Compressor(); // Destructor - ~Compressor(); + ~Compressor() override; - // Explicitly disable copy and move constructor/assignment + // Explicitly disable copy constructor/assignment and enable the move version Compressor(Compressor const&) = delete; - Compressor& operator=(Compressor const&) = delete; + auto operator=(Compressor const&) -> Compressor& = delete; + + Compressor(Compressor&&) noexcept = default; + auto operator=(Compressor&&) -> Compressor& = default; // Methods implementing the WriterInterface /** @@ -44,11 +47,11 @@ class Compressor : public ::clp::streaming_compression::Compressor { * @param data * @param data_length */ - void write(char const* data, size_t data_length) override; + auto write(char const* data, size_t data_length) -> void override; /** * Writes any internally buffered data to file and ends the current frame */ - void flush() override; + auto flush() -> void override; /** * Tries to get the current position of the write head @@ -56,26 +59,26 @@ class Compressor : public ::clp::streaming_compression::Compressor { * @return ErrorCode_NotInit if the compressor is not open * @return ErrorCode_Success on success */ - ErrorCode try_get_pos(size_t& pos) const override; + [[nodiscard]] auto try_get_pos(size_t& pos) const -> ErrorCode override; // Methods implementing the Compressor interface /** * Closes the compressor */ - void close() override; + auto close() -> void override; - // Methods /** - * Initialize streaming compressor + * Initializes the compression stream with the given compression level * @param file_writer * @param compression_level */ - void open(FileWriter& file_writer, int compression_level = cDefaultCompressionLevel); + auto open(FileWriter& file_writer, int compression_level = cDefaultCompressionLevel) + -> void override; /** * Flushes the stream without ending the current frame */ - void flush_without_ending_frame(); + auto flush_without_ending_frame() -> void; private: // Variables @@ -86,9 +89,14 @@ class Compressor : public ::clp::streaming_compression::Compressor { bool m_compression_stream_contains_data; ZSTD_outBuffer m_compressed_stream_block; - std::unique_ptr m_compressed_stream_block_buffer; + std::vector m_compressed_stream_block_buffer; size_t m_uncompressed_stream_pos; + + /** + * Tells if a `size_t` ZStd function result is an error code and is not `ZSTD_error_no_error` + */ + [[nodiscard]] static auto zstd_is_error(size_t size_t_function_result) -> bool; }; } // namespace clp::streaming_compression::zstd diff --git a/components/core/src/clp/streaming_compression/zstd/Constants.hpp b/components/core/src/clp/streaming_compression/zstd/Constants.hpp index a0e57e3e1..153478377 100644 --- a/components/core/src/clp/streaming_compression/zstd/Constants.hpp +++ b/components/core/src/clp/streaming_compression/zstd/Constants.hpp @@ -1,11 +1,8 @@ #ifndef CLP_STREAMING_COMPRESSION_ZSTD_CONSTANTS_HPP #define CLP_STREAMING_COMPRESSION_ZSTD_CONSTANTS_HPP -#include -#include - namespace clp::streaming_compression::zstd { -constexpr int cDefaultCompressionLevel = 3; +constexpr int cDefaultCompressionLevel{3}; } // namespace clp::streaming_compression::zstd #endif // CLP_STREAMING_COMPRESSION_ZSTD_CONSTANTS_HPP diff --git a/components/core/tests/test-StreamingCompression.cpp b/components/core/tests/test-StreamingCompression.cpp index 747a38a05..2849f20a9 100644 --- a/components/core/tests/test-StreamingCompression.cpp +++ b/components/core/tests/test-StreamingCompression.cpp @@ -1,11 +1,18 @@ +#include #include +#include #include +#include -#include +#include #include #include +#include "../src/clp/ErrorCode.hpp" +#include "../src/clp/FileWriter.hpp" #include "../src/clp/ReadOnlyMemoryMappedFile.hpp" +#include "../src/clp/streaming_compression/Compressor.hpp" +#include "../src/clp/streaming_compression/Decompressor.hpp" #include "../src/clp/streaming_compression/passthrough/Compressor.hpp" #include "../src/clp/streaming_compression/passthrough/Decompressor.hpp" #include "../src/clp/streaming_compression/zstd/Compressor.hpp" @@ -13,218 +20,83 @@ using clp::ErrorCode_Success; using clp::FileWriter; +using clp::streaming_compression::Compressor; +using clp::streaming_compression::Decompressor; + +namespace { +constexpr size_t cUncompressedDataSize{128L * 1024 * 1024}; // 128MB +constexpr auto cCompressionChunkSizes = std::to_array( + {cUncompressedDataSize / 100, + cUncompressedDataSize / 50, + cUncompressedDataSize / 25, + cUncompressedDataSize / 10, + cUncompressedDataSize / 5, + cUncompressedDataSize / 2, + cUncompressedDataSize} +); +constexpr size_t cUncompressedDataPatternPeriod = 26; // lower-case alphabet +} // namespace TEST_CASE("StreamingCompression", "[StreamingCompression]") { - // Initialize data to test compression and decompression - size_t uncompressed_data_size = 128L * 1024 * 1024; // 128MB - char* uncompressed_data = new char[uncompressed_data_size]; - for (size_t i = 0; i < uncompressed_data_size; ++i) { - uncompressed_data[i] = (char)('a' + (i % 26)); + std::string const compressed_file_path{"test_streaming_compressed_file.bin"}; + std::vector compression_chunk_sizes{ + cCompressionChunkSizes.begin(), + cCompressionChunkSizes.end() + }; + std::unique_ptr compressor{}; + std::unique_ptr decompressor{}; + + SECTION("Initiate zstd single phase compression") { + compression_chunk_sizes.insert(compression_chunk_sizes.begin(), ZSTD_CStreamInSize()); + compressor = std::make_unique(); + decompressor = std::make_unique(); } - // Create output buffer - char* decompressed_data = new char[uncompressed_data_size]; - - SECTION("zstd single phase compression") { - // Clear output buffer - memset(decompressed_data, 0, uncompressed_data_size); - std::string compressed_file_path = "compressed_file.zstd.bin.1"; - - // Compress - FileWriter file_writer; - file_writer.open(compressed_file_path, FileWriter::OpenMode::CREATE_FOR_WRITING); - clp::streaming_compression::zstd::Compressor compressor; - compressor.open(file_writer); - compressor.write(uncompressed_data, ZSTD_CStreamInSize()); - compressor.write(uncompressed_data, uncompressed_data_size / 100); - compressor.write(uncompressed_data, uncompressed_data_size / 50); - compressor.write(uncompressed_data, uncompressed_data_size / 25); - compressor.write(uncompressed_data, uncompressed_data_size / 10); - compressor.write(uncompressed_data, uncompressed_data_size / 5); - compressor.write(uncompressed_data, uncompressed_data_size / 2); - compressor.write(uncompressed_data, uncompressed_data_size); - compressor.close(); - file_writer.close(); - - // Decompress - clp::streaming_compression::zstd::Decompressor decompressor; - REQUIRE(ErrorCode_Success == decompressor.open(compressed_file_path)); - size_t uncompressed_bytes = 0; - REQUIRE(ErrorCode_Success - == decompressor.get_decompressed_stream_region( - uncompressed_bytes, - decompressed_data, - ZSTD_CStreamInSize() - )); - REQUIRE(memcmp(uncompressed_data, decompressed_data, ZSTD_CStreamInSize()) == 0); - memset(decompressed_data, 0, uncompressed_data_size); - uncompressed_bytes += ZSTD_CStreamInSize(); - - REQUIRE(ErrorCode_Success - == decompressor.get_decompressed_stream_region( - uncompressed_bytes, - decompressed_data, - uncompressed_data_size / 100 - )); - REQUIRE(memcmp(uncompressed_data, decompressed_data, uncompressed_data_size / 100) == 0); - memset(decompressed_data, 0, uncompressed_data_size); - uncompressed_bytes += uncompressed_data_size / 100; - REQUIRE(ErrorCode_Success - == decompressor.get_decompressed_stream_region( - uncompressed_bytes, - decompressed_data, - uncompressed_data_size / 50 - )); - REQUIRE(memcmp(uncompressed_data, decompressed_data, uncompressed_data_size / 50) == 0); - memset(decompressed_data, 0, uncompressed_data_size); - uncompressed_bytes += uncompressed_data_size / 50; - REQUIRE(ErrorCode_Success - == decompressor.get_decompressed_stream_region( - uncompressed_bytes, - decompressed_data, - uncompressed_data_size / 25 - )); - REQUIRE(memcmp(uncompressed_data, decompressed_data, uncompressed_data_size / 25) == 0); - memset(decompressed_data, 0, uncompressed_data_size); - uncompressed_bytes += uncompressed_data_size / 25; - REQUIRE(ErrorCode_Success - == decompressor.get_decompressed_stream_region( - uncompressed_bytes, - decompressed_data, - uncompressed_data_size / 10 - )); - REQUIRE(memcmp(uncompressed_data, decompressed_data, uncompressed_data_size / 10) == 0); - memset(decompressed_data, 0, uncompressed_data_size); - uncompressed_bytes += uncompressed_data_size / 10; - REQUIRE(ErrorCode_Success - == decompressor.get_decompressed_stream_region( - uncompressed_bytes, - decompressed_data, - uncompressed_data_size / 5 - )); - REQUIRE(memcmp(uncompressed_data, decompressed_data, uncompressed_data_size / 5) == 0); - memset(decompressed_data, 0, uncompressed_data_size); - uncompressed_bytes += uncompressed_data_size / 5; - REQUIRE(ErrorCode_Success - == decompressor.get_decompressed_stream_region( - uncompressed_bytes, - decompressed_data, - uncompressed_data_size / 2 - )); - REQUIRE(memcmp(uncompressed_data, decompressed_data, uncompressed_data_size / 2) == 0); - memset(decompressed_data, 0, uncompressed_data_size); - uncompressed_bytes += uncompressed_data_size / 2; - REQUIRE(ErrorCode_Success - == decompressor.get_decompressed_stream_region( - uncompressed_bytes, - decompressed_data, - uncompressed_data_size - )); - REQUIRE(memcmp(uncompressed_data, decompressed_data, uncompressed_data_size) == 0); - memset(decompressed_data, 0, uncompressed_data_size); - uncompressed_bytes += uncompressed_data_size; - - // Cleanup - boost::filesystem::remove(compressed_file_path); + SECTION("Initiate passthrough compression") { + compressor = std::make_unique(); + decompressor = std::make_unique(); } - SECTION("passthrough compression") { - // Clear output buffer - memset(decompressed_data, 0, uncompressed_data_size); - std::string compressed_file_path = "compressed_file.passthrough.bin"; - - // Compress - FileWriter file_writer; - file_writer.open(compressed_file_path, FileWriter::OpenMode::CREATE_FOR_WRITING); - clp::streaming_compression::passthrough::Compressor compressor; - compressor.open(file_writer); - compressor.write(uncompressed_data, uncompressed_data_size / 100); - compressor.write(uncompressed_data, uncompressed_data_size / 50); - compressor.write(uncompressed_data, uncompressed_data_size / 25); - compressor.write(uncompressed_data, uncompressed_data_size / 10); - compressor.write(uncompressed_data, uncompressed_data_size / 5); - compressor.write(uncompressed_data, uncompressed_data_size / 2); - compressor.write(uncompressed_data, uncompressed_data_size); - compressor.close(); - file_writer.close(); - - // Decompress - // Memory map compressed file - clp::ReadOnlyMemoryMappedFile const memory_mapped_compressed_file{compressed_file_path}; - clp::streaming_compression::passthrough::Decompressor decompressor; - auto const compressed_file_view{memory_mapped_compressed_file.get_view()}; - decompressor.open(compressed_file_view.data(), compressed_file_view.size()); + // Initialize buffers + std::vector uncompressed_buffer{}; + uncompressed_buffer.reserve(cUncompressedDataSize); + for (size_t i{0}; i < cUncompressedDataSize; ++i) { + uncompressed_buffer.push_back((char)('a' + (i % cUncompressedDataPatternPeriod))); + } - size_t uncompressed_bytes = 0; - REQUIRE(ErrorCode_Success - == decompressor.get_decompressed_stream_region( - uncompressed_bytes, - decompressed_data, - uncompressed_data_size / 100 - )); - REQUIRE(memcmp(uncompressed_data, decompressed_data, uncompressed_data_size / 100) == 0); - memset(decompressed_data, 0, uncompressed_data_size); - uncompressed_bytes += uncompressed_data_size / 100; - REQUIRE(ErrorCode_Success - == decompressor.get_decompressed_stream_region( - uncompressed_bytes, - decompressed_data, - uncompressed_data_size / 50 - )); - REQUIRE(memcmp(uncompressed_data, decompressed_data, uncompressed_data_size / 50) == 0); - memset(decompressed_data, 0, uncompressed_data_size); - uncompressed_bytes += uncompressed_data_size / 50; - REQUIRE(ErrorCode_Success - == decompressor.get_decompressed_stream_region( - uncompressed_bytes, - decompressed_data, - uncompressed_data_size / 25 - )); - REQUIRE(memcmp(uncompressed_data, decompressed_data, uncompressed_data_size / 25) == 0); - memset(decompressed_data, 0, uncompressed_data_size); - uncompressed_bytes += uncompressed_data_size / 25; - REQUIRE(ErrorCode_Success - == decompressor.get_decompressed_stream_region( - uncompressed_bytes, - decompressed_data, - uncompressed_data_size / 10 - )); - REQUIRE(memcmp(uncompressed_data, decompressed_data, uncompressed_data_size / 10) == 0); - memset(decompressed_data, 0, uncompressed_data_size); - uncompressed_bytes += uncompressed_data_size / 10; - REQUIRE(ErrorCode_Success - == decompressor.get_decompressed_stream_region( - uncompressed_bytes, - decompressed_data, - uncompressed_data_size / 5 - )); - REQUIRE(memcmp(uncompressed_data, decompressed_data, uncompressed_data_size / 5) == 0); - memset(decompressed_data, 0, uncompressed_data_size); - uncompressed_bytes += uncompressed_data_size / 5; - REQUIRE(ErrorCode_Success - == decompressor.get_decompressed_stream_region( - uncompressed_bytes, - decompressed_data, - uncompressed_data_size / 2 - )); - REQUIRE(memcmp(uncompressed_data, decompressed_data, uncompressed_data_size / 2) == 0); - memset(decompressed_data, 0, uncompressed_data_size); - uncompressed_bytes += uncompressed_data_size / 2; - REQUIRE(ErrorCode_Success - == decompressor.get_decompressed_stream_region( - uncompressed_bytes, - decompressed_data, - uncompressed_data_size - )); - REQUIRE(memcmp(uncompressed_data, decompressed_data, uncompressed_data_size) == 0); - memset(decompressed_data, 0, uncompressed_data_size); - uncompressed_bytes += uncompressed_data_size; + std::vector decompressed_buffer{}; + decompressed_buffer.reserve(cUncompressedDataSize); - // Cleanup - boost::filesystem::remove(compressed_file_path); + // Compress + FileWriter file_writer; + file_writer.open(compressed_file_path, FileWriter::OpenMode::CREATE_FOR_WRITING); + compressor->open(file_writer); + for (auto const chunk_size : compression_chunk_sizes) { + compressor->write(uncompressed_buffer.data(), chunk_size); + } + compressor->close(); + file_writer.close(); + + // Decompress and compare + clp::ReadOnlyMemoryMappedFile const memory_mapped_compressed_file{compressed_file_path}; + auto const compressed_file_view{memory_mapped_compressed_file.get_view()}; + decompressor->open(compressed_file_view.data(), compressed_file_view.size()); + + size_t uncompressed_bytes{0}; + for (auto chunk_size : compression_chunk_sizes) { + memset(decompressed_buffer.data(), 0, cUncompressedDataSize); + REQUIRE( + (ErrorCode_Success + == decompressor->get_decompressed_stream_region( + uncompressed_bytes, + decompressed_buffer.data(), + chunk_size + )) + ); + REQUIRE((memcmp(uncompressed_buffer.data(), decompressed_buffer.data(), chunk_size) == 0)); + uncompressed_bytes += chunk_size; } // Cleanup - delete[] uncompressed_data; - delete[] decompressed_data; + boost::filesystem::remove(compressed_file_path); }