First import of lzma in clp

Bill-hbrhbr · Nov 14, 2024 · 5378481 · 5378481
1 parent 9d28d57
commit 5378481
Show file tree

Hide file tree

Showing 18 changed files with 1,040 additions and 2 deletions.
diff --git a/components/core/.clang-format b/components/core/.clang-format
@@ -4,7 +4,7 @@ IncludeCategories:
   # NOTE: A header is grouped by first matching regex
   # Library headers. Update when adding new libraries.
   # NOTE: clang-format retains leading white-space on a line in violation of the YAML spec.
-  - Regex: "<(absl|antlr4|archive|boost|bsoncxx|catch2|curl|date|fmt|json|log_surgeon|mongocxx\
+  - Regex: "<(absl|antlr4|archive|boost|bsoncxx|catch2|curl|date|fmt|json|log_surgeon|lzma|mongocxx\
 |msgpack|mysql|openssl|outcome|regex_utils|simdjson|spdlog|sqlite3|string_utils|yaml-cpp|zstd)"
     Priority: 3
   # C system headers

diff --git a/components/core/CMakeLists.txt b/components/core/CMakeLists.txt
@@ -11,13 +11,16 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
 # Set general compressor
 set(GENERAL_COMPRESSOR "zstd" CACHE STRING "The general-purpose compressor used as the 2nd-stage compressor")
-set_property(CACHE GENERAL_COMPRESSOR PROPERTY STRINGS passthrough zstd)
+set_property(CACHE GENERAL_COMPRESSOR PROPERTY STRINGS passthrough zstd lzma)
 if ("${GENERAL_COMPRESSOR}" STREQUAL "passthrough")
     add_definitions(-DUSE_PASSTHROUGH_COMPRESSION=1)
     message(STATUS "Using passthrough compression")
 elseif ("${GENERAL_COMPRESSOR}" STREQUAL "zstd")
     add_definitions(-DUSE_ZSTD_COMPRESSION=1)
     message(STATUS "Using Zstandard compression")
+elseif ("${GENERAL_COMPRESSOR}" STREQUAL "lzma")
+    add_definitions(-DUSE_LZMA_COMPRESSION=1)
+    message(STATUS "Using Lempel–Ziv–Markov chain Algorithm compression")
 else()
     message(SEND_ERROR "GENERAL_COMPRESSOR=${GENERAL_COMPRESSOR} is unimplemented.")
 endif()
@@ -224,6 +227,19 @@ else()
     message(FATAL_ERROR "Could not find ${CLP_LIBS_STRING} libraries for ZStd")
 endif()
 
+# Find and setup LZMA Library
+# Notice that we don't have support to switch between static and shared libraries.
+# TODO: add a script in ./cmake/Modules to resolve .a vs. .so
+find_package(LibLZMA REQUIRED)
+if(LIBLZMA_FOUND)
+    message(STATUS "Found LIBLZMA_FOUND ${LIBLZMA_VERSION_STRING}")
+    message(STATUS "Lzma library location: ${LIBLZMA_LIBRARIES}")
+else()
+    message(FATAL_ERROR "Could not find ${CLP_LIBS_STRING} libraries for LIBLZMA_FOUND")
+endif()
+include_directories(${LIBLZMA_INCLUDE_DIRS})
+message("LZMA Include Dir: ${LIBLZMA_INCLUDE_DIRS}")
+
 # sqlite dependencies
 set(sqlite_DYNAMIC_LIBS "dl;m;pthread")
 include(cmake/Modules/FindLibraryDependencies.cmake)

diff --git a/components/core/src/clp/DictionaryReader.hpp b/components/core/src/clp/DictionaryReader.hpp
@@ -10,6 +10,7 @@
 #include "dictionary_utils.hpp"
 #include "DictionaryEntry.hpp"
 #include "FileReader.hpp"
+#include "streaming_compression/lzma/Decompressor.hpp"
 #include "streaming_compression/passthrough/Decompressor.hpp"
 #include "streaming_compression/zstd/Decompressor.hpp"
 #include "Utils.hpp"
@@ -115,6 +116,9 @@ class DictionaryReader {
 #elif USE_ZSTD_COMPRESSION
     streaming_compression::zstd::Decompressor m_dictionary_decompressor;
     streaming_compression::zstd::Decompressor m_segment_index_decompressor;
+#elif USE_LZMA_COMPRESSION
+    streaming_compression::lzma::Decompressor m_dictionary_decompressor;
+    streaming_compression::lzma::Decompressor m_segment_index_decompressor;
 #else
     static_assert(false, "Unsupported compression mode.");
 #endif

diff --git a/components/core/src/clp/DictionaryWriter.hpp b/components/core/src/clp/DictionaryWriter.hpp
@@ -8,6 +8,8 @@
 #include "Defs.h"
 #include "FileWriter.hpp"
 #include "spdlog_with_specializations.hpp"
+#include "streaming_compression/lzma/Compressor.hpp"
+#include "streaming_compression/lzma/Decompressor.hpp"
 #include "streaming_compression/passthrough/Compressor.hpp"
 #include "streaming_compression/passthrough/Decompressor.hpp"
 #include "streaming_compression/zstd/Compressor.hpp"
@@ -97,6 +99,9 @@ class DictionaryWriter {
 #elif USE_ZSTD_COMPRESSION
     streaming_compression::zstd::Compressor m_dictionary_compressor;
     streaming_compression::zstd::Compressor m_segment_index_compressor;
+#elif USE_LZMA_COMPRESSION
+    streaming_compression::lzma::Compressor m_dictionary_compressor;
+    streaming_compression::lzma::Compressor m_segment_index_compressor;
 #else
     static_assert(false, "Unsupported compression mode.");
 #endif

diff --git a/components/core/src/clp/clg/CMakeLists.txt b/components/core/src/clp/clg/CMakeLists.txt
@@ -91,6 +91,11 @@ set(
         ../streaming_archive/writer/Segment.hpp
         ../streaming_compression/Constants.hpp
         ../streaming_compression/Decompressor.hpp
+        ../streaming_compression/lzma/Compressor.cpp
+        ../streaming_compression/lzma/Compressor.hpp
+        ../streaming_compression/lzma/Constants.hpp
+        ../streaming_compression/lzma/Decompressor.cpp
+        ../streaming_compression/lzma/Decompressor.hpp
         ../streaming_compression/passthrough/Compressor.cpp
         ../streaming_compression/passthrough/Compressor.hpp
         ../streaming_compression/passthrough/Decompressor.cpp

diff --git a/components/core/src/clp/clo/CMakeLists.txt b/components/core/src/clp/clo/CMakeLists.txt
@@ -91,6 +91,11 @@ set(
         ../streaming_archive/writer/Segment.hpp
         ../streaming_compression/Constants.hpp
         ../streaming_compression/Decompressor.hpp
+        ../streaming_compression/lzma/Compressor.cpp
+        ../streaming_compression/lzma/Compressor.hpp
+        ../streaming_compression/lzma/Constants.hpp
+        ../streaming_compression/lzma/Decompressor.cpp
+        ../streaming_compression/lzma/Decompressor.hpp
         ../streaming_compression/passthrough/Compressor.cpp
         ../streaming_compression/passthrough/Compressor.hpp
         ../streaming_compression/passthrough/Decompressor.cpp

diff --git a/components/core/src/clp/clp/CMakeLists.txt b/components/core/src/clp/clp/CMakeLists.txt
@@ -116,6 +116,11 @@ set(
         ../streaming_archive/writer/Segment.hpp
         ../streaming_archive/writer/utils.cpp
         ../streaming_archive/writer/utils.hpp
+        ../streaming_compression/lzma/Compressor.cpp
+        ../streaming_compression/lzma/Compressor.hpp
+        ../streaming_compression/lzma/Constants.hpp
+        ../streaming_compression/lzma/Decompressor.cpp
+        ../streaming_compression/lzma/Decompressor.hpp
         ../streaming_compression/Compressor.hpp
         ../streaming_compression/Constants.hpp
         ../streaming_compression/Decompressor.hpp

diff --git a/components/core/src/clp/make_dictionaries_readable/CMakeLists.txt b/components/core/src/clp/make_dictionaries_readable/CMakeLists.txt
@@ -23,6 +23,8 @@ set(
         ../ReadOnlyMemoryMappedFile.hpp
         ../spdlog_with_specializations.hpp
         ../streaming_compression/Decompressor.hpp
+        ../streaming_compression/lzma/Decompressor.cpp
+        ../streaming_compression/lzma/Decompressor.hpp
         ../streaming_compression/passthrough/Decompressor.cpp
         ../streaming_compression/passthrough/Decompressor.hpp
         ../streaming_compression/zstd/Decompressor.cpp

diff --git a/components/core/src/clp/streaming_archive/reader/Segment.hpp b/components/core/src/clp/streaming_archive/reader/Segment.hpp
@@ -8,6 +8,7 @@
 
 #include "../../Defs.h"
 #include "../../ErrorCode.hpp"
+#include "../../streaming_compression/lzma/Decompressor.hpp"
 #include "../../streaming_compression/passthrough/Decompressor.hpp"
 #include "../../streaming_compression/zstd/Decompressor.hpp"
 #include "../Constants.hpp"
@@ -59,6 +60,8 @@ class Segment {
     streaming_compression::passthrough::Decompressor m_decompressor;
 #elif USE_ZSTD_COMPRESSION
     streaming_compression::zstd::Decompressor m_decompressor;
+#elif USE_LZMA_COMPRESSION
+    streaming_compression::lzma::Decompressor m_decompressor;
 #else
     static_assert(false, "Unsupported compression mode.");
 #endif

diff --git a/components/core/src/clp/streaming_archive/writer/Segment.cpp b/components/core/src/clp/streaming_archive/writer/Segment.cpp
@@ -45,6 +45,8 @@ void Segment::open(string const& segments_dir_path, segment_id_t id, int compres
     m_compressor.open(m_file_writer);
 #elif USE_ZSTD_COMPRESSION
     m_compressor.open(m_file_writer, compression_level);
+#elif USE_LZMA_COMPRESSION
+    m_compressor.open(m_file_writer, compression_level);
 #else
     static_assert(false, "Unsupported compression mode.");
 #endif

diff --git a/components/core/src/clp/streaming_archive/writer/Segment.hpp b/components/core/src/clp/streaming_archive/writer/Segment.hpp
@@ -6,6 +6,7 @@
 
 #include "../../Defs.h"
 #include "../../ErrorCode.hpp"
+#include "../../streaming_compression/lzma/Compressor.hpp"
 #include "../../streaming_compression/passthrough/Compressor.hpp"
 #include "../../streaming_compression/zstd/Compressor.hpp"
 #include "../../TraceableException.hpp"
@@ -90,6 +91,8 @@ class Segment {
     streaming_compression::passthrough::Compressor m_compressor;
 #elif USE_ZSTD_COMPRESSION
     streaming_compression::zstd::Compressor m_compressor;
+#elif USE_LZMA_COMPRESSION
+    streaming_compression::lzma::Compressor m_compressor;
 #else
     static_assert(false, "Unsupported compression mode.");
 #endif

diff --git a/components/core/src/clp/streaming_compression/Constants.hpp b/components/core/src/clp/streaming_compression/Constants.hpp
@@ -7,6 +7,7 @@
 namespace clp::streaming_compression {
 enum class CompressorType : uint8_t {
     ZSTD = 0x10,
+    LZMA = 0x20,
     Passthrough = 0xFF,
 };
 }  // namespace clp::streaming_compression