Replace compressor_frontend with log-surgeon submodule. (#131)

y-scope · Dec 6, 2023 · 6571d8d · 6571d8d
1 parent 96c577b
commit 6571d8d
Show file tree

Hide file tree

Showing 62 changed files with 658 additions and 4,917 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -11,6 +11,9 @@
 [submodule "components/core/submodules/yaml-cpp"]
 	path = components/core/submodules/yaml-cpp
 	url = https://github.com/jbeder/yaml-cpp.git
+[submodule "components/core/submodules/log-surgeon"]
+	path = components/core/submodules/log-surgeon
+	url = https://github.com/y-scope/log-surgeon.git
 [submodule "components/core/submodules/boost-outcome"]
 	path = components/core/submodules/boost-outcome
 	url = https://github.com/boostorg/outcome.git
diff --git a/components/core/.clang-format b/components/core/.clang-format
@@ -68,7 +68,7 @@ IncludeBlocks: Regroup
 IncludeCategories:
   # NOTE: A header is grouped by first matching regex
   # Third-party headers. Update when adding new third-party libraries.
-  - Regex: '^<(archive|boost|catch2|date|fmt|json|mariadb|spdlog|sqlite3|yaml-cpp|zstd)'
+  - Regex: '^<(archive|boost|catch2|date|fmt|json|log_surgeon|mariadb|spdlog|sqlite3|yaml-cpp|zstd)'
     Priority: 3
   # C system headers
   - Regex: '^<.+.h>'

diff --git a/components/core/CMakeLists.txt b/components/core/CMakeLists.txt
@@ -128,6 +128,9 @@ else()
     message(FATAL_ERROR "Could not find ${CLP_LIBS_STRING} libraries for LibArchive")
 endif()
 
+# Add log surgeon
+add_subdirectory(submodules/log-surgeon EXCLUDE_FROM_ALL)
+
 # Find and setup MariaDBClient library
 if(CLP_USE_STATIC_LIBS)
     # NOTE: We can't statically link to MariaDBClient since it's GPL
@@ -193,28 +196,6 @@ set(SOURCE_FILES_clp
         src/clp/StructuredFileToCompress.hpp
         src/clp/utils.cpp
         src/clp/utils.hpp
-        src/compressor_frontend/Constants.hpp
-        src/compressor_frontend/finite_automata/RegexAST.hpp
-        src/compressor_frontend/finite_automata/RegexAST.inc
-        src/compressor_frontend/finite_automata/RegexDFA.hpp
-        src/compressor_frontend/finite_automata/RegexDFA.inc
-        src/compressor_frontend/finite_automata/RegexNFA.hpp
-        src/compressor_frontend/finite_automata/RegexNFA.inc
-        src/compressor_frontend/finite_automata/UnicodeIntervalTree.hpp
-        src/compressor_frontend/finite_automata/UnicodeIntervalTree.inc
-        src/compressor_frontend/LALR1Parser.cpp
-        src/compressor_frontend/LALR1Parser.hpp
-        src/compressor_frontend/LALR1Parser.inc
-        src/compressor_frontend/Lexer.hpp
-        src/compressor_frontend/Lexer.inc
-        src/compressor_frontend/LogParser.cpp
-        src/compressor_frontend/LogParser.hpp
-        src/compressor_frontend/SchemaParser.cpp
-        src/compressor_frontend/SchemaParser.hpp
-        src/compressor_frontend/Token.cpp
-        src/compressor_frontend/Token.hpp
-        src/compressor_frontend/utils.cpp
-        src/compressor_frontend/utils.hpp
         src/database_utils.cpp
         src/database_utils.hpp
         src/Defs.h
@@ -262,6 +243,8 @@ set(SOURCE_FILES_clp
         src/LibarchiveFileReader.hpp
         src/LibarchiveReader.cpp
         src/LibarchiveReader.hpp
+        src/LogSurgeonReader.cpp
+        src/LogSurgeonReader.hpp
         src/LogTypeDictionaryEntry.cpp
         src/LogTypeDictionaryEntry.hpp
         src/LogTypeDictionaryReader.cpp
@@ -360,6 +343,7 @@ target_link_libraries(clp
         PRIVATE
         Boost::filesystem Boost::iostreams Boost::program_options
         fmt::fmt
+        log_surgeon::log_surgeon
         spdlog::spdlog
         ${sqlite_LIBRARY_DEPENDENCIES}
         LibArchive::LibArchive
@@ -384,26 +368,6 @@ set(SOURCE_FILES_clg
         src/clg/clg.cpp
         src/clg/CommandLineArguments.cpp
         src/clg/CommandLineArguments.hpp
-        src/compressor_frontend/Constants.hpp
-        src/compressor_frontend/finite_automata/RegexAST.hpp
-        src/compressor_frontend/finite_automata/RegexAST.inc
-        src/compressor_frontend/finite_automata/RegexDFA.hpp
-        src/compressor_frontend/finite_automata/RegexDFA.inc
-        src/compressor_frontend/finite_automata/RegexNFA.hpp
-        src/compressor_frontend/finite_automata/RegexNFA.inc
-        src/compressor_frontend/finite_automata/UnicodeIntervalTree.hpp
-        src/compressor_frontend/finite_automata/UnicodeIntervalTree.inc
-        src/compressor_frontend/LALR1Parser.cpp
-        src/compressor_frontend/LALR1Parser.hpp
-        src/compressor_frontend/LALR1Parser.inc
-        src/compressor_frontend/Lexer.hpp
-        src/compressor_frontend/Lexer.inc
-        src/compressor_frontend/SchemaParser.cpp
-        src/compressor_frontend/SchemaParser.hpp
-        src/compressor_frontend/Token.cpp
-        src/compressor_frontend/Token.hpp
-        src/compressor_frontend/utils.cpp
-        src/compressor_frontend/utils.hpp
         src/database_utils.cpp
         src/database_utils.hpp
         src/Defs.h
@@ -439,6 +403,8 @@ set(SOURCE_FILES_clg
         src/ir/LogEvent.hpp
         src/ir/parsing.cpp
         src/ir/parsing.hpp
+        src/LogSurgeonReader.cpp
+        src/LogSurgeonReader.hpp
         src/ir/parsing.inc
         src/LogTypeDictionaryEntry.cpp
         src/LogTypeDictionaryEntry.hpp
@@ -532,6 +498,7 @@ target_link_libraries(clg
         Boost::filesystem Boost::iostreams Boost::program_options
         fmt::fmt
         KQL
+        log_surgeon::log_surgeon
         MariaDBClient::MariaDBClient
         spdlog::spdlog
         ${sqlite_LIBRARY_DEPENDENCIES}
@@ -551,26 +518,6 @@ set(SOURCE_FILES_clo
         src/clo/CommandLineArguments.hpp
         src/clo/ControllerMonitoringThread.cpp
         src/clo/ControllerMonitoringThread.hpp
-        src/compressor_frontend/Constants.hpp
-        src/compressor_frontend/finite_automata/RegexAST.hpp
-        src/compressor_frontend/finite_automata/RegexAST.inc
-        src/compressor_frontend/finite_automata/RegexDFA.hpp
-        src/compressor_frontend/finite_automata/RegexDFA.inc
-        src/compressor_frontend/finite_automata/RegexNFA.hpp
-        src/compressor_frontend/finite_automata/RegexNFA.inc
-        src/compressor_frontend/finite_automata/UnicodeIntervalTree.hpp
-        src/compressor_frontend/finite_automata/UnicodeIntervalTree.inc
-        src/compressor_frontend/LALR1Parser.cpp
-        src/compressor_frontend/LALR1Parser.hpp
-        src/compressor_frontend/LALR1Parser.inc
-        src/compressor_frontend/Lexer.hpp
-        src/compressor_frontend/Lexer.inc
-        src/compressor_frontend/SchemaParser.cpp
-        src/compressor_frontend/SchemaParser.hpp
-        src/compressor_frontend/Token.cpp
-        src/compressor_frontend/Token.hpp
-        src/compressor_frontend/utils.cpp
-        src/compressor_frontend/utils.hpp
         src/database_utils.cpp
         src/database_utils.hpp
         src/Defs.h
@@ -598,6 +545,8 @@ set(SOURCE_FILES_clo
         src/ir/LogEvent.hpp
         src/ir/parsing.cpp
         src/ir/parsing.hpp
+        src/LogSurgeonReader.cpp
+        src/LogSurgeonReader.hpp
         src/ir/parsing.inc
         src/LogTypeDictionaryEntry.cpp
         src/LogTypeDictionaryEntry.hpp
@@ -690,6 +639,7 @@ target_link_libraries(clo
         PRIVATE
         Boost::filesystem Boost::iostreams Boost::program_options
         fmt::fmt
+        log_surgeon::log_surgeon
         msgpack-cxx
         spdlog::spdlog
         ${sqlite_LIBRARY_DEPENDENCIES}
@@ -723,28 +673,6 @@ set(SOURCE_FILES_unitTest
         src/clp/StructuredFileToCompress.hpp
         src/clp/utils.cpp
         src/clp/utils.hpp
-        src/compressor_frontend/Constants.hpp
-        src/compressor_frontend/finite_automata/RegexAST.hpp
-        src/compressor_frontend/finite_automata/RegexAST.inc
-        src/compressor_frontend/finite_automata/RegexDFA.hpp
-        src/compressor_frontend/finite_automata/RegexDFA.inc
-        src/compressor_frontend/finite_automata/RegexNFA.hpp
-        src/compressor_frontend/finite_automata/RegexNFA.inc
-        src/compressor_frontend/finite_automata/UnicodeIntervalTree.hpp
-        src/compressor_frontend/finite_automata/UnicodeIntervalTree.inc
-        src/compressor_frontend/LALR1Parser.cpp
-        src/compressor_frontend/LALR1Parser.hpp
-        src/compressor_frontend/LALR1Parser.inc
-        src/compressor_frontend/Lexer.hpp
-        src/compressor_frontend/Lexer.inc
-        src/compressor_frontend/LogParser.cpp
-        src/compressor_frontend/LogParser.hpp
-        src/compressor_frontend/SchemaParser.cpp
-        src/compressor_frontend/SchemaParser.hpp
-        src/compressor_frontend/Token.cpp
-        src/compressor_frontend/Token.hpp
-        src/compressor_frontend/utils.cpp
-        src/compressor_frontend/utils.hpp
         src/database_utils.cpp
         src/database_utils.hpp
         src/Defs.h
@@ -809,6 +737,8 @@ set(SOURCE_FILES_unitTest
         src/LibarchiveFileReader.hpp
         src/LibarchiveReader.cpp
         src/LibarchiveReader.hpp
+        src/LogSurgeonReader.cpp
+        src/LogSurgeonReader.hpp
         src/LogTypeDictionaryEntry.cpp
         src/LogTypeDictionaryEntry.hpp
         src/LogTypeDictionaryReader.cpp
@@ -925,6 +855,7 @@ target_link_libraries(unitTest
         PRIVATE
         Boost::filesystem Boost::iostreams Boost::program_options
         fmt::fmt
+        log_surgeon::log_surgeon
         LibArchive::LibArchive
         MariaDBClient::MariaDBClient
         spdlog::spdlog

diff --git a/components/core/README-Schema.md b/components/core/README-Schema.md
@@ -17,7 +17,7 @@ delimiters: \t\r\n:,!;%
 timestamp:\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3}){0,1}
 timestamp:\[\d{8}\-\d{2}:\d{2}:\d{2}\]
 int:\-{0,1}[0-9]+
-double:\-{0,1}[0-9]+\.[0-9]+
+float:\-{0,1}[0-9]+\.[0-9]+
 
 // Custom variables
 hex:[a-fA-F]+
@@ -49,7 +49,7 @@ equals:.*=.*[a-zA-Z0-9].*
   start of the file then a newline is used to indicate the beginning of a new
   log message. Timestamp patterns are not matched midline and are not stored as
   dictionary variables as they may contain delimiters.
-* `int` and `double` are keywords. These are encoded specially for compression
+* `int` and `float` are keywords. These are encoded specially for compression
   performance.
 
 ## Supported Regex

diff --git a/components/core/README.md b/components/core/README.md
@@ -22,7 +22,7 @@ CLP core is the low-level component that performs compression, decompression, an
 * We have built and tested CLP on the OSes listed 
   [below](https://github.com/y-scope/clp/tree/main/components/core#native-environment).
   * If you have trouble building for another OS, file an issue, and we may be able to help.
-* A compiler that supports C++17 (e.g., gcc-8)
+* A compiler that supports C++17 and std::span (e.g., gcc-10)
 
 ## Building
 

diff --git a/components/core/cmake/utils.cmake b/components/core/cmake/utils.cmake
@@ -41,9 +41,14 @@ set(SOURCE_FILES_make-dictionaries-readable
         ${CMAKE_CURRENT_SOURCE_DIR}/submodules/date/include/date/date.h
         )
 add_executable(make-dictionaries-readable ${SOURCE_FILES_make-dictionaries-readable})
+target_include_directories(make-dictionaries-readable
+                           PRIVATE
+                           ${CMAKE_SOURCE_DIR}/submodules
+                           )
 target_link_libraries(make-dictionaries-readable
         PRIVATE
         Boost::filesystem Boost::iostreams Boost::program_options
+        log_surgeon::log_surgeon
         spdlog::spdlog
         ZStd::ZStd
         )

diff --git a/components/core/config/schemas.txt b/components/core/config/schemas.txt
@@ -9,9 +9,9 @@ timestamp:\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3}){0,1}
 // E.g. [20150131-15:50:45]
 timestamp:\[\d{8}\-\d{2}:\d{2}:\d{2}\]
 
-// Specially-encoded variables (using the `int` and `double` keywords)
+// Specially-encoded variables (using the `int` and `float` keywords)
 int:\-{0,1}[0-9]+
-double:\-{0,1}[0-9]+\.[0-9]+
+float:\-{0,1}[0-9]+\.[0-9]+
 
 // Dictionary variables
 hex:[a-fA-F]+