core: Add unit test for LogEventSerializer. (#431)

components/core/CMakeLists.txt

@@ -357,6 +357,7 @@ set(SOURCE_FILES_unitTest
         src/clp/hash_utils.cpp
         src/clp/hash_utils.hpp
         src/clp/ir/constants.hpp
+        src/clp/ir/EncodedTextAst.cpp
         src/clp/ir/EncodedTextAst.hpp
         src/clp/ir/LogEvent.hpp
         src/clp/ir/LogEventDeserializer.cpp
@@ -478,6 +479,7 @@ set(SOURCE_FILES_unitTest
         tests/test-hash_utils.cpp
         tests/test-ir_encoding_methods.cpp
         tests/test-ir_parsing.cpp
+        tests/test-ir_serializer.cpp
         tests/test-kql.cpp
         tests/test-main.cpp
         tests/test-math_utils.cpp

components/core/src/clp/clg/CMakeLists.txt

@@ -33,6 +33,7 @@ set(
         ../GlobalSQLiteMetadataDB.hpp
         ../Grep.cpp
         ../Grep.hpp
+        ../ir/EncodedTextAst.cpp
         ../ir/EncodedTextAst.hpp
         ../ir/LogEvent.hpp
         ../ir/parsing.cpp

components/core/src/clp/clo/CMakeLists.txt

@@ -34,6 +34,7 @@ set(
         ../FileWriter.hpp
         ../Grep.cpp
         ../Grep.hpp
+        ../ir/EncodedTextAst.cpp
         ../ir/EncodedTextAst.hpp
         ../ir/LogEvent.hpp
         ../ir/LogEventSerializer.cpp

components/core/src/clp/clp/CMakeLists.txt

@@ -41,6 +41,7 @@ set(
         ../GlobalSQLiteMetadataDB.cpp
         ../GlobalSQLiteMetadataDB.hpp
         ../ir/constants.hpp
+        ../ir/EncodedTextAst.cpp
         ../ir/EncodedTextAst.hpp
         ../ir/LogEvent.hpp
         ../ir/LogEventDeserializer.cpp

components/core/src/clp/ir/EncodedTextAst.cpp

@@ -0,0 +1,57 @@
+#include "EncodedTextAst.hpp"
+
+#include <cstddef>
+#include <optional>
+#include <string>
+
+#include "../ffi/encoding_methods.hpp"
+#include "ffi/ir_stream/decoding_methods.hpp"
+
+using clp::ffi::decode_float_var;
+using clp::ffi::decode_integer_var;
+using clp::ffi::ir_stream::DecodingException;
+using clp::ffi::ir_stream::generic_decode_message;
+using std::optional;
+using std::string;
+
+namespace clp::ir {
+template <typename encoded_variable_t>
+auto EncodedTextAst<encoded_variable_t>::decode_and_unparse() const -> optional<string> {
+    string decoded_string;
+
+    auto constant_handler = [&](string const& value, size_t begin_pos, size_t length) {
+        decoded_string.append(value, begin_pos, length);
+    };
+
+    auto encoded_int_handler
+            = [&](encoded_variable_t value) { decoded_string.append(decode_integer_var(value)); };
+
+    auto encoded_float_handler = [&](encoded_variable_t encoded_float) {
+        decoded_string.append(decode_float_var(encoded_float));
+    };
+
+    auto dict_var_handler = [&](string const& dict_var) { decoded_string.append(dict_var); };
+
+    try {
+        generic_decode_message<true>(
+                m_logtype,
+                m_encoded_vars,
+                m_dict_vars,
+                constant_handler,
+                encoded_int_handler,
+                encoded_float_handler,
+                dict_var_handler
+        );
+    } catch (DecodingException const& e) {
+        return std::nullopt;
+    }
+    return std::make_optional<string>(decoded_string);
+}
+
+// Explicitly declare template specializations so that we can define the template methods in this
+// file
+template auto EncodedTextAst<eight_byte_encoded_variable_t>::decode_and_unparse(
+) const -> optional<string>;
+template auto EncodedTextAst<four_byte_encoded_variable_t>::decode_and_unparse(
+) const -> optional<string>;
+}  // namespace clp::ir

components/core/src/clp/ir/EncodedTextAst.hpp

@@ -1,6 +1,7 @@
 #ifndef CLP_IR_ENCODEDTEXTAST_HPP
 #define CLP_IR_ENCODEDTEXTAST_HPP
 
+#include <optional>
 #include <string>
 #include <utility>
 #include <vector>
@@ -47,6 +48,13 @@ class EncodedTextAst {
         return m_encoded_vars;
     }
 
+    /**
+     * Decodes and un-parses the EncodedTextAst into its string form.
+     * @return The string corresponding to the EncodedTextAst on success.
+     * @return std::nullopt if decoding fails.
+     */
+    [[nodiscard]] auto decode_and_unparse() const -> std::optional<std::string>;
+
 private:
     // Variables
     std::string m_logtype;

components/core/tests/test-ir_serializer.cpp

@@ -0,0 +1,120 @@
+#include <chrono>
+#include <cstdint>
+#include <filesystem>
+#include <numeric>
+#include <string>
+#include <vector>
+
+#include <Catch2/single_include/catch2/catch.hpp>
+
+#include "../src/clp/ffi/ir_stream/decoding_methods.hpp"
+#include "../src/clp/ir/constants.hpp"
+#include "../src/clp/ir/LogEventDeserializer.hpp"
+#include "../src/clp/ir/LogEventSerializer.hpp"
+#include "../src/clp/ir/types.hpp"
+#include "../src/clp/streaming_compression/zstd/Decompressor.hpp"
+
+using clp::ffi::ir_stream::IRErrorCode::IRErrorCode_Success;
+using clp::ir::cIrFileExtension;
+using clp::ir::eight_byte_encoded_variable_t;
+using clp::ir::epoch_time_ms_t;
+using clp::ir::four_byte_encoded_variable_t;
+using clp::ir::LogEventDeserializer;
+using clp::ir::LogEventSerializer;
+using clp::streaming_compression::zstd::Decompressor;
+using std::chrono::milliseconds;
+using std::chrono::system_clock;
+using std::is_same_v;
+using std::string;
+using std::vector;
+
+namespace {
+struct TestLogEvent {
+    epoch_time_ms_t timestamp;
+    string msg;
+};
+}  // namespace
+
+TEMPLATE_TEST_CASE(
+        "Encode and serialize log events",
+        "[ir][serialize-log-event]",
+        four_byte_encoded_variable_t,
+        eight_byte_encoded_variable_t
+) {
+    vector<TestLogEvent> test_log_events;
+
+    auto const ts_1 = duration_cast<milliseconds>(system_clock::now().time_since_epoch()).count();
+    vector<string> const log_event_1_tokens
+            = {"Here is the first string with a small int ",
+               "4938",
+               " and a medium int ",
+               std::to_string(INT32_MAX),
+               " and a very large int ",
+               std::to_string(INT64_MAX),
+               " and a small float ",
+               "0.1",
+               "\n"};
+    auto const log_event_1
+            = std::accumulate(log_event_1_tokens.begin(), log_event_1_tokens.end(), string(""));
+    test_log_events.push_back({ts_1, log_event_1});
+
+    auto const ts_2 = duration_cast<milliseconds>(system_clock::now().time_since_epoch()).count();
+    vector<string> const log_event_2_tokens
+            = {"Here is the second string with a medium float ",
+               "-25.519686",
+               " and a high precision float ",
+               "-25.5196868642755",
+               " and a weird float ",
+               "-00.00",
+               " and a string with numbers ",
+               "bin/python2.7.3",
+               " and another string with numbers ",
+               "abc123",
+               "\n"};
+    auto const log_event_2
+            = std::accumulate(log_event_2_tokens.begin(), log_event_2_tokens.end(), string(""));
+    test_log_events.push_back({ts_2, log_event_2});
+
+    string ir_test_file = "ir_serializer_test";
+    ir_test_file += cIrFileExtension;
+
+    LogEventSerializer<TestType> serializer;
+    REQUIRE(serializer.open(ir_test_file));
+    // Test serializing log events
+    for (auto const& test_log_event : test_log_events) {
+        REQUIRE(serializer.serialize_log_event(test_log_event.timestamp, test_log_event.msg));
+    }
+    serializer.close();
+
+    Decompressor ir_reader;
+    ir_reader.open(ir_test_file);
+
+    bool uses_four_byte_encoding{false};
+    REQUIRE(
+            (IRErrorCode_Success
+             == clp::ffi::ir_stream::get_encoding_type(ir_reader, uses_four_byte_encoding))
+    );
+    REQUIRE((is_same_v<TestType, four_byte_encoded_variable_t> == uses_four_byte_encoding));
+
+    auto result = LogEventDeserializer<TestType>::create(ir_reader);
+    REQUIRE((false == result.has_error()));
+    auto& deserializer = result.value();
+
+    // Decode and deserialize all expected log events
+    for (auto const& test_log_event : test_log_events) {
+        auto deserialized_result = deserializer.deserialize_log_event();
+        REQUIRE((false == deserialized_result.has_error()));
+
+        auto& log_event = deserialized_result.value();
+        auto const decoded_message = log_event.get_message().decode_and_unparse();
+        REQUIRE(decoded_message.has_value());
+
+        REQUIRE((decoded_message.value() == test_log_event.msg));
+        REQUIRE((log_event.get_timestamp() == test_log_event.timestamp));
+    }
+    // Try decoding a nonexistent log event
+    auto deserialized_result = deserializer.deserialize_log_event();
+    REQUIRE(deserialized_result.has_error());
+
+    std::filesystem::remove(ir_test_file);
+}

components/core/tests/test-query_methods.cpp

@@ -165,6 +165,7 @@ TEMPLATE_TEST_CASE(
         message += " and a very large int " + var_strs[var_ix++];
         message += " and a small double " + var_strs[var_ix++];
         message += " and a medium double " + var_strs[var_ix++];
+        message += " and a high precison double " + var_strs[var_ix++];
         message += " and a weird double " + var_strs[var_ix++];
         message += " and a string with numbers " + var_strs[var_ix++];
         message += " and another string with numbers " + var_strs[var_ix++];