From 046056201af1b1d8995efdae110f4dd5c2210b7d Mon Sep 17 00:00:00 2001 From: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Fri, 29 Dec 2023 18:14:50 -0500 Subject: [PATCH] Move CLP IR types from the ffi to the ir namespace. (#199) --- components/core/CMakeLists.txt | 8 ++- .../core/src/EncodedVariableInterpreter.cpp | 54 ++++++++++--------- .../core/src/EncodedVariableInterpreter.hpp | 3 +- components/core/src/Grep.cpp | 1 + .../core/src/LogTypeDictionaryEntry.cpp | 16 +++--- .../core/src/LogTypeDictionaryEntry.hpp | 2 +- components/core/src/clp/FileCompressor.cpp | 15 +++--- components/core/src/ffi/encoding_methods.cpp | 4 ++ components/core/src/ffi/encoding_methods.hpp | 19 +++---- components/core/src/ffi/encoding_methods.inc | 53 +++++++++--------- .../src/ffi/ir_stream/decoding_methods.cpp | 4 ++ .../src/ffi/ir_stream/decoding_methods.hpp | 7 +-- .../src/ffi/ir_stream/decoding_methods.inc | 8 +-- .../src/ffi/ir_stream/encoding_methods.cpp | 4 ++ .../src/ffi/ir_stream/encoding_methods.hpp | 9 ++-- .../src/ffi/search/CompositeWildcardToken.cpp | 5 +- .../src/ffi/search/ExactVariableToken.cpp | 7 ++- .../src/ffi/search/ExactVariableToken.hpp | 5 +- components/core/src/ffi/search/Subquery.cpp | 5 +- .../core/src/ffi/search/WildcardToken.cpp | 8 ++- .../core/src/ffi/search/query_methods.cpp | 7 ++- components/core/src/ir/LogEvent.hpp | 8 +-- .../core/src/ir/LogEventDeserializer.cpp | 35 ++++++------ .../core/src/ir/LogEventDeserializer.hpp | 9 ++-- components/core/src/ir/parsing.cpp | 3 +- components/core/src/ir/parsing.hpp | 7 --- components/core/src/ir/parsing.inc | 1 + components/core/src/ir/types.hpp | 19 +++++++ .../src/streaming_archive/writer/Archive.cpp | 13 +++-- .../make-dictionaries-readable.cpp | 13 ++--- .../tests/test-EncodedVariableInterpreter.cpp | 31 +++++------ .../core/tests/test-encoding_methods.cpp | 5 +- .../core/tests/test-ir_encoding_methods.cpp | 38 ++++++------- components/core/tests/test-ir_parsing.cpp | 1 + components/core/tests/test-query_methods.cpp | 14 ++--- 35 files changed, 248 insertions(+), 193 deletions(-) create mode 100644 components/core/src/ir/types.hpp diff --git a/components/core/CMakeLists.txt b/components/core/CMakeLists.txt index a50d22c82..630d2ad7f 100644 --- a/components/core/CMakeLists.txt +++ b/components/core/CMakeLists.txt @@ -229,6 +229,7 @@ set(SOURCE_FILES_clp src/ir/parsing.cpp src/ir/parsing.hpp src/ir/parsing.inc + src/ir/types.hpp src/ir/utils.cpp src/ir/utils.hpp src/LibarchiveFileReader.cpp @@ -388,9 +389,10 @@ set(SOURCE_FILES_clg src/ir/LogEvent.hpp src/ir/parsing.cpp src/ir/parsing.hpp + src/ir/parsing.inc + src/ir/types.hpp src/LogSurgeonReader.cpp src/LogSurgeonReader.hpp - src/ir/parsing.inc src/LogTypeDictionaryEntry.cpp src/LogTypeDictionaryEntry.hpp src/LogTypeDictionaryReader.hpp @@ -524,9 +526,10 @@ set(SOURCE_FILES_clo src/ir/LogEvent.hpp src/ir/parsing.cpp src/ir/parsing.hpp + src/ir/parsing.inc + src/ir/types.hpp src/LogSurgeonReader.cpp src/LogSurgeonReader.hpp - src/ir/parsing.inc src/LogTypeDictionaryEntry.cpp src/LogTypeDictionaryEntry.hpp src/LogTypeDictionaryReader.hpp @@ -698,6 +701,7 @@ set(SOURCE_FILES_unitTest src/ir/parsing.cpp src/ir/parsing.hpp src/ir/parsing.inc + src/ir/types.hpp src/ir/utils.cpp src/ir/utils.hpp src/LibarchiveFileReader.cpp diff --git a/components/core/src/EncodedVariableInterpreter.cpp b/components/core/src/EncodedVariableInterpreter.cpp index e137dee69..128202f8a 100644 --- a/components/core/src/EncodedVariableInterpreter.cpp +++ b/components/core/src/EncodedVariableInterpreter.cpp @@ -5,14 +5,18 @@ #include #include "Defs.h" -#include "ffi/encoding_methods.hpp" #include "ffi/ir_stream/decoding_methods.hpp" -#include "ir/parsing.hpp" +#include "ir/LogEvent.hpp" +#include "ir/types.hpp" #include "spdlog_with_specializations.hpp" #include "string_utils.hpp" #include "type_utils.hpp" using ffi::cEightByteEncodedFloatDigitsBitMask; +using ir::eight_byte_encoded_variable_t; +using ir::four_byte_encoded_variable_t; +using ir::LogEvent; +using ir::VariablePlaceholder; using std::string; using std::unordered_set; using std::vector; @@ -221,10 +225,10 @@ void EncodedVariableInterpreter::encode_and_add_to_dictionary( template void EncodedVariableInterpreter::encode_and_add_to_dictionary( - ir::LogEvent const& log_event, + LogEvent const& log_event, LogTypeDictionaryEntry& logtype_dict_entry, VariableDictionaryWriter& var_dict, - std::vector& encoded_vars, + std::vector& encoded_vars, std::vector& var_ids, size_t& raw_num_bytes ) { @@ -242,23 +246,23 @@ void EncodedVariableInterpreter::encode_and_add_to_dictionary( raw_num_bytes += ffi::decode_integer_var(encoded_var).length(); logtype_dict_entry.add_int_var(); - ffi::eight_byte_encoded_variable_t eight_byte_encoded_var{}; - if constexpr (std::is_same_v) { + eight_byte_encoded_variable_t eight_byte_encoded_var{}; + if constexpr (std::is_same_v) { eight_byte_encoded_var = encoded_var; - } else { // std::is_same_v + } else { // std::is_same_v eight_byte_encoded_var = ffi::encode_four_byte_integer_as_eight_byte(encoded_var); } encoded_vars.push_back(eight_byte_encoded_var); }; - auto encoded_float_handler = [&](ffi::four_byte_encoded_variable_t encoded_var) { + auto encoded_float_handler = [&](four_byte_encoded_variable_t encoded_var) { raw_num_bytes += ffi::decode_float_var(encoded_var).length(); logtype_dict_entry.add_float_var(); - ffi::eight_byte_encoded_variable_t eight_byte_encoded_var{}; - if constexpr (std::is_same_v) { + eight_byte_encoded_variable_t eight_byte_encoded_var{}; + if constexpr (std::is_same_v) { eight_byte_encoded_var = encoded_var; - } else { // std::is_same_v + } else { // std::is_same_v eight_byte_encoded_var = ffi::encode_four_byte_float_as_eight_byte(encoded_var); } encoded_vars.push_back(eight_byte_encoded_var); @@ -267,12 +271,12 @@ void EncodedVariableInterpreter::encode_and_add_to_dictionary( auto dict_var_handler = [&](string const& dict_var) { raw_num_bytes += dict_var.length(); - ffi::eight_byte_encoded_variable_t encoded_var{}; - if constexpr (std::is_same_v) { + eight_byte_encoded_variable_t encoded_var{}; + if constexpr (std::is_same_v) { encoded_var = encode_var_dict_id( add_dict_var(dict_var, logtype_dict_entry, var_dict, var_ids) ); - } else { // std::is_same_v + } else { // std::is_same_v encoded_var = encode_var(dict_var, logtype_dict_entry, var_dict, var_ids); } encoded_vars.push_back(encoded_var); @@ -309,7 +313,7 @@ bool EncodedVariableInterpreter::decode_variables_into_message( return false; } - ir::VariablePlaceholder var_placeholder; + VariablePlaceholder var_placeholder; size_t constant_begin_pos = 0; string float_str; variable_dictionary_id_t var_dict_id; @@ -327,18 +331,18 @@ bool EncodedVariableInterpreter::decode_variables_into_message( placeholder_position - constant_begin_pos ); switch (var_placeholder) { - case ir::VariablePlaceholder::Integer: + case VariablePlaceholder::Integer: decompressed_msg += std::to_string(encoded_vars[var_ix++]); break; - case ir::VariablePlaceholder::Float: + case VariablePlaceholder::Float: convert_encoded_float_to_string(encoded_vars[var_ix++], float_str); decompressed_msg += float_str; break; - case ir::VariablePlaceholder::Dictionary: + case VariablePlaceholder::Dictionary: var_dict_id = decode_var_dict_id(encoded_vars[var_ix++]); decompressed_msg += var_dict.get_value(var_dict_id); break; - case ir::VariablePlaceholder::Escape: + case VariablePlaceholder::Escape: break; default: SPDLOG_ERROR( @@ -459,21 +463,21 @@ variable_dictionary_id_t EncodedVariableInterpreter::add_dict_var( // Explicitly declare template specializations so that we can define the template methods in this // file template void -EncodedVariableInterpreter::encode_and_add_to_dictionary( - ir::LogEvent const& log_event, +EncodedVariableInterpreter::encode_and_add_to_dictionary( + LogEvent const& log_event, LogTypeDictionaryEntry& logtype_dict_entry, VariableDictionaryWriter& var_dict, - std::vector& encoded_vars, + std::vector& encoded_vars, std::vector& var_ids, size_t& raw_num_bytes ); template void -EncodedVariableInterpreter::encode_and_add_to_dictionary( - ir::LogEvent const& log_event, +EncodedVariableInterpreter::encode_and_add_to_dictionary( + LogEvent const& log_event, LogTypeDictionaryEntry& logtype_dict_entry, VariableDictionaryWriter& var_dict, - std::vector& encoded_vars, + std::vector& encoded_vars, std::vector& var_ids, size_t& raw_num_bytes ); diff --git a/components/core/src/EncodedVariableInterpreter.hpp b/components/core/src/EncodedVariableInterpreter.hpp index 7abcfa4fe..07af82061 100644 --- a/components/core/src/EncodedVariableInterpreter.hpp +++ b/components/core/src/EncodedVariableInterpreter.hpp @@ -5,6 +5,7 @@ #include #include "ir/LogEvent.hpp" +#include "ir/types.hpp" #include "Query.hpp" #include "TraceableException.hpp" #include "VariableDictionaryReader.hpp" @@ -107,7 +108,7 @@ class EncodedVariableInterpreter { ir::LogEvent const& log_event, LogTypeDictionaryEntry& logtype_dict_entry, VariableDictionaryWriter& var_dict, - std::vector& encoded_vars, + std::vector& encoded_vars, std::vector& var_ids, size_t& raw_num_bytes ); diff --git a/components/core/src/Grep.cpp b/components/core/src/Grep.cpp index 75c9c63ca..a56aecbca 100644 --- a/components/core/src/Grep.cpp +++ b/components/core/src/Grep.cpp @@ -6,6 +6,7 @@ #include "EncodedVariableInterpreter.hpp" #include "ir/parsing.hpp" +#include "ir/types.hpp" #include "LogSurgeonReader.hpp" #include "StringReader.hpp" #include "Utils.hpp" diff --git a/components/core/src/LogTypeDictionaryEntry.cpp b/components/core/src/LogTypeDictionaryEntry.cpp index 4f6cffd53..ba9fd3e3b 100644 --- a/components/core/src/LogTypeDictionaryEntry.cpp +++ b/components/core/src/LogTypeDictionaryEntry.cpp @@ -1,22 +1,24 @@ #include "LogTypeDictionaryEntry.hpp" #include "ir/parsing.hpp" +#include "ir/types.hpp" #include "type_utils.hpp" #include "Utils.hpp" +using ir::VariablePlaceholder; using std::string; using std::string_view; size_t LogTypeDictionaryEntry::get_placeholder_info( size_t placeholder_ix, - ir::VariablePlaceholder& placeholder + VariablePlaceholder& placeholder ) const { if (placeholder_ix >= m_placeholder_positions.size()) { return SIZE_MAX; } auto var_position = m_placeholder_positions[placeholder_ix]; - placeholder = static_cast(m_value[var_position]); + placeholder = static_cast(m_value[var_position]); return m_placeholder_positions[placeholder_ix]; } @@ -73,7 +75,7 @@ bool LogTypeDictionaryEntry::parse_next_var( ) -> void { m_placeholder_positions.push_back(logtype.size()); ++m_num_escaped_placeholders; - logtype += enum_to_underlying_type(ir::VariablePlaceholder::Escape); + logtype += enum_to_underlying_type(VariablePlaceholder::Escape); }; // clang-format on if (ir::get_bounds_of_next_var(msg, var_begin_pos, var_end_pos)) { @@ -144,21 +146,21 @@ ErrorCode LogTypeDictionaryEntry::try_read_from_file( if (is_escaped) { constant += c; is_escaped = false; - } else if (enum_to_underlying_type(ir::VariablePlaceholder::Escape) == c) { + } else if (enum_to_underlying_type(VariablePlaceholder::Escape) == c) { is_escaped = true; add_constant(constant, 0, constant.length()); constant.clear(); add_escape(); } else { - if (enum_to_underlying_type(ir::VariablePlaceholder::Integer) == c) { + if (enum_to_underlying_type(VariablePlaceholder::Integer) == c) { add_constant(constant, 0, constant.length()); constant.clear(); add_int_var(); - } else if (enum_to_underlying_type(ir::VariablePlaceholder::Float) == c) { + } else if (enum_to_underlying_type(VariablePlaceholder::Float) == c) { add_constant(constant, 0, constant.length()); constant.clear(); add_float_var(); - } else if (enum_to_underlying_type(ir::VariablePlaceholder::Dictionary) == c) { + } else if (enum_to_underlying_type(VariablePlaceholder::Dictionary) == c) { add_constant(constant, 0, constant.length()); constant.clear(); add_dictionary_var(); diff --git a/components/core/src/LogTypeDictionaryEntry.hpp b/components/core/src/LogTypeDictionaryEntry.hpp index d0f256f02..b154beae6 100644 --- a/components/core/src/LogTypeDictionaryEntry.hpp +++ b/components/core/src/LogTypeDictionaryEntry.hpp @@ -7,7 +7,7 @@ #include "DictionaryEntry.hpp" #include "ErrorCode.hpp" #include "FileReader.hpp" -#include "ir/parsing.hpp" +#include "ir/types.hpp" #include "streaming_compression/zstd/Compressor.hpp" #include "streaming_compression/zstd/Decompressor.hpp" #include "TraceableException.hpp" diff --git a/components/core/src/clp/FileCompressor.cpp b/components/core/src/clp/FileCompressor.cpp index bf5504b70..c4aa08483 100644 --- a/components/core/src/clp/FileCompressor.cpp +++ b/components/core/src/clp/FileCompressor.cpp @@ -11,11 +11,14 @@ #include #include "../ffi/ir_stream/decoding_methods.hpp" +#include "../ir/types.hpp" #include "../ir/utils.hpp" #include "../LogSurgeonReader.hpp" #include "../Profiler.hpp" #include "utils.hpp" +using ir::eight_byte_encoded_variable_t; +using ir::four_byte_encoded_variable_t; using ir::has_ir_stream_magic_number; using ir::LogEventDeserializer; using log_surgeon::LogEventView; @@ -432,7 +435,7 @@ bool FileCompressor::compress_ir_stream( try { std::error_code error_code{}; if (uses_four_byte_encoding) { - auto result = LogEventDeserializer::create(reader); + auto result = LogEventDeserializer::create(reader); if (result.has_error()) { error_code = result.error(); } else { @@ -447,7 +450,7 @@ bool FileCompressor::compress_ir_stream( ); } } else { - auto result = LogEventDeserializer::create(reader); + auto result = LogEventDeserializer::create(reader); if (result.has_error()) { error_code = result.error(); } else { @@ -548,23 +551,23 @@ std::error_code FileCompressor::compress_ir_stream_by_encoding( // Explicitly declare template specializations so that we can define the template methods in this // file template std::error_code -FileCompressor::compress_ir_stream_by_encoding( +FileCompressor::compress_ir_stream_by_encoding( size_t target_data_size_of_dicts, streaming_archive::writer::Archive::UserConfig& archive_user_config, size_t target_encoded_file_size, string const& path, group_id_t group_id, streaming_archive::writer::Archive& archive, - LogEventDeserializer& log_event_deserializer + LogEventDeserializer& log_event_deserializer ); template std::error_code -FileCompressor::compress_ir_stream_by_encoding( +FileCompressor::compress_ir_stream_by_encoding( size_t target_data_size_of_dicts, streaming_archive::writer::Archive::UserConfig& archive_user_config, size_t target_encoded_file_size, string const& path, group_id_t group_id, streaming_archive::writer::Archive& archive, - LogEventDeserializer& log_event_deserializer + LogEventDeserializer& log_event_deserializer ); } // namespace clp diff --git a/components/core/src/ffi/encoding_methods.cpp b/components/core/src/ffi/encoding_methods.cpp index 821c5177f..0b98af626 100644 --- a/components/core/src/ffi/encoding_methods.cpp +++ b/components/core/src/ffi/encoding_methods.cpp @@ -3,6 +3,10 @@ #include #include +#include "../ir/types.hpp" + +using ir::eight_byte_encoded_variable_t; +using ir::four_byte_encoded_variable_t; using std::string_view; namespace ffi { diff --git a/components/core/src/ffi/encoding_methods.hpp b/components/core/src/ffi/encoding_methods.hpp index 2b6335191..5e190ace7 100644 --- a/components/core/src/ffi/encoding_methods.hpp +++ b/components/core/src/ffi/encoding_methods.hpp @@ -5,17 +5,12 @@ #include #include "../ir/parsing.hpp" +#include "../ir/types.hpp" #include "../TraceableException.hpp" // TODO Some of the methods in this file are mostly duplicated from code that exists elsewhere in // the repo. They should be consolidated in a future commit. namespace ffi { -// Types -using epoch_time_ms_t = int64_t; - -using eight_byte_encoded_variable_t = int64_t; -using four_byte_encoded_variable_t = int32_t; - class EncodingException : public TraceableException { public: // Constructors @@ -77,8 +72,8 @@ bool encode_float_string(std::string_view str, encoded_variable_t& encoded_var); * @param four_byte_encoded_var * @return The float using the eight-byte encoding */ -eight_byte_encoded_variable_t encode_four_byte_float_as_eight_byte( - four_byte_encoded_variable_t four_byte_encoded_var +ir::eight_byte_encoded_variable_t encode_four_byte_float_as_eight_byte( + ir::four_byte_encoded_variable_t four_byte_encoded_var ); /** @@ -95,7 +90,7 @@ template encoded_variable_t encode_float_properties( bool is_negative, std::conditional_t< - std::is_same_v, + std::is_same_v, uint32_t, uint64_t> digits, size_t num_digits, @@ -116,7 +111,7 @@ void decode_float_properties( encoded_variable_t encoded_var, bool& is_negative, std::conditional_t< - std::is_same_v, + std::is_same_v, uint32_t, uint64_t>& digits, uint8_t& num_digits, @@ -147,8 +142,8 @@ bool encode_integer_string(std::string_view str, encoded_variable_t& encoded_var * @param four_byte_encoded_var * @return The integer using the eight-byte encoding */ -eight_byte_encoded_variable_t encode_four_byte_integer_as_eight_byte( - four_byte_encoded_variable_t four_byte_encoded_var +ir::eight_byte_encoded_variable_t encode_four_byte_integer_as_eight_byte( + ir::four_byte_encoded_variable_t four_byte_encoded_var ); /** diff --git a/components/core/src/ffi/encoding_methods.inc b/components/core/src/ffi/encoding_methods.inc index bec957343..dbbb45901 100644 --- a/components/core/src/ffi/encoding_methods.inc +++ b/components/core/src/ffi/encoding_methods.inc @@ -4,11 +4,10 @@ #include #include "../ir/parsing.hpp" +#include "../ir/types.hpp" #include "../string_utils.hpp" #include "../type_utils.hpp" -using ir::VariablePlaceholder; - namespace ffi { template bool encode_float_string(std::string_view str, encoded_variable_t& encoded_var) { @@ -20,7 +19,7 @@ bool encode_float_string(std::string_view str, encoded_variable_t& encoded_var) size_t pos = 0; constexpr size_t cMaxDigitsInRepresentableFloatVar - = std::is_same_v + = std::is_same_v ? cMaxDigitsInRepresentableFourByteFloatVar : cMaxDigitsInRepresentableEightByteFloatVar; // +1 for decimal point @@ -43,7 +42,7 @@ bool encode_float_string(std::string_view str, encoded_variable_t& encoded_var) size_t num_digits = 0; size_t decimal_point_pos = std::string::npos; std::conditional_t< - std::is_same_v, + std::is_same_v, uint32_t, uint64_t> digits = 0; @@ -64,7 +63,7 @@ bool encode_float_string(std::string_view str, encoded_variable_t& encoded_var) // No decimal point found, decimal point is after all digits, or no digits found return false; } - if constexpr (std::is_same_v) { + if constexpr (std::is_same_v) { if (cFourByteEncodedFloatDigitsBitMask < digits) { // digits is larger than maximum representable return false; @@ -85,17 +84,17 @@ template encoded_variable_t encode_float_properties( bool is_negative, std::conditional_t< - std::is_same_v, + std::is_same_v, uint32_t, uint64_t> digits, size_t num_digits, size_t decimal_point_pos ) { static_assert( - (std::is_same_v - || std::is_same_v) + (std::is_same_v + || std::is_same_v) ); - if constexpr (std::is_same_v) { + if constexpr (std::is_same_v) { // Encode into 64 bits with the following format (from MSB to LSB): // - 1 bit : is negative // - 1 bit : unused @@ -169,17 +168,17 @@ void decode_float_properties( encoded_variable_t encoded_var, bool& is_negative, std::conditional_t< - std::is_same_v, + std::is_same_v, uint32_t, uint64_t>& digits, uint8_t& num_digits, uint8_t& decimal_point_pos ) { static_assert( - (std::is_same_v - || std::is_same_v) + (std::is_same_v + || std::is_same_v) ); - if constexpr (std::is_same_v) { + if constexpr (std::is_same_v) { auto encoded_float = bit_cast(encoded_var); // Decode according to the format described in encode_float_string @@ -223,7 +222,7 @@ std::string decode_float_var(encoded_variable_t encoded_var) { uint8_t decimal_point_pos; uint8_t num_digits; std::conditional_t< - std::is_same_v, + std::is_same_v, uint32_t, uint64_t> digits; @@ -368,13 +367,13 @@ bool encode_message_generically( std::string_view var_string{&message[var_begin_pos], var_end_pos - var_begin_pos}; encoded_variable_t encoded_variable; if (encode_float_string(var_string, encoded_variable)) { - logtype += enum_to_underlying_type(VariablePlaceholder::Float); + logtype += enum_to_underlying_type(ir::VariablePlaceholder::Float); encoded_variable_handler(encoded_variable); } else if (encode_integer_string(var_string, encoded_variable)) { - logtype += enum_to_underlying_type(VariablePlaceholder::Integer); + logtype += enum_to_underlying_type(ir::VariablePlaceholder::Integer); encoded_variable_handler(encoded_variable); } else { - logtype += enum_to_underlying_type(VariablePlaceholder::Dictionary); + logtype += enum_to_underlying_type(ir::VariablePlaceholder::Dictionary); if (false == dictionary_variable_handler(message, var_begin_pos, var_end_pos)) { return false; } @@ -444,7 +443,7 @@ std::string decode_message( size_t encoded_vars_ix = 0; for (size_t i = 0; i < logtype.length(); ++i) { auto c = logtype[i]; - if (enum_to_underlying_type(VariablePlaceholder::Float) == c) { + if (enum_to_underlying_type(ir::VariablePlaceholder::Float) == c) { message.append(logtype, last_variable_end_pos, i - last_variable_end_pos); last_variable_end_pos = i + 1; if (encoded_vars_ix >= encoded_vars_length) { @@ -457,7 +456,7 @@ std::string decode_message( } message.append(decode_float_var(encoded_vars[encoded_vars_ix])); ++encoded_vars_ix; - } else if (enum_to_underlying_type(VariablePlaceholder::Integer) == c) { + } else if (enum_to_underlying_type(ir::VariablePlaceholder::Integer) == c) { message.append(logtype, last_variable_end_pos, i - last_variable_end_pos); last_variable_end_pos = i + 1; if (encoded_vars_ix >= encoded_vars_length) { @@ -470,7 +469,7 @@ std::string decode_message( } message.append(decode_integer_var(encoded_vars[encoded_vars_ix])); ++encoded_vars_ix; - } else if (enum_to_underlying_type(VariablePlaceholder::Dictionary) == c) { + } else if (enum_to_underlying_type(ir::VariablePlaceholder::Dictionary) == c) { message.append(logtype, last_variable_end_pos, i - last_variable_end_pos); last_variable_end_pos = i + 1; if (dictionary_var_bounds_ix >= dictionary_var_end_offsets_length) { @@ -499,7 +498,7 @@ std::string decode_message( return message; } -template +template bool wildcard_query_matches_any_encoded_var( std::string_view wildcard_query, std::string_view logtype, @@ -508,7 +507,7 @@ bool wildcard_query_matches_any_encoded_var( ) { size_t encoded_vars_ix = 0; for (auto c : logtype) { - if (enum_to_underlying_type(VariablePlaceholder::Float) == c) { + if (enum_to_underlying_type(ir::VariablePlaceholder::Float) == c) { if (encoded_vars_ix >= encoded_vars_length) { throw EncodingException( ErrorCode_Corrupt, @@ -518,7 +517,7 @@ bool wildcard_query_matches_any_encoded_var( ); } - if constexpr (VariablePlaceholder::Float == var_placeholder) { + if constexpr (ir::VariablePlaceholder::Float == var_placeholder) { auto decoded_var = decode_float_var(encoded_vars[encoded_vars_ix]); if (wildcard_match_unsafe(decoded_var, wildcard_query)) { return true; @@ -526,7 +525,7 @@ bool wildcard_query_matches_any_encoded_var( } ++encoded_vars_ix; - } else if (enum_to_underlying_type(VariablePlaceholder::Integer) == c) { + } else if (enum_to_underlying_type(ir::VariablePlaceholder::Integer) == c) { if (encoded_vars_ix >= encoded_vars_length) { throw EncodingException( ErrorCode_Corrupt, @@ -536,7 +535,7 @@ bool wildcard_query_matches_any_encoded_var( ); } - if constexpr (VariablePlaceholder::Integer == var_placeholder) { + if constexpr (ir::VariablePlaceholder::Integer == var_placeholder) { auto decoded_var = decode_integer_var(encoded_vars[encoded_vars_ix]); if (wildcard_match_unsafe(decoded_var, wildcard_query)) { return true; @@ -580,7 +579,7 @@ bool wildcard_match_encoded_vars( size_t var_ix = 0; size_t wildcard_var_ix = 0; for (auto c : logtype) { - if (enum_to_underlying_type(VariablePlaceholder::Float) == c) { + if (enum_to_underlying_type(ir::VariablePlaceholder::Float) == c) { if (var_ix >= encoded_vars_length) { throw EncodingException( ErrorCode_Corrupt, @@ -601,7 +600,7 @@ bool wildcard_match_encoded_vars( } ++var_ix; - } else if (enum_to_underlying_type(VariablePlaceholder::Integer) == c) { + } else if (enum_to_underlying_type(ir::VariablePlaceholder::Integer) == c) { if (var_ix >= encoded_vars_length) { throw EncodingException( ErrorCode_Corrupt, diff --git a/components/core/src/ffi/ir_stream/decoding_methods.cpp b/components/core/src/ffi/ir_stream/decoding_methods.cpp index 51e433914..32f02f44d 100644 --- a/components/core/src/ffi/ir_stream/decoding_methods.cpp +++ b/components/core/src/ffi/ir_stream/decoding_methods.cpp @@ -2,9 +2,13 @@ #include +#include "../../ir/types.hpp" #include "byteswap.hpp" #include "protocol_constants.hpp" +using ir::eight_byte_encoded_variable_t; +using ir::epoch_time_ms_t; +using ir::four_byte_encoded_variable_t; using std::is_same_v; using std::string; using std::vector; diff --git a/components/core/src/ffi/ir_stream/decoding_methods.hpp b/components/core/src/ffi/ir_stream/decoding_methods.hpp index 1a2d8c738..ec90405d5 100644 --- a/components/core/src/ffi/ir_stream/decoding_methods.hpp +++ b/components/core/src/ffi/ir_stream/decoding_methods.hpp @@ -4,6 +4,7 @@ #include #include +#include "../../ir/types.hpp" #include "../../ReaderInterface.hpp" #include "../encoding_methods.hpp" @@ -74,7 +75,7 @@ auto deserialize_log_event( std::string& logtype, std::vector& encoded_vars, std::vector& dict_vars, - epoch_time_ms_t& timestamp_or_timestamp_delta + ir::epoch_time_ms_t& timestamp_or_timestamp_delta ) -> IRErrorCode; /** @@ -176,7 +177,7 @@ namespace eight_byte_encoding { IRErrorCode deserialize_log_event( ReaderInterface& reader, std::string& message, - epoch_time_ms_t& timestamp + ir::epoch_time_ms_t& timestamp ); } // namespace eight_byte_encoding @@ -195,7 +196,7 @@ namespace four_byte_encoding { IRErrorCode deserialize_log_event( ReaderInterface& reader, std::string& message, - epoch_time_ms_t& timestamp_delta + ir::epoch_time_ms_t& timestamp_delta ); } // namespace four_byte_encoding } // namespace ffi::ir_stream diff --git a/components/core/src/ffi/ir_stream/decoding_methods.inc b/components/core/src/ffi/ir_stream/decoding_methods.inc index 04a5d883b..0829b0dad 100644 --- a/components/core/src/ffi/ir_stream/decoding_methods.inc +++ b/components/core/src/ffi/ir_stream/decoding_methods.inc @@ -4,7 +4,7 @@ #include #include -#include "../../ir/parsing.hpp" +#include "../../ir/types.hpp" #include "../encoding_methods.hpp" #include "decoding_methods.hpp" #include "protocol_constants.hpp" @@ -36,7 +36,7 @@ void generic_decode_message( for (size_t cur_pos = 0; cur_pos < logtype_length; ++cur_pos) { auto c = logtype[cur_pos]; switch (c) { - case enum_to_underlying_type(VariablePlaceholder::Float): { + case enum_to_underlying_type(ir::VariablePlaceholder::Float): { constant_handler( logtype, next_static_text_begin_pos, @@ -57,7 +57,7 @@ void generic_decode_message( break; } - case enum_to_underlying_type(VariablePlaceholder::Integer): { + case enum_to_underlying_type(ir::VariablePlaceholder::Integer): { constant_handler( logtype, next_static_text_begin_pos, @@ -78,7 +78,7 @@ void generic_decode_message( break; } - case enum_to_underlying_type(VariablePlaceholder::Dictionary): { + case enum_to_underlying_type(ir::VariablePlaceholder::Dictionary): { constant_handler( logtype, next_static_text_begin_pos, diff --git a/components/core/src/ffi/ir_stream/encoding_methods.cpp b/components/core/src/ffi/ir_stream/encoding_methods.cpp index 17c0730ac..aff29db72 100644 --- a/components/core/src/ffi/ir_stream/encoding_methods.cpp +++ b/components/core/src/ffi/ir_stream/encoding_methods.cpp @@ -3,9 +3,13 @@ #include #include "../../ir/parsing.hpp" +#include "../../ir/types.hpp" #include "byteswap.hpp" #include "protocol_constants.hpp" +using ir::eight_byte_encoded_variable_t; +using ir::epoch_time_ms_t; +using ir::four_byte_encoded_variable_t; using std::string; using std::string_view; using std::vector; diff --git a/components/core/src/ffi/ir_stream/encoding_methods.hpp b/components/core/src/ffi/ir_stream/encoding_methods.hpp index c1437adf6..5716d16f2 100644 --- a/components/core/src/ffi/ir_stream/encoding_methods.hpp +++ b/components/core/src/ffi/ir_stream/encoding_methods.hpp @@ -4,6 +4,7 @@ #include #include +#include "../../ir/types.hpp" #include "../encoding_methods.hpp" namespace ffi::ir_stream { @@ -32,7 +33,7 @@ namespace eight_byte_encoding { * @return true on success, false otherwise */ bool serialize_log_event( - epoch_time_ms_t timestamp, + ir::epoch_time_ms_t timestamp, std::string_view message, std::string& logtype, std::vector& ir_buf @@ -53,7 +54,7 @@ namespace four_byte_encoding { std::string_view timestamp_pattern, std::string_view timestamp_pattern_syntax, std::string_view time_zone_id, - epoch_time_ms_t reference_timestamp, + ir::epoch_time_ms_t reference_timestamp, std::vector& ir_buf ); @@ -66,7 +67,7 @@ namespace four_byte_encoding { * @return true on success, false otherwise */ bool serialize_log_event( - epoch_time_ms_t timestamp_delta, + ir::epoch_time_ms_t timestamp_delta, std::string_view message, std::string& logtype, std::vector& ir_buf @@ -89,7 +90,7 @@ namespace four_byte_encoding { * @param ir_buf * @return true on success, false otherwise */ - bool serialize_timestamp(epoch_time_ms_t timestamp_delta, std::vector& ir_buf); + bool serialize_timestamp(ir::epoch_time_ms_t timestamp_delta, std::vector& ir_buf); } // namespace four_byte_encoding } // namespace ffi::ir_stream diff --git a/components/core/src/ffi/search/CompositeWildcardToken.cpp b/components/core/src/ffi/search/CompositeWildcardToken.cpp index 9520322c3..4d821c41b 100644 --- a/components/core/src/ffi/search/CompositeWildcardToken.cpp +++ b/components/core/src/ffi/search/CompositeWildcardToken.cpp @@ -1,6 +1,7 @@ #include "CompositeWildcardToken.hpp" #include "../../ir/parsing.hpp" +#include "../../ir/types.hpp" using std::string; using std::string_view; @@ -262,6 +263,6 @@ void CompositeWildcardToken::try_add_wildcard_variable( // Explicitly declare specializations to avoid having to validate that the template parameters are // supported -template class ffi::search::CompositeWildcardToken; -template class ffi::search::CompositeWildcardToken; +template class ffi::search::CompositeWildcardToken; +template class ffi::search::CompositeWildcardToken; } // namespace ffi::search diff --git a/components/core/src/ffi/search/ExactVariableToken.cpp b/components/core/src/ffi/search/ExactVariableToken.cpp index ea2421117..868612e46 100644 --- a/components/core/src/ffi/search/ExactVariableToken.cpp +++ b/components/core/src/ffi/search/ExactVariableToken.cpp @@ -1,5 +1,8 @@ #include "ExactVariableToken.hpp" +#include "../../ir/types.hpp" + +using ir::VariablePlaceholder; using std::string_view; namespace ffi::search { @@ -26,6 +29,6 @@ ExactVariableToken::ExactVariableToken( // Explicitly declare specializations to avoid having to validate that the template parameters are // supported -template class ExactVariableToken; -template class ExactVariableToken; +template class ExactVariableToken; +template class ExactVariableToken; } // namespace ffi::search diff --git a/components/core/src/ffi/search/ExactVariableToken.hpp b/components/core/src/ffi/search/ExactVariableToken.hpp index 62c7c40bf..9e461b35b 100644 --- a/components/core/src/ffi/search/ExactVariableToken.hpp +++ b/components/core/src/ffi/search/ExactVariableToken.hpp @@ -2,6 +2,7 @@ #define FFI_SEARCH_EXACTVARIABLETOKEN_HPP #include "../../Defs.h" +#include "../../ir/types.hpp" #include "../encoding_methods.hpp" #include "QueryToken.hpp" @@ -39,11 +40,11 @@ class ExactVariableToken : public QueryToken { [[nodiscard]] encoded_variable_t get_encoded_value() const { return m_encoded_value; } - [[nodiscard]] VariablePlaceholder get_placeholder() const { return m_placeholder; } + [[nodiscard]] ir::VariablePlaceholder get_placeholder() const { return m_placeholder; } private: encoded_variable_t m_encoded_value; - VariablePlaceholder m_placeholder; + ir::VariablePlaceholder m_placeholder; }; } // namespace ffi::search diff --git a/components/core/src/ffi/search/Subquery.cpp b/components/core/src/ffi/search/Subquery.cpp index d94d1e358..961a8b249 100644 --- a/components/core/src/ffi/search/Subquery.cpp +++ b/components/core/src/ffi/search/Subquery.cpp @@ -1,6 +1,7 @@ #include "Subquery.hpp" #include "../../ir/parsing.hpp" +#include "../../ir/types.hpp" #include "QueryWildcard.hpp" using std::string; @@ -56,6 +57,6 @@ Subquery::Subquery(string logtype_query, Subquery::QueryVari // Explicitly declare specializations to avoid having to validate that the template parameters are // supported -template class Subquery; -template class Subquery; +template class Subquery; +template class Subquery; } // namespace ffi::search diff --git a/components/core/src/ffi/search/WildcardToken.cpp b/components/core/src/ffi/search/WildcardToken.cpp index 2193266a6..5f48ae838 100644 --- a/components/core/src/ffi/search/WildcardToken.cpp +++ b/components/core/src/ffi/search/WildcardToken.cpp @@ -2,11 +2,15 @@ #include +#include "../../ir/types.hpp" #include "../../string_utils.hpp" #include "../../type_utils.hpp" #include "../encoding_methods.hpp" #include "QueryWildcard.hpp" +using ir::eight_byte_encoded_variable_t; +using ir::four_byte_encoded_variable_t; +using ir::VariablePlaceholder; using std::string; using std::string_view; @@ -214,6 +218,6 @@ bool WildcardToken::next_interpretation() { // Explicitly declare specializations to avoid having to validate that the template parameters are // supported -template class WildcardToken; -template class WildcardToken; +template class WildcardToken; +template class WildcardToken; } // namespace ffi::search diff --git a/components/core/src/ffi/search/query_methods.cpp b/components/core/src/ffi/search/query_methods.cpp index 060bd14b4..cffc5bcc1 100644 --- a/components/core/src/ffi/search/query_methods.cpp +++ b/components/core/src/ffi/search/query_methods.cpp @@ -1,9 +1,12 @@ #include "query_methods.hpp" #include "../../ir/parsing.hpp" +#include "../../ir/types.hpp" #include "CompositeWildcardToken.hpp" #include "QueryMethodFailed.hpp" +using ir::eight_byte_encoded_variable_t; +using ir::four_byte_encoded_variable_t; using ir::is_delim; using std::pair; using std::string; @@ -296,14 +299,14 @@ template void generate_subqueries( string_view wildcard_query, vector>& sub_queries ); -template void tokenize_query( +template void tokenize_query( string_view wildcard_query, vector< variant, CompositeWildcardToken>>& tokens, vector& composite_wildcard_token_indexes ); -template void tokenize_query( +template void tokenize_query( string_view wildcard_query, vector< variant, diff --git a/components/core/src/ir/LogEvent.hpp b/components/core/src/ir/LogEvent.hpp index c84eba2f8..b7fcd18e4 100644 --- a/components/core/src/ir/LogEvent.hpp +++ b/components/core/src/ir/LogEvent.hpp @@ -5,7 +5,7 @@ #include #include "../Defs.h" -#include "../ffi/encoding_methods.hpp" +#include "types.hpp" namespace ir { /** @@ -17,7 +17,7 @@ class LogEvent { public: // Constructors LogEvent( - ffi::epoch_time_ms_t timestamp, + epoch_time_ms_t timestamp, std::string logtype, std::vector dict_vars, std::vector encoded_vars @@ -28,7 +28,7 @@ class LogEvent { m_encoded_vars{std::move(encoded_vars)} {} // Methods - [[nodiscard]] auto get_timestamp() const -> ffi::epoch_time_ms_t { return m_timestamp; } + [[nodiscard]] auto get_timestamp() const -> epoch_time_ms_t { return m_timestamp; } [[nodiscard]] auto get_logtype() const -> std::string const& { return m_logtype; } @@ -42,7 +42,7 @@ class LogEvent { private: // Variables - ffi::epoch_time_ms_t m_timestamp; + epoch_time_ms_t m_timestamp; std::string m_logtype; std::vector m_dict_vars; std::vector m_encoded_vars; diff --git a/components/core/src/ir/LogEventDeserializer.cpp b/components/core/src/ir/LogEventDeserializer.cpp index 262d64b7c..d7fceafa3 100644 --- a/components/core/src/ir/LogEventDeserializer.cpp +++ b/components/core/src/ir/LogEventDeserializer.cpp @@ -5,6 +5,7 @@ #include #include "../ffi/ir_stream/decoding_methods.hpp" +#include "types.hpp" namespace ir { template @@ -43,10 +44,10 @@ auto LogEventDeserializer::create(ReaderInterface& reader) return std::errc::protocol_not_supported; } - if constexpr (std::is_same_v) { + if constexpr (std::is_same_v) { return LogEventDeserializer{reader}; } - if constexpr (std::is_same_v) { + if constexpr (std::is_same_v) { // Get reference timestamp auto ref_timestamp_iter = metadata_json.find(ffi::ir_stream::cProtocol::Metadata::ReferenceTimestampKey); @@ -54,7 +55,7 @@ auto LogEventDeserializer::create(ReaderInterface& reader) return std::errc::protocol_error; } auto ref_timestamp_str = ref_timestamp_iter->get_ref(); - ffi::epoch_time_ms_t ref_timestamp{}; + epoch_time_ms_t ref_timestamp{}; if (false == convert_string_to_int(ref_timestamp_str, ref_timestamp)) { return std::errc::protocol_error; } @@ -66,7 +67,7 @@ auto LogEventDeserializer::create(ReaderInterface& reader) template auto LogEventDeserializer::deserialize_log_event() -> BOOST_OUTCOME_V2_NAMESPACE::std_result> { - ffi::epoch_time_ms_t timestamp_or_timestamp_delta{}; + epoch_time_ms_t timestamp_or_timestamp_delta{}; std::string logtype; std::vector dict_vars; std::vector encoded_vars; @@ -90,10 +91,10 @@ auto LogEventDeserializer::deserialize_log_event() } } - ffi::epoch_time_ms_t timestamp{}; - if constexpr (std::is_same_v) { + epoch_time_ms_t timestamp{}; + if constexpr (std::is_same_v) { timestamp = timestamp_or_timestamp_delta; - } else { // std::is_same_v + } else { // std::is_same_v m_prev_msg_timestamp += timestamp_or_timestamp_delta; timestamp = m_prev_msg_timestamp; } @@ -103,16 +104,12 @@ auto LogEventDeserializer::deserialize_log_event() // Explicitly declare template specializations so that we can define the template methods in this // file -template auto -LogEventDeserializer::create(ReaderInterface& reader) - -> BOOST_OUTCOME_V2_NAMESPACE::std_result< - LogEventDeserializer>; -template auto -LogEventDeserializer::create(ReaderInterface& reader) - -> BOOST_OUTCOME_V2_NAMESPACE::std_result< - LogEventDeserializer>; -template auto LogEventDeserializer::deserialize_log_event() - -> BOOST_OUTCOME_V2_NAMESPACE::std_result>; -template auto LogEventDeserializer::deserialize_log_event() - -> BOOST_OUTCOME_V2_NAMESPACE::std_result>; +template auto LogEventDeserializer::create(ReaderInterface& reader +) -> BOOST_OUTCOME_V2_NAMESPACE::std_result>; +template auto LogEventDeserializer::create(ReaderInterface& reader +) -> BOOST_OUTCOME_V2_NAMESPACE::std_result>; +template auto LogEventDeserializer::deserialize_log_event() + -> BOOST_OUTCOME_V2_NAMESPACE::std_result>; +template auto LogEventDeserializer::deserialize_log_event() + -> BOOST_OUTCOME_V2_NAMESPACE::std_result>; } // namespace ir diff --git a/components/core/src/ir/LogEventDeserializer.hpp b/components/core/src/ir/LogEventDeserializer.hpp index 410ff6458..e2d3d5dab 100644 --- a/components/core/src/ir/LogEventDeserializer.hpp +++ b/components/core/src/ir/LogEventDeserializer.hpp @@ -5,11 +5,12 @@ #include -#include "../ffi/encoding_methods.hpp" #include "../ReaderInterface.hpp" #include "../TimestampPattern.hpp" #include "../TraceableException.hpp" +#include "../type_utils.hpp" #include "LogEvent.hpp" +#include "types.hpp" namespace ir { /** @@ -64,15 +65,15 @@ class LogEventDeserializer { // Constructors explicit LogEventDeserializer(ReaderInterface& reader) : m_reader{reader} {} - LogEventDeserializer(ReaderInterface& reader, ffi::epoch_time_ms_t ref_timestamp) + LogEventDeserializer(ReaderInterface& reader, epoch_time_ms_t ref_timestamp) : m_reader{reader}, m_prev_msg_timestamp{ref_timestamp} {} // Variables TimestampPattern m_timestamp_pattern{0, "%Y-%m-%dT%H:%M:%S.%3"}; [[no_unique_address]] std::conditional_t< - std::is_same_v, - ffi::epoch_time_ms_t, + std::is_same_v, + epoch_time_ms_t, EmptyType> m_prev_msg_timestamp{}; ReaderInterface& m_reader; diff --git a/components/core/src/ir/parsing.cpp b/components/core/src/ir/parsing.cpp index 30d3bc0a8..50c738f5c 100644 --- a/components/core/src/ir/parsing.cpp +++ b/components/core/src/ir/parsing.cpp @@ -2,6 +2,7 @@ #include "../string_utils.hpp" #include "../type_utils.hpp" +#include "types.hpp" using std::string; using std::string_view; @@ -94,7 +95,7 @@ void escape_and_append_const_to_logtype(string_view constant, string& logtype) { [[maybe_unused]] size_t char_to_escape_pos, string& logtype ) -> void { - logtype += enum_to_underlying_type(ir::VariablePlaceholder::Escape); + logtype += enum_to_underlying_type(VariablePlaceholder::Escape); }; // clang-format on append_constant_to_logtype(constant, escape_handler, logtype); diff --git a/components/core/src/ir/parsing.hpp b/components/core/src/ir/parsing.hpp index 972023881..0fbea3e92 100644 --- a/components/core/src/ir/parsing.hpp +++ b/components/core/src/ir/parsing.hpp @@ -13,13 +13,6 @@ #include namespace ir { -enum class VariablePlaceholder : char { - Integer = 0x11, - Dictionary = 0x12, - Float = 0x13, - Escape = '\\', -}; - /** * Checks if the given character is a delimiter * We treat everything *except* the following quoted characters as a delimiter: "+-.0-9A-Z\_a-z" diff --git a/components/core/src/ir/parsing.inc b/components/core/src/ir/parsing.inc index efab9d397..af40f89eb 100644 --- a/components/core/src/ir/parsing.inc +++ b/components/core/src/ir/parsing.inc @@ -5,6 +5,7 @@ #include #include "../type_utils.hpp" +#include "types.hpp" namespace ir { template diff --git a/components/core/src/ir/types.hpp b/components/core/src/ir/types.hpp new file mode 100644 index 000000000..2d37dabf3 --- /dev/null +++ b/components/core/src/ir/types.hpp @@ -0,0 +1,19 @@ +#ifndef IR_TYPES_HPP +#define IR_TYPES_HPP + +#include + +namespace ir { +using epoch_time_ms_t = int64_t; +using eight_byte_encoded_variable_t = int64_t; +using four_byte_encoded_variable_t = int32_t; + +enum class VariablePlaceholder : char { + Integer = 0x11, + Dictionary = 0x12, + Float = 0x13, + Escape = '\\', +}; +} // namespace ir + +#endif // IR_TYPES_HPP diff --git a/components/core/src/streaming_archive/writer/Archive.cpp b/components/core/src/streaming_archive/writer/Archive.cpp index e0f3738f1..996b0b219 100644 --- a/components/core/src/streaming_archive/writer/Archive.cpp +++ b/components/core/src/streaming_archive/writer/Archive.cpp @@ -16,10 +16,13 @@ #include "../../clp/utils.hpp" #include "../../EncodedVariableInterpreter.hpp" +#include "../../ir/types.hpp" #include "../../spdlog_with_specializations.hpp" #include "../../Utils.hpp" #include "../Constants.hpp" +using ir::eight_byte_encoded_variable_t; +using ir::four_byte_encoded_variable_t; using log_surgeon::LogEventView; using std::list; using std::make_unique; @@ -440,7 +443,7 @@ void Archive::write_msg_using_schema(LogEventView const& log_view) { template void Archive::write_log_event_ir(ir::LogEvent const& log_event) { - vector encoded_vars; + vector encoded_vars; vector var_ids; size_t original_num_bytes{0}; EncodedVariableInterpreter::encode_and_add_to_dictionary( @@ -650,10 +653,10 @@ void Archive::update_metadata() { // Explicitly declare template specializations so that we can define the template methods in this // file -template void Archive::write_log_event_ir( - ir::LogEvent const& log_event +template void Archive::write_log_event_ir( + ir::LogEvent const& log_event ); -template void Archive::write_log_event_ir( - ir::LogEvent const& log_event +template void Archive::write_log_event_ir( + ir::LogEvent const& log_event ); } // namespace streaming_archive::writer diff --git a/components/core/src/utils/make_dictionaries_readable/make-dictionaries-readable.cpp b/components/core/src/utils/make_dictionaries_readable/make-dictionaries-readable.cpp index 2057812e1..c0cccd658 100644 --- a/components/core/src/utils/make_dictionaries_readable/make-dictionaries-readable.cpp +++ b/components/core/src/utils/make_dictionaries_readable/make-dictionaries-readable.cpp @@ -5,7 +5,7 @@ #include #include "../../FileWriter.hpp" -#include "../../ir/parsing.hpp" +#include "../../ir/types.hpp" #include "../../LogTypeDictionaryReader.hpp" #include "../../spdlog_with_specializations.hpp" #include "../../streaming_archive/Constants.hpp" @@ -13,6 +13,7 @@ #include "../../VariableDictionaryReader.hpp" #include "CommandLineArguments.hpp" +using ir::VariablePlaceholder; using std::string; int main(int argc, char const* argv[]) { @@ -74,7 +75,7 @@ int main(int argc, char const* argv[]) { for (size_t placeholder_ix = 0; placeholder_ix < entry.get_num_placeholders(); ++placeholder_ix) { - ir::VariablePlaceholder var_placeholder; + VariablePlaceholder var_placeholder; size_t const placeholder_pos = entry.get_placeholder_info(placeholder_ix, var_placeholder); @@ -83,16 +84,16 @@ int main(int argc, char const* argv[]) { .append(value, constant_begin_pos, placeholder_pos - constant_begin_pos); switch (var_placeholder) { - case ir::VariablePlaceholder::Integer: + case VariablePlaceholder::Integer: human_readable_value += "\\i"; break; - case ir::VariablePlaceholder::Float: + case VariablePlaceholder::Float: human_readable_value += "\\f"; break; - case ir::VariablePlaceholder::Dictionary: + case VariablePlaceholder::Dictionary: human_readable_value += "\\d"; break; - case ir::VariablePlaceholder::Escape: + case VariablePlaceholder::Escape: break; default: SPDLOG_ERROR( diff --git a/components/core/tests/test-EncodedVariableInterpreter.cpp b/components/core/tests/test-EncodedVariableInterpreter.cpp index cf09a08c6..e3f5c61fd 100644 --- a/components/core/tests/test-EncodedVariableInterpreter.cpp +++ b/components/core/tests/test-EncodedVariableInterpreter.cpp @@ -3,9 +3,10 @@ #include #include "../src/EncodedVariableInterpreter.hpp" -#include "../src/ir/parsing.hpp" +#include "../src/ir/types.hpp" #include "../src/streaming_archive/Constants.hpp" +using ir::VariablePlaceholder; using std::string; using std::to_string; using std::vector; @@ -395,13 +396,13 @@ TEST_CASE("EncodedVariableInterpreter", "[EncodedVariableInterpreter]") { + " and a weird double " + var_strs[3] + " and a str with numbers " + var_strs[4] + " and an escape " - + enum_to_underlying_type(ir::VariablePlaceholder::Escape) + + enum_to_underlying_type(VariablePlaceholder::Escape) + " and an int placeholder " - + enum_to_underlying_type(ir::VariablePlaceholder::Integer) + + enum_to_underlying_type(VariablePlaceholder::Integer) + " and a float placeholder " - + enum_to_underlying_type(ir::VariablePlaceholder::Float) + + enum_to_underlying_type(VariablePlaceholder::Float) + " and a dictionary placeholder " - + enum_to_underlying_type(ir::VariablePlaceholder::Dictionary); + + enum_to_underlying_type(VariablePlaceholder::Dictionary); // clang-format on LogTypeDictionaryEntry logtype_dict_entry; @@ -416,12 +417,12 @@ TEST_CASE("EncodedVariableInterpreter", "[EncodedVariableInterpreter]") { // Test var_ids is correctly populated size_t encoded_var_id_ix = 0; - ir::VariablePlaceholder var_placeholder; + VariablePlaceholder var_placeholder; for (auto placeholder_ix = 0; placeholder_ix < logtype_dict_entry.get_num_placeholders(); placeholder_ix++) { std::ignore = logtype_dict_entry.get_placeholder_info(placeholder_ix, var_placeholder); - if (ir::VariablePlaceholder::Dictionary == var_placeholder) { + if (VariablePlaceholder::Dictionary == var_placeholder) { auto var = encoded_vars[placeholder_ix]; REQUIRE(var_ids.size() > encoded_var_id_ix); REQUIRE(EncodedVariableInterpreter::decode_var_dict_id(var) @@ -479,17 +480,17 @@ TEST_CASE("EncodedVariableInterpreter", "[EncodedVariableInterpreter]") { sub_query )); search_logtype += " and an escape "; - search_logtype += enum_to_underlying_type(ir::VariablePlaceholder::Escape); - search_logtype += enum_to_underlying_type(ir::VariablePlaceholder::Escape); + search_logtype += enum_to_underlying_type(VariablePlaceholder::Escape); + search_logtype += enum_to_underlying_type(VariablePlaceholder::Escape); search_logtype += " and an int placeholder "; - search_logtype += enum_to_underlying_type(ir::VariablePlaceholder::Escape); - search_logtype += enum_to_underlying_type(ir::VariablePlaceholder::Integer); + search_logtype += enum_to_underlying_type(VariablePlaceholder::Escape); + search_logtype += enum_to_underlying_type(VariablePlaceholder::Integer); search_logtype += " and a float placeholder "; - search_logtype += enum_to_underlying_type(ir::VariablePlaceholder::Escape); - search_logtype += enum_to_underlying_type(ir::VariablePlaceholder::Float); + search_logtype += enum_to_underlying_type(VariablePlaceholder::Escape); + search_logtype += enum_to_underlying_type(VariablePlaceholder::Float); search_logtype += " and a dictionary placeholder "; - search_logtype += enum_to_underlying_type(ir::VariablePlaceholder::Escape); - search_logtype += enum_to_underlying_type(ir::VariablePlaceholder::Dictionary); + search_logtype += enum_to_underlying_type(VariablePlaceholder::Escape); + search_logtype += enum_to_underlying_type(VariablePlaceholder::Dictionary); auto& vars = sub_query.get_vars(); REQUIRE(vars.size() == encoded_vars.size()); for (size_t i = 0; i < vars.size(); ++i) { diff --git a/components/core/tests/test-encoding_methods.cpp b/components/core/tests/test-encoding_methods.cpp index d86fede6e..0e430efae 100644 --- a/components/core/tests/test-encoding_methods.cpp +++ b/components/core/tests/test-encoding_methods.cpp @@ -1,17 +1,18 @@ #include #include "../src/ffi/encoding_methods.hpp" +#include "../src/ir/types.hpp" using ffi::decode_float_var; using ffi::decode_integer_var; using ffi::decode_message; -using ffi::eight_byte_encoded_variable_t; using ffi::encode_float_string; using ffi::encode_integer_string; using ffi::encode_message; -using ffi::four_byte_encoded_variable_t; using ffi::wildcard_match_encoded_vars; using ffi::wildcard_query_matches_any_encoded_var; +using ir::eight_byte_encoded_variable_t; +using ir::four_byte_encoded_variable_t; using ir::get_bounds_of_next_var; using ir::VariablePlaceholder; using std::string; diff --git a/components/core/tests/test-ir_encoding_methods.cpp b/components/core/tests/test-ir_encoding_methods.cpp index 4bef71af7..8d890a8f9 100644 --- a/components/core/tests/test-ir_encoding_methods.cpp +++ b/components/core/tests/test-ir_encoding_methods.cpp @@ -6,17 +6,14 @@ #include "../src/ffi/ir_stream/decoding_methods.hpp" #include "../src/ffi/ir_stream/encoding_methods.hpp" #include "../src/ffi/ir_stream/protocol_constants.hpp" -#include "../src/ir/parsing.hpp" +#include "../src/ir/types.hpp" using ffi::decode_float_var; using ffi::decode_integer_var; using ffi::decode_message; -using ffi::eight_byte_encoded_variable_t; using ffi::encode_float_string; using ffi::encode_integer_string; using ffi::encode_message; -using ffi::epoch_time_ms_t; -using ffi::four_byte_encoded_variable_t; using ffi::ir_stream::cProtocol::EightByteEncodingMagicNumber; using ffi::ir_stream::cProtocol::FourByteEncodingMagicNumber; using ffi::ir_stream::cProtocol::MagicNumberLength; @@ -26,6 +23,9 @@ using ffi::ir_stream::get_encoding_type; using ffi::ir_stream::IRErrorCode; using ffi::ir_stream::validate_protocol_version; using ffi::wildcard_query_matches_any_encoded_var; +using ir::eight_byte_encoded_variable_t; +using ir::epoch_time_ms_t; +using ir::four_byte_encoded_variable_t; using ir::VariablePlaceholder; using std::chrono::duration_cast; using std::chrono::milliseconds; @@ -506,21 +506,21 @@ TEST_CASE("decode_next_message_four_byte_timestamp_delta", "[ffi][deserialize_lo string const message = "Static <\text>, dictVar1, 123, 456345232.7234223, " "dictVar2, 987, 654.3, end of static text"; auto ts_delta = GENERATE( - static_cast(0), - static_cast(INT8_MIN), - static_cast(INT8_MIN + 1), - static_cast(INT8_MAX - 1), - static_cast(INT8_MAX), - static_cast(INT16_MIN), - static_cast(INT16_MIN + 1), - static_cast(INT16_MAX - 1), - static_cast(INT16_MAX), - static_cast(INT32_MIN), - static_cast(INT32_MIN + 1), - static_cast(INT32_MAX - 1), - static_cast(INT32_MAX), - static_cast(INT64_MIN), - static_cast(INT64_MAX) + static_cast(0), + static_cast(INT8_MIN), + static_cast(INT8_MIN + 1), + static_cast(INT8_MAX - 1), + static_cast(INT8_MAX), + static_cast(INT16_MIN), + static_cast(INT16_MIN + 1), + static_cast(INT16_MAX - 1), + static_cast(INT16_MAX), + static_cast(INT32_MIN), + static_cast(INT32_MIN + 1), + static_cast(INT32_MAX - 1), + static_cast(INT32_MAX), + static_cast(INT64_MIN), + static_cast(INT64_MAX) ); vector ir_buf; string logtype; diff --git a/components/core/tests/test-ir_parsing.cpp b/components/core/tests/test-ir_parsing.cpp index 9a8286d22..bb1667aa8 100644 --- a/components/core/tests/test-ir_parsing.cpp +++ b/components/core/tests/test-ir_parsing.cpp @@ -1,6 +1,7 @@ #include #include "../src/ir/parsing.hpp" +#include "../src/ir/types.hpp" #include "../src/type_utils.hpp" using ir::get_bounds_of_next_var; diff --git a/components/core/tests/test-query_methods.cpp b/components/core/tests/test-query_methods.cpp index 245117bff..a35a8b727 100644 --- a/components/core/tests/test-query_methods.cpp +++ b/components/core/tests/test-query_methods.cpp @@ -7,15 +7,15 @@ #include "../src/ffi/search/query_methods.hpp" #include "../src/ffi/search/QueryMethodFailed.hpp" #include "../src/ffi/search/WildcardToken.hpp" -#include "../src/ir/parsing.hpp" +#include "../src/ir/types.hpp" -using ffi::eight_byte_encoded_variable_t; -using ffi::four_byte_encoded_variable_t; using ffi::search::ExactVariableToken; using ffi::search::generate_subqueries; using ffi::search::Subquery; using ffi::search::TokenType; using ffi::search::WildcardToken; +using ir::eight_byte_encoded_variable_t; +using ir::four_byte_encoded_variable_t; using ir::VariablePlaceholder; using std::string; using std::variant; @@ -168,13 +168,13 @@ TEMPLATE_TEST_CASE( message += " and a string with numbers " + var_strs[var_ix++]; message += " and another string with numbers " + var_strs[var_ix++]; message += " and an escape "; - message += enum_to_underlying_type(ir::VariablePlaceholder::Escape); + message += enum_to_underlying_type(VariablePlaceholder::Escape); message += " and an int placeholder "; - message += enum_to_underlying_type(ir::VariablePlaceholder::Integer); + message += enum_to_underlying_type(VariablePlaceholder::Integer); message += " and a float placeholder "; - message += enum_to_underlying_type(ir::VariablePlaceholder::Float); + message += enum_to_underlying_type(VariablePlaceholder::Float); message += " and a dictionary placeholder "; - message += enum_to_underlying_type(ir::VariablePlaceholder::Dictionary); + message += enum_to_underlying_type(VariablePlaceholder::Dictionary); REQUIRE(ffi::encode_message(message, logtype, encoded_vars, dictionary_var_bounds)); wildcard_query = message;