From 8ebf5dedb39af7fd7b9b85066586e1f4c8f518ae Mon Sep 17 00:00:00 2001 From: LittleStar <59785146+LinZhihao-723@users.noreply.github.com> Date: Mon, 18 Sep 2023 14:41:48 -0400 Subject: [PATCH] Fix var tracking --- .../core/src/EncodedVariableInterpreter.cpp | 21 +++++---- .../core/src/LogTypeDictionaryEntry.cpp | 46 +++++-------------- .../core/src/LogTypeDictionaryEntry.hpp | 31 +++++++------ .../src/streaming_archive/reader/File.cpp | 6 +-- .../make-dictionaries-readable.cpp | 10 ++-- .../tests/test-EncodedVariableInterpreter.cpp | 6 +-- 6 files changed, 53 insertions(+), 67 deletions(-) diff --git a/components/core/src/EncodedVariableInterpreter.cpp b/components/core/src/EncodedVariableInterpreter.cpp index 62fa78e34..40326d994 100644 --- a/components/core/src/EncodedVariableInterpreter.cpp +++ b/components/core/src/EncodedVariableInterpreter.cpp @@ -287,13 +287,14 @@ void EncodedVariableInterpreter::encode_and_add_to_dictionary( bool EncodedVariableInterpreter::decode_variables_into_message (const LogTypeDictionaryEntry& logtype_dict_entry, const VariableDictionaryReader& var_dict, const vector& encoded_vars, string& decompressed_msg) { - size_t num_vars_in_logtype = logtype_dict_entry.get_num_vars(); + size_t const num_placeholders_in_logtype = logtype_dict_entry.get_num_placeholders(); + size_t const num_vars = logtype_dict_entry.get_num_variables(); // Ensure the number of variables in the logtype matches the number of encoded variables given const auto& logtype_value = logtype_dict_entry.get_value(); - if (num_vars_in_logtype != encoded_vars.size()) { + if (num_vars != encoded_vars.size()) { SPDLOG_ERROR("EncodedVariableInterpreter: Logtype '{}' contains {} variables, but {} were given for decoding.", logtype_value.c_str(), - num_vars_in_logtype, encoded_vars.size()); + num_vars, encoded_vars.size()); return false; } @@ -301,22 +302,22 @@ bool EncodedVariableInterpreter::decode_variables_into_message (const LogTypeDic size_t constant_begin_pos = 0; string float_str; variable_dictionary_id_t var_dict_id; - for (size_t i = 0; i < num_vars_in_logtype; ++i) { - size_t var_position = logtype_dict_entry.get_var_info(i, var_placeholder); + for (size_t placeholder_ix = 0, var_ix = 0; placeholder_ix < num_placeholders_in_logtype; ++placeholder_ix) { + size_t placeholder_position = logtype_dict_entry.get_placeholder_info(placeholder_ix, var_placeholder); // Add the constant that's between the last variable and this one decompressed_msg.append(logtype_value, constant_begin_pos, - var_position - constant_begin_pos); + placeholder_position - constant_begin_pos); switch (var_placeholder) { case ir::VariablePlaceholder::Integer: - decompressed_msg += std::to_string(encoded_vars[i]); + decompressed_msg += std::to_string(encoded_vars[var_ix++]); break; case ir::VariablePlaceholder::Float: - convert_encoded_float_to_string(encoded_vars[i], float_str); + convert_encoded_float_to_string(encoded_vars[var_ix++], float_str); decompressed_msg += float_str; break; case ir::VariablePlaceholder::Dictionary: - var_dict_id = decode_var_dict_id(encoded_vars[i]); + var_dict_id = decode_var_dict_id(encoded_vars[var_ix++]); decompressed_msg += var_dict.get_value(var_dict_id); break; case ir::VariablePlaceholder::Escape: @@ -330,7 +331,7 @@ bool EncodedVariableInterpreter::decode_variables_into_message (const LogTypeDic return false; } // Move past the variable placeholder - constant_begin_pos = var_position + 1; + constant_begin_pos = placeholder_position + 1; } // Append remainder of logtype, if any if (constant_begin_pos < logtype_value.length()) { diff --git a/components/core/src/LogTypeDictionaryEntry.cpp b/components/core/src/LogTypeDictionaryEntry.cpp index 9166892eb..9c3b31314 100644 --- a/components/core/src/LogTypeDictionaryEntry.cpp +++ b/components/core/src/LogTypeDictionaryEntry.cpp @@ -7,23 +7,23 @@ using std::string; -size_t LogTypeDictionaryEntry::get_var_info( +size_t LogTypeDictionaryEntry::get_placeholder_info( size_t var_ix, ir::VariablePlaceholder& var_placeholder ) const { - if (var_ix >= m_var_positions.size()) { + if (var_ix >= m_placeholder_positions.size()) { return SIZE_MAX; } - auto var_position = m_var_positions[var_ix]; + auto var_position = m_placeholder_positions[var_ix]; var_placeholder = static_cast(m_value[var_position]); - return m_var_positions[var_ix]; + return m_placeholder_positions[var_ix]; } size_t LogTypeDictionaryEntry::get_data_size () const { // NOTE: sizeof(vector[0]) is executed at compile time so there's no risk of an exception at runtime - return sizeof(m_id) + m_value.length() + m_var_positions.size() * sizeof(m_var_positions[0]) + + return sizeof(m_id) + m_value.length() + m_placeholder_positions.size() * sizeof(m_placeholder_positions[0]) + m_ids_of_segments_containing_entry.size() * sizeof(segment_id_t); } @@ -32,23 +32,24 @@ void LogTypeDictionaryEntry::add_constant (const string& value_containing_consta } void LogTypeDictionaryEntry::add_dictionary_var () { - m_var_positions.push_back(m_value.length()); + m_placeholder_positions.push_back(m_value.length()); add_dict_var(m_value); } void LogTypeDictionaryEntry::add_int_var () { - m_var_positions.push_back(m_value.length()); + m_placeholder_positions.push_back(m_value.length()); add_int_var(m_value); } void LogTypeDictionaryEntry::add_float_var () { - m_var_positions.push_back(m_value.length()); + m_placeholder_positions.push_back(m_value.length()); add_float_var(m_value); } void LogTypeDictionaryEntry::add_escape_var() { - m_var_positions.push_back(m_value.length()); + m_placeholder_positions.push_back(m_value.length()); add_escape_var(m_value); + ++m_num_escape_placeholders; } bool LogTypeDictionaryEntry::parse_next_var (const string& msg, size_t& var_begin_pos, size_t& var_end_pos, string& var) { @@ -72,7 +73,8 @@ bool LogTypeDictionaryEntry::parse_next_var (const string& msg, size_t& var_begi void LogTypeDictionaryEntry::clear () { m_value.clear(); - m_var_positions.clear(); + m_placeholder_positions.clear(); + m_num_escape_placeholders = 0; } void LogTypeDictionaryEntry::write_to_file (streaming_compression::Compressor& compressor) const { @@ -149,27 +151,3 @@ void LogTypeDictionaryEntry::read_from_file (streaming_compression::Decompressor throw OperationFailed(error_code, __FILENAME__, __LINE__); } } - -void LogTypeDictionaryEntry::get_value_with_unfounded_variables_escaped (string& escaped_logtype_value) const { - auto value_view = static_cast(m_value); - size_t begin_ix = 0; - // Reset escaped value and reserve enough space to at least contain the whole value - escaped_logtype_value.clear(); - escaped_logtype_value.reserve(value_view.length()); - for (auto var_position : m_var_positions) { - size_t end_ix = var_position; - - ir::escape_and_append_constant_to_logtype( - value_view.substr(begin_ix, end_ix - begin_ix), - escaped_logtype_value - ); - - // Add variable placeholder - escaped_logtype_value += value_view[end_ix]; - - // Move begin to start of next portion of logtype between variables - begin_ix = end_ix + 1; - } - // Escape any variable placeholders in remainder of value - ir::escape_and_append_constant_to_logtype(value_view.substr(begin_ix), escaped_logtype_value); -} diff --git a/components/core/src/LogTypeDictionaryEntry.hpp b/components/core/src/LogTypeDictionaryEntry.hpp index 35bef1e60..25168f9d8 100644 --- a/components/core/src/LogTypeDictionaryEntry.hpp +++ b/components/core/src/LogTypeDictionaryEntry.hpp @@ -33,7 +33,7 @@ class LogTypeDictionaryEntry : public DictionaryEntry { }; // Constructors - LogTypeDictionaryEntry () = default; + LogTypeDictionaryEntry () : m_num_escape_placeholders{0} {}; // Use default copy constructor LogTypeDictionaryEntry (const LogTypeDictionaryEntry&) = default; @@ -71,14 +71,26 @@ class LogTypeDictionaryEntry : public DictionaryEntry { logtype += enum_to_underlying_type(ir::VariablePlaceholder::Escape); } - size_t get_num_vars () const { return m_var_positions.size(); } /** - * Gets all info about a variable in the logtype + * @return The total number of placeholders on record. The placeholders may + * represent an escape characters. + */ + size_t get_num_placeholders () const { return m_placeholder_positions.size(); } + + /** + * @return The total number of variables on record. + */ + size_t get_num_variables () const { + return m_placeholder_positions.size() - m_num_escape_placeholders; + } + + /** + * Gets all info about a placeholder in the logtype * @param var_ix The index of the variable to get the info for * @param var_placeholder * @return The variable's position in the logtype, or SIZE_MAX if var_ix is out of bounds */ - size_t get_var_info (size_t var_ix, ir::VariablePlaceholder& var_placeholder) const; + size_t get_placeholder_info (size_t var_ix, ir::VariablePlaceholder& var_placeholder) const; /** * Gets the size (in-memory) of the data contained in this entry @@ -148,16 +160,9 @@ class LogTypeDictionaryEntry : public DictionaryEntry { void read_from_file (streaming_compression::Decompressor& decompressor); private: - // Methods - /** - * Escapes any variable placeholders that don't correspond to the positions - * of variables in the logtype entry's value - * @param escaped_logtype_value - */ - void get_value_with_unfounded_variables_escaped (std::string& escaped_logtype_value) const; - // Variables - std::vector m_var_positions; + std::vector m_placeholder_positions; + size_t m_num_escape_placeholders; }; #endif // LOGTYPEDICTIONARYENTRY_HPP diff --git a/components/core/src/streaming_archive/reader/File.cpp b/components/core/src/streaming_archive/reader/File.cpp index 346585fbc..afd0f048c 100644 --- a/components/core/src/streaming_archive/reader/File.cpp +++ b/components/core/src/streaming_archive/reader/File.cpp @@ -212,7 +212,7 @@ namespace streaming_archive::reader { // Get number of variables in logtype const auto& logtype_dictionary_entry = m_archive_logtype_dict->get_entry(logtype_id); - auto num_vars = logtype_dictionary_entry.get_num_vars(); + auto const num_vars = logtype_dictionary_entry.get_num_variables(); auto timestamp = m_timestamps[m_msgs_ix]; if (search_begin_timestamp <= timestamp && timestamp <= search_end_timestamp) { @@ -253,7 +253,7 @@ namespace streaming_archive::reader { // Get number of variables in logtype const auto& logtype_dictionary_entry = m_archive_logtype_dict->get_entry(logtype_id); - auto num_vars = logtype_dictionary_entry.get_num_vars(); + auto const num_vars = logtype_dictionary_entry.get_num_variables(); for (auto sub_query : query.get_relevant_sub_queries()) { // Check if logtype matches search @@ -315,7 +315,7 @@ namespace streaming_archive::reader { // Get variables msg.clear_vars(); const auto& logtype_dictionary_entry = m_archive_logtype_dict->get_entry(logtype_id); - auto num_vars = logtype_dictionary_entry.get_num_vars(); + auto const num_vars = logtype_dictionary_entry.get_num_variables(); if (m_variables_ix + num_vars > m_num_variables) { return false; } diff --git a/components/core/src/utils/make_dictionaries_readable/make-dictionaries-readable.cpp b/components/core/src/utils/make_dictionaries_readable/make-dictionaries-readable.cpp index d1785038f..4d992de32 100644 --- a/components/core/src/utils/make_dictionaries_readable/make-dictionaries-readable.cpp +++ b/components/core/src/utils/make_dictionaries_readable/make-dictionaries-readable.cpp @@ -66,12 +66,12 @@ int main (int argc, const char* argv[]) { human_readable_value.clear(); size_t constant_begin_pos = 0; - for (size_t var_ix = 0; var_ix < entry.get_num_vars(); ++var_ix) { + for (size_t placeholder_ix = 0; placeholder_ix < entry.get_num_placeholders(); ++placeholder_ix) { ir::VariablePlaceholder var_placeholder; - size_t var_pos = entry.get_var_info(var_ix, var_placeholder); + size_t placeholder_pos = entry.get_placeholder_info(placeholder_ix, var_placeholder); // Add the constant that's between the last variable and this one, with newlines escaped - human_readable_value.append(value, constant_begin_pos, var_pos - constant_begin_pos); + human_readable_value.append(value, constant_begin_pos, placeholder_pos - constant_begin_pos); switch (var_placeholder) { case ir::VariablePlaceholder::Integer: @@ -83,13 +83,15 @@ int main (int argc, const char* argv[]) { case ir::VariablePlaceholder::Dictionary: human_readable_value += "\\d"; break; + case ir::VariablePlaceholder::Escape: + break; default: SPDLOG_ERROR("Logtype '{}' contains unexpected variable placeholder 0x{:x}", value, enum_to_underlying_type(var_placeholder)); return -1; } // Move past the variable placeholder - constant_begin_pos = var_pos + 1; + constant_begin_pos = placeholder_pos + 1; } // Append remainder of value, if any if (constant_begin_pos < value.length()) { diff --git a/components/core/tests/test-EncodedVariableInterpreter.cpp b/components/core/tests/test-EncodedVariableInterpreter.cpp index c95971204..97d9247d1 100644 --- a/components/core/tests/test-EncodedVariableInterpreter.cpp +++ b/components/core/tests/test-EncodedVariableInterpreter.cpp @@ -246,10 +246,10 @@ TEST_CASE("EncodedVariableInterpreter", "[EncodedVariableInterpreter]") { // Test var_ids is correctly populated size_t encoded_var_id_ix = 0; ir::VariablePlaceholder var_placeholder; - for (auto var_ix = 0; var_ix < logtype_dict_entry.get_num_vars(); var_ix++) { - std::ignore = logtype_dict_entry.get_var_info(var_ix, var_placeholder); + for (auto placeholder_ix = 0; placeholder_ix < logtype_dict_entry.get_num_placeholders(); placeholder_ix++) { + std::ignore = logtype_dict_entry.get_placeholder_info(placeholder_ix, var_placeholder); if (ir::VariablePlaceholder::Dictionary == var_placeholder) { - auto var = encoded_vars[var_ix]; + auto var = encoded_vars[placeholder_ix]; REQUIRE(var_ids.size() > encoded_var_id_ix); REQUIRE(EncodedVariableInterpreter::decode_var_dict_id(var) == var_ids[encoded_var_id_ix]);