Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update the CLP dependency to the lastest main branch commit #29

Merged
merged 6 commits into from
Nov 6, 2023
Merged

Update the CLP dependency to the lastest main branch commit #29

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
7eb41d6
Update the submodule to the latest CLP commit
LinZhihao-723 Nov 1, 2023
d8c97c8
Compile with the lastest dependency
LinZhihao-723 Nov 1, 2023
0183c2d
Implement the version check for the ir stream metadata
LinZhihao-723 Nov 1, 2023
c6ac2a7
Add missing break statement
LinZhihao-723 Nov 1, 2023
aa54661
Include .h files for building clp core files
LinZhihao-723 Nov 1, 2023
0fe8f00
Adapt code review comments
LinZhihao-723 Nov 6, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions MANIFEST.in
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
recursive-include src/clp/components/core/src *.h
recursive-include src/clp/components/core/src *.hpp
recursive-include src/clp/components/core/src *.tpp
recursive-include src/clp/components/core/src *.inc
LinZhihao-723 marked this conversation as resolved.
Show resolved Hide resolved
recursive-include src/clp_ffi_py *.hpp
recursive-include src/clp_ffi_py *.tpp
recursive-include clp_ffi_py *.py
Expand Down
3 changes: 3 additions & 0 deletions setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,12 @@
"src/clp/components/core/submodules",
],
sources=[
"src/clp/components/core/src/BufferReader.cpp",
"src/clp/components/core/src/ffi/ir_stream/decoding_methods.cpp",
"src/clp/components/core/src/ffi/ir_stream/encoding_methods.cpp",
"src/clp/components/core/src/ffi/encoding_methods.cpp",
"src/clp/components/core/src/ir/parsing.cpp",
"src/clp/components/core/src/ReaderInterface.cpp",
"src/clp/components/core/src/string_utils.cpp",
"src/clp/components/core/src/TraceableException.cpp",
"src/clp_ffi_py/ir/native/decoding_methods.cpp",
Expand Down
2 changes: 1 addition & 1 deletion src/clp
Open in desktop
Submodule clp updated 73 files
+3 −0 .gitmodules
+85 −26 components/core/CMakeLists.txt
+2 −0 components/core/cmake/utils.cmake
+100 −0 components/core/src/BufferReader.cpp
+111 −0 components/core/src/BufferReader.hpp
+371 −0 components/core/src/BufferedFileReader.cpp
+276 −0 components/core/src/BufferedFileReader.hpp
+135 −23 components/core/src/EncodedVariableInterpreter.cpp
+61 −0 components/core/src/EncodedVariableInterpreter.hpp
+4 −1 components/core/src/Grep.cpp
+40 −0 components/core/src/LibarchiveFileReader.cpp
+22 −0 components/core/src/LibarchiveFileReader.hpp
+11 −69 components/core/src/LibarchiveReader.cpp
+5 −18 components/core/src/LibarchiveReader.hpp
+21 −44 components/core/src/LogTypeDictionaryEntry.cpp
+14 −21 components/core/src/LogTypeDictionaryEntry.hpp
+7 −2 components/core/src/TraceableException.hpp
+0 −49 components/core/src/Utils.cpp
+0 −43 components/core/src/Utils.hpp
+224 −35 components/core/src/clp/FileCompressor.cpp
+52 −7 components/core/src/clp/FileCompressor.hpp
+1 −1 components/core/src/compressor_frontend/LALR1Parser.hpp
+0 −0 components/core/src/compressor_frontend/LALR1Parser.inc
+1 −1 components/core/src/compressor_frontend/Lexer.hpp
+0 −0 components/core/src/compressor_frontend/Lexer.inc
+1 −1 components/core/src/compressor_frontend/finite_automata/RegexAST.hpp
+0 −0 components/core/src/compressor_frontend/finite_automata/RegexAST.inc
+2 −2 components/core/src/compressor_frontend/finite_automata/RegexDFA.hpp
+0 −0 components/core/src/compressor_frontend/finite_automata/RegexDFA.inc
+2 −2 components/core/src/compressor_frontend/finite_automata/RegexNFA.hpp
+0 −0 components/core/src/compressor_frontend/finite_automata/RegexNFA.inc
+2 −2 components/core/src/compressor_frontend/finite_automata/UnicodeIntervalTree.hpp
+0 −0 components/core/src/compressor_frontend/finite_automata/UnicodeIntervalTree.inc
+18 −101 components/core/src/ffi/encoding_methods.cpp
+48 −72 components/core/src/ffi/encoding_methods.hpp
+39 −37 components/core/src/ffi/encoding_methods.inc
+230 −280 components/core/src/ffi/ir_stream/decoding_methods.cpp
+141 −92 components/core/src/ffi/ir_stream/decoding_methods.hpp
+142 −0 components/core/src/ffi/ir_stream/decoding_methods.inc
+9 −42 components/core/src/ffi/ir_stream/encoding_methods.cpp
+9 −1 components/core/src/ffi/ir_stream/protocol_constants.hpp
+3 −1 components/core/src/ffi/search/CompositeWildcardToken.cpp
+3 −2 components/core/src/ffi/search/query_methods.cpp
+52 −0 components/core/src/ir/LogEvent.hpp
+118 −0 components/core/src/ir/LogEventDeserializer.cpp
+85 −0 components/core/src/ir/LogEventDeserializer.hpp
+104 −0 components/core/src/ir/parsing.cpp
+91 −0 components/core/src/ir/parsing.hpp
+13 −0 components/core/src/ir/utils.cpp
+15 −0 components/core/src/ir/utils.hpp
+21 −0 components/core/src/math_utils.hpp
+1 −1 components/core/src/streaming_archive/Constants.hpp
+1 −0 components/core/src/streaming_archive/MetadataDB.cpp
+4 −0 components/core/src/streaming_archive/reader/Archive.cpp
+52 −16 components/core/src/streaming_archive/writer/Archive.cpp
+18 −2 components/core/src/streaming_archive/writer/Archive.hpp
+66 −73 components/core/src/string_utils.cpp
+27 −25 components/core/src/string_utils.hpp
+3 −4 components/core/src/string_utils.inc
+6 −1 components/core/src/type_utils.hpp
+3 −3 components/core/src/utils/make_dictionaries_readable/README.md
+9 −8 components/core/src/utils/make_dictionaries_readable/make-dictionaries-readable.cpp
+1 −0 components/core/submodules/boost-outcome
+292 −0 components/core/tests/test-BufferedFileReader.cpp
+3 −3 components/core/tests/test-EncodedVariableInterpreter.cpp
+0 −82 components/core/tests/test-Utils.cpp
+2 −119 components/core/tests/test-encoding_methods.cpp
+330 −181 components/core/tests/test-ir_encoding_methods.cpp
+101 −0 components/core/tests/test-ir_parsing.cpp
+22 −0 components/core/tests/test-math_utils.cpp
+2 −1 components/core/tests/test-query_methods.cpp
+10 −0 components/core/tools/scripts/deps-download/boost-outcome.json
+1 −0 components/core/tools/scripts/deps-download/download-all.sh
51 changes: 40 additions & 11 deletions src/clp_ffi_py/ir/native/decoding_methods.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,9 @@

#include "decoding_methods.hpp"

#include <clp/components/core/src/BufferReader.hpp>
#include <clp/components/core/src/ffi/ir_stream/decoding_methods.hpp>
#include <clp/components/core/src/ffi/ir_stream/protocol_constants.hpp>
#include <clp/components/core/src/type_utils.hpp>
#include <gsl/span>
#include <json/single_include/nlohmann/json.hpp>
Expand Down Expand Up @@ -46,7 +48,9 @@ auto decode(
bool reached_eof{false};
while (true) {
auto const unconsumed_bytes{decoder_buffer->get_unconsumed_bytes()};
ffi::ir_stream::IrBuffer ir_buffer{unconsumed_bytes.data(), unconsumed_bytes.size()};
BufferReader ir_buffer{
size_checked_pointer_cast<char const>(unconsumed_bytes.data()),
unconsumed_bytes.size()};
auto const err{ffi::ir_stream::four_byte_encoding::decode_next_message(
ir_buffer,
decoded_message,
Expand Down Expand Up @@ -77,8 +81,7 @@ auto decode(

timestamp += timestamp_delta;
current_log_event_idx = decoder_buffer->get_and_increment_decoded_message_count();
decoder_buffer->commit_read_buffer_consumption(
static_cast<Py_ssize_t>(ir_buffer.get_cursor_pos())
decoder_buffer->commit_read_buffer_consumption(static_cast<Py_ssize_t>(ir_buffer.get_pos())
);

if (nullptr == py_query) {
Expand Down Expand Up @@ -124,10 +127,12 @@ auto decode_preamble(PyObject* Py_UNUSED(self), PyObject* py_decoder_buffer) ->
size_t ir_buffer_cursor_pos{0};
while (true) {
auto const unconsumed_bytes{decoder_buffer->get_unconsumed_bytes()};
ffi::ir_stream::IrBuffer ir_buffer{unconsumed_bytes.data(), unconsumed_bytes.size()};
BufferReader ir_buffer{
size_checked_pointer_cast<char const>(unconsumed_bytes.data()),
unconsumed_bytes.size()};
auto const err{ffi::ir_stream::get_encoding_type(ir_buffer, is_four_byte_encoding)};
if (ffi::ir_stream::IRErrorCode_Success == err) {
ir_buffer_cursor_pos = ir_buffer.get_cursor_pos();
ir_buffer_cursor_pos = ir_buffer.get_pos();
break;
}
if (ffi::ir_stream::IRErrorCode_Incomplete_IR != err) {
Expand All @@ -149,15 +154,17 @@ auto decode_preamble(PyObject* Py_UNUSED(self), PyObject* py_decoder_buffer) ->
uint16_t metadata_size{0};
while (true) {
auto const unconsumed_bytes = decoder_buffer->get_unconsumed_bytes();
ffi::ir_stream::IrBuffer ir_buffer{unconsumed_bytes.data(), unconsumed_bytes.size()};
BufferReader ir_buffer{
size_checked_pointer_cast<char const>(unconsumed_bytes.data()),
unconsumed_bytes.size()};
auto const err{ffi::ir_stream::decode_preamble(
ir_buffer,
metadata_type_tag,
metadata_pos,
metadata_size
)};
if (ffi::ir_stream::IRErrorCode_Success == err) {
ir_buffer_cursor_pos = ir_buffer.get_cursor_pos();
ir_buffer_cursor_pos = ir_buffer.get_pos();
break;
}
if (ffi::ir_stream::IRErrorCode_Incomplete_IR != err) {
Expand All @@ -171,8 +178,7 @@ auto decode_preamble(PyObject* Py_UNUSED(self), PyObject* py_decoder_buffer) ->

auto const unconsumed_bytes = decoder_buffer->get_unconsumed_bytes();
auto const metadata_buffer{
unconsumed_bytes.subspan(metadata_pos, static_cast<size_t>(metadata_size))
};
unconsumed_bytes.subspan(metadata_pos, static_cast<size_t>(metadata_size))};
decoder_buffer->commit_read_buffer_consumption(static_cast<Py_ssize_t>(ir_buffer_cursor_pos));
PyMetadata* metadata{nullptr};
try {
Expand All @@ -181,6 +187,30 @@ auto decode_preamble(PyObject* Py_UNUSED(self), PyObject* py_decoder_buffer) ->
nlohmann::json const metadata_json(
nlohmann::json::parse(metadata_buffer.begin(), metadata_buffer.end())
);
std::string const version{
metadata_json.at(ffi::ir_stream::cProtocol::Metadata::VersionKey)};
auto const error_code{ffi::ir_stream::validate_protocol_version(version)};
if (ffi::ir_stream::IRProtocolErrorCode_Supported != error_code) {
switch (error_code) {
case ffi::ir_stream::IRProtocolErrorCode_Invalid:
PyErr_Format(PyExc_RuntimeError, "Invalid version number: %s", version.c_str());
break;
case ffi::ir_stream::IRProtocolErrorCode_Too_New:
PyErr_Format(PyExc_RuntimeError, "Version too new: %s", version.c_str());
break;
case ffi::ir_stream::IRProtocolErrorCode_Too_Old:
PyErr_Format(PyExc_RuntimeError, "Version too old: %s", version.c_str());
break;
default:
PyErr_Format(
PyExc_NotImplementedError,
"Unrecognized return code %d with version value %s",
LinZhihao-723 marked this conversation as resolved.
Show resolved Hide resolved
error_code,
version.c_str()
);
LinZhihao-723 marked this conversation as resolved.
Show resolved Hide resolved
}
return nullptr;
}
metadata = PyMetadata::create_new_from_json(metadata_json, is_four_byte_encoding);
} catch (nlohmann::json::exception& ex) {
PyErr_Format(PyExc_RuntimeError, "Json Parsing Error: %s", ex.what());
Expand All @@ -201,8 +231,7 @@ auto decode_next_log_event(PyObject* Py_UNUSED(self), PyObject* args, PyObject*
static_cast<char*>(keyword_decoder_buffer),
static_cast<char*>(keyword_query),
static_cast<char*>(keyword_allow_incomplete_stream),
nullptr
};
nullptr};

PyDecoderBuffer* decoder_buffer{nullptr};
PyObject* query{Py_None};
Expand Down