From a8d57ca1968f41a0d4e121e1dbcf653f01249f8f Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Thu, 4 Jul 2024 18:16:04 -0400 Subject: [PATCH] Update src with the latest clp --- setup.py | 7 +- src/clp_ffi_py/ExceptionFFI.hpp | 2 +- src/clp_ffi_py/Py_utils.hpp | 2 +- src/clp_ffi_py/ir/native/LogEvent.hpp | 2 +- src/clp_ffi_py/ir/native/Metadata.cpp | 2 +- src/clp_ffi_py/ir/native/Metadata.hpp | 4 +- src/clp_ffi_py/ir/native/PyDecoderBuffer.hpp | 2 +- src/clp_ffi_py/ir/native/PyQuery.cpp | 2 +- src/clp_ffi_py/ir/native/Query.cpp | 2 +- src/clp_ffi_py/ir/native/Query.hpp | 4 +- src/clp_ffi_py/ir/native/decoding_methods.cpp | 104 +++++++++++++----- src/clp_ffi_py/ir/native/encoding_methods.cpp | 8 +- src/clp_ffi_py/utils.inc | 2 +- 13 files changed, 98 insertions(+), 45 deletions(-) diff --git a/setup.py b/setup.py index 0c6cc036..01c41540 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,8 @@ from typing import List, Optional, Tuple cpp_src_root: str = "src" -clp_src_root: str = f"{cpp_src_root}/clp/components/core/src/clp" +clp_core_src_root: str = f"{cpp_src_root}/clp/components/core/src" +clp_src_root: str = f"{clp_core_src_root}/clp" clp_submodule_root: str = f"{cpp_src_root}/clp/components/core/submodules" clp_ffi_py_src_root: str = f"{cpp_src_root}/clp_ffi_py" @@ -15,14 +16,16 @@ name="clp_ffi_py.ir.native", language="c++", include_dirs=[ - cpp_src_root, + clp_core_src_root, clp_src_root, clp_submodule_root, + cpp_src_root, ], sources=[ f"{clp_src_root}/BufferReader.cpp", f"{clp_src_root}/ffi/ir_stream/decoding_methods.cpp", f"{clp_src_root}/ffi/ir_stream/encoding_methods.cpp", + f"{clp_src_root}/ffi/ir_stream/utils.cpp", f"{clp_src_root}/ffi/encoding_methods.cpp", f"{clp_src_root}/ir/parsing.cpp", f"{clp_src_root}/ReaderInterface.cpp", diff --git a/src/clp_ffi_py/ExceptionFFI.hpp b/src/clp_ffi_py/ExceptionFFI.hpp index 472ffed1..03a6a642 100644 --- a/src/clp_ffi_py/ExceptionFFI.hpp +++ b/src/clp_ffi_py/ExceptionFFI.hpp @@ -3,7 +3,7 @@ #include -#include +#include namespace clp_ffi_py { /** diff --git a/src/clp_ffi_py/Py_utils.hpp b/src/clp_ffi_py/Py_utils.hpp index f9e7dfa5..f9bb4a62 100644 --- a/src/clp_ffi_py/Py_utils.hpp +++ b/src/clp_ffi_py/Py_utils.hpp @@ -3,7 +3,7 @@ #include // Must always be included before any other header files -#include +#include namespace clp_ffi_py { /** diff --git a/src/clp_ffi_py/ir/native/LogEvent.hpp b/src/clp_ffi_py/ir/native/LogEvent.hpp index 07132a65..326973b8 100644 --- a/src/clp_ffi_py/ir/native/LogEvent.hpp +++ b/src/clp_ffi_py/ir/native/LogEvent.hpp @@ -3,7 +3,7 @@ #include -#include +#include namespace clp_ffi_py::ir::native { /** diff --git a/src/clp_ffi_py/ir/native/Metadata.cpp b/src/clp_ffi_py/ir/native/Metadata.cpp index bf082546..f2a867ef 100644 --- a/src/clp_ffi_py/ir/native/Metadata.cpp +++ b/src/clp_ffi_py/ir/native/Metadata.cpp @@ -1,6 +1,6 @@ #include "Metadata.hpp" -#include +#include #include #include diff --git a/src/clp_ffi_py/ir/native/Metadata.hpp b/src/clp_ffi_py/ir/native/Metadata.hpp index b3419b20..0eecadaf 100644 --- a/src/clp_ffi_py/ir/native/Metadata.hpp +++ b/src/clp_ffi_py/ir/native/Metadata.hpp @@ -3,8 +3,8 @@ #include -#include -#include +#include +#include namespace clp_ffi_py::ir::native { /** diff --git a/src/clp_ffi_py/ir/native/PyDecoderBuffer.hpp b/src/clp_ffi_py/ir/native/PyDecoderBuffer.hpp index 5081cfa3..cb8b1e6f 100644 --- a/src/clp_ffi_py/ir/native/PyDecoderBuffer.hpp +++ b/src/clp_ffi_py/ir/native/PyDecoderBuffer.hpp @@ -5,7 +5,7 @@ #include -#include +#include #include #include diff --git a/src/clp_ffi_py/ir/native/PyQuery.cpp b/src/clp_ffi_py/ir/native/PyQuery.cpp index 733cf04b..04ebdd7f 100644 --- a/src/clp_ffi_py/ir/native/PyQuery.cpp +++ b/src/clp_ffi_py/ir/native/PyQuery.cpp @@ -3,7 +3,7 @@ #include "PyQuery.hpp" -#include +#include #include #include diff --git a/src/clp_ffi_py/ir/native/Query.cpp b/src/clp_ffi_py/ir/native/Query.cpp index 5585f9eb..6eef4692 100644 --- a/src/clp_ffi_py/ir/native/Query.cpp +++ b/src/clp_ffi_py/ir/native/Query.cpp @@ -3,7 +3,7 @@ #include #include -#include +#include namespace clp_ffi_py::ir::native { auto Query::matches_wildcard_queries(std::string_view log_message) const -> bool { diff --git a/src/clp_ffi_py/ir/native/Query.hpp b/src/clp_ffi_py/ir/native/Query.hpp index e10d57b1..24d62d8b 100644 --- a/src/clp_ffi_py/ir/native/Query.hpp +++ b/src/clp_ffi_py/ir/native/Query.hpp @@ -6,8 +6,8 @@ #include #include -#include -#include +#include +#include #include #include diff --git a/src/clp_ffi_py/ir/native/decoding_methods.cpp b/src/clp_ffi_py/ir/native/decoding_methods.cpp index 6e1b5ee9..fc990cca 100644 --- a/src/clp_ffi_py/ir/native/decoding_methods.cpp +++ b/src/clp_ffi_py/ir/native/decoding_methods.cpp @@ -2,12 +2,13 @@ #include "decoding_methods.hpp" +#include #include -#include -#include -#include -#include +#include +#include +#include +#include #include #include @@ -21,6 +22,10 @@ #include namespace clp_ffi_py::ir::native { +using clp::ffi::ir_stream::encoded_tag_t; +using clp::ffi::ir_stream::IRErrorCode; +using clp::ffi::ir_stream::IRProtocolErrorCode; + namespace { /** * This template defines the function signature of a termination handler @@ -42,6 +47,35 @@ concept TerminateHandlerSignature = requires(TerminateHandler handler) { } -> std::same_as; }; +/** + * Handles the error when IRErrorCode::IRErrorCode_Incomplete_IR is seen. The handler will first + * try to load more data into `decoder_buffer`. If it fails, `allow_incomplete_stream` will be used + * to determine whether to swallow the incomplete IR exception. + * @param decoder_buffer + * @param allow_incomplete_stream A flag to indicate whether the incomplete stream error should be + * ignored. If it is set to true, incomplete stream error + * @param std::nullopt if more data is loaded. + * @param PyNone if the IR stream is incomplete and allowed. + * @param nullptr if the IR stream is incomplete not allowed, with the relevant Python exceptions + * and error set. + */ +[[nodiscard]] auto handle_incomplete_ir_error( + PyDecoderBuffer* decoder_buffer, + bool allow_incomplete_stream +) -> std::optional { + if (decoder_buffer->try_read()) { + return std::nullopt; + } + if (allow_incomplete_stream + && static_cast(PyErr_ExceptionMatches(PyDecoderBuffer::get_py_incomplete_stream_error( + )))) + { + PyErr_Clear(); + Py_RETURN_NONE; + } + return nullptr; +} + /** * Decodes the next log event from the CLP IR buffer `decoder_buffer` until * terminate handler returns true. @@ -67,6 +101,7 @@ auto generic_decode_log_events( auto timestamp{decoder_buffer->get_ref_timestamp()}; size_t current_log_event_idx{0}; PyObject* return_value{nullptr}; + clp::ffi::ir_stream::encoded_tag_t tag{}; while (true) { auto const unconsumed_bytes{decoder_buffer->get_unconsumed_bytes()}; @@ -74,29 +109,44 @@ auto generic_decode_log_events( clp::size_checked_pointer_cast(unconsumed_bytes.data()), unconsumed_bytes.size() }; + + if (auto const err{clp::ffi::ir_stream::deserialize_tag(ir_buffer, tag)}; + IRErrorCode::IRErrorCode_Success != err) + { + if (IRErrorCode::IRErrorCode_Incomplete_IR != err) { + PyErr_Format(PyExc_RuntimeError, cDecoderErrorCodeFormatStr, err); + return nullptr; + } + if (auto const ret_val{ + handle_incomplete_ir_error(decoder_buffer, allow_incomplete_stream) + }; + ret_val.has_value()) + { + return ret_val.value(); + } + continue; + } + if (clp::ffi::ir_stream::cProtocol::Eof == tag) { + Py_RETURN_NONE; + } + auto const err{clp::ffi::ir_stream::four_byte_encoding::deserialize_log_event( ir_buffer, + tag, decoded_message, timestamp_delta )}; - if (clp::ffi::ir_stream::IRErrorCode_Incomplete_IR == err) { - if (decoder_buffer->try_read()) { - continue; - } - if (allow_incomplete_stream - && static_cast( - PyErr_ExceptionMatches(PyDecoderBuffer::get_py_incomplete_stream_error()) - )) + if (IRErrorCode::IRErrorCode_Incomplete_IR == err) { + if (auto const ret_val{ + handle_incomplete_ir_error(decoder_buffer, allow_incomplete_stream) + }; + ret_val.has_value()) { - PyErr_Clear(); - Py_RETURN_NONE; + return ret_val.value(); } - return nullptr; - } - if (clp::ffi::ir_stream::IRErrorCode_Eof == err) { - Py_RETURN_NONE; + continue; } - if (clp::ffi::ir_stream::IRErrorCode_Success != err) { + if (IRErrorCode::IRErrorCode_Success != err) { PyErr_Format(PyExc_RuntimeError, cDecoderErrorCodeFormatStr, err); return nullptr; } @@ -143,11 +193,11 @@ auto decode_preamble(PyObject* Py_UNUSED(self), PyObject* py_decoder_buffer) -> unconsumed_bytes.size() }; auto const err{clp::ffi::ir_stream::get_encoding_type(ir_buffer, is_four_byte_encoding)}; - if (clp::ffi::ir_stream::IRErrorCode_Success == err) { + if (IRErrorCode::IRErrorCode_Success == err) { ir_buffer_cursor_pos = ir_buffer.get_pos(); break; } - if (clp::ffi::ir_stream::IRErrorCode_Incomplete_IR != err) { + if (IRErrorCode::IRErrorCode_Incomplete_IR != err) { PyErr_Format(PyExc_RuntimeError, cDecoderErrorCodeFormatStr, err); return nullptr; } @@ -176,11 +226,11 @@ auto decode_preamble(PyObject* Py_UNUSED(self), PyObject* py_decoder_buffer) -> metadata_pos, metadata_size )}; - if (clp::ffi::ir_stream::IRErrorCode_Success == err) { + if (IRErrorCode::IRErrorCode_Success == err) { ir_buffer_cursor_pos = ir_buffer.get_pos(); break; } - if (clp::ffi::ir_stream::IRErrorCode_Incomplete_IR != err) { + if (IRErrorCode ::IRErrorCode_Incomplete_IR != err) { PyErr_Format(PyExc_RuntimeError, cDecoderErrorCodeFormatStr, err); return nullptr; } @@ -205,15 +255,15 @@ auto decode_preamble(PyObject* Py_UNUSED(self), PyObject* py_decoder_buffer) -> static_cast(clp::ffi::ir_stream::cProtocol::Metadata::VersionKey) )}; auto const error_code{clp::ffi::ir_stream::validate_protocol_version(version)}; - if (clp::ffi::ir_stream::IRProtocolErrorCode_Supported != error_code) { + if (IRProtocolErrorCode::IRProtocolErrorCode_Supported != error_code) { switch (error_code) { - case clp::ffi::ir_stream::IRProtocolErrorCode_Invalid: + case IRProtocolErrorCode::IRProtocolErrorCode_Invalid: PyErr_Format(PyExc_RuntimeError, "Invalid version number: %s", version.c_str()); break; - case clp::ffi::ir_stream::IRProtocolErrorCode_Too_New: + case IRProtocolErrorCode::IRProtocolErrorCode_Too_New: PyErr_Format(PyExc_RuntimeError, "Version too new: %s", version.c_str()); break; - case clp::ffi::ir_stream::IRProtocolErrorCode_Too_Old: + case IRProtocolErrorCode::IRProtocolErrorCode_Too_Old: PyErr_Format(PyExc_RuntimeError, "Version too old: %s", version.c_str()); break; default: diff --git a/src/clp_ffi_py/ir/native/encoding_methods.cpp b/src/clp_ffi_py/ir/native/encoding_methods.cpp index 5f0a6678..57406fc0 100644 --- a/src/clp_ffi_py/ir/native/encoding_methods.cpp +++ b/src/clp_ffi_py/ir/native/encoding_methods.cpp @@ -2,10 +2,10 @@ #include "encoding_methods.hpp" -#include -#include -#include -#include +#include +#include +#include +#include #include diff --git a/src/clp_ffi_py/utils.inc b/src/clp_ffi_py/utils.inc index da673e57..6f3012fb 100644 --- a/src/clp_ffi_py/utils.inc +++ b/src/clp_ffi_py/utils.inc @@ -5,7 +5,7 @@ #include -#include +#include namespace clp_ffi_py { namespace {