From 6f20e574e9ba40bab3986f9730b4e0868a2ce877 Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Wed, 30 Oct 2024 21:18:32 -0400 Subject: [PATCH 01/50] Add template to start --- CMakeLists.txt | 2 ++ .../ir/native/PyKeyValuePairLogEvent.cpp | 1 + .../ir/native/PyKeyValuePairLogEvent.hpp | 14 ++++++++++++++ 3 files changed, 17 insertions(+) create mode 100644 src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp create mode 100644 src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 12913763..527f035e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -86,6 +86,8 @@ set(CLP_FFI_PY_LIB_IR_SOURCES ${CLP_FFI_PY_LIB_SRC_DIR}/ir/native/PyFourByteDeserializer.hpp ${CLP_FFI_PY_LIB_SRC_DIR}/ir/native/PyFourByteSerializer.cpp ${CLP_FFI_PY_LIB_SRC_DIR}/ir/native/PyFourByteSerializer.hpp + ${CLP_FFI_PY_LIB_SRC_DIR}/ir/native/PyKeyValuePairLogEvent.cpp + ${CLP_FFI_PY_LIB_SRC_DIR}/ir/native/PyKeyValuePairLogEvent.hpp ${CLP_FFI_PY_LIB_SRC_DIR}/ir/native/PyLogEvent.cpp ${CLP_FFI_PY_LIB_SRC_DIR}/ir/native/PyLogEvent.hpp ${CLP_FFI_PY_LIB_SRC_DIR}/ir/native/PyMetadata.cpp diff --git a/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp b/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp new file mode 100644 index 00000000..10693ac4 --- /dev/null +++ b/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp @@ -0,0 +1 @@ +#include "PyKeyValuePairLogEvent.hpp" diff --git a/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.hpp b/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.hpp new file mode 100644 index 00000000..2a6be514 --- /dev/null +++ b/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.hpp @@ -0,0 +1,14 @@ +#ifndef CLP_FFI_PY_IR_NATIVE_PYKEYVALUEPAIRLOGEVENT_HPP +#define CLP_FFI_PY_IR_NATIVE_PYKEYVALUEPAIRLOGEVENT_HPP + +namespace clp_ffi_py::ir::native { +/** + * A PyObject structure functioning as a Python-compatible interface to retrieve a key-value pair + * log event. The underlying data is pointed to by `m_kv_pair_log_event`. + */ +class PyKeyValuePairLogEvent { + +}; +} + +#endif // CLP_FFI_PY_IR_NATIVE_PYKEYVALUEPAIRLOGEVENT_HPP From 28eed53fe7a2abf09e260adf1fb91bb3fd228841 Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Sat, 2 Nov 2024 00:43:12 -0400 Subject: [PATCH 02/50] Add msgpack --- .gitmodules | 3 +++ CMakeLists.txt | 11 ++++++++++- src/msgpack | 1 + 3 files changed, 14 insertions(+), 1 deletion(-) create mode 160000 src/msgpack diff --git a/.gitmodules b/.gitmodules index 7452a0d7..05db3733 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "src/clp"] path = src/clp url = https://github.com/y-scope/clp.git +[submodule "src/msgpack"] + path = src/msgpack + url = https://github.com/msgpack/msgpack-c diff --git a/CMakeLists.txt b/CMakeLists.txt index 527f035e..23be1041 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -59,14 +59,23 @@ set(CLP_FFI_PY_CLP_CORE_DIR "${CLP_FFI_PY_SRC_DIR}/clp/components/core") add_subdirectory(${CLP_FFI_PY_CLP_CORE_DIR}/src/clp/string_utils) +set(MSGPACK_CXX20 ON) +set(MSGPACK_USE_BOOST OFF) +add_subdirectory(${CLP_FFI_PY_SRC_DIR}/msgpack EXCLUDE_FROM_ALL) + # NOTE: We don't add headers here since CLP core is technically a library we're using, not a part of # this project. set(CLP_FFI_PY_CLP_CORE_SOURCES ${CLP_FFI_PY_CLP_CORE_DIR}/src/clp/BufferReader.cpp ${CLP_FFI_PY_CLP_CORE_DIR}/src/clp/ffi/ir_stream/decoding_methods.cpp ${CLP_FFI_PY_CLP_CORE_DIR}/src/clp/ffi/ir_stream/encoding_methods.cpp + ${CLP_FFI_PY_CLP_CORE_DIR}/src/clp/ffi/ir_stream/ir_unit_deserialization_methods.cpp + ${CLP_FFI_PY_CLP_CORE_DIR}/src/clp/ffi/ir_stream/Serializer.cpp ${CLP_FFI_PY_CLP_CORE_DIR}/src/clp/ffi/ir_stream/utils.cpp ${CLP_FFI_PY_CLP_CORE_DIR}/src/clp/ffi/encoding_methods.cpp + ${CLP_FFI_PY_CLP_CORE_DIR}/src/clp/ffi/SchemaTree.cpp + ${CLP_FFI_PY_CLP_CORE_DIR}/src/clp/ffi/KeyValuePairLogEvent.cpp + ${CLP_FFI_PY_CLP_CORE_DIR}/src/clp/ir/EncodedTextAst.cpp ${CLP_FFI_PY_CLP_CORE_DIR}/src/clp/ir/parsing.cpp ${CLP_FFI_PY_CLP_CORE_DIR}/src/clp/ReaderInterface.cpp ) @@ -127,7 +136,7 @@ target_include_directories( target_include_directories(${CLP_FFI_PY_LIB_IR} PRIVATE ${CLP_FFI_PY_SRC_DIR}) -target_link_libraries(${CLP_FFI_PY_LIB_IR} PRIVATE clp::string_utils) +target_link_libraries(${CLP_FFI_PY_LIB_IR} PRIVATE clp::string_utils msgpack-cxx) if(CLP_FFI_PY_INSTALL_LIBS) install( diff --git a/src/msgpack b/src/msgpack new file mode 160000 index 00000000..f9f2917c --- /dev/null +++ b/src/msgpack @@ -0,0 +1 @@ +Subproject commit f9f2917cc01ce6431dacf35986f53058e1857080 From 45767a186561e62774c35d8d44deb33cbe1580e1 Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Sat, 2 Nov 2024 00:47:15 -0400 Subject: [PATCH 03/50] Refactoring cmake --- CMakeLists.txt | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 23be1041..2b544b81 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -136,7 +136,12 @@ target_include_directories( target_include_directories(${CLP_FFI_PY_LIB_IR} PRIVATE ${CLP_FFI_PY_SRC_DIR}) -target_link_libraries(${CLP_FFI_PY_LIB_IR} PRIVATE clp::string_utils msgpack-cxx) +target_link_libraries( + ${CLP_FFI_PY_LIB_IR} + PRIVATE + clp::string_utils + msgpack-cxx +) if(CLP_FFI_PY_INSTALL_LIBS) install( From ead58b8c3b35838fb1a9aff77d4b0ad4f8e96cb7 Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Sat, 2 Nov 2024 00:49:22 -0400 Subject: [PATCH 04/50] Fix --- clp_ffi_py/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/clp_ffi_py/utils.py b/clp_ffi_py/utils.py index 46a342b8..d9a148e5 100644 --- a/clp_ffi_py/utils.py +++ b/clp_ffi_py/utils.py @@ -2,6 +2,7 @@ from typing import Any, Dict, Optional import dateutil.tz + import msgpack From 384459505b2ecbf1566b6763c6f4e889bbab1111 Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Sat, 2 Nov 2024 16:10:29 -0400 Subject: [PATCH 05/50] Add gsl --- .gitmodules | 3 +++ src/GSL | 1 + 2 files changed, 4 insertions(+) create mode 160000 src/GSL diff --git a/.gitmodules b/.gitmodules index 05db3733..d2958bff 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "src/msgpack"] path = src/msgpack url = https://github.com/msgpack/msgpack-c +[submodule "src/GSL"] + path = src/GSL + url = https://github.com/microsoft/GSL.git diff --git a/src/GSL b/src/GSL new file mode 160000 index 00000000..87f9d768 --- /dev/null +++ b/src/GSL @@ -0,0 +1 @@ +Subproject commit 87f9d768866548b5b86e72be66c60c5abd4d9b37 From 467c030a4e5443fd91ad4a4bd8cffd904565f9fb Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Sun, 3 Nov 2024 15:47:18 -0500 Subject: [PATCH 06/50] Implement PyKeyValuePairLogEvent --- .clang-format | 4 +- CMakeLists.txt | 10 + clp_ffi_py/ir/__init__.py | 1 + clp_ffi_py/ir/native.pyi | 4 + clp_ffi_py/utils.py | 11 + src/clp_ffi_py/PyObjectCast.hpp | 4 + src/clp_ffi_py/Py_utils.cpp | 40 +- src/clp_ffi_py/Py_utils.hpp | 17 +- src/clp_ffi_py/Python.hpp | 17 + src/clp_ffi_py/api_decoration.hpp | 11 + .../ir/native/PyKeyValuePairLogEvent.cpp | 343 +++++++++++++++++- .../ir/native/PyKeyValuePairLogEvent.hpp | 87 ++++- src/clp_ffi_py/ir/native/error_messages.hpp | 17 + src/clp_ffi_py/modules/ir_native.cpp | 6 + src/clp_ffi_py/utils.cpp | 16 + src/clp_ffi_py/utils.hpp | 12 + 16 files changed, 576 insertions(+), 24 deletions(-) create mode 100644 src/clp_ffi_py/api_decoration.hpp diff --git a/.clang-format b/.clang-format index 0564c962..bf35c622 100644 --- a/.clang-format +++ b/.clang-format @@ -77,9 +77,7 @@ IncludeCategories: - Regex: "^$" diff --git a/CMakeLists.txt b/CMakeLists.txt index 2b544b81..676fd56b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,6 +18,7 @@ if(DEFINED SKBUILD_PROJECT_NAME) "The version of the project parsed by scikit-build-core." FORCE ) + set(CLP_FFI_PY_LINT OFF CACHE BOOL "Disable linting." FORCE) else() set(CLP_FFI_PY_INSTALL_LIBS OFF CACHE BOOL "Disable installing the built libraries." FORCE) set(CLP_FFI_PY_PROJECT_NAME "clp-ffi-py" CACHE STRING "Use a placeholder project name." FORCE) @@ -28,6 +29,7 @@ else() "Enable/Disable output of compile commands during generation." FORCE ) + set(CLP_FFI_PY_LINT ON CACHE BOOL "Enable linting." FORCE) endif() project(${CLP_FFI_PY_PROJECT_NAME} LANGUAGES CXX VERSION ${CLP_FFI_PY_VERSION}) @@ -63,6 +65,8 @@ set(MSGPACK_CXX20 ON) set(MSGPACK_USE_BOOST OFF) add_subdirectory(${CLP_FFI_PY_SRC_DIR}/msgpack EXCLUDE_FROM_ALL) +add_subdirectory(${CLP_FFI_PY_SRC_DIR}/GSL) + # NOTE: We don't add headers here since CLP core is technically a library we're using, not a part of # this project. set(CLP_FFI_PY_CLP_CORE_SOURCES @@ -82,6 +86,7 @@ set(CLP_FFI_PY_CLP_CORE_SOURCES # NOTE: We include headers to ensure IDEs like CLion load the project properly. set(CLP_FFI_PY_LIB_IR_SOURCES + ${CLP_FFI_PY_LIB_SRC_DIR}/api_decoration.hpp ${CLP_FFI_PY_LIB_SRC_DIR}/error_messages.hpp ${CLP_FFI_PY_LIB_SRC_DIR}/ExceptionFFI.hpp ${CLP_FFI_PY_LIB_SRC_DIR}/ir/native/deserialization_methods.cpp @@ -136,10 +141,15 @@ target_include_directories( target_include_directories(${CLP_FFI_PY_LIB_IR} PRIVATE ${CLP_FFI_PY_SRC_DIR}) +if(CLP_FFI_PY_LINT) + target_compile_definitions(${CLP_FFI_PY_LIB_IR} PRIVATE CLP_FFI_PY_ENABLE_LINTING) +endif() + target_link_libraries( ${CLP_FFI_PY_LIB_IR} PRIVATE clp::string_utils + Microsoft.GSL::GSL msgpack-cxx ) diff --git a/clp_ffi_py/ir/__init__.py b/clp_ffi_py/ir/__init__.py index 9811e948..defc702e 100644 --- a/clp_ffi_py/ir/__init__.py +++ b/clp_ffi_py/ir/__init__.py @@ -13,6 +13,7 @@ "FourByteEncoder", # native_deprecated "FourByteSerializer", # native "IncompleteStreamError", # native + "KeyValuePairLogEvent", # native "LogEvent", # native "Metadata", # native "Query", # native diff --git a/clp_ffi_py/ir/native.pyi b/clp_ffi_py/ir/native.pyi index b991617d..16e2e9c2 100644 --- a/clp_ffi_py/ir/native.pyi +++ b/clp_ffi_py/ir/native.pyi @@ -82,4 +82,8 @@ class FourByteDeserializer: allow_incomplete_stream: bool = False, ) -> Optional[LogEvent]: ... +class KeyValuePairLogEvent: + def __init__(self, dictionary: Dict[Any, Any]): ... + def to_dict(self) -> Dict[Any, Any]: ... + class IncompleteStreamError(Exception): ... diff --git a/clp_ffi_py/utils.py b/clp_ffi_py/utils.py index d9a148e5..2a8264eb 100644 --- a/clp_ffi_py/utils.py +++ b/clp_ffi_py/utils.py @@ -1,3 +1,4 @@ +import json from datetime import datetime, tzinfo from typing import Any, Dict, Optional @@ -46,3 +47,13 @@ def serialize_dict_to_msgpack(dictionary: Dict[Any, Any]) -> bytes: if not isinstance(dictionary, dict): raise TypeError("The type of the input object must be a dictionary.") return msgpack.packb(dictionary) + + +def parse_json_str(json_str: str) -> Any: + """ + Wrapper of `json.loads`, which parses a JSON string into a Python object. + + :param json_str: The JSON string to parse. + :return: The parsed JSON object. + """ + return json.loads(json_str) diff --git a/src/clp_ffi_py/PyObjectCast.hpp b/src/clp_ffi_py/PyObjectCast.hpp index e9125d12..909793cf 100644 --- a/src/clp_ffi_py/PyObjectCast.hpp +++ b/src/clp_ffi_py/PyObjectCast.hpp @@ -114,6 +114,7 @@ auto py_reinterpret_cast(Src* src) noexcept -> Dst* { namespace ir::native { class PyDeserializerBuffer; class PyFourByteDeserializer; +class PyKeyValuePairLogEvent; class PyLogEvent; class PyMetadata; class PyQuery; @@ -121,9 +122,12 @@ class PyQuery; CLP_FFI_PY_MARK_AS_PYOBJECT(ir::native::PyDeserializerBuffer); CLP_FFI_PY_MARK_AS_PYOBJECT(ir::native::PyFourByteDeserializer); +CLP_FFI_PY_MARK_AS_PYOBJECT(ir::native::PyKeyValuePairLogEvent); CLP_FFI_PY_MARK_AS_PYOBJECT(ir::native::PyLogEvent); CLP_FFI_PY_MARK_AS_PYOBJECT(ir::native::PyMetadata); CLP_FFI_PY_MARK_AS_PYOBJECT(ir::native::PyQuery); +CLP_FFI_PY_MARK_AS_PYOBJECT(PyBytesObject); +CLP_FFI_PY_MARK_AS_PYOBJECT(PyDictObject); CLP_FFI_PY_MARK_AS_PYOBJECT(PyTypeObject); } // namespace clp_ffi_py diff --git a/src/clp_ffi_py/Py_utils.cpp b/src/clp_ffi_py/Py_utils.cpp index 6ae1cb44..266e0753 100644 --- a/src/clp_ffi_py/Py_utils.cpp +++ b/src/clp_ffi_py/Py_utils.cpp @@ -4,6 +4,7 @@ #include +#include #include namespace clp_ffi_py { @@ -17,6 +18,9 @@ PyObjectStaticPtr Py_func_get_timezone_from_timezone_id{nullptr}; constexpr std::string_view cPyFuncNameSerializeDictToMsgpack{"serialize_dict_to_msgpack"}; PyObjectStaticPtr Py_func_serialize_dict_to_msgpack{nullptr}; +constexpr std::string_view cPyFuncNameParseJsonStr{"parse_json_str"}; +PyObjectStaticPtr Py_func_parse_json_str{nullptr}; + /** * Wrapper of PyObject_CallObject. * @param func PyObject that points to the calling function. @@ -55,6 +59,12 @@ auto py_utils_init() -> bool { if (nullptr == Py_func_serialize_dict_to_msgpack.get()) { return false; } + + Py_func_parse_json_str.reset(PyObject_GetAttrString(py_utils, cPyFuncNameParseJsonStr.data())); + if (nullptr == Py_func_parse_json_str.get()) { + return false; + } + return true; } @@ -77,12 +87,36 @@ auto py_utils_get_timezone_from_timezone_id(std::string const& timezone_id) -> P return py_utils_function_call_wrapper(Py_func_get_timezone_from_timezone_id.get(), func_args); } -auto py_utils_serialize_dict_to_msgpack(PyObject* dictionary) -> PyObject* { - PyObjectPtr const func_args_ptr{Py_BuildValue("O", dictionary)}; +auto py_utils_serialize_dict_to_msgpack(PyDictObject* py_dict) -> PyBytesObject* { + PyObjectPtr const func_args_ptr{ + Py_BuildValue("(O)", py_reinterpret_cast(py_dict)) + }; + auto* func_args{func_args_ptr.get()}; + if (nullptr == func_args) { + return nullptr; + } + auto* result{py_utils_function_call_wrapper(Py_func_serialize_dict_to_msgpack.get(), func_args) + }; + if (nullptr == result) { + return nullptr; + } + if (false == static_cast(PyBytes_Check(result))) { + PyErr_SetString( + PyExc_TypeError, + "`serialize_dict_to_msgpack` is supposed to return a `bytes` object" + ); + return nullptr; + } + + return py_reinterpret_cast(result); +} + +auto py_utils_parse_json_str(std::string_view json_str) -> PyObject* { + PyObjectPtr const func_args_ptr{Py_BuildValue("(s)", json_str.data())}; auto* func_args{func_args_ptr.get()}; if (nullptr == func_args) { return nullptr; } - return py_utils_function_call_wrapper(Py_func_serialize_dict_to_msgpack.get(), func_args); + return py_utils_function_call_wrapper(Py_func_parse_json_str.get(), func_args); } } // namespace clp_ffi_py diff --git a/src/clp_ffi_py/Py_utils.hpp b/src/clp_ffi_py/Py_utils.hpp index b98f93fe..bd687661 100644 --- a/src/clp_ffi_py/Py_utils.hpp +++ b/src/clp_ffi_py/Py_utils.hpp @@ -3,6 +3,9 @@ #include // Must always be included before any other header files +#include +#include + #include namespace clp_ffi_py { @@ -37,11 +40,19 @@ namespace clp_ffi_py { /** * CPython wrapper of `clp_ffi_py.utils.serialize_dict_to_msgpack`. - * @param dictionary - * @return a new reference of a Python `bytes` object containing msgpack-serialized dictionary. + * @param py_dict + * @return a new reference of a `PyByteObject` containing msgpack-serialized dictionary. + * @return nullptr on failure with the relevant Python exception and error set. + */ +[[nodiscard]] auto py_utils_serialize_dict_to_msgpack(PyDictObject* py_dict) -> PyBytesObject*; + +/** + * CPython wrapper of `clp_ffi_py.utils.parse_json_str_to_dict`. + * @param json_str + * @return a new reference of the parsed JSON object. * @return nullptr on failure with the relevant Python exception and error set. */ -[[nodiscard]] auto py_utils_serialize_dict_to_msgpack(PyObject* dictionary) -> PyObject*; +[[nodiscard]] auto py_utils_parse_json_str(std::string_view json_str) -> PyObject*; } // namespace clp_ffi_py #endif // CLP_FFI_PY_PY_UTILS_HPP diff --git a/src/clp_ffi_py/Python.hpp b/src/clp_ffi_py/Python.hpp index 75ac3609..9c626e19 100644 --- a/src/clp_ffi_py/Python.hpp +++ b/src/clp_ffi_py/Python.hpp @@ -8,3 +8,20 @@ #define PY_SSIZE_T_CLEAN #include #include + +// clang-format off +#ifdef CLP_FFI_PY_ENABLE_LINTING +// The following headers are added to export Python headers to get rid of clang-tidy warnings. +// IWYU pragma: begin_exports +#include +#include +#include +#include +#include +#include +#include +#include +#include +// IWYU pragma: end_exports +#endif +// clang-format on diff --git a/src/clp_ffi_py/api_decoration.hpp b/src/clp_ffi_py/api_decoration.hpp new file mode 100644 index 00000000..d3950cb8 --- /dev/null +++ b/src/clp_ffi_py/api_decoration.hpp @@ -0,0 +1,11 @@ +#ifndef CLP_FFI_PY_API_DECORATION_HPP +#define CLP_FFI_PY_API_DECORATION_HPP + +/** + * `CLP_FFI_PY_METHOD` should be added at the beginning of a function's declaration/implementation + * to decorate any APIs that are directly invoked by Python's interpreter. The macro expands to + * `extern "C"` to ensure C linkage. + */ +#define CLP_FFI_PY_METHOD extern "C" + +#endif // CLP_FFI_PY_API_DECORATION_HPP diff --git a/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp b/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp index 10693ac4..c287322c 100644 --- a/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp +++ b/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp @@ -1 +1,342 @@ -#include "PyKeyValuePairLogEvent.hpp" +#include // Must always be included before any other header files + +#include "PyKeyValuePairLogEvent.hpp" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +using clp::ffi::ir_stream::IRErrorCode; + +namespace clp_ffi_py::ir::native { +namespace { +/** + * Class that implements `clp::ffi::ir_stream::IrUnitHandlerInterface` for deserializing log events. + */ +class IrUnitHandler { +public: + // Implements `clp::ffi::ir_stream::IrUnitHandlerInterface` interface + [[nodiscard]] auto handle_log_event(clp::ffi::KeyValuePairLogEvent&& log_event) -> IRErrorCode { + m_log_event.emplace(std::move(log_event)); + return IRErrorCode::IRErrorCode_Success; + } + + [[nodiscard]] static auto handle_utc_offset_change( + [[maybe_unused]] clp::UtcOffset utc_offset_old, + [[maybe_unused]] clp::UtcOffset utc_offset_new + ) -> IRErrorCode { + return IRErrorCode::IRErrorCode_Success; + } + + [[nodiscard]] static auto handle_schema_tree_node_insertion( + [[maybe_unused]] clp::ffi::SchemaTree::NodeLocator schema_tree_node_locator + ) -> IRErrorCode { + return IRErrorCode::IRErrorCode_Success; + } + + [[nodiscard]] static auto handle_end_of_stream() -> IRErrorCode { + return IRErrorCode::IRErrorCode_Success; + } + + // NOLINTNEXTLINE(*) + std::optional m_log_event; +}; + +// NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays) +PyDoc_STRVAR( + cPyKeyValuePairLogEventDoc, + "This class represents a key-value pair log event and provides ways to access the" + " underlying log data. This class is designed to be instantiated by the IR deserializer." + " However, direct instantiation using the `__init__` method is also supported for testing" + " purposes, although this may not be as efficient as emitting from the IR deserializer.\n\n" + "The signature of `__init__` method is shown as following:\n\n" + "__init__(self, dictionary)\n\n" + "Initializes an object that represents a key-value pair log event from the given Python" + " dictionary. Notice that each object should be strictly initialized only once. Double" + " initialization will result in memory leak.\n\n" + ":param dictionary: A dictionary representing the key-value log event, where all keys are" + " expected to be of string type, including keys inside any sub-dictionaries.\n" +); + +/** + * Callback of `PyKeyValuePairLogEvent`'s `__init__` method: + * @param self + * @param args + * @param keywords + * @return 0 on success. + * @return -1 on failure with the relevant Python exception and error set. + */ +CLP_FFI_PY_METHOD auto PyKeyValuePairLogEvent_init( + PyKeyValuePairLogEvent* self, + PyObject* args, + PyObject* keywords +) -> int; + +/** + * Callback of `PyKeyValuePairLogEvent`'s deallocator. + * @param self + */ +CLP_FFI_PY_METHOD auto PyKeyValuePairLogEvent_dealloc(PyKeyValuePairLogEvent* self) -> void; + +// NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays) +PyDoc_STRVAR( + cPyKeyValuePairLogEventToDictDoc, + "to_dict(self)\n" + "--\n\n" + "Converts the underlying key-value pair log event into a Python dictionary.\n\n" + ":return: Serialized log event in a Python dictionary.\n" +); + +CLP_FFI_PY_METHOD auto PyKeyValuePairLogEvent_to_dict(PyKeyValuePairLogEvent* self) -> PyObject*; + +// NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays) +PyMethodDef PyKeyValuePairLogEvent_method_table[]{ + {"to_dict", + py_c_function_cast(PyKeyValuePairLogEvent_to_dict), + METH_NOARGS, + static_cast(cPyKeyValuePairLogEventToDictDoc)}, + + {nullptr} +}; + +// NOLINTBEGIN(cppcoreguidelines-avoid-c-arrays, cppcoreguidelines-pro-type-*-cast) +PyType_Slot PyKeyValuePairLogEvent_slots[]{ + {Py_tp_alloc, reinterpret_cast(PyType_GenericAlloc)}, + {Py_tp_dealloc, reinterpret_cast(PyKeyValuePairLogEvent_dealloc)}, + {Py_tp_new, reinterpret_cast(PyType_GenericNew)}, + {Py_tp_init, reinterpret_cast(PyKeyValuePairLogEvent_init)}, + {Py_tp_methods, static_cast(PyKeyValuePairLogEvent_method_table)}, + {Py_tp_doc, const_cast(static_cast(cPyKeyValuePairLogEventToDictDoc))}, + {0, nullptr} +}; +// NOLINTEND(cppcoreguidelines-avoid-c-arrays, cppcoreguidelines-pro-type-*-cast) + +/** + * `PyKeyValuePairLogEvent`'s Python type specifications. + */ +PyType_Spec PyKeyValuePairLogEvent_type_spec{ + "clp_ffi_py.ir.native.KeyValuePairLogEvent", + sizeof(PyKeyValuePairLogEvent), + 0, + Py_TPFLAGS_DEFAULT, + static_cast(PyKeyValuePairLogEvent_slots) +}; + +/** + * Converts the given Python dictionary to `clp::ffi::KeyValuePairLogEvent`. + * NOTE: Currently, there is no direct utility for constructing a `clp::ffi::KeyValuePairLogEvent` + * from a dictionary-like structure. As a workaround, this function serializes the dictionary + * using the key-value pair IR format, then deserializes it to create a `KeyValuePairLogEvent` + * instance. This approach is inefficient and intended solely for testing purposes, as it allows + * instance creation without a full IR stream. Future implementations should replace this method + * with a more efficient conversion once a direct utility is available. + * @param py_dict + * @return The converted key-value log event of the given dictionary on success. + * @return std::nullopt on failure with the relevant Python exception and error set. + */ +[[nodiscard]] auto convert_py_dict_to_key_value_pair_log_event(PyDictObject* py_dict +) -> std::optional; + +CLP_FFI_PY_METHOD auto PyKeyValuePairLogEvent_init( + PyKeyValuePairLogEvent* self, + PyObject* args, + PyObject* keywords +) -> int { + static char keyword_dictionary[]{"dictionary"}; + static char* keyword_table[]{static_cast(keyword_dictionary), nullptr}; + + PyObject* dictionary{Py_None}; + if (false + == static_cast(PyArg_ParseTupleAndKeywords( + args, + keywords, + "O", + static_cast(keyword_table), + &dictionary + ))) + { + return -1; + } + + // If the argument parsing fails, `self` will be deallocated. We must reset all pointers to + // nullptr in advance, otherwise the deallocator might trigger segmentation fault. + self->default_init(); + + if (false == static_cast(PyDict_Check(dictionary))) { + PyErr_SetString(PyExc_TypeError, "`dictionary` must be a Python dictionary object"); + return -1; + } + PyDictObject* py_dict{py_reinterpret_cast(dictionary)}; + + auto optional_kv_pair_log_event{convert_py_dict_to_key_value_pair_log_event(py_dict)}; + if (false == optional_kv_pair_log_event.has_value()) { + return -1; + } + + if (false == self->init(std::move(optional_kv_pair_log_event.value()))) { + return -1; + } + + return 0; +} + +CLP_FFI_PY_METHOD auto PyKeyValuePairLogEvent_dealloc(PyKeyValuePairLogEvent* self) -> void { + self->clean(); + PyObject_Del(self); +} + +CLP_FFI_PY_METHOD auto PyKeyValuePairLogEvent_to_dict(PyKeyValuePairLogEvent* self) -> PyObject* { + // TODO: use an efficient algorithm to turn the underlying log event to a Python dictionary + auto const& kv_pair_log_event{self->get_kv_pair_log_event()}; + auto const serialized_json_result{kv_pair_log_event.serialize_to_json()}; + if (serialized_json_result.has_error()) { + PyErr_Format( + PyExc_RuntimeError, + cKeyValuePairLogEventSerializeToStringErrorFormatStr.data(), + serialized_json_result.error().message().c_str() + ); + return nullptr; + } + auto const json_str{serialized_json_result.value().dump()}; + auto* parsed_json{py_utils_parse_json_str(json_str)}; + if (nullptr == parsed_json) { + return nullptr; + } + if (false == static_cast(PyDict_Check(parsed_json))) { + PyErr_SetString(PyExc_TypeError, "Serialized JSON object is not a dictionary"); + } + return parsed_json; +} + +auto convert_py_dict_to_key_value_pair_log_event(PyDictObject* py_dict +) -> std::optional { + auto* serialized_msgpack_byte_sequence{py_utils_serialize_dict_to_msgpack(py_dict)}; + if (nullptr == serialized_msgpack_byte_sequence) { + return std::nullopt; + } + + // Since the type is already checked, we can use the macro to avoid duplicated type checking. + std::span const data_view{ + PyBytes_AS_STRING(serialized_msgpack_byte_sequence), + static_cast(PyBytes_GET_SIZE(serialized_msgpack_byte_sequence)) + }; + auto const unpack_result{unpack_msgpack(data_view)}; + if (unpack_result.has_error()) { + PyErr_SetString(PyExc_RuntimeError, unpack_result.error().c_str()); + return std::nullopt; + } + auto const& msgpack_obj{unpack_result.value().get()}; + if (msgpack::type::MAP != msgpack_obj.type) { + PyErr_SetString(PyExc_TypeError, "Unpacked msgpack is not a map"); + return std::nullopt; + } + + auto serializer_result{ + clp::ffi::ir_stream::Serializer::create() + }; + if (serializer_result.has_error()) { + PyErr_Format( + PyExc_RuntimeError, + cSerializerCreateErrorFormatStr.data(), + serializer_result.error().message().c_str() + ); + return std::nullopt; + } + + auto& serializer{serializer_result.value()}; + if (false == serializer.serialize_msgpack_map(msgpack_obj.via.map)) { + PyErr_SetString(PyExc_RuntimeError, cSerializerSerializeMsgpackMapError.data()); + return std::nullopt; + } + + auto const ir_buf{serializer.get_ir_buf_view()}; + clp::BufferReader buf_reader{ + clp::size_checked_pointer_cast(ir_buf.data()), + ir_buf.size() + }; + + auto deserializer_result{ + clp::ffi::ir_stream::Deserializer::create(buf_reader, IrUnitHandler{}) + }; + if (deserializer_result.has_error()) { + PyErr_Format( + PyExc_RuntimeError, + cDeserializerCreateErrorFormatStr.data(), + deserializer_result.error().message().c_str() + ); + return std::nullopt; + } + + auto& deserializer{deserializer_result.value()}; + auto& ir_unit_handler{deserializer.get_ir_unit_handler()}; + while (true) { + auto const result{deserializer.deserialize_next_ir_unit(buf_reader)}; + if (result.has_error()) { + PyErr_Format( + PyExc_RuntimeError, + cDeserializerDeserializeNextIrUnitErrorFormatStr.data(), + result.error().message().c_str() + ); + return std::nullopt; + } + auto const ir_unit_type{result.value()}; + if (ir_unit_type == clp::ffi::ir_stream::IrUnitType::SchemaTreeNodeInsertion) { + continue; + } + if (ir_unit_type != clp::ffi::ir_stream::IrUnitType::LogEvent) { + PyErr_SetString(PyExc_RuntimeError, "Unexpected Ir Unit Type"); + return std::nullopt; + } + break; + } + + if (false == ir_unit_handler.m_log_event.has_value()) { + PyErr_SetString(PyExc_RuntimeError, "No log event has been deserialized"); + return std::nullopt; + } + + return std::move(ir_unit_handler.m_log_event); +} +} // namespace + +auto PyKeyValuePairLogEvent::init(clp::ffi::KeyValuePairLogEvent kv_pair_log_event) -> bool { + m_kv_pair_log_event = new clp::ffi::KeyValuePairLogEvent{std::move(kv_pair_log_event)}; + return nullptr != m_kv_pair_log_event; +} + +auto PyKeyValuePairLogEvent::get_py_type() -> PyTypeObject* { + return m_py_type.get(); +} + +auto PyKeyValuePairLogEvent::module_level_init(PyObject* py_module) -> bool { + // TODO: complete this function + static_assert(std::is_trivially_destructible()); + auto* type{py_reinterpret_cast(PyType_FromSpec(&PyKeyValuePairLogEvent_type_spec)) + }; + m_py_type.reset(type); + if (nullptr == type) { + return false; + } + return add_python_type(get_py_type(), "KeyValuePairLogEvent", py_module); +} +} // namespace clp_ffi_py::ir::native diff --git a/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.hpp b/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.hpp index 2a6be514..b52cce66 100644 --- a/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.hpp +++ b/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.hpp @@ -1,14 +1,73 @@ -#ifndef CLP_FFI_PY_IR_NATIVE_PYKEYVALUEPAIRLOGEVENT_HPP -#define CLP_FFI_PY_IR_NATIVE_PYKEYVALUEPAIRLOGEVENT_HPP - -namespace clp_ffi_py::ir::native { -/** - * A PyObject structure functioning as a Python-compatible interface to retrieve a key-value pair - * log event. The underlying data is pointed to by `m_kv_pair_log_event`. - */ -class PyKeyValuePairLogEvent { - -}; -} - -#endif // CLP_FFI_PY_IR_NATIVE_PYKEYVALUEPAIRLOGEVENT_HPP +#ifndef CLP_FFI_PY_IR_NATIVE_PYKEYVALUEPAIRLOGEVENT_HPP +#define CLP_FFI_PY_IR_NATIVE_PYKEYVALUEPAIRLOGEVENT_HPP + +#include // Must always be included before any other header files + +#include + +#include + +#include + +namespace clp_ffi_py::ir::native { +/** + * A PyObject structure functioning as a Python-compatible interface to retrieve a key-value pair + * log event. The underlying data is pointed to by `m_kv_pair_log_event`. + */ +class PyKeyValuePairLogEvent { +public: + /** + * Initializes the underlying data with the given inputs. Since the memory allocation of + * `PyKeyValuePairLogEvent` is handled by CPython's allocator, cpp constructors will not be + * explicitly called. This function serves as the default constructor initialize the underlying + * key-value pair log event. It has to be manually called whenever creating a new + * `PyKeyValuePairLogEvent` object through CPython APIs. + * @param kv_pair_log_event + * @return true on success. + * @return false on failure with the relevant Python exception and error set. + */ + [[nodiscard]] auto init(clp::ffi::KeyValuePairLogEvent kv_pair_log_event) -> bool; + + /** + * Initializes the pointers to nullptr by default. Should be called once the object is + * allocated. + */ + auto default_init() -> void { m_kv_pair_log_event = nullptr; } + + /** + * Releases the memory allocated for underlying data fields. + */ + auto clean() -> void { delete m_kv_pair_log_event; } + + [[nodiscard]] auto get_kv_pair_log_event() const -> clp::ffi::KeyValuePairLogEvent const& { + return *m_kv_pair_log_event; + } + + /** + * Gets the `PyTypeObject` that represents `PyKeyValuePair`'s Python type. This type is + * dynamically created and initialized during the execution of `module_level_init`. + * @return Python type object associated with `PyKeyValuePairLogEvent`. + */ + [[nodiscard]] static auto get_py_type() -> PyTypeObject*; + + /** + * Creates and initializes `PyKeyValuePairLogEvent` as a Python type, and then incorporates this + * type as a Python object into the py_module module. + * @param py_module This is the Python module where the initialized `PyKeyValuePairLogEvent` + * will be incorporated. + * @return true on success. + * @return false on failure with the relevant Python exception and error set. + */ + [[nodiscard]] static auto module_level_init(PyObject* py_module) -> bool; + +private: + PyObject_HEAD; + + // NOLINTNEXTLINE(cppcoreguidelines-owning-memory) + gsl::owner m_kv_pair_log_event; + + static inline PyObjectStaticPtr m_py_type{nullptr}; +}; +} // namespace clp_ffi_py::ir::native + +#endif // CLP_FFI_PY_IR_NATIVE_PYKEYVALUEPAIRLOGEVENT_HPP diff --git a/src/clp_ffi_py/ir/native/error_messages.hpp b/src/clp_ffi_py/ir/native/error_messages.hpp index c598475f..fc13720c 100644 --- a/src/clp_ffi_py/ir/native/error_messages.hpp +++ b/src/clp_ffi_py/ir/native/error_messages.hpp @@ -1,6 +1,8 @@ #ifndef CLP_FFI_PY_IR_NATIVE_ERROR_MESSAGES #define CLP_FFI_PY_IR_NATIVE_ERROR_MESSAGES +#include + namespace clp_ffi_py::ir::native { constexpr char const* cDeserializerBufferOverflowError = "DeserializerBuffer internal read buffer overflows."; @@ -13,6 +15,21 @@ constexpr char const* cSerializePreambleError = "Native serializer cannot serialize the given preamble"; constexpr char const* cSerializeMessageError = "Native serializer cannot serialize the given message"; + +constexpr std::string_view cSerializerCreateErrorFormatStr{"Native `Serializer::create` failed: %s" +}; +constexpr std::string_view cSerializerSerializeMsgpackMapError{ + "Native `Serializer::serialize_msgpack_map` failed" +}; +constexpr std::string_view cDeserializerCreateErrorFormatStr{ + "Native `Deserializer::create` failed: %s" +}; +constexpr std::string_view cDeserializerDeserializeNextIrUnitErrorFormatStr{ + "Native `Deserializer::deserialize_next_ir_unit` failed: %s" +}; +constexpr std::string_view cKeyValuePairLogEventSerializeToStringErrorFormatStr{ + "Native `KeyValuePairLogEvent::serialize_to_json` failed: %s" +}; } // namespace clp_ffi_py::ir::native #endif // CLP_FFI_PY_IR_NATIVE_ERROR_MESSAGES diff --git a/src/clp_ffi_py/modules/ir_native.cpp b/src/clp_ffi_py/modules/ir_native.cpp index b0b509c7..c362d7be 100644 --- a/src/clp_ffi_py/modules/ir_native.cpp +++ b/src/clp_ffi_py/modules/ir_native.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -69,5 +70,10 @@ PyMODINIT_FUNC PyInit_native() { return nullptr; } + if (false == clp_ffi_py::ir::native::PyKeyValuePairLogEvent::module_level_init(new_module)) { + Py_DECREF(new_module); + return nullptr; + } + return new_module; } diff --git a/src/clp_ffi_py/utils.cpp b/src/clp_ffi_py/utils.cpp index e5caa604..eae478e3 100644 --- a/src/clp_ffi_py/utils.cpp +++ b/src/clp_ffi_py/utils.cpp @@ -3,6 +3,11 @@ #include "utils.hpp" #include +#include +#include + +#include +#include #include @@ -59,4 +64,15 @@ auto get_py_bool(bool is_true) -> PyObject* { } Py_RETURN_FALSE; } + +auto unpack_msgpack(std::span msgpack_byte_sequence +) -> outcome_v2::std_result { + msgpack::object_handle handle; + try { + msgpack::unpack(handle, msgpack_byte_sequence.data(), msgpack_byte_sequence.size()); + } catch (msgpack::unpack_error const& error) { + return std::string{error.what()}; + } + return handle; +} } // namespace clp_ffi_py diff --git a/src/clp_ffi_py/utils.hpp b/src/clp_ffi_py/utils.hpp index ad24b7b4..5c2395a7 100644 --- a/src/clp_ffi_py/utils.hpp +++ b/src/clp_ffi_py/utils.hpp @@ -4,11 +4,14 @@ #include // Must always be included before any other header files #include +#include #include #include #include #include +#include +#include namespace clp_ffi_py { /** @@ -60,6 +63,15 @@ auto get_py_bool(bool is_true) -> PyObject*; template auto parse_py_int(PyObject* py_int, int_type& val) -> bool; +/** + * Unpacks the given msgpack byte sequence. + * @param msgpack_byte_sequence + * @return A result containing the unpack msgpack object handle on success or an error string + * indicating the unpack failure (forwarded from the thrown `msgpack::unpack_error`). + */ +[[nodiscard]] auto unpack_msgpack(std::span msgpack_byte_sequence +) -> outcome_v2::std_result; + /** * A template that always evaluates as false. */ From 273aed693b05b368b90c89f633ca8528c8d50fc4 Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Sun, 3 Nov 2024 16:42:12 -0500 Subject: [PATCH 07/50] Add unit tests --- tests/test_ir/__init__.py | 1 + tests/test_ir/test_json/elasticsearch.json | 100 ++++++ tests/test_ir/test_json/spark-event-logs.json | 332 ++++++++++++++++++ .../test_ir/test_key_value_pair_log_event.py | 31 ++ tests/test_ir/test_utils.py | 30 +- tests/test_utils.py | 3 +- 6 files changed, 494 insertions(+), 3 deletions(-) create mode 100644 tests/test_ir/test_json/elasticsearch.json create mode 100644 tests/test_ir/test_json/spark-event-logs.json create mode 100644 tests/test_ir/test_key_value_pair_log_event.py diff --git a/tests/test_ir/__init__.py b/tests/test_ir/__init__.py index 8c2b8968..88e25bf9 100644 --- a/tests/test_ir/__init__.py +++ b/tests/test_ir/__init__.py @@ -6,6 +6,7 @@ from test_ir.test_deserializer import * # noqa from test_ir.test_deserializer_buffer import * # noqa from test_ir.test_encoder import * # noqa +from test_ir.test_key_value_pair_log_event import * # noqa from test_ir.test_log_event import * # noqa from test_ir.test_metadata import * # noqa from test_ir.test_query import * # noqa diff --git a/tests/test_ir/test_json/elasticsearch.json b/tests/test_ir/test_json/elasticsearch.json new file mode 100644 index 00000000..c6368011 --- /dev/null +++ b/tests/test_ir/test_json/elasticsearch.json @@ -0,0 +1,100 @@ +{"@timestamp":"2023-03-28T04:00:00.040Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#7]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [abacus_go]"]} +{"@timestamp":"2023-03-28T04:00:00.202Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#1]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [rider_product_cored]"]} +{"@timestamp":"2023-03-28T04:00:00.202Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#4]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [fares_management]"]} +{"@timestamp":"2023-03-28T04:00:00.201Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#8]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [fulfillment_compatibled]"]} +{"@timestamp":"2023-03-28T04:00:00.201Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#3]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [k8s_apiserver]"]} +{"@timestamp":"2023-03-28T04:00:00.202Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#6]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [event_logs]"]} +{"@timestamp":"2023-03-28T04:00:00.202Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#2]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [k8s_kubelet]"]} +{"@timestamp":"2023-03-28T04:00:00.203Z", "log.level":"TRACE", "message":"Processing scheduled tasks started", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][generic][T#9]","log.logger":"org.elasticsearch.xpack.transform.transforms.scheduling.TransformScheduler","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:00.240Z", "log.level":"TRACE", "message":"checking jobs [2023-03-28T04:00:00.240275019Z]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"ticker-schedule-trigger-engine","log.logger":"org.elasticsearch.xpack.watcher.trigger.schedule.engine.TickerScheduleTriggerEngine","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:00.069Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#5]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [driverapp]"]} +{"@timestamp":"2023-03-28T04:00:01.119Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#7]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [michelangelo_gateway]"]} +{"@timestamp":"2023-03-28T04:00:01.119Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#6]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [finprod_rewards_eligibility]"]} +{"@timestamp":"2023-03-28T04:00:01.119Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#8]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [.geoip_databases]"]} +{"@timestamp":"2023-03-28T04:00:01.119Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#2]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [fares_intelligence]"]} +{"@timestamp":"2023-03-28T04:00:01.892Z", "log.level":"TRACE", "message":"Processing scheduled tasks finished, took 0ms", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][generic][T#9]","log.logger":"org.elasticsearch.xpack.transform.transforms.scheduling.TransformScheduler","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:01.892Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#6]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [access_platform_core]"]} +{"@timestamp":"2023-03-28T04:00:01.320Z", "log.level":"TRACE", "message":"scheduling global_checkpoint_sync every 30s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][generic][T#7]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [k8s_kubelet]"]} +{"@timestamp":"2023-03-28T04:00:01.329Z", "log.level":"DEBUG", "message":"no retention leases are expired from current retention leases [RetentionLeases{primaryTerm=1, version=1, leases={peer_recovery/iKPGCkp9RVOKXOj20uOt4g=RetentionLease{id='peer_recovery/iKPGCkp9RVOKXOj20uOt4g', retainingSequenceNumber=0, timestamp=1679973869034, source='peer recovery'}}}]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][management][T#2]","log.logger":"org.elasticsearch.index.seqno.ReplicationTracker","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [k8s_kubelet][0]"]} +{"@timestamp":"2023-03-28T04:00:01.619Z", "log.level":"TRACE", "message":"checking jobs [2023-03-28T04:00:01.619921956Z]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"ticker-schedule-trigger-engine","log.logger":"org.elasticsearch.xpack.watcher.trigger.schedule.engine.TickerScheduleTriggerEngine","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:01.892Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#8]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [web_wok]"]} +{"@timestamp":"2023-03-28T04:00:01.893Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#1]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [fare_route]"]} +{"@timestamp":"2023-03-28T04:00:01.119Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#5]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [invoice_gen]"]} +{"@timestamp":"2023-03-28T04:00:01.893Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#7]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [ad_bidder]"]} +{"@timestamp":"2023-03-28T04:00:01.893Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#3]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [apphealth]"]} +{"@timestamp":"2023-03-28T04:00:01.893Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#4]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [eats_store]"]} +{"@timestamp":"2023-03-28T04:00:01.894Z", "log.level":"TRACE", "message":"background syncing retention leases [RetentionLeases{primaryTerm=1, version=1, leases={peer_recovery/iKPGCkp9RVOKXOj20uOt4g=RetentionLease{id='peer_recovery/iKPGCkp9RVOKXOj20uOt4g', retainingSequenceNumber=0, timestamp=1679973869034, source='peer recovery'}}}] after expiration check", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][management][T#2]","log.logger":"org.elasticsearch.index.shard.IndexShard","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [k8s_kubelet][0]"]} +{"@timestamp":"2023-03-28T04:00:01.895Z", "log.level":"TRACE", "message":"register 617 [transport] [retention_lease_background_sync] [retention_lease_background_sync shardId=[k8s_kubelet][0]]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][management][T#2]","log.logger":"org.elasticsearch.tasks.TaskManager","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:01.895Z", "log.level":"TRACE", "message":"[324][indices:admin/seq_no/retention_lease_background_sync[p]] sent to [{hostb9}{iKPGCkp9RVOKXOj20uOt4g}{cPRh9FqiSlGP9jClhm8Ugg}{hostb9}{127.0.0.1}{127.0.0.1:9300}{cdfhilmrstw}{ml.max_jvm_size=33285996544, ml.allocated_processors_double=16.0, xpack.installed=true, ml.machine_memory=135062048768, ml.allocated_processors=16}] (timeout: [null])", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][management][T#2]","log.logger":"org.elasticsearch.transport.TransportService.tracer","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:01.896Z", "log.level":"TRACE", "message":"[324][indices:admin/seq_no/retention_lease_background_sync[p]] received request", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][management][T#2]","log.logger":"org.elasticsearch.transport.TransportService.tracer","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:01.896Z", "log.level":"TRACE", "message":"scheduling retention_lease_sync every 30s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][management][T#2]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [k8s_kubelet]"]} +{"@timestamp":"2023-03-28T04:00:01.896Z", "log.level":"TRACE", "message":"register 618 [direct] [indices:admin/seq_no/retention_lease_background_sync[p]] [retention_lease_background_sync shardId=[k8s_kubelet][0]]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][management][T#1]","log.logger":"org.elasticsearch.tasks.TaskManager","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:01.896Z", "log.level":"TRACE", "message":"skipping persisting retention leases [RetentionLeases{primaryTerm=1, version=1, leases={peer_recovery/iKPGCkp9RVOKXOj20uOt4g=RetentionLease{id='peer_recovery/iKPGCkp9RVOKXOj20uOt4g', retainingSequenceNumber=0, timestamp=1679973869034, source='peer recovery'}}}], already persisted", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][management][T#1]","log.logger":"org.elasticsearch.index.seqno.ReplicationTracker","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [k8s_kubelet][0]"]} +{"@timestamp":"2023-03-28T04:00:01.897Z", "log.level":"TRACE", "message":"[[k8s_kubelet][0]] op [indices:admin/seq_no/retention_lease_background_sync] completed on primary for request [RetentionLeaseBackgroundSyncAction.Request{retentionLeases=RetentionLeases{primaryTerm=1, version=1, leases={peer_recovery/iKPGCkp9RVOKXOj20uOt4g=RetentionLease{id='peer_recovery/iKPGCkp9RVOKXOj20uOt4g', retainingSequenceNumber=0, timestamp=1679973869034, source='peer recovery'}}}, shardId=[k8s_kubelet][0], timeout=1m, index='k8s_kubelet', waitForActiveShards=0}]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][management][T#1]","log.logger":"org.elasticsearch.index.seqno.RetentionLeaseBackgroundSyncAction","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:01.897Z", "log.level":"TRACE", "message":"skipped updating local checkpoint of [Q4Xb11JATPaFZqmgTQHcVQ] from [-1] to [-1], current checkpoint is higher", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][management][T#1]","log.logger":"org.elasticsearch.index.seqno.ReplicationTracker","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [k8s_kubelet][0]"]} +{"@timestamp":"2023-03-28T04:00:01.897Z", "log.level":"TRACE", "message":"unregister task for id: 618", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][management][T#1]","log.logger":"org.elasticsearch.tasks.TaskManager","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:01.898Z", "log.level":"TRACE", "message":"[324][indices:admin/seq_no/retention_lease_background_sync[p]] sent response", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][management][T#1]","log.logger":"org.elasticsearch.transport.TransportService.tracer","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:01.898Z", "log.level":"TRACE", "message":"[324][indices:admin/seq_no/retention_lease_background_sync[p]] received response from [{hostb9}{iKPGCkp9RVOKXOj20uOt4g}{cPRh9FqiSlGP9jClhm8Ugg}{hostb9}{127.0.0.1}{127.0.0.1:9300}{cdfhilmrstw}{ml.max_jvm_size=33285996544, ml.allocated_processors_double=16.0, xpack.installed=true, ml.machine_memory=135062048768, ml.allocated_processors=16}]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][management][T#1]","log.logger":"org.elasticsearch.transport.TransportService.tracer","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:01.898Z", "log.level":"TRACE", "message":"unregister task for id: 617", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][management][T#1]","log.logger":"org.elasticsearch.tasks.TaskManager","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:02.039Z", "log.level":"TRACE", "message":"[apphealth][0] top warming [ElasticsearchDirectoryReader()]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][generic][T#3]","log.logger":"org.elasticsearch.index.IndexWarmer","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:02.040Z", "log.level":"TRACE", "message":"top warming took [43.9micros]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][generic][T#3]","log.logger":"org.elasticsearch.index.warmer.ShardIndexWarmerService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [apphealth][0]"]} +{"@timestamp":"2023-03-28T04:00:02.040Z", "log.level":"DEBUG", "message":"state: [RECOVERING]->[POST_RECOVERY], reason [post recovery from shard_store]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][generic][T#3]","log.logger":"org.elasticsearch.index.shard.IndexShard","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [apphealth][0]"]} +{"@timestamp":"2023-03-28T04:00:02.040Z", "log.level":"TRACE", "message":"recovery completed from [shard_store], took [13.4s]\n index : files [0] with total_size [0b], took[9.5s]\n : recovered_files [0] with total_size [0b]\n : reusing_files [0] with total_size [0b]\n verify_index : took [0s], check_index [0s]\n translog : number_of_operations [0], took [3.7s]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][generic][T#3]","log.logger":"org.elasticsearch.index.shard.IndexShard","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [apphealth][0]"]} +{"@timestamp":"2023-03-28T04:00:02.041Z", "log.level":"DEBUG", "message":"sending [internal:cluster/shard/started] to [iKPGCkp9RVOKXOj20uOt4g] for shard entry [StartedShardEntry{shardId [[apphealth][0]], allocationId [Abip50RxQTmavqi4NTwSqQ], primary term [1], message [after new shard recovery]}]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][generic][T#3]","log.logger":"org.elasticsearch.cluster.action.shard.ShardStateAction","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:02.041Z", "log.level":"TRACE", "message":"[325][internal:cluster/shard/started] sent to [{hostb9}{iKPGCkp9RVOKXOj20uOt4g}{cPRh9FqiSlGP9jClhm8Ugg}{hostb9}{127.0.0.1}{127.0.0.1:9300}{cdfhilmrstw}{ml.max_jvm_size=33285996544, ml.allocated_processors_double=16.0, xpack.installed=true, ml.machine_memory=135062048768, ml.allocated_processors=16}] (timeout: [null])", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][generic][T#3]","log.logger":"org.elasticsearch.transport.TransportService.tracer","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:02.041Z", "log.level":"TRACE", "message":"[325][internal:cluster/shard/started] received request", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][generic][T#3]","log.logger":"org.elasticsearch.transport.TransportService.tracer","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:02.041Z", "log.level":"TRACE", "message":"register 619 [direct] [internal:cluster/shard/started] []", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][generic][T#3]","log.logger":"org.elasticsearch.tasks.TaskManager","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:02.042Z", "log.level":"DEBUG", "message":"[apphealth][0] received shard started for [StartedShardEntry{shardId [[apphealth][0]], allocationId [Abip50RxQTmavqi4NTwSqQ], primary term [1], message [after new shard recovery]}]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][generic][T#3]","log.logger":"org.elasticsearch.cluster.action.shard.ShardStateAction","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:02.042Z", "log.level":"TRACE", "message":"will process [shard-started StartedShardEntry{shardId [[apphealth][0]], allocationId [Abip50RxQTmavqi4NTwSqQ], primary term [1], message [after new shard recovery]}[StartedShardUpdateTask{entry=StartedShardEntry{shardId [[apphealth][0]], allocationId [Abip50RxQTmavqi4NTwSqQ], primary term [1], message [after new shard recovery]}, listener=ChannelActionListener{org.elasticsearch.transport.TaskTransportChannel@1aeede97}{StartedShardEntry{shardId [[apphealth][0]], allocationId [Abip50RxQTmavqi4NTwSqQ], primary term [1], message [after new shard recovery]}}{internal:cluster/shard/started}}]]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.service.MasterService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:02.042Z", "log.level":"DEBUG", "message":"executing cluster state update for [shard-started StartedShardEntry{shardId [[apphealth][0]], allocationId [Abip50RxQTmavqi4NTwSqQ], primary term [1], message [after new shard recovery]}[StartedShardUpdateTask{entry=StartedShardEntry{shardId [[apphealth][0]], allocationId [Abip50RxQTmavqi4NTwSqQ], primary term [1], message [after new shard recovery]}, listener=ChannelActionListener{org.elasticsearch.transport.TaskTransportChannel@1aeede97}{StartedShardEntry{shardId [[apphealth][0]], allocationId [Abip50RxQTmavqi4NTwSqQ], primary term [1], message [after new shard recovery]}}{internal:cluster/shard/started}}]]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.service.MasterService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:02.043Z", "log.level":"DEBUG", "message":"[apphealth][0] starting shard [apphealth][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], recovery_source[new shard recovery], s[INITIALIZING], a[id=Abip50RxQTmavqi4NTwSqQ], unassigned_info[[reason=INDEX_CREATED], at[2023-03-28T03:59:38.780Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0], expected_shard_size[225] (shard started task: [StartedShardEntry{shardId [[apphealth][0]], allocationId [Abip50RxQTmavqi4NTwSqQ], primary term [1], message [after new shard recovery]}])", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.action.shard.ShardStateAction","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:02.043Z", "log.level":"TRACE", "message":"[apphealth][0] marked shard as started (routing: [apphealth][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], recovery_source[new shard recovery], s[INITIALIZING], a[id=Abip50RxQTmavqi4NTwSqQ], unassigned_info[[reason=INDEX_CREATED], at[2023-03-28T03:59:38.780Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0], expected_shard_size[225])", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.routing.allocation.AllocationService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:02.045Z", "log.level":"DEBUG", "message":"took [2ms] to compute cluster state update for [shard-started StartedShardEntry{shardId [[apphealth][0]], allocationId [Abip50RxQTmavqi4NTwSqQ], primary term [1], message [after new shard recovery]}[StartedShardUpdateTask{entry=StartedShardEntry{shardId [[apphealth][0]], allocationId [Abip50RxQTmavqi4NTwSqQ], primary term [1], message [after new shard recovery]}, listener=ChannelActionListener{org.elasticsearch.transport.TaskTransportChannel@1aeede97}{StartedShardEntry{shardId [[apphealth][0]], allocationId [Abip50RxQTmavqi4NTwSqQ], primary term [1], message [after new shard recovery]}}{internal:cluster/shard/started}}]]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.service.MasterService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:02.045Z", "log.level":"TRACE", "message":"register 620 [master] [publish_cluster_state_update] [publication of cluster state [154]]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.tasks.TaskManager","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:02.054Z", "log.level":"TRACE", "message":"cluster state updated, source [shard-started StartedShardEntry{shardId [[apphealth][0]], allocationId [Abip50RxQTmavqi4NTwSqQ], primary term [1], message [after new shard recovery]}[StartedShardUpdateTask{entry=StartedShardEntry{shardId [[apphealth][0]], allocationId [Abip50RxQTmavqi4NTwSqQ], primary term [1], message [after new shard recovery]}, listener=ChannelActionListener{org.elasticsearch.transport.TaskTransportChannel@1aeede97}{StartedShardEntry{shardId [[apphealth][0]], allocationId [Abip50RxQTmavqi4NTwSqQ], primary term [1], message [after new shard recovery]}}{internal:cluster/shard/started}}]]\ncluster uuid: bp-xkJD1S1iyBlj6I_JRHA [committed: true]\nversion: 154\nstate uuid: Xx549IAcSMCY30xGiugtFg\nfrom_diff: false\nmeta data version: 56\n coordination_metadata:\n term: 6\n last_committed_config: VotingConfiguration{iKPGCkp9RVOKXOj20uOt4g}\n last_accepted_config: VotingConfiguration{iKPGCkp9RVOKXOj20uOt4g}\n voting tombstones: []\n [k8s_kubelet/fQ9Iy7wlTKahsWLbs-oE_Q]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [Q4Xb11JATPaFZqmgTQHcVQ]\n [fares_intelligence/pSRJWxyXSqefhwLVhmT9nQ]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [-dD8ROsgS6qaYMoalciocA]\n [event_logs/DcomuHoGQzOjZq2bX67TFg]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [1uCl_LIDSBCWh6koVA0fjQ]\n [rider_product_cored/AdGiEBLBQbicsXDomsa3mw]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [GwRH_ZlFRtCV7m_n626z7g]\n [access_platform_core/l4kdkOEQRaKfWGLpTZURCQ]: v[6], mv[1], sv[2], av[1]\n 0: p_term [1], isa_ids [6ajNSTQZSr6tP3YtmRaxgw]\n [.geoip_databases/5k1GHwKvQBKersi12DFh4A]: v[6], mv[1], sv[1], av[1]\n 0: p_term [3], isa_ids [UdvCKUGqTEmYFzZlnSu_tA]\n [web_wok/R1IULAXSQ4SC0pLFCU5oJA]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [JXruVMhdRVGvP1kQCbUNSQ]\n [abacus_go/cfvmY-Y_Su63-qSkPODDzg]: v[6], mv[1], sv[2], av[1]\n 0: p_term [1], isa_ids [AMhgJMlSTPOoKSsL7s3OKA]\n [k8s_apiserver/qQasynZgQ6e6AmD5ZWqh3w]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [Aaa0F0jiStCkteGDYR8yRQ]\n [michelangelo_gateway/ZV2vZ8PmS0uysg-xzKO7QA]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [k5vDE54CSTSakhfgtGaTcA]\n [driverapp/cqbade6KQFqLEK-wPKlkBg]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [b5_Bhd3ITjOkl-xLF8H1jg]\n [apphealth/rcPrX1aWRmeED9W5PU6A4g]: v[4], mv[1], sv[1], av[1]\n 0: p_term [1], isa_ids [Abip50RxQTmavqi4NTwSqQ]\n [fares_management/F1lH-QvyTkS5hnbDFwE2Xg]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [sOxfKX3nReynYN5HN88cuQ]\n [finprod_rewards_eligibility/dJNRGPwrQS2bIPlpsXgUoQ]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [Hg4Dm9O8QEqXWKY7rH1X0A]\n [fare_route/a4fjivWhQRee8JW7ZbXiXw]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [ltSLFgu4Rry0_0vXq28DbA]\n [ad_bidder/tLHfw5r2QH-o7lG5tapKxw]: v[6], mv[1], sv[2], av[1]\n 0: p_term [1], isa_ids [YtEUwPgfQDGM0Cm_C4hXlw]\n [fulfillment_compatibled/tMgrGBWeR3irgR5PyN8Oig]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [D_3C_3kMSNKa-XJk7pyoGg]\n [eats_store/hF-kN4XFRy--z_gUhYLwdA]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [WRNwRQ6gTNGfSNQok4iSyQ]\n [invoice_gen/RPu5EzJVRfuqyI8MQ7iTjQ]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [onWbjhHuTD2C3AEayRJq1w]\nmetadata customs:\n licenses: LicensesMetadata{license={\"uid\":\"8e65ab16-3f3f-48cc-ab5f-e20097ceb81e\",\"type\":\"basic\",\"issue_date_in_millis\":1679968689560,\"max_nodes\":1000,\"max_resource_units\":null,\"issued_to\":\"elasticsearch\",\"issuer\":\"elasticsearch\",\"signature\":\"////+wAAAOBYLHp18OvjZrRTi0AmCZ6DOpkrd2cXJB/ffBB+AxaDsF+z7lkfdPbgYofMvUB7XbiD5j95mIfUcdlTJHVdk9NpPTFMKL5wC7BXn4CTS6kNox+Xi0+qSLpcgHzelOT63alJr1L5j/2pqffHnuMLG88Ye15eFhAb9H2A5MS3hX0HG9uTxyEidFCyu4gWV4qrZJDQKLpzC49MMnpP+ob2bNH/pA7dGVdWqEJpVNPe5/hAtGtCdVdUX6DvV0Q3dgwuejc0hRg/vl0YUtp45bQPBHDHECNKgxWmJCZrfstl/prRHA==\",\"start_date_in_millis\":-1}, trialVersion=null} component_template: {\"component_template\":{\"metrics-mappings\":{\"template\":{\"mappings\":{\"dynamic_templates\":[{\"long_metrics\":{\"mapping\":{\"index\":false,\"type\":\"long\"},\"match_mapping_type\":\"long\"}},{\"double_metrics\":{\"mapping\":{\"index\":false,\"type\":\"float\"},\"match_mapping_type\":\"double\"}}],\"properties\":{\"data_stream\":{\"properties\":{\"type\":{\"type\":\"constant_keyword\",\"value\":\"metrics\"}}}}}},\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default mappings for the metrics index template installed by x-pack\"}},\".deprecation-indexing-mappings\":{\"template\":{\"mappings\":{\"dynamic_templates\":[{\"strings_as_keyword\":{\"mapping\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"match_mapping_type\":\"string\"}}],\"date_detection\":false,\"properties\":{\"@timestamp\":{\"type\":\"date\"},\"ecs\":{\"properties\":{\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"data_stream\":{\"properties\":{\"namespace\":{\"type\":\"constant_keyword\"},\"type\":{\"type\":\"constant_keyword\",\"value\":\"logs\"},\"dataset\":{\"type\":\"constant_keyword\"}}},\"host\":{\"properties\":{\"ip\":{\"type\":\"ip\"}}},\"message\":{\"type\":\"text\"}}}},\"version\":1,\"_meta\":{\"managed\":true,\"description\":\"default mappings for Stack deprecation logs index template installed by x-pack\"}},\"synthetics-mappings\":{\"template\":{\"mappings\":{\"properties\":{\"data_stream\":{\"properties\":{\"type\":{\"type\":\"constant_keyword\",\"value\":\"synthetics\"}}}}}},\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default mappings for the synthetics index template installed by x-pack\"}},\"metrics-settings\":{\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\"metrics\"},\"codec\":\"best_compression\",\"query\":{\"default_field\":[\"message\"]}}}},\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default settings for the metrics index template installed by x-pack\"}},\"logs-settings\":{\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\"logs\"},\"codec\":\"best_compression\",\"query\":{\"default_field\":[\"message\"]}}}},\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default settings for the logs index template installed by x-pack\"}},\".deprecation-indexing-settings\":{\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\".deprecation-indexing-ilm-policy\"},\"codec\":\"best_compression\",\"hidden\":\"true\",\"auto_expand_replicas\":\"0-1\",\"query\":{\"default_field\":[\"message\"]}}}},\"version\":1,\"_meta\":{\"managed\":true,\"description\":\"default settings for Stack deprecation logs index template installed by x-pack\"}},\"synthetics-settings\":{\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\"synthetics\"},\"codec\":\"best_compression\"}}},\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default settings for the synthetics index template installed by x-pack\"}},\"logs-mappings\":{\"template\":{\"mappings\":{\"properties\":{\"data_stream\":{\"properties\":{\"type\":{\"type\":\"constant_keyword\",\"value\":\"logs\"}}}}}},\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default mappings for the logs index template installed by x-pack\"}},\"data-streams-mappings\":{\"template\":{\"mappings\":{\"dynamic_templates\":[{\"match_ip\":{\"mapping\":{\"type\":\"ip\"},\"match_mapping_type\":\"string\",\"match\":\"ip\"}},{\"match_message\":{\"mapping\":{\"type\":\"match_only_text\"},\"match_mapping_type\":\"string\",\"match\":\"message\"}},{\"strings_as_keyword\":{\"mapping\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"match_mapping_type\":\"string\"}}],\"date_detection\":false,\"properties\":{\"@timestamp\":{\"type\":\"date\"},\"ecs\":{\"properties\":{\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"data_stream\":{\"properties\":{\"namespace\":{\"type\":\"constant_keyword\"},\"dataset\":{\"type\":\"constant_keyword\"}}},\"host\":{\"type\":\"object\"}}}},\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"general mapping conventions for data streams\"}}}} persistent_tasks: {\"last_allocation_id\":6,\"tasks\":[{\"id\":\"health-node\",\"task\":{\"health-node\":{\"params\":{}}},\"allocation_id\":5,\"assignment\":{\"executor_node\":\"iKPGCkp9RVOKXOj20uOt4g\",\"explanation\":\"\"},\"allocation_id_on_last_status_update\":0},{\"id\":\"geoip-downloader\",\"task\":{\"geoip-downloader\":{\"params\":{},\"state\":{\"databases\":{\"GeoLite2-ASN.mmdb\":{\"last_update\":1679968651799,\"last_check\":1679968651799,\"first_chunk\":0,\"last_chunk\":4,\"md5\":\"33268a0d7cb3ba8d9d581bf1e5dc0980\"},\"GeoLite2-Country.mmdb\":{\"last_update\":1679968809323,\"last_check\":1679968809323,\"first_chunk\":0,\"last_chunk\":2,\"md5\":\"da6d05a7b044ea35910f20f756c057ad\"},\"GeoLite2-City.mmdb\":{\"last_update\":1679968693385,\"last_check\":1679968693385,\"first_chunk\":0,\"last_chunk\":33,\"md5\":\"2d91f3e42689a12c2f496199befdfccf\"}}}}},\"allocation_id\":6,\"assignment\":{\"executor_node\":\"iKPGCkp9RVOKXOj20uOt4g\",\"explanation\":\"\"},\"allocation_id_on_last_status_update\":0}]} index_lifecycle: {\n \"policies\" : {\n \"90-days-default\" : {\n \"policy\" : {\n \"phases\" : {\n \"warm\" : {\n \"min_age\" : \"2d\",\n \"actions\" : {\n \"shrink\" : {\n \"number_of_shards\" : 1\n },\n \"forcemerge\" : {\n \"max_num_segments\" : 1\n }\n }\n },\n \"cold\" : {\n \"min_age\" : \"30d\",\n \"actions\" : { }\n },\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"90d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"built-in ILM policy using the hot, warm, and cold phases with a retention of 90 days\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968678193,\n \"modified_date_string\" : \"2023-03-28T01:57:58.193Z\"\n },\n \"ilm-history-ilm-policy\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"90d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"default policy for the ILM history indices\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968685095,\n \"modified_date_string\" : \"2023-03-28T01:58:05.095Z\"\n },\n \"watch-history-ilm-policy-16\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"3d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"4d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"default policy for the watcher history indices\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968647327,\n \"modified_date_string\" : \"2023-03-28T01:57:27.327Z\"\n },\n \"30-days-default\" : {\n \"policy\" : {\n \"phases\" : {\n \"warm\" : {\n \"min_age\" : \"2d\",\n \"actions\" : {\n \"shrink\" : {\n \"number_of_shards\" : 1\n },\n \"forcemerge\" : {\n \"max_num_segments\" : 1\n }\n }\n },\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"30d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"built-in ILM policy using the hot and warm phases with a retention of 30 days\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968662496,\n \"modified_date_string\" : \"2023-03-28T01:57:42.496Z\"\n },\n \"synthetics\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"default policy for the synthetics index template installed by x-pack\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968682454,\n \"modified_date_string\" : \"2023-03-28T01:58:02.454Z\"\n },\n \".monitoring-8-ilm-policy\" : {\n \"policy\" : {\n \"phases\" : {\n \"warm\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"forcemerge\" : {\n \"max_num_segments\" : 1\n }\n }\n },\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"3d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"3d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"defaults\" : {\n \"delete_min_age\" : \"Using value of [3d] based on the monitoring plugin default\"\n },\n \"description\" : \"Index lifecycle policy generated for [monitoring-*-8] data streams\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968676492,\n \"modified_date_string\" : \"2023-03-28T01:57:56.492Z\"\n },\n \"slm-history-ilm-policy\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"90d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"default policy for the SLM history indices\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968644040,\n \"modified_date_string\" : \"2023-03-28T01:57:24.040Z\"\n },\n \".fleet-actions-results-ilm-policy\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_size\" : \"300gb\",\n \"max_age\" : \"30d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"90d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"default policy for fleet action results indices\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968686060,\n \"modified_date_string\" : \"2023-03-28T01:58:06.060Z\"\n },\n \"365-days-default\" : {\n \"policy\" : {\n \"phases\" : {\n \"warm\" : {\n \"min_age\" : \"2d\",\n \"actions\" : {\n \"shrink\" : {\n \"number_of_shards\" : 1\n },\n \"forcemerge\" : {\n \"max_num_segments\" : 1\n }\n }\n },\n \"cold\" : {\n \"min_age\" : \"30d\",\n \"actions\" : { }\n },\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"365d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"built-in ILM policy using the hot, warm, and cold phases with a retention of 365 days\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968675542,\n \"modified_date_string\" : \"2023-03-28T01:57:55.542Z\"\n },\n \"7-days-default\" : {\n \"policy\" : {\n \"phases\" : {\n \"warm\" : {\n \"min_age\" : \"2d\",\n \"actions\" : {\n \"shrink\" : {\n \"number_of_shards\" : 1\n },\n \"forcemerge\" : {\n \"max_num_segments\" : 1\n }\n }\n },\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"7d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"7d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"built-in ILM policy using the hot and warm phases with a retention of 7 days\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968672404,\n \"modified_date_string\" : \"2023-03-28T01:57:52.404Z\"\n },\n \".deprecation-indexing-ilm-policy\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"10gb\",\n \"max_age\" : \"30d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"30d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"ILM policy used for deprecation logs\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968670134,\n \"modified_date_string\" : \"2023-03-28T01:57:50.134Z\"\n },\n \"metrics\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"default policy for the metrics index template installed by x-pack\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968660707,\n \"modified_date_string\" : \"2023-03-28T01:57:40.707Z\"\n },\n \"ml-size-based-ilm-policy\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\"\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"default policy for machine learning state and stats indices\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968650499,\n \"modified_date_string\" : \"2023-03-28T01:57:30.499Z\"\n },\n \"180-days-default\" : {\n \"policy\" : {\n \"phases\" : {\n \"warm\" : {\n \"min_age\" : \"2d\",\n \"actions\" : {\n \"shrink\" : {\n \"number_of_shards\" : 1\n },\n \"forcemerge\" : {\n \"max_num_segments\" : 1\n }\n }\n },\n \"cold\" : {\n \"min_age\" : \"30d\",\n \"actions\" : { }\n },\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"180d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"built-in ILM policy using the hot, warm, and cold phases with a retention of 180 days\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968681386,\n \"modified_date_string\" : \"2023-03-28T01:58:01.386Z\"\n },\n \"logs\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"default policy for the logs index template installed by x-pack\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968679651,\n \"modified_date_string\" : \"2023-03-28T01:57:59.651Z\"\n }\n },\n \"operation_mode\" : \"RUNNING\"\n} index_template: {\"index_template\":{\".watch-history-16\":{\"index_patterns\":[\".watcher-history-16*\"],\"template\":{\"settings\":{\"index\":{\"format\":\"6\",\"lifecycle\":{\"name\":\"watch-history-ilm-policy-16\"},\"hidden\":\"true\",\"number_of_shards\":\"1\",\"auto_expand_replicas\":\"0-1\",\"number_of_replicas\":\"0\"}},\"mappings\":{\"_meta\":{\"watcher-history-version\":\"16\"},\"dynamic\":false,\"dynamic_templates\":[{\"disabled_payload_fields\":{\"match_pattern\":\"regex\",\"path_match\":\"result\\\\.(input(\\\\..+)*|(transform(\\\\..+)*)|(actions\\\\.transform(\\\\..+)*))\\\\.payload\",\"mapping\":{\"type\":\"object\",\"enabled\":false}}},{\"disabled_search_request_body_fields\":{\"match_pattern\":\"regex\",\"path_match\":\"result\\\\.(input(\\\\..+)*|(transform(\\\\..+)*)|(actions\\\\.transform(\\\\..+)*))\\\\.search\\\\.request\\\\.(body|template)\",\"mapping\":{\"type\":\"object\",\"enabled\":false}}},{\"disabled_exception_fields\":{\"match_pattern\":\"regex\",\"path_match\":\"result\\\\.(input(\\\\..+)*|(transform(\\\\..+)*)|(actions\\\\.transform(\\\\..+)*)|actions)\\\\.error\",\"mapping\":{\"type\":\"object\",\"enabled\":false}}},{\"disabled_jira_custom_fields\":{\"path_match\":\"result.actions.jira.fields.customfield_*\",\"mapping\":{\"type\":\"object\",\"enabled\":false}}}],\"properties\":{\"exception\":{\"type\":\"object\",\"enabled\":false},\"metadata\":{\"dynamic\":true,\"type\":\"object\"},\"trigger_event\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"schedule\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"scheduled_time\":{\"type\":\"date\"}}},\"triggered_time\":{\"type\":\"date\"},\"type\":{\"type\":\"keyword\"},\"manual\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"schedule\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"scheduled_time\":{\"type\":\"date\"}}}}}}},\"result\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"input\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"search\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"request\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"indices\":{\"type\":\"keyword\"},\"types\":{\"type\":\"keyword\"},\"search_type\":{\"type\":\"keyword\"}}}}},\"payload\":{\"type\":\"object\",\"enabled\":false},\"http\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"request\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"path\":{\"type\":\"keyword\"},\"host\":{\"type\":\"keyword\"}}}}},\"type\":{\"type\":\"keyword\"},\"status\":{\"type\":\"keyword\"}}},\"condition\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"compare\":{\"type\":\"object\",\"enabled\":false},\"array_compare\":{\"type\":\"object\",\"enabled\":false},\"type\":{\"type\":\"keyword\"},\"met\":{\"type\":\"boolean\"},\"script\":{\"type\":\"object\",\"enabled\":false},\"status\":{\"type\":\"keyword\"}}},\"transform\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"search\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"request\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"indices\":{\"type\":\"keyword\"},\"types\":{\"type\":\"keyword\"}}}}},\"type\":{\"type\":\"keyword\"}}},\"execution_duration\":{\"type\":\"long\"},\"actions\":{\"include_in_parent\":true,\"dynamic\":true,\"type\":\"nested\",\"properties\":{\"reason\":{\"type\":\"keyword\"},\"foreach\":{\"type\":\"object\",\"enabled\":false},\"webhook\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"request\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"path\":{\"type\":\"keyword\"},\"host\":{\"type\":\"keyword\"}}}}},\"number_of_actions_executed\":{\"type\":\"integer\"},\"slack\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"sent_messages\":{\"include_in_parent\":true,\"dynamic\":true,\"type\":\"nested\",\"properties\":{\"reason\":{\"type\":\"text\"},\"request\":{\"type\":\"object\",\"enabled\":false},\"response\":{\"type\":\"object\",\"enabled\":false},\"to\":{\"type\":\"keyword\"},\"message\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"attachments\":{\"include_in_parent\":true,\"dynamic\":true,\"type\":\"nested\",\"properties\":{\"color\":{\"type\":\"keyword\"},\"fields\":{\"properties\":{\"value\":{\"type\":\"text\"}}}}},\"icon\":{\"type\":\"keyword\"},\"from\":{\"type\":\"text\"},\"text\":{\"type\":\"text\"}}},\"status\":{\"type\":\"keyword\"}}},\"account\":{\"type\":\"keyword\"}}},\"index\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"response\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"index\":{\"type\":\"keyword\"},\"id\":{\"type\":\"keyword\"},\"type\":{\"type\":\"keyword\"}}}}},\"pagerduty\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"sent_event\":{\"include_in_parent\":true,\"dynamic\":true,\"type\":\"nested\",\"properties\":{\"reason\":{\"type\":\"text\"},\"request\":{\"type\":\"object\",\"enabled\":false},\"response\":{\"type\":\"object\",\"enabled\":false},\"event\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"client_url\":{\"type\":\"keyword\"},\"context\":{\"include_in_parent\":true,\"dynamic\":true,\"type\":\"nested\",\"properties\":{\"src\":{\"type\":\"keyword\"},\"alt\":{\"type\":\"text\"},\"href\":{\"type\":\"keyword\"},\"type\":{\"type\":\"keyword\"}}},\"client\":{\"type\":\"text\"},\"description\":{\"type\":\"text\"},\"attach_payload\":{\"type\":\"boolean\"},\"incident_key\":{\"type\":\"keyword\"},\"type\":{\"type\":\"keyword\"},\"account\":{\"type\":\"keyword\"}}}}},\"account\":{\"type\":\"keyword\"}}},\"id\":{\"type\":\"keyword\"},\"type\":{\"type\":\"keyword\"},\"email\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"message\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"cc\":{\"type\":\"keyword\"},\"bcc\":{\"type\":\"keyword\"},\"reply_to\":{\"type\":\"keyword\"},\"from\":{\"type\":\"keyword\"},\"id\":{\"type\":\"keyword\"},\"to\":{\"type\":\"keyword\"}}}}},\"status\":{\"type\":\"keyword\"},\"jira\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"result\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"self\":{\"type\":\"keyword\"},\"id\":{\"type\":\"keyword\"},\"key\":{\"type\":\"keyword\"}}},\"reason\":{\"type\":\"text\"},\"request\":{\"type\":\"object\",\"enabled\":false},\"response\":{\"type\":\"object\",\"enabled\":false},\"fields\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"summary\":{\"type\":\"text\"},\"issuetype\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"name\":{\"type\":\"keyword\"},\"id\":{\"type\":\"keyword\"}}},\"description\":{\"type\":\"text\"},\"project\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"id\":{\"type\":\"keyword\"},\"key\":{\"type\":\"keyword\"}}},\"labels\":{\"type\":\"text\"}}},\"account\":{\"type\":\"keyword\"}}}}},\"execution_time\":{\"type\":\"date\"}}},\"node\":{\"type\":\"keyword\"},\"input\":{\"type\":\"object\",\"enabled\":false},\"condition\":{\"type\":\"object\",\"enabled\":false},\"watch_id\":{\"type\":\"keyword\"},\"messages\":{\"type\":\"text\"},\"vars\":{\"type\":\"object\",\"enabled\":false},\"state\":{\"type\":\"keyword\"},\"user\":{\"type\":\"text\"},\"status\":{\"dynamic\":true,\"type\":\"object\",\"enabled\":false}}}},\"composed_of\":[],\"priority\":2147483647,\"version\":16,\"_meta\":{\"managed\":true,\"description\":\"index template for watcher history indices\"},\"data_stream\":{\"hidden\":true,\"allow_custom_routing\":false},\"allow_auto_create\":true},\".monitoring-beats-mb\":{\"index_patterns\":[\".monitoring-beats-8-*\"],\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\".monitoring-8-ilm-policy\"},\"mapping\":{\"total_fields\":{\"limit\":\"2000\"}}}},\"mappings\":{\"dynamic\":false,\"properties\":{\"@timestamp\":{\"type\":\"date\"},\"cluster_uuid\":{\"path\":\"beat.elasticsearch.cluster.id\",\"type\":\"alias\"},\"service\":{\"properties\":{\"node\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"environment\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"hostname\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"origin\":{\"properties\":{\"node\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"environment\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"target\":{\"properties\":{\"node\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"environment\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"beat\":{\"properties\":{\"elasticsearch\":{\"properties\":{\"cluster\":{\"properties\":{\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"stats\":{\"properties\":{\"handles\":{\"properties\":{\"limit\":{\"properties\":{\"hard\":{\"type\":\"long\"},\"soft\":{\"type\":\"long\"}}},\"open\":{\"type\":\"long\"}}},\"apm_server\":{\"properties\":{\"server\":{\"properties\":{\"request\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"response\":{\"properties\":{\"valid\":{\"properties\":{\"count\":{\"type\":\"long\"},\"accepted\":{\"type\":\"long\"},\"ok\":{\"type\":\"long\"}}},\"count\":{\"type\":\"long\"},\"errors\":{\"properties\":{\"ratelimit\":{\"type\":\"long\"},\"internal\":{\"type\":\"long\"},\"toolarge\":{\"type\":\"long\"},\"method\":{\"type\":\"long\"},\"unauthorized\":{\"type\":\"long\"},\"forbidden\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"},\"closed\":{\"type\":\"long\"},\"decode\":{\"type\":\"long\"},\"queue\":{\"type\":\"long\"},\"concurrency\":{\"type\":\"long\"},\"validate\":{\"type\":\"long\"}}}}},\"concurrent\":{\"properties\":{\"wait\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}}}},\"acm\":{\"properties\":{\"request\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"response\":{\"properties\":{\"valid\":{\"properties\":{\"notmodified\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"},\"accepted\":{\"type\":\"long\"},\"ok\":{\"type\":\"long\"}}},\"request\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"count\":{\"type\":\"long\"},\"unset\":{\"type\":\"long\"},\"errors\":{\"properties\":{\"ratelimit\":{\"type\":\"long\"},\"internal\":{\"type\":\"long\"},\"toolarge\":{\"type\":\"long\"},\"method\":{\"type\":\"long\"},\"unavailable\":{\"type\":\"long\"},\"forbidden\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"},\"decode\":{\"type\":\"long\"},\"unauthorized\":{\"type\":\"long\"},\"notfound\":{\"type\":\"long\"},\"closed\":{\"type\":\"long\"},\"invalidquery\":{\"type\":\"long\"},\"queue\":{\"type\":\"long\"},\"validate\":{\"type\":\"long\"}}}}}}},\"decoder\":{\"properties\":{\"deflate\":{\"properties\":{\"content-length\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"reader\":{\"properties\":{\"size\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"gzip\":{\"properties\":{\"content-length\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"uncompressed\":{\"properties\":{\"content-length\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"missing-content-length\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"processor\":{\"properties\":{\"metric\":{\"properties\":{\"transformations\":{\"type\":\"long\"},\"decoding\":{\"properties\":{\"count\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}},\"validation\":{\"properties\":{\"count\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}}}},\"sourcemap\":{\"properties\":{\"counter\":{\"type\":\"long\"},\"decoding\":{\"properties\":{\"count\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}},\"validation\":{\"properties\":{\"count\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}}}},\"error\":{\"properties\":{\"stacktraces\":{\"type\":\"long\"},\"spans\":{\"type\":\"long\"},\"frames\":{\"type\":\"long\"},\"transformations\":{\"type\":\"long\"},\"decoding\":{\"properties\":{\"count\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}},\"validation\":{\"properties\":{\"count\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}}}},\"transaction\":{\"properties\":{\"stacktraces\":{\"type\":\"long\"},\"spans\":{\"type\":\"long\"},\"frames\":{\"type\":\"long\"},\"transformations\":{\"type\":\"long\"},\"transactions\":{\"type\":\"long\"},\"decoding\":{\"properties\":{\"count\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}},\"validation\":{\"properties\":{\"count\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}}}},\"span\":{\"properties\":{\"transformations\":{\"type\":\"long\"}}}}}}},\"system\":{\"properties\":{\"load\":{\"properties\":{\"1\":{\"type\":\"double\"},\"15\":{\"type\":\"double\"},\"5\":{\"type\":\"double\"},\"norm\":{\"properties\":{\"1\":{\"type\":\"double\"},\"15\":{\"type\":\"double\"},\"5\":{\"type\":\"double\"}}}}},\"cpu\":{\"properties\":{\"cores\":{\"type\":\"long\"}}}}},\"beat\":{\"properties\":{\"host\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"cpu\":{\"properties\":{\"total\":{\"properties\":{\"ticks\":{\"type\":\"long\"},\"time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"value\":{\"type\":\"long\"}}},\"system\":{\"properties\":{\"ticks\":{\"type\":\"long\"},\"time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"user\":{\"properties\":{\"ticks\":{\"type\":\"long\"},\"time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}}}},\"runtime\":{\"properties\":{\"goroutines\":{\"type\":\"long\"}}},\"memstats\":{\"properties\":{\"memory\":{\"properties\":{\"total\":{\"type\":\"long\"},\"alloc\":{\"type\":\"long\"}}},\"rss\":{\"type\":\"long\"},\"gc_next\":{\"type\":\"long\"}}},\"libbeat\":{\"properties\":{\"output\":{\"properties\":{\"read\":{\"properties\":{\"bytes\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"write\":{\"properties\":{\"bytes\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}},\"events\":{\"properties\":{\"batches\":{\"type\":\"long\"},\"duplicates\":{\"type\":\"long\"},\"total\":{\"type\":\"long\"},\"dropped\":{\"type\":\"long\"},\"toomany\":{\"type\":\"long\"},\"active\":{\"type\":\"long\"},\"failed\":{\"type\":\"long\"},\"acked\":{\"type\":\"long\"}}}}},\"pipeline\":{\"properties\":{\"clients\":{\"type\":\"long\"},\"events\":{\"properties\":{\"total\":{\"type\":\"long\"},\"filtered\":{\"type\":\"long\"},\"dropped\":{\"type\":\"long\"},\"active\":{\"type\":\"long\"},\"failed\":{\"type\":\"long\"},\"published\":{\"type\":\"long\"},\"retry\":{\"type\":\"long\"}}},\"queue\":{\"properties\":{\"acked\":{\"type\":\"long\"}}}}},\"config\":{\"properties\":{\"running\":{\"type\":\"long\"},\"stops\":{\"type\":\"long\"},\"starts\":{\"type\":\"long\"},\"reloads\":{\"type\":\"long\"}}}}},\"cgroup\":{\"properties\":{\"memory\":{\"properties\":{\"mem\":{\"properties\":{\"usage\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"limit\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"cpu\":{\"properties\":{\"cfs\":{\"properties\":{\"period\":{\"properties\":{\"us\":{\"type\":\"long\"}}},\"quota\":{\"properties\":{\"us\":{\"type\":\"long\"}}}}},\"stats\":{\"properties\":{\"periods\":{\"type\":\"long\"},\"throttled\":{\"properties\":{\"ns\":{\"type\":\"long\"},\"periods\":{\"type\":\"long\"}}}}},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"cpuacct\":{\"properties\":{\"total\":{\"properties\":{\"ns\":{\"type\":\"long\"}}},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"info\":{\"properties\":{\"host\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"uptime\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"uptime\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"properties\":{\"output\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"cluster\":{\"properties\":{\"uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"input\":{\"properties\":{\"names\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"count\":{\"type\":\"long\"}}},\"management\":{\"properties\":{\"enabled\":{\"type\":\"boolean\"}}},\"service\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"module\":{\"properties\":{\"names\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"count\":{\"type\":\"long\"}}},\"beat\":{\"properties\":{\"host\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"host\":{\"properties\":{\"os\":{\"properties\":{\"kernel\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"platform\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"containerized\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"queue\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"host\":{\"properties\":{\"hostname\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"architecture\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"metricset\":{\"properties\":{\"period\":{\"type\":\"long\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"event\":{\"properties\":{\"reason\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"code\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"timezone\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"agent_id_status\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"duration\":{\"type\":\"long\"},\"reference\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ingested\":{\"type\":\"date\"},\"provider\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"action\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"end\":{\"type\":\"date\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"outcome\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"severity\":{\"type\":\"long\"},\"original\":{\"ignore_above\":1024,\"index\":false,\"type\":\"keyword\",\"doc_values\":false},\"risk_score\":{\"type\":\"float\"},\"created\":{\"type\":\"date\"},\"kind\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"module\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"start\":{\"type\":\"date\"},\"url\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"sequence\":{\"type\":\"long\"},\"risk_score_norm\":{\"type\":\"float\"},\"category\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"dataset\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"hash\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"beats_stats\":{\"properties\":{\"beat\":{\"properties\":{\"host\":{\"path\":\"beat.stats.beat.host\",\"type\":\"alias\"},\"name\":{\"path\":\"beat.stats.beat.name\",\"type\":\"alias\"},\"type\":{\"path\":\"beat.stats.beat.type\",\"type\":\"alias\"},\"uuid\":{\"path\":\"beat.stats.beat.uuid\",\"type\":\"alias\"},\"version\":{\"path\":\"beat.stats.beat.version\",\"type\":\"alias\"}}},\"metrics\":{\"properties\":{\"system\":{\"properties\":{\"load\":{\"properties\":{\"1\":{\"path\":\"beat.stats.system.load.1\",\"type\":\"alias\"},\"15\":{\"path\":\"beat.stats.system.load.15\",\"type\":\"alias\"},\"5\":{\"path\":\"beat.stats.system.load.5\",\"type\":\"alias\"},\"norm\":{\"properties\":{\"1\":{\"path\":\"beat.stats.system.load.norm.1\",\"type\":\"alias\"},\"15\":{\"path\":\"beat.stats.system.load.norm.15\",\"type\":\"alias\"},\"5\":{\"path\":\"beat.stats.system.load.norm.5\",\"type\":\"alias\"}}}}},\"cpu\":{\"properties\":{\"cores\":{\"path\":\"beat.stats.system.cpu.cores\",\"type\":\"alias\"}}}}},\"beat\":{\"properties\":{\"handles\":{\"properties\":{\"limit\":{\"properties\":{\"hard\":{\"path\":\"beat.stats.handles.limit.hard\",\"type\":\"alias\"},\"soft\":{\"path\":\"beat.stats.handles.limit.soft\",\"type\":\"alias\"}}},\"open\":{\"path\":\"beat.stats.handles.open\",\"type\":\"alias\"}}},\"cpu\":{\"properties\":{\"total\":{\"properties\":{\"ticks\":{\"path\":\"beat.stats.cpu.total.ticks\",\"type\":\"alias\"},\"time\":{\"properties\":{\"ms\":{\"path\":\"beat.stats.cpu.total.time.ms\",\"type\":\"alias\"}}},\"value\":{\"path\":\"beat.stats.cpu.total.value\",\"type\":\"alias\"}}},\"system\":{\"properties\":{\"ticks\":{\"path\":\"beat.stats.cpu.system.ticks\",\"type\":\"alias\"},\"time\":{\"properties\":{\"ms\":{\"path\":\"beat.stats.cpu.system.time.ms\",\"type\":\"alias\"}}}}},\"user\":{\"properties\":{\"ticks\":{\"path\":\"beat.stats.cpu.user.ticks\",\"type\":\"alias\"},\"time\":{\"properties\":{\"ms\":{\"path\":\"beat.stats.cpu.user.time.ms\",\"type\":\"alias\"}}}}}}},\"memstats\":{\"properties\":{\"rss\":{\"path\":\"beat.stats.memstats.rss\",\"type\":\"alias\"},\"gc_next\":{\"path\":\"beat.stats.memstats.gc_next\",\"type\":\"alias\"},\"memory_alloc\":{\"path\":\"beat.stats.memstats.memory.alloc\",\"type\":\"alias\"},\"memory_total\":{\"path\":\"beat.stats.memstats.memory.total\",\"type\":\"alias\"}}},\"cgroup\":{\"properties\":{\"memory\":{\"properties\":{\"id\":{\"path\":\"beat.stats.cgroup.memory.id\",\"type\":\"alias\"}}},\"mem\":{\"properties\":{\"usage\":{\"properties\":{\"bytes\":{\"path\":\"beat.stats.cgroup.memory.mem.usage.bytes\",\"type\":\"alias\"}}},\"limit\":{\"properties\":{\"bytes\":{\"path\":\"beat.stats.cgroup.memory.mem.limit.bytes\",\"type\":\"alias\"}}}}},\"cpu\":{\"properties\":{\"cfs\":{\"properties\":{\"period\":{\"properties\":{\"us\":{\"path\":\"beat.stats.cgroup.cpu.cfs.period.us\",\"type\":\"alias\"}}},\"quota\":{\"properties\":{\"us\":{\"path\":\"beat.stats.cgroup.cpu.cfs.quota.us\",\"type\":\"alias\"}}}}},\"stats\":{\"properties\":{\"periods\":{\"path\":\"beat.stats.cgroup.cpu.stats.periods\",\"type\":\"alias\"},\"throttled\":{\"properties\":{\"ns\":{\"path\":\"beat.stats.cgroup.cpu.stats.throttled.ns\",\"type\":\"alias\"},\"periods\":{\"path\":\"beat.stats.cgroup.cpu.stats.throttled.periods\",\"type\":\"alias\"}}}}},\"id\":{\"path\":\"beat.stats.cgroup.cpu.id\",\"type\":\"alias\"}}},\"cpuacct\":{\"properties\":{\"total\":{\"properties\":{\"ns\":{\"path\":\"beat.stats.cgroup.cpuacct.total.ns\",\"type\":\"alias\"}}},\"id\":{\"path\":\"beat.stats.cgroup.cpuacct.id\",\"type\":\"alias\"}}}}},\"info\":{\"properties\":{\"ephemeral_id\":{\"path\":\"beat.stats.info.ephemeral_id\",\"type\":\"alias\"},\"uptime\":{\"properties\":{\"ms\":{\"path\":\"beat.stats.info.uptime.ms\",\"type\":\"alias\"}}}}}}},\"libbeat\":{\"properties\":{\"output\":{\"properties\":{\"read\":{\"properties\":{\"bytes\":{\"path\":\"beat.stats.libbeat.output.read.bytes\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.libbeat.output.read.errors\",\"type\":\"alias\"}}},\"type\":{\"path\":\"beat.stats.libbeat.output.type\",\"type\":\"alias\"},\"write\":{\"properties\":{\"bytes\":{\"path\":\"beat.stats.libbeat.output.write.bytes\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.libbeat.output.write.errors\",\"type\":\"alias\"}}},\"events\":{\"properties\":{\"duplicated\":{\"path\":\"beat.stats.libbeat.output.events.duplicates\",\"type\":\"alias\"},\"batches\":{\"path\":\"beat.stats.libbeat.output.events.batches\",\"type\":\"alias\"},\"total\":{\"path\":\"beat.stats.libbeat.output.events.total\",\"type\":\"alias\"},\"dropped\":{\"path\":\"beat.stats.libbeat.output.events.dropped\",\"type\":\"alias\"},\"toomany\":{\"path\":\"beat.stats.libbeat.output.events.toomany\",\"type\":\"alias\"},\"active\":{\"path\":\"beat.stats.libbeat.output.events.active\",\"type\":\"alias\"},\"failed\":{\"path\":\"beat.stats.libbeat.output.events.failed\",\"type\":\"alias\"},\"acked\":{\"path\":\"beat.stats.libbeat.output.events.acked\",\"type\":\"alias\"}}}}},\"pipeline\":{\"properties\":{\"clients\":{\"path\":\"beat.stats.libbeat.pipeline.clients\",\"type\":\"alias\"},\"events\":{\"properties\":{\"total\":{\"path\":\"beat.stats.libbeat.pipeline.events.total\",\"type\":\"alias\"},\"filtered\":{\"path\":\"beat.stats.libbeat.pipeline.events.filtered\",\"type\":\"alias\"},\"dropped\":{\"path\":\"beat.stats.libbeat.pipeline.events.dropped\",\"type\":\"alias\"},\"active\":{\"path\":\"beat.stats.libbeat.pipeline.events.active\",\"type\":\"alias\"},\"failed\":{\"path\":\"beat.stats.libbeat.pipeline.events.failed\",\"type\":\"alias\"},\"published\":{\"path\":\"beat.stats.libbeat.pipeline.events.published\",\"type\":\"alias\"},\"retry\":{\"path\":\"beat.stats.libbeat.pipeline.events.retry\",\"type\":\"alias\"}}},\"queue\":{\"properties\":{\"acked\":{\"path\":\"beat.stats.libbeat.pipeline.queue.acked\",\"type\":\"alias\"}}}}},\"config\":{\"properties\":{\"module\":{\"properties\":{\"running\":{\"path\":\"beat.stats.libbeat.config.running\",\"type\":\"alias\"},\"stops\":{\"path\":\"beat.stats.libbeat.config.stops\",\"type\":\"alias\"},\"starts\":{\"path\":\"beat.stats.libbeat.config.starts\",\"type\":\"alias\"}}},\"reloads\":{\"path\":\"beat.stats.libbeat.config.reloads\",\"type\":\"alias\"}}}}},\"apm-server\":{\"properties\":{\"server\":{\"properties\":{\"request\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.server.request.count\",\"type\":\"alias\"}}},\"response\":{\"properties\":{\"valid\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.server.response.valid.count\",\"type\":\"alias\"},\"accepted\":{\"path\":\"beat.stats.apm_server.server.response.valid.accepted\",\"type\":\"alias\"},\"ok\":{\"path\":\"beat.stats.apm_server.server.response.valid.ok\",\"type\":\"alias\"}}},\"count\":{\"path\":\"beat.stats.apm_server.server.response.count\",\"type\":\"alias\"},\"errors\":{\"properties\":{\"ratelimit\":{\"path\":\"beat.stats.apm_server.server.response.errors.ratelimit\",\"type\":\"alias\"},\"internal\":{\"path\":\"beat.stats.apm_server.server.response.errors.internal\",\"type\":\"alias\"},\"toolarge\":{\"path\":\"beat.stats.apm_server.server.response.errors.toolarge\",\"type\":\"alias\"},\"method\":{\"path\":\"beat.stats.apm_server.server.response.errors.method\",\"type\":\"alias\"},\"unauthorized\":{\"path\":\"beat.stats.apm_server.server.response.errors.unauthorized\",\"type\":\"alias\"},\"forbidden\":{\"path\":\"beat.stats.apm_server.server.response.errors.forbidden\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.stats.apm_server.server.response.errors.count\",\"type\":\"alias\"},\"closed\":{\"path\":\"beat.stats.apm_server.server.response.errors.closed\",\"type\":\"alias\"},\"decode\":{\"path\":\"beat.stats.apm_server.server.response.errors.decode\",\"type\":\"alias\"},\"queue\":{\"path\":\"beat.stats.apm_server.server.response.errors.queue\",\"type\":\"alias\"},\"concurrency\":{\"path\":\"beat.stats.apm_server.server.response.errors.concurrency\",\"type\":\"alias\"},\"validate\":{\"path\":\"beat.stats.apm_server.server.response.errors.validate\",\"type\":\"alias\"}}}}},\"concurrent\":{\"properties\":{\"wait\":{\"properties\":{\"ms\":{\"path\":\"beat.stats.apm_server.server.concurrent.wait.ms\",\"type\":\"alias\"}}}}}}},\"sampling\":{\"properties\":{\"tail\":{\"properties\":{\"dynamic_service_groups\":{\"type\":\"long\"},\"storage\":{\"properties\":{\"lsm_size\":{\"type\":\"long\"},\"value_log_size\":{\"type\":\"long\"}}},\"events\":{\"properties\":{\"processed\":{\"type\":\"long\"},\"head_unsampled\":{\"type\":\"long\"},\"stored\":{\"type\":\"long\"},\"dropped\":{\"type\":\"long\"},\"sampled\":{\"type\":\"long\"}}}}},\"transactions_dropped\":{\"type\":\"long\"}}},\"acm\":{\"properties\":{\"request\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.acm.request.count\",\"type\":\"alias\"}}},\"response\":{\"properties\":{\"valid\":{\"properties\":{\"notmodified\":{\"path\":\"beat.stats.apm_server.acm.response.valid.notmodified\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.stats.apm_server.acm.response.valid.count\",\"type\":\"alias\"},\"accepted\":{\"path\":\"beat.stats.apm_server.acm.response.valid.accepted\",\"type\":\"alias\"},\"ok\":{\"path\":\"beat.stats.apm_server.acm.response.valid.ok\",\"type\":\"alias\"}}},\"request\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.acm.response.request.count\",\"type\":\"alias\"}}},\"count\":{\"path\":\"beat.stats.apm_server.acm.response.count\",\"type\":\"alias\"},\"unset\":{\"path\":\"beat.stats.apm_server.acm.response.unset\",\"type\":\"alias\"},\"errors\":{\"properties\":{\"ratelimit\":{\"path\":\"beat.stats.apm_server.acm.response.errors.ratelimit\",\"type\":\"alias\"},\"internal\":{\"path\":\"beat.stats.apm_server.acm.response.errors.internal\",\"type\":\"alias\"},\"toolarge\":{\"path\":\"beat.stats.apm_server.acm.response.errors.toolarge\",\"type\":\"alias\"},\"method\":{\"path\":\"beat.stats.apm_server.acm.response.errors.method\",\"type\":\"alias\"},\"unavailable\":{\"path\":\"beat.stats.apm_server.acm.response.errors.unavailable\",\"type\":\"alias\"},\"forbidden\":{\"path\":\"beat.stats.apm_server.acm.response.errors.forbidden\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.stats.apm_server.acm.response.errors.count\",\"type\":\"alias\"},\"decode\":{\"path\":\"beat.stats.apm_server.acm.response.errors.decode\",\"type\":\"alias\"},\"unauthorized\":{\"path\":\"beat.stats.apm_server.acm.response.errors.unauthorized\",\"type\":\"alias\"},\"notfound\":{\"path\":\"beat.stats.apm_server.acm.response.errors.notfound\",\"type\":\"alias\"},\"closed\":{\"path\":\"beat.stats.apm_server.acm.response.errors.closed\",\"type\":\"alias\"},\"invalidquery\":{\"path\":\"beat.stats.apm_server.acm.response.errors.invalidquery\",\"type\":\"alias\"},\"queue\":{\"path\":\"beat.stats.apm_server.acm.response.errors.queue\",\"type\":\"alias\"},\"validate\":{\"path\":\"beat.stats.apm_server.acm.response.errors.validate\",\"type\":\"alias\"}}}}}}},\"decoder\":{\"properties\":{\"deflate\":{\"properties\":{\"content-length\":{\"path\":\"beat.stats.apm_server.decoder.deflate.content-length\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.stats.apm_server.decoder.deflate.count\",\"type\":\"alias\"}}},\"reader\":{\"properties\":{\"size\":{\"path\":\"beat.stats.apm_server.decoder.reader.size\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.stats.apm_server.decoder.reader.count\",\"type\":\"alias\"}}},\"gzip\":{\"properties\":{\"content-length\":{\"path\":\"beat.stats.apm_server.decoder.gzip.content-length\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.stats.apm_server.decoder.gzip.count\",\"type\":\"alias\"}}},\"uncompressed\":{\"properties\":{\"content-length\":{\"path\":\"beat.stats.apm_server.decoder.uncompressed.content-length\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.stats.apm_server.decoder.uncompressed.count\",\"type\":\"alias\"}}},\"missing-content-length\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.decoder.missing-content-length.count\",\"type\":\"alias\"}}}}},\"processor\":{\"properties\":{\"metric\":{\"properties\":{\"transformations\":{\"path\":\"beat.stats.apm_server.processor.metric.transformations\",\"type\":\"alias\"},\"decoding\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.processor.metric.decoding.count\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.apm_server.processor.metric.decoding.errors\",\"type\":\"alias\"}}},\"validation\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.processor.metric.validation.count\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.apm_server.processor.metric.validation.errors\",\"type\":\"alias\"}}}}},\"sourcemap\":{\"properties\":{\"counter\":{\"path\":\"beat.stats.apm_server.processor.sourcemap.counter\",\"type\":\"alias\"},\"decoding\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.processor.sourcemap.decoding.count\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.apm_server.processor.sourcemap.decoding.errors\",\"type\":\"alias\"}}},\"validation\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.processor.sourcemap.validation.count\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.apm_server.processor.sourcemap.validation.errors\",\"type\":\"alias\"}}}}},\"error\":{\"properties\":{\"stacktraces\":{\"path\":\"beat.stats.apm_server.processor.error.stacktraces\",\"type\":\"alias\"},\"spans\":{\"path\":\"beat.stats.apm_server.processor.error.spans\",\"type\":\"alias\"},\"frames\":{\"path\":\"beat.stats.apm_server.processor.error.frames\",\"type\":\"alias\"},\"transformations\":{\"path\":\"beat.stats.apm_server.processor.error.transformations\",\"type\":\"alias\"},\"decoding\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.processor.error.decoding.count\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.apm_server.processor.error.decoding.errors\",\"type\":\"alias\"}}},\"validation\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.processor.error.validation.count\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.apm_server.processor.error.validation.errors\",\"type\":\"alias\"}}}}},\"transaction\":{\"properties\":{\"stacktraces\":{\"path\":\"beat.stats.apm_server.processor.transaction.stacktraces\",\"type\":\"alias\"},\"spans\":{\"path\":\"beat.stats.apm_server.processor.transaction.spans\",\"type\":\"alias\"},\"frames\":{\"path\":\"beat.stats.apm_server.processor.transaction.frames\",\"type\":\"alias\"},\"transformations\":{\"path\":\"beat.stats.apm_server.processor.transaction.transformations\",\"type\":\"alias\"},\"transactions\":{\"path\":\"beat.stats.apm_server.processor.transaction.transactions\",\"type\":\"alias\"},\"decoding\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.processor.transaction.decoding.count\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.apm_server.processor.transaction.decoding.errors\",\"type\":\"alias\"}}},\"validation\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.processor.transaction.validation.count\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.apm_server.processor.transaction.validation.errors\",\"type\":\"alias\"}}}}},\"span\":{\"properties\":{\"transformations\":{\"path\":\"beat.stats.apm_server.processor.span.transformations\",\"type\":\"alias\"}}}}}}}}},\"timestamp\":{\"path\":\"@timestamp\",\"type\":\"alias\"}}},\"beats_state\":{\"properties\":{\"beat\":{\"properties\":{\"host\":{\"path\":\"beat.state.beat.host\",\"type\":\"alias\"},\"name\":{\"path\":\"beat.state.beat.name\",\"type\":\"alias\"},\"type\":{\"path\":\"beat.state.beat.type\",\"type\":\"alias\"},\"uuid\":{\"path\":\"beat.state.beat.uuid\",\"type\":\"alias\"},\"version\":{\"path\":\"beat.state.beat.version\",\"type\":\"alias\"}}},\"state\":{\"properties\":{\"output\":{\"properties\":{\"name\":{\"path\":\"beat.state.output.name\",\"type\":\"alias\"}}},\"input\":{\"properties\":{\"names\":{\"path\":\"beat.state.input.names\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.state.input.count\",\"type\":\"alias\"}}},\"service\":{\"properties\":{\"name\":{\"path\":\"beat.state.service.name\",\"type\":\"alias\"},\"id\":{\"path\":\"beat.state.service.id\",\"type\":\"alias\"},\"version\":{\"path\":\"beat.state.service.version\",\"type\":\"alias\"}}},\"module\":{\"properties\":{\"names\":{\"path\":\"beat.state.module.names\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.state.module.count\",\"type\":\"alias\"}}},\"beat\":{\"properties\":{\"name\":{\"path\":\"beat.state.beat.name\",\"type\":\"alias\"}}},\"host\":{\"properties\":{\"hostname\":{\"path\":\"host.hostname\",\"type\":\"alias\"},\"os\":{\"properties\":{\"version\":{\"path\":\"beat.state.host.os.version\",\"type\":\"alias\"},\"platform\":{\"path\":\"beat.state.host.os.platform\",\"type\":\"alias\"}}},\"name\":{\"path\":\"host.name\",\"type\":\"alias\"},\"architecture\":{\"path\":\"host.architecture\",\"type\":\"alias\"}}}}},\"timestamp\":{\"path\":\"@timestamp\",\"type\":\"alias\"}}},\"timestamp\":{\"path\":\"@timestamp\",\"type\":\"alias\"}}}},\"composed_of\":[],\"version\":8000103,\"data_stream\":{\"hidden\":false,\"allow_custom_routing\":false}},\".monitoring-kibana-mb\":{\"index_patterns\":[\".monitoring-kibana-8-*\"],\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\".monitoring-8-ilm-policy\"},\"mapping\":{\"total_fields\":{\"limit\":\"2000\"}}}},\"mappings\":{\"dynamic\":false,\"properties\":{\"process\":{\"properties\":{\"pid\":{\"type\":\"long\"}}},\"@timestamp\":{\"type\":\"date\"},\"ecs\":{\"properties\":{\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"cluster_uuid\":{\"path\":\"kibana.elasticsearch.cluster.id\",\"type\":\"alias\"},\"service\":{\"properties\":{\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"host\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"metricset\":{\"properties\":{\"period\":{\"type\":\"long\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"kibana\":{\"properties\":{\"elasticsearch\":{\"properties\":{\"cluster\":{\"properties\":{\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"stats\":{\"properties\":{\"request\":{\"properties\":{\"total\":{\"type\":\"long\"},\"disconnects\":{\"type\":\"long\"}}},\"process\":{\"properties\":{\"memory\":{\"properties\":{\"resident_set_size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"heap\":{\"properties\":{\"total\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"used\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"size_limit\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"uptime\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}}}},\"event_loop_delay\":{\"properties\":{\"ms\":{\"scaling_factor\":1000,\"type\":\"scaled_float\"}}},\"uptime\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"os\":{\"properties\":{\"distroRelease\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"distro\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"memory\":{\"properties\":{\"used_in_bytes\":{\"type\":\"long\"},\"total_in_bytes\":{\"type\":\"long\"},\"free_in_bytes\":{\"type\":\"long\"}}},\"load\":{\"properties\":{\"5m\":{\"type\":\"half_float\"},\"15m\":{\"type\":\"half_float\"},\"1m\":{\"type\":\"half_float\"}}},\"platformRelease\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"platform\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"usage\":{\"properties\":{\"index\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"host\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"index\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"response_time\":{\"properties\":{\"avg\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"max\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"kibana\":{\"properties\":{\"status\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"concurrent_connections\":{\"type\":\"long\"},\"snapshot\":{\"type\":\"boolean\"},\"status\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"node_rules\":{\"properties\":{\"failures\":{\"type\":\"long\"},\"executions\":{\"type\":\"long\"},\"timeouts\":{\"type\":\"long\"}}},\"cluster_actions\":{\"properties\":{\"overdue\":{\"properties\":{\"delay\":{\"properties\":{\"p99\":{\"type\":\"float\"},\"p50\":{\"type\":\"float\"}}},\"count\":{\"type\":\"long\"}}}}},\"cluster_rules\":{\"properties\":{\"overdue\":{\"properties\":{\"delay\":{\"properties\":{\"p99\":{\"type\":\"float\"},\"p50\":{\"type\":\"float\"}}},\"count\":{\"type\":\"long\"}}}}},\"node_actions\":{\"properties\":{\"failures\":{\"type\":\"long\"},\"executions\":{\"type\":\"long\"},\"timeouts\":{\"type\":\"long\"}}},\"status\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"metrics\":{\"properties\":{\"requests\":{\"properties\":{\"total\":{\"type\":\"long\"},\"disconnects\":{\"type\":\"long\"}}},\"concurrent_connections\":{\"type\":\"long\"}}},\"status\":{\"properties\":{\"overall\":{\"properties\":{\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}}}}}},\"event\":{\"properties\":{\"duration\":{\"type\":\"long\"},\"module\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"dataset\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"kibana_stats\":{\"properties\":{\"process\":{\"properties\":{\"memory\":{\"properties\":{\"resident_set_size_in_bytes\":{\"path\":\"kibana.stats.process.memory.resident_set_size.bytes\",\"type\":\"alias\"},\"heap\":{\"properties\":{\"size_limit\":{\"path\":\"kibana.stats.process.memory.heap.size_limit.bytes\",\"type\":\"alias\"}}}}},\"event_loop_delay\":{\"path\":\"kibana.stats.process.event_loop_delay.ms\",\"type\":\"alias\"},\"uptime_in_millis\":{\"path\":\"kibana.stats.process.uptime.ms\",\"type\":\"alias\"}}},\"os\":{\"properties\":{\"memory\":{\"properties\":{\"free_in_bytes\":{\"path\":\"kibana.stats.os.memory.free_in_bytes\",\"type\":\"alias\"}}},\"load\":{\"properties\":{\"5m\":{\"path\":\"kibana.stats.os.load.5m\",\"type\":\"alias\"},\"15m\":{\"path\":\"kibana.stats.os.load.15m\",\"type\":\"alias\"},\"1m\":{\"path\":\"kibana.stats.os.load.1m\",\"type\":\"alias\"}}}}},\"response_times\":{\"properties\":{\"average\":{\"path\":\"kibana.stats.response_time.avg.ms\",\"type\":\"alias\"},\"max\":{\"path\":\"kibana.stats.response_time.max.ms\",\"type\":\"alias\"}}},\"requests\":{\"properties\":{\"total\":{\"path\":\"kibana.stats.request.total\",\"type\":\"alias\"},\"disconnects\":{\"path\":\"kibana.stats.request.disconnects\",\"type\":\"alias\"}}},\"kibana\":{\"properties\":{\"response_time\":{\"properties\":{\"max\":{\"path\":\"kibana.stats.response_time.max.ms\",\"type\":\"alias\"}}},\"uuid\":{\"path\":\"service.id\",\"type\":\"alias\"},\"version\":{\"path\":\"service.version\",\"type\":\"alias\"},\"status\":{\"path\":\"kibana.stats.status\",\"type\":\"alias\"}}},\"concurrent_connections\":{\"path\":\"kibana.stats.concurrent_connections\",\"type\":\"alias\"},\"timestamp\":{\"path\":\"@timestamp\",\"type\":\"alias\"}}},\"timestamp\":{\"path\":\"@timestamp\",\"type\":\"alias\"}}}},\"composed_of\":[],\"version\":8000103,\"data_stream\":{\"hidden\":false,\"allow_custom_routing\":false}},\".monitoring-ent-search-mb\":{\"index_patterns\":[\".monitoring-ent-search-8-*\"],\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\".monitoring-8-ilm-policy\"},\"mapping\":{\"total_fields\":{\"limit\":\"2000\"}}}},\"mappings\":{\"dynamic\":false,\"properties\":{\"enterprisesearch\":{\"properties\":{\"cluster_uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"stats\":{\"properties\":{\"connectors\":{\"properties\":{\"pool\":{\"properties\":{\"publish_worker_pool\":{\"properties\":{\"total_completed\":{\"type\":\"long\"},\"size\":{\"type\":\"long\"},\"idle\":{\"type\":\"long\"},\"busy\":{\"type\":\"long\"},\"queue_depth\":{\"type\":\"long\"},\"total_scheduled\":{\"type\":\"long\"}}},\"extract_worker_pool\":{\"properties\":{\"total_completed\":{\"type\":\"long\"},\"size\":{\"type\":\"long\"},\"idle\":{\"type\":\"long\"},\"busy\":{\"type\":\"long\"},\"queue_depth\":{\"type\":\"long\"},\"total_scheduled\":{\"type\":\"long\"}}},\"subextract_worker_pool\":{\"properties\":{\"total_completed\":{\"type\":\"long\"},\"size\":{\"type\":\"long\"},\"idle\":{\"type\":\"long\"},\"busy\":{\"type\":\"long\"},\"queue_depth\":{\"type\":\"long\"},\"total_scheduled\":{\"type\":\"long\"}}}}},\"job_store\":{\"properties\":{\"waiting\":{\"type\":\"long\"},\"job_types\":{\"properties\":{\"permissions\":{\"type\":\"long\"},\"incremental\":{\"type\":\"long\"},\"delete\":{\"type\":\"long\"},\"full\":{\"type\":\"long\"}}},\"working\":{\"type\":\"long\"}}}}},\"queues\":{\"properties\":{\"engine_destroyer\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"process_crawl\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"failed\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"mailer\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"http\":{\"properties\":{\"request_duration\":{\"properties\":{\"max\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"std_dev\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"mean\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"responses\":{\"properties\":{\"1xx\":{\"type\":\"long\"},\"2xx\":{\"type\":\"long\"},\"3xx\":{\"type\":\"long\"},\"4xx\":{\"type\":\"long\"},\"5xx\":{\"type\":\"long\"}}},\"connections\":{\"properties\":{\"current\":{\"type\":\"long\"},\"total\":{\"type\":\"long\"},\"max\":{\"type\":\"long\"}}},\"network\":{\"properties\":{\"received\":{\"properties\":{\"bytes_per_sec\":{\"type\":\"long\"},\"bytes\":{\"type\":\"long\"}}},\"sent\":{\"properties\":{\"bytes_per_sec\":{\"type\":\"long\"},\"bytes\":{\"type\":\"long\"}}}}}}},\"product_usage\":{\"properties\":{\"app_search\":{\"properties\":{\"total_engines\":{\"type\":\"long\"}}},\"workplace_search\":{\"properties\":{\"total_org_sources\":{\"type\":\"long\"},\"total_private_sources\":{\"type\":\"long\"}}}}}}},\"health\":{\"properties\":{\"jvm\":{\"properties\":{\"memory_usage\":{\"properties\":{\"heap_committed\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"heap_used\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"non_heap_committed\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"non_heap_init\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"heap_init\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"heap_max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"object_pending_finalization_count\":{\"type\":\"long\"}}},\"threads\":{\"properties\":{\"current\":{\"type\":\"long\"},\"total_started\":{\"type\":\"long\"},\"max\":{\"type\":\"long\"},\"daemon\":{\"type\":\"long\"}}},\"gc\":{\"properties\":{\"collection_count\":{\"type\":\"long\"},\"collection_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"process\":{\"properties\":{\"pid\":{\"type\":\"long\"},\"filebeat\":{\"properties\":{\"restart_count\":{\"type\":\"long\"},\"pid\":{\"type\":\"long\"},\"time_since_last_restart\":{\"properties\":{\"sec\":{\"type\":\"long\"}}}}},\"uptime\":{\"properties\":{\"sec\":{\"type\":\"long\"}}}}},\"crawler\":{\"properties\":{\"workers\":{\"properties\":{\"pool_size\":{\"type\":\"long\"},\"available\":{\"type\":\"long\"},\"active\":{\"type\":\"long\"}}}}},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"properties\":{\"number\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"build_hash\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}}}},\"agent\":{\"properties\":{\"id\":{\"type\":\"keyword\"},\"ephemeral_id\":{\"type\":\"keyword\"},\"type\":{\"type\":\"keyword\"},\"version\":{\"type\":\"keyword\"}}},\"process\":{\"properties\":{\"pid\":{\"type\":\"long\"}}},\"@timestamp\":{\"type\":\"date\"},\"ecs\":{\"properties\":{\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"service\":{\"properties\":{\"node\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"hostname\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"environment\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"origin\":{\"properties\":{\"node\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"environment\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"target\":{\"properties\":{\"node\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"environment\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"host\":{\"properties\":{\"hostname\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"architecture\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"metricset\":{\"properties\":{\"period\":{\"type\":\"long\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"event\":{\"properties\":{\"reason\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"code\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"timezone\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"agent_id_status\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"duration\":{\"type\":\"long\"},\"reference\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ingested\":{\"type\":\"date\"},\"provider\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"action\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"end\":{\"type\":\"date\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"outcome\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"severity\":{\"type\":\"long\"},\"original\":{\"ignore_above\":1024,\"index\":false,\"type\":\"keyword\",\"doc_values\":false},\"risk_score\":{\"type\":\"float\"},\"created\":{\"type\":\"date\"},\"kind\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"module\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"start\":{\"type\":\"date\"},\"url\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"sequence\":{\"type\":\"long\"},\"risk_score_norm\":{\"type\":\"float\"},\"category\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"dataset\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"hash\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}}},\"composed_of\":[],\"version\":8000103,\"_meta\":{\"managed\":true,\"description\":\"Template used by Enterprise Search Metricbeat module monitoring information for Stack Monitoring\"},\"data_stream\":{\"hidden\":false,\"allow_custom_routing\":false}},\"synthetics\":{\"index_patterns\":[\"synthetics-*-*\"],\"composed_of\":[\"synthetics-mappings\",\"data-streams-mappings\",\"synthetics-settings\"],\"priority\":100,\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default synthetics template installed by x-pack\"},\"data_stream\":{\"hidden\":false,\"allow_custom_routing\":false},\"allow_auto_create\":true},\"ilm-history\":{\"index_patterns\":[\"ilm-history-5*\"],\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\"ilm-history-ilm-policy\"},\"number_of_shards\":\"1\",\"auto_expand_replicas\":\"0-1\",\"number_of_replicas\":\"0\"}},\"mappings\":{\"dynamic\":false,\"properties\":{\"index_age\":{\"type\":\"long\"},\"@timestamp\":{\"format\":\"epoch_millis\",\"type\":\"date\"},\"error_details\":{\"type\":\"text\"},\"success\":{\"type\":\"boolean\"},\"index\":{\"type\":\"keyword\"},\"state\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"phase\":{\"type\":\"keyword\"},\"failed_step\":{\"type\":\"keyword\"},\"phase_definition\":{\"type\":\"text\"},\"action_time\":{\"format\":\"epoch_millis\",\"type\":\"date\"},\"phase_time\":{\"format\":\"epoch_millis\",\"type\":\"date\"},\"step_info\":{\"type\":\"text\"},\"action\":{\"type\":\"keyword\"},\"step\":{\"type\":\"keyword\"},\"is_auto-retryable_error\":{\"type\":\"keyword\"},\"creation_date\":{\"format\":\"epoch_millis\",\"type\":\"date\"},\"step_time\":{\"format\":\"epoch_millis\",\"type\":\"date\"}}},\"policy\":{\"type\":\"keyword\"}}}},\"composed_of\":[],\"priority\":2147483647,\"version\":5,\"_meta\":{\"managed\":true,\"description\":\"index template for ILM history indices\"},\"data_stream\":{\"hidden\":true,\"allow_custom_routing\":false},\"allow_auto_create\":true},\".ml-state\":{\"index_patterns\":[\".ml-state*\"],\"template\":{\"settings\":{\"index\":{\"hidden\":\"true\",\"lifecycle\":{\"name\":\"ml-size-based-ilm-policy\",\"rollover_alias\":\".ml-state-write\"},\"auto_expand_replicas\":\"0-1\"}},\"mappings\":{\"_meta\":{\"version\":\"8060299\"},\"enabled\":false},\"aliases\":{}},\"composed_of\":[],\"priority\":2147483647,\"version\":8060299,\"_meta\":{\"managed\":true,\"description\":\"index template for ML state indices\"}},\".monitoring-es-mb\":{\"index_patterns\":[\".monitoring-es-8-*\"],\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\".monitoring-8-ilm-policy\"},\"mapping\":{\"total_fields\":{\"limit\":\"2000\"}}}},\"mappings\":{\"dynamic\":false,\"properties\":{\"ccr_auto_follow_stats\":{\"properties\":{\"number_of_failed_remote_cluster_state_requests\":{\"path\":\"elasticsearch.ccr.auto_follow.failed.remote_cluster_state_requests.count\",\"type\":\"alias\"},\"follower\":{\"properties\":{\"failed_read_requests\":{\"path\":\"elasticsearch.ccr.requests.failed.read.count\",\"type\":\"alias\"}}},\"number_of_failed_follow_indices\":{\"path\":\"elasticsearch.ccr.auto_follow.failed.follow_indices.count\",\"type\":\"alias\"},\"number_of_successful_follow_indices\":{\"path\":\"elasticsearch.ccr.auto_follow.success.follow_indices.count\",\"type\":\"alias\"}}},\"ccr_stats\":{\"properties\":{\"write_buffer_size_in_bytes\":{\"path\":\"elasticsearch.ccr.write_buffer.size.bytes\",\"type\":\"alias\"},\"leader_global_checkpoint\":{\"path\":\"elasticsearch.ccr.leader.global_checkpoint\",\"type\":\"alias\"},\"follower_index\":{\"path\":\"elasticsearch.ccr.follower.index\",\"type\":\"alias\"},\"leader_max_seq_no\":{\"path\":\"elasticsearch.ccr.leader.max_seq_no\",\"type\":\"alias\"},\"last_requested_seq_no\":{\"path\":\"elasticsearch.ccr.last_requested_seq_no\",\"type\":\"alias\"},\"follower_settings_version\":{\"path\":\"elasticsearch.ccr.follower.settings_version\",\"type\":\"alias\"},\"successful_write_requests\":{\"path\":\"elasticsearch.ccr.requests.successful.write.count\",\"type\":\"alias\"},\"remote_cluster\":{\"path\":\"elasticsearch.ccr.remote_cluster\",\"type\":\"alias\"},\"outstanding_write_requests\":{\"path\":\"elasticsearch.ccr.requests.outstanding.write.count\",\"type\":\"alias\"},\"total_read_time_millis\":{\"path\":\"elasticsearch.ccr.total_time.read.ms\",\"type\":\"alias\"},\"outstanding_read_requests\":{\"path\":\"elasticsearch.ccr.requests.outstanding.read.count\",\"type\":\"alias\"},\"total_write_time_millis\":{\"path\":\"elasticsearch.ccr.total_time.write.ms\",\"type\":\"alias\"},\"failed_write_requests\":{\"path\":\"elasticsearch.ccr.requests.failed.write.count\",\"type\":\"alias\"},\"failed_read_requests\":{\"path\":\"elasticsearch.ccr.requests.failed.read.count\",\"type\":\"alias\"},\"bytes_read\":{\"path\":\"elasticsearch.ccr.bytes_read\",\"type\":\"alias\"},\"leader_index\":{\"path\":\"elasticsearch.ccr.leader.index\",\"type\":\"alias\"},\"follower_max_seq_no\":{\"path\":\"elasticsearch.ccr.follower.max_seq_no\",\"type\":\"alias\"},\"operations_written\":{\"path\":\"elasticsearch.ccr.follower.operations_written\",\"type\":\"alias\"},\"write_buffer_operation_count\":{\"path\":\"elasticsearch.ccr.write_buffer.operation.count\",\"type\":\"alias\"},\"successful_read_requests\":{\"path\":\"elasticsearch.ccr.requests.successful.read.count\",\"type\":\"alias\"},\"shard_id\":{\"path\":\"elasticsearch.ccr.follower.shard.number\",\"type\":\"alias\"},\"follower_mapping_version\":{\"path\":\"elasticsearch.ccr.follower.mapping_version\",\"type\":\"alias\"},\"follower_aliases_version\":{\"path\":\"elasticsearch.ccr.follower.aliases_version\",\"type\":\"alias\"},\"follower_global_checkpoint\":{\"path\":\"elasticsearch.ccr.follower.global_checkpoint\",\"type\":\"alias\"},\"total_read_remote_exec_time_millis\":{\"path\":\"elasticsearch.ccr.total_time.read.remote_exec.ms\",\"type\":\"alias\"},\"operations_read\":{\"path\":\"elasticsearch.ccr.follower.operations.read.count\",\"type\":\"alias\"}}},\"cluster_state\":{\"properties\":{\"nodes_hash\":{\"path\":\"elasticsearch.cluster.stats.state.nodes_hash\",\"type\":\"alias\"},\"master_node\":{\"path\":\"elasticsearch.cluster.stats.state.master_node\",\"type\":\"alias\"},\"state_uuid\":{\"path\":\"elasticsearch.cluster.stats.state.state_uuid\",\"type\":\"alias\"},\"version\":{\"path\":\"elasticsearch.cluster.stats.state.version\",\"type\":\"alias\"},\"status\":{\"path\":\"elasticsearch.cluster.stats.status\",\"type\":\"alias\"}}},\"indices_stats\":{\"properties\":{\"_all\":{\"properties\":{\"primaries\":{\"properties\":{\"indexing\":{\"properties\":{\"index_time_in_millis\":{\"path\":\"elasticsearch.index.summary.primaries.indexing.index.time.ms\",\"type\":\"alias\"},\"index_total\":{\"path\":\"elasticsearch.index.summary.primaries.indexing.index.count\",\"type\":\"alias\"}}}}},\"total\":{\"properties\":{\"search\":{\"properties\":{\"query_total\":{\"path\":\"elasticsearch.index.summary.total.search.query.count\",\"type\":\"alias\"},\"query_time_in_millis\":{\"path\":\"elasticsearch.index.summary.total.search.query.time.ms\",\"type\":\"alias\"}}},\"indexing\":{\"properties\":{\"index_total\":{\"path\":\"elasticsearch.index.summary.total.indexing.index.count\",\"type\":\"alias\"}}}}}}}}},\"node_stats\":{\"properties\":{\"jvm\":{\"properties\":{\"mem\":{\"properties\":{\"heap_used_percent\":{\"path\":\"elasticsearch.node.stats.jvm.mem.heap.used.pct\",\"type\":\"alias\"},\"heap_max_in_bytes\":{\"path\":\"elasticsearch.node.stats.jvm.mem.heap.max.bytes\",\"type\":\"alias\"},\"heap_used_in_bytes\":{\"path\":\"elasticsearch.node.stats.jvm.mem.heap.used.bytes\",\"type\":\"alias\"}}},\"gc\":{\"properties\":{\"collectors\":{\"properties\":{\"young\":{\"properties\":{\"collection_count\":{\"path\":\"elasticsearch.node.stats.jvm.gc.collectors.young.collection.count\",\"type\":\"alias\"},\"collection_time_in_millis\":{\"path\":\"elasticsearch.node.stats.jvm.gc.collectors.young.collection.ms\",\"type\":\"alias\"}}},\"old\":{\"properties\":{\"collection_count\":{\"path\":\"elasticsearch.node.stats.jvm.gc.collectors.old.collection.count\",\"type\":\"alias\"},\"collection_time_in_millis\":{\"path\":\"elasticsearch.node.stats.jvm.gc.collectors.old.collection.ms\",\"type\":\"alias\"}}}}}}}}},\"indices\":{\"properties\":{\"search\":{\"properties\":{\"query_total\":{\"path\":\"elasticsearch.node.stats.indices.search.query_total.count\",\"type\":\"alias\"},\"query_time_in_millis\":{\"path\":\"elasticsearch.node.stats.indices.search.query_time.ms\",\"type\":\"alias\"}}},\"query_cache\":{\"properties\":{\"memory_size_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.query_cache.memory.bytes\",\"type\":\"alias\"}}},\"docs\":{\"properties\":{\"count\":{\"path\":\"elasticsearch.node.stats.indices.docs.count\",\"type\":\"alias\"}}},\"indexing\":{\"properties\":{\"throttle_time_in_millis\":{\"path\":\"elasticsearch.node.stats.indices.indexing.throttle_time.ms\",\"type\":\"alias\"},\"index_time_in_millis\":{\"path\":\"elasticsearch.node.stats.indices.indexing.index_time.ms\",\"type\":\"alias\"},\"index_total\":{\"path\":\"elasticsearch.node.stats.indices.indexing.index_total.count\",\"type\":\"alias\"}}},\"fielddata\":{\"properties\":{\"memory_size_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.fielddata.memory.bytes\",\"type\":\"alias\"}}},\"store\":{\"properties\":{\"size\":{\"properties\":{\"bytes\":{\"path\":\"elasticsearch.node.stats.indices.store.size.bytes\",\"type\":\"alias\"}}},\"size_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.store.size.bytes\",\"type\":\"alias\"}}},\"request_cache\":{\"properties\":{\"memory_size_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.request_cache.memory.bytes\",\"type\":\"alias\"}}},\"segments\":{\"properties\":{\"version_map_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.version_map.memory.bytes\",\"type\":\"alias\"},\"norms_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.norms.memory.bytes\",\"type\":\"alias\"},\"count\":{\"path\":\"elasticsearch.node.stats.indices.segments.count\",\"type\":\"alias\"},\"term_vectors_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.term_vectors.memory.bytes\",\"type\":\"alias\"},\"points_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.points.memory.bytes\",\"type\":\"alias\"},\"index_writer_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.index_writer.memory.bytes\",\"type\":\"alias\"},\"memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.memory.bytes\",\"type\":\"alias\"},\"doc_values_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.doc_values.memory.bytes\",\"type\":\"alias\"},\"terms_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.terms.memory.bytes\",\"type\":\"alias\"},\"fixed_bit_set_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.fixed_bit_set.memory.bytes\",\"type\":\"alias\"},\"stored_fields_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.stored_fields.memory.bytes\",\"type\":\"alias\"}}}}},\"process\":{\"properties\":{\"cpu\":{\"properties\":{\"percent\":{\"path\":\"elasticsearch.node.stats.process.cpu.pct\",\"type\":\"alias\"}}}}},\"os\":{\"properties\":{\"cpu\":{\"properties\":{\"load_average\":{\"properties\":{\"1m\":{\"path\":\"elasticsearch.node.stats.os.cpu.load_avg.1m\",\"type\":\"alias\"}}}}},\"cgroup\":{\"properties\":{\"memory\":{\"properties\":{\"usage_in_bytes\":{\"path\":\"elasticsearch.node.stats.os.cgroup.memory.usage.bytes\",\"type\":\"alias\"},\"control_group\":{\"path\":\"elasticsearch.node.stats.os.cgroup.memory.control_group\",\"type\":\"alias\"},\"limit_in_bytes\":{\"path\":\"elasticsearch.node.stats.os.cgroup.memory.limit.bytes\",\"type\":\"alias\"}}},\"cpu\":{\"properties\":{\"stat\":{\"properties\":{\"number_of_elapsed_periods\":{\"path\":\"elasticsearch.node.stats.os.cgroup.cpu.stat.elapsed_periods.count\",\"type\":\"alias\"},\"number_of_times_throttled\":{\"path\":\"elasticsearch.node.stats.os.cgroup.cpu.stat.times_throttled.count\",\"type\":\"alias\"},\"time_throttled_nanos\":{\"path\":\"elasticsearch.node.stats.os.cgroup.cpu.stat.time_throttled.ns\",\"type\":\"alias\"}}},\"cfs_quota_micros\":{\"path\":\"elasticsearch.node.stats.os.cgroup.cpu.cfs.quota.us\",\"type\":\"alias\"}}},\"cpuacct\":{\"properties\":{\"usage_nanos\":{\"path\":\"elasticsearch.node.stats.os.cgroup.cpuacct.usage.ns\",\"type\":\"alias\"}}}}}}},\"thread_pool\":{\"properties\":{\"search\":{\"properties\":{\"rejected\":{\"path\":\"elasticsearch.node.stats.thread_pool.search.rejected.count\",\"type\":\"alias\"},\"queue\":{\"path\":\"elasticsearch.node.stats.thread_pool.search.queue.count\",\"type\":\"alias\"}}},\"get\":{\"properties\":{\"rejected\":{\"path\":\"elasticsearch.node.stats.thread_pool.get.rejected.count\",\"type\":\"alias\"},\"queue\":{\"path\":\"elasticsearch.node.stats.thread_pool.get.queue.count\",\"type\":\"alias\"}}},\"index\":{\"properties\":{\"rejected\":{\"path\":\"elasticsearch.node.stats.thread_pool.index.rejected.count\",\"type\":\"alias\"},\"queue\":{\"path\":\"elasticsearch.node.stats.thread_pool.index.queue.count\",\"type\":\"alias\"}}},\"bulk\":{\"properties\":{\"rejected\":{\"path\":\"elasticsearch.node.stats.thread_pool.bulk.rejected.count\",\"type\":\"alias\"},\"queue\":{\"path\":\"elasticsearch.node.stats.thread_pool.bulk.queue.count\",\"type\":\"alias\"}}},\"write\":{\"properties\":{\"rejected\":{\"path\":\"elasticsearch.node.stats.thread_pool.write.rejected.count\",\"type\":\"alias\"},\"queue\":{\"path\":\"elasticsearch.node.stats.thread_pool.write.queue.count\",\"type\":\"alias\"}}}}},\"fs\":{\"properties\":{\"summary\":{\"properties\":{\"total\":{\"properties\":{\"bytes\":{\"path\":\"elasticsearch.node.stats.fs.summary.total.bytes\",\"type\":\"alias\"}}},\"available\":{\"properties\":{\"bytes\":{\"path\":\"elasticsearch.node.stats.fs.summary.available.bytes\",\"type\":\"alias\"}}}}},\"io_stats\":{\"properties\":{\"total\":{\"properties\":{\"write_operations\":{\"path\":\"elasticsearch.node.stats.fs.io_stats.total.write.operations.count\",\"type\":\"alias\"},\"operations\":{\"path\":\"elasticsearch.node.stats.fs.io_stats.total.operations.count\",\"type\":\"alias\"},\"read_operations\":{\"path\":\"elasticsearch.node.stats.fs.io_stats.total.read.operations.count\",\"type\":\"alias\"}}}}},\"total\":{\"properties\":{\"total_in_bytes\":{\"path\":\"elasticsearch.node.stats.fs.summary.total.bytes\",\"type\":\"alias\"},\"available_in_bytes\":{\"path\":\"elasticsearch.node.stats.fs.summary.available.bytes\",\"type\":\"alias\"}}}}},\"node_id\":{\"path\":\"elasticsearch.node.id\",\"type\":\"alias\"}}},\"stack_stats\":{\"properties\":{\"xpack\":{\"properties\":{\"ccr\":{\"properties\":{\"available\":{\"path\":\"elasticsearch.cluster.stats.stack.xpack.ccr.available\",\"type\":\"alias\"},\"enabled\":{\"path\":\"elasticsearch.cluster.stats.stack.xpack.ccr.enabled\",\"type\":\"alias\"}}}}},\"apm\":{\"properties\":{\"found\":{\"path\":\"elasticsearch.cluster.stats.stack.apm.found\",\"type\":\"alias\"}}}}},\"license\":{\"properties\":{\"type\":{\"path\":\"elasticsearch.cluster.stats.license.type\",\"type\":\"alias\"},\"status\":{\"path\":\"elasticsearch.cluster.stats.license.status\",\"type\":\"alias\"}}},\"index_stats\":{\"properties\":{\"primaries\":{\"properties\":{\"docs\":{\"properties\":{\"count\":{\"path\":\"elasticsearch.index.primaries.docs.count\",\"type\":\"alias\"}}},\"indexing\":{\"properties\":{\"throttle_time_in_millis\":{\"path\":\"elasticsearch.index.primaries.indexing.throttle_time_in_millis\",\"type\":\"alias\"},\"index_time_in_millis\":{\"path\":\"elasticsearch.index.primaries.indexing.index_time_in_millis\",\"type\":\"alias\"},\"index_total\":{\"path\":\"elasticsearch.index.primaries.indexing.index_total\",\"type\":\"alias\"}}},\"refresh\":{\"properties\":{\"total_time_in_millis\":{\"path\":\"elasticsearch.index.primaries.refresh.total_time_in_millis\",\"type\":\"alias\"}}},\"store\":{\"properties\":{\"size_in_bytes\":{\"path\":\"elasticsearch.index.primaries.store.size_in_bytes\",\"type\":\"alias\"}}},\"merges\":{\"properties\":{\"total_size_in_bytes\":{\"path\":\"elasticsearch.index.primaries.merges.total_size_in_bytes\",\"type\":\"alias\"}}},\"segments\":{\"properties\":{\"count\":{\"path\":\"elasticsearch.index.primaries.segments.count\",\"type\":\"alias\"}}}}},\"total\":{\"properties\":{\"search\":{\"properties\":{\"query_total\":{\"path\":\"elasticsearch.index.total.search.query_total\",\"type\":\"alias\"},\"query_time_in_millis\":{\"path\":\"elasticsearch.index.total.search.query_time_in_millis\",\"type\":\"alias\"}}},\"query_cache\":{\"properties\":{\"memory_size_in_bytes\":{\"path\":\"elasticsearch.index.total.query_cache.memory_size_in_bytes\",\"type\":\"alias\"}}},\"fielddata\":{\"properties\":{\"memory_size_in_bytes\":{\"path\":\"elasticsearch.index.total.fielddata.memory_size_in_bytes\",\"type\":\"alias\"}}},\"indexing\":{\"properties\":{\"throttle_time_in_millis\":{\"path\":\"elasticsearch.index.total.indexing.throttle_time_in_millis\",\"type\":\"alias\"},\"index_time_in_millis\":{\"path\":\"elasticsearch.index.total.indexing.index_time_in_millis\",\"type\":\"alias\"},\"index_total\":{\"path\":\"elasticsearch.index.total.indexing.index_total\",\"type\":\"alias\"}}},\"refresh\":{\"properties\":{\"total_time_in_millis\":{\"path\":\"elasticsearch.index.total.refresh.total_time_in_millis\",\"type\":\"alias\"}}},\"request_cache\":{\"properties\":{\"memory_size_in_bytes\":{\"path\":\"elasticsearch.index.total.request_cache.memory_size_in_bytes\",\"type\":\"alias\"}}},\"store\":{\"properties\":{\"size_in_bytes\":{\"path\":\"elasticsearch.index.total.store.size_in_bytes\",\"type\":\"alias\"}}},\"merges\":{\"properties\":{\"total_size_in_bytes\":{\"path\":\"elasticsearch.index.total.merges.total_size_in_bytes\",\"type\":\"alias\"}}},\"segments\":{\"properties\":{\"version_map_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.version_map_memory_in_bytes\",\"type\":\"alias\"},\"norms_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.norms_memory_in_bytes\",\"type\":\"alias\"},\"count\":{\"path\":\"elasticsearch.index.total.segments.count\",\"type\":\"alias\"},\"term_vectors_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.term_vectors_memory_in_bytes\",\"type\":\"alias\"},\"points_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.points_memory_in_bytes\",\"type\":\"alias\"},\"index_writer_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.index_writer_memory_in_bytes\",\"type\":\"alias\"},\"memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.memory_in_bytes\",\"type\":\"alias\"},\"doc_values_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.doc_values_memory_in_bytes\",\"type\":\"alias\"},\"terms_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.terms_memory_in_bytes\",\"type\":\"alias\"},\"fixed_bit_set_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.fixed_bit_set_memory_in_bytes\",\"type\":\"alias\"},\"stored_fields_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.stored_fields_memory_in_bytes\",\"type\":\"alias\"}}}}},\"index\":{\"path\":\"elasticsearch.index.name\",\"type\":\"alias\"}}},\"@timestamp\":{\"type\":\"date\"},\"elasticsearch\":{\"properties\":{\"cluster\":{\"properties\":{\"stats\":{\"properties\":{\"license\":{\"properties\":{\"expiry_date_in_millis\":{\"type\":\"long\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"status\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"indices\":{\"properties\":{\"shards\":{\"properties\":{\"primaries\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"total\":{\"type\":\"long\"},\"docs\":{\"properties\":{\"total\":{\"type\":\"long\"}}},\"fielddata\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"store\":{\"properties\":{\"size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}}}},\"stack\":{\"properties\":{\"xpack\":{\"properties\":{\"ccr\":{\"properties\":{\"available\":{\"type\":\"boolean\"},\"enabled\":{\"type\":\"boolean\"}}}}},\"apm\":{\"properties\":{\"found\":{\"type\":\"boolean\"}}}}},\"nodes\":{\"properties\":{\"jvm\":{\"properties\":{\"memory\":{\"properties\":{\"heap\":{\"properties\":{\"max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"used\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}}}},\"max_uptime\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"data\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"},\"fs\":{\"properties\":{\"total\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"available\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"master\":{\"type\":\"long\"}}},\"state\":{\"properties\":{\"nodes_hash\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"master_node\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state_uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"status\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"properties\":{\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"node\":{\"properties\":{\"jvm\":{\"properties\":{\"memory\":{\"properties\":{\"heap\":{\"properties\":{\"init\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"nonheap\":{\"properties\":{\"init\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}}}},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"process\":{\"properties\":{\"mlockall\":{\"type\":\"boolean\"}}},\"stats\":{\"properties\":{\"jvm\":{\"properties\":{\"mem\":{\"properties\":{\"pools\":{\"properties\":{\"young\":{\"properties\":{\"max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"peak\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"peak_max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"used\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"old\":{\"properties\":{\"max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"peak\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"peak_max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"used\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"survivor\":{\"properties\":{\"max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"peak\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"peak_max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"used\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}}}},\"heap\":{\"properties\":{\"max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"used\":{\"properties\":{\"pct\":{\"type\":\"double\"},\"bytes\":{\"type\":\"long\"}}}}}}},\"gc\":{\"properties\":{\"collectors\":{\"properties\":{\"young\":{\"properties\":{\"collection\":{\"properties\":{\"ms\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}}}},\"old\":{\"properties\":{\"collection\":{\"properties\":{\"ms\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}}}}}}}}}},\"indices\":{\"properties\":{\"search\":{\"properties\":{\"query_total\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"query_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"query_cache\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"docs\":{\"properties\":{\"deleted\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"fielddata\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"indexing\":{\"properties\":{\"index_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"index_total\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"throttle_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"store\":{\"properties\":{\"size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"request_cache\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"bulk\":{\"properties\":{\"operations\":{\"properties\":{\"total\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"total_size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"avg_size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"total_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"avg_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"segments\":{\"properties\":{\"version_map\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"norms\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"terms\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"stored_fields\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"index_writer\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"count\":{\"type\":\"long\"},\"fixed_bit_set\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"term_vectors\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"doc_values\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"points\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}}}}}},\"process\":{\"properties\":{\"cpu\":{\"properties\":{\"pct\":{\"type\":\"double\"}}}}},\"os\":{\"properties\":{\"cpu\":{\"properties\":{\"load_avg\":{\"properties\":{\"1m\":{\"type\":\"half_float\"}}}}},\"cgroup\":{\"properties\":{\"memory\":{\"properties\":{\"control_group\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"usage\":{\"properties\":{\"bytes\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"limit\":{\"properties\":{\"bytes\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"cpu\":{\"properties\":{\"cfs\":{\"properties\":{\"quota\":{\"properties\":{\"us\":{\"type\":\"long\"}}}}},\"stat\":{\"properties\":{\"elapsed_periods\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"times_throttled\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"time_throttled\":{\"properties\":{\"ns\":{\"type\":\"long\"}}}}}}},\"cpuacct\":{\"properties\":{\"usage\":{\"properties\":{\"ns\":{\"type\":\"long\"}}}}}}}}},\"thread_pool\":{\"properties\":{\"search\":{\"properties\":{\"rejected\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"queue\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"get\":{\"properties\":{\"rejected\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"queue\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"index\":{\"properties\":{\"rejected\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"queue\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"bulk\":{\"properties\":{\"rejected\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"queue\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"write\":{\"properties\":{\"rejected\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"queue\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}}}},\"fs\":{\"properties\":{\"summary\":{\"properties\":{\"total\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"available\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"free\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"io_stats\":{\"properties\":{\"total\":{\"properties\":{\"operations\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"read\":{\"properties\":{\"operations\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"write\":{\"properties\":{\"operations\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}}}}}},\"total\":{\"properties\":{\"total_in_bytes\":{\"type\":\"long\"},\"available_in_bytes\":{\"type\":\"long\"}}}}}}},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"mlockall\":{\"type\":\"boolean\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"master\":{\"type\":\"boolean\"}}},\"ccr\":{\"properties\":{\"leader\":{\"properties\":{\"index\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"max_seq_no\":{\"type\":\"long\"},\"global_checkpoint\":{\"type\":\"long\"}}},\"follower\":{\"properties\":{\"operations\":{\"properties\":{\"read\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"time_since_last_read\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"index\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"settings_version\":{\"type\":\"long\"},\"shard\":{\"properties\":{\"number\":{\"type\":\"long\"}}},\"max_seq_no\":{\"type\":\"long\"},\"mapping_version\":{\"type\":\"long\"},\"aliases_version\":{\"type\":\"long\"},\"operations_written\":{\"type\":\"long\"},\"global_checkpoint\":{\"type\":\"long\"}}},\"remote_cluster\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"read_exceptions\":{\"type\":\"nested\"},\"shard_id\":{\"type\":\"long\"},\"bytes_read\":{\"type\":\"long\"},\"requests\":{\"properties\":{\"outstanding\":{\"properties\":{\"read\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"write\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"failed\":{\"properties\":{\"read\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"write\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"successful\":{\"properties\":{\"read\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"write\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}}}},\"auto_follow\":{\"properties\":{\"success\":{\"properties\":{\"follow_indices\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"failed\":{\"properties\":{\"follow_indices\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"remote_cluster_state_requests\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}}}},\"total_time\":{\"properties\":{\"read\":{\"properties\":{\"ms\":{\"type\":\"long\"},\"remote_exec\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"write\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"last_requested_seq_no\":{\"type\":\"long\"},\"write_buffer\":{\"properties\":{\"size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"operation\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}}}},\"pending_tasks\":{\"properties\":{\"time_in_queue\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"source\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"priority\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"insert_order\":{\"type\":\"long\"}}},\"enrich\":{\"properties\":{\"executed_searches\":{\"properties\":{\"total\":{\"type\":\"long\"}}},\"remote_requests\":{\"properties\":{\"current\":{\"type\":\"long\"},\"total\":{\"type\":\"long\"}}},\"executing_policy\":{\"properties\":{\"task\":{\"properties\":{\"task\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"parent_task_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"action\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"type\":\"long\"},\"time\":{\"properties\":{\"running\":{\"properties\":{\"nano\":{\"type\":\"long\"}}},\"start\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"cancellable\":{\"type\":\"boolean\"}}},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"queue\":{\"properties\":{\"size\":{\"type\":\"long\"}}}}},\"index\":{\"properties\":{\"summary\":{\"properties\":{\"primaries\":{\"properties\":{\"search\":{\"properties\":{\"query\":{\"properties\":{\"count\":{\"type\":\"long\"},\"time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}}}},\"docs\":{\"properties\":{\"deleted\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"indexing\":{\"properties\":{\"index\":{\"properties\":{\"count\":{\"type\":\"long\"},\"time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}}}},\"store\":{\"properties\":{\"size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"bulk\":{\"properties\":{\"operations\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"time\":{\"properties\":{\"avg\":{\"properties\":{\"ms\":{\"type\":\"long\"},\"bytes\":{\"type\":\"long\"}}},\"count\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}}}},\"segments\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"count\":{\"type\":\"long\"}}}}},\"total\":{\"properties\":{\"search\":{\"properties\":{\"query\":{\"properties\":{\"count\":{\"type\":\"long\"},\"time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}}}},\"docs\":{\"properties\":{\"deleted\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"indexing\":{\"properties\":{\"index\":{\"properties\":{\"count\":{\"type\":\"long\"},\"time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"is_throttled\":{\"type\":\"boolean\"},\"throttle_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"store\":{\"properties\":{\"size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"bulk\":{\"properties\":{\"operations\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"time\":{\"properties\":{\"avg\":{\"properties\":{\"ms\":{\"type\":\"long\"},\"bytes\":{\"type\":\"long\"}}}}}}},\"segments\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"count\":{\"type\":\"long\"}}}}}}},\"shards\":{\"properties\":{\"primaries\":{\"type\":\"long\"},\"total\":{\"type\":\"long\"}}},\"primaries\":{\"properties\":{\"search\":{\"properties\":{\"query_total\":{\"type\":\"long\"},\"query_time_in_millis\":{\"type\":\"long\"}}},\"query_cache\":{\"properties\":{\"miss_count\":{\"type\":\"long\"},\"memory_size_in_bytes\":{\"type\":\"long\"},\"hit_count\":{\"type\":\"long\"}}},\"docs\":{\"properties\":{\"deleted\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"indexing\":{\"properties\":{\"throttle_time_in_millis\":{\"type\":\"long\"},\"index_time_in_millis\":{\"type\":\"long\"},\"index_total\":{\"type\":\"long\"}}},\"refresh\":{\"properties\":{\"external_total_time_in_millis\":{\"type\":\"long\"},\"total_time_in_millis\":{\"type\":\"long\"}}},\"request_cache\":{\"properties\":{\"miss_count\":{\"type\":\"long\"},\"memory_size_in_bytes\":{\"type\":\"long\"},\"evictions\":{\"type\":\"long\"},\"hit_count\":{\"type\":\"long\"}}},\"store\":{\"properties\":{\"size_in_bytes\":{\"type\":\"long\"}}},\"segments\":{\"properties\":{\"version_map_memory_in_bytes\":{\"type\":\"long\"},\"norms_memory_in_bytes\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"},\"term_vectors_memory_in_bytes\":{\"type\":\"long\"},\"points_memory_in_bytes\":{\"type\":\"long\"},\"index_writer_memory_in_bytes\":{\"type\":\"long\"},\"memory_in_bytes\":{\"type\":\"long\"},\"terms_memory_in_bytes\":{\"type\":\"long\"},\"doc_values_memory_in_bytes\":{\"type\":\"long\"},\"stored_fields_memory_in_bytes\":{\"type\":\"long\"},\"fixed_bit_set_memory_in_bytes\":{\"type\":\"long\"}}},\"merges\":{\"properties\":{\"total_size_in_bytes\":{\"type\":\"long\"}}}}},\"total\":{\"properties\":{\"search\":{\"properties\":{\"query_total\":{\"type\":\"long\"},\"query_time_in_millis\":{\"type\":\"long\"}}},\"query_cache\":{\"properties\":{\"miss_count\":{\"type\":\"long\"},\"memory_size_in_bytes\":{\"type\":\"long\"},\"evictions\":{\"type\":\"long\"},\"hit_count\":{\"type\":\"long\"}}},\"docs\":{\"properties\":{\"deleted\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"fielddata\":{\"properties\":{\"memory_size_in_bytes\":{\"type\":\"long\"},\"evictions\":{\"type\":\"long\"}}},\"indexing\":{\"properties\":{\"throttle_time_in_millis\":{\"type\":\"long\"},\"index_time_in_millis\":{\"type\":\"long\"},\"index_total\":{\"type\":\"long\"}}},\"refresh\":{\"properties\":{\"external_total_time_in_millis\":{\"type\":\"long\"},\"total_time_in_millis\":{\"type\":\"long\"}}},\"store\":{\"properties\":{\"size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"size_in_bytes\":{\"type\":\"long\"}}},\"request_cache\":{\"properties\":{\"miss_count\":{\"type\":\"long\"},\"memory_size_in_bytes\":{\"type\":\"long\"},\"evictions\":{\"type\":\"long\"},\"hit_count\":{\"type\":\"long\"}}},\"bulk\":{\"properties\":{\"avg_time_in_millis\":{\"type\":\"long\"},\"avg_size_in_bytes\":{\"type\":\"long\"},\"total_time_in_millis\":{\"type\":\"long\"},\"total_operations\":{\"type\":\"long\"},\"total_size_in_bytes\":{\"type\":\"long\"}}},\"merges\":{\"properties\":{\"total_size_in_bytes\":{\"type\":\"long\"}}},\"segments\":{\"properties\":{\"version_map_memory_in_bytes\":{\"type\":\"long\"},\"norms_memory_in_bytes\":{\"type\":\"long\"},\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"count\":{\"type\":\"long\"},\"term_vectors_memory_in_bytes\":{\"type\":\"long\"},\"points_memory_in_bytes\":{\"type\":\"long\"},\"index_writer_memory_in_bytes\":{\"type\":\"long\"},\"memory_in_bytes\":{\"type\":\"long\"},\"terms_memory_in_bytes\":{\"type\":\"long\"},\"doc_values_memory_in_bytes\":{\"type\":\"long\"},\"stored_fields_memory_in_bytes\":{\"type\":\"long\"},\"fixed_bit_set_memory_in_bytes\":{\"type\":\"long\"}}}}},\"hidden\":{\"type\":\"boolean\"},\"created\":{\"type\":\"long\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"recovery\":{\"properties\":{\"stop_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"translog\":{\"properties\":{\"total_on_start\":{\"type\":\"long\"},\"total\":{\"type\":\"long\"},\"percent\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"verify_index\":{\"properties\":{\"total_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"check_index_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"index\":{\"properties\":{\"size\":{\"properties\":{\"total_in_bytes\":{\"type\":\"long\"},\"reused_in_bytes\":{\"type\":\"long\"},\"recovered_in_bytes\":{\"type\":\"long\"}}},\"files\":{\"properties\":{\"recovered\":{\"type\":\"long\"},\"total\":{\"type\":\"long\"},\"percent\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"reused\":{\"type\":\"long\"}}}}},\"source\":{\"properties\":{\"transport_address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"host\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"target\":{\"properties\":{\"transport_address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"host\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"start_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"stage\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"type\":\"long\"},\"total_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"primary\":{\"type\":\"boolean\"}}},\"uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"status\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"shard\":{\"properties\":{\"number\":{\"type\":\"long\"},\"source_node\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"relocating_node\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"primary\":{\"type\":\"boolean\"}}},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ml\":{\"properties\":{\"job\":{\"properties\":{\"data\":{\"properties\":{\"invalid_date\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"data_counts\":{\"properties\":{\"invalid_date_count\":{\"type\":\"long\"},\"processed_record_count\":{\"type\":\"long\"}}},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"forecasts_stats\":{\"properties\":{\"total\":{\"type\":\"long\"}}},\"model_size\":{\"properties\":{\"memory_status\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}}}}}},\"cluster_uuid\":{\"path\":\"elasticsearch.cluster.id\",\"type\":\"alias\"},\"ecs\":{\"properties\":{\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"service\":{\"properties\":{\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"index_recovery\":{\"properties\":{\"shards\":{\"properties\":{\"stop_time_in_millis\":{\"path\":\"elasticsearch.index.recovery.stop_time.ms\",\"type\":\"alias\"},\"start_time_in_millis\":{\"path\":\"elasticsearch.index.recovery.start_time.ms\",\"type\":\"alias\"}}}}},\"source_node\":{\"properties\":{\"name\":{\"path\":\"elasticsearch.node.name\",\"type\":\"alias\"},\"uuid\":{\"path\":\"elasticsearch.node.id\",\"type\":\"alias\"}}},\"host\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"metricset\":{\"properties\":{\"period\":{\"type\":\"long\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"shard\":{\"properties\":{\"node\":{\"path\":\"elasticsearch.node.id\",\"type\":\"alias\"},\"index\":{\"path\":\"elasticsearch.index.name\",\"type\":\"alias\"},\"state\":{\"path\":\"elasticsearch.shard.state\",\"type\":\"alias\"},\"shard\":{\"path\":\"elasticsearch.shard.number\",\"type\":\"alias\"},\"primary\":{\"path\":\"elasticsearch.shard.primary\",\"type\":\"alias\"}}},\"event\":{\"properties\":{\"duration\":{\"type\":\"long\"},\"module\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"dataset\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"cluster_stats\":{\"properties\":{\"indices\":{\"properties\":{\"shards\":{\"properties\":{\"total\":{\"path\":\"elasticsearch.cluster.stats.indices.shards.count\",\"type\":\"alias\"}}},\"count\":{\"path\":\"elasticsearch.cluster.stats.indices.total\",\"type\":\"alias\"}}},\"nodes\":{\"properties\":{\"jvm\":{\"properties\":{\"max_uptime_in_millis\":{\"path\":\"elasticsearch.cluster.stats.nodes.jvm.max_uptime.ms\",\"type\":\"alias\"},\"mem\":{\"properties\":{\"heap_max_in_bytes\":{\"path\":\"elasticsearch.cluster.stats.nodes.jvm.memory.heap.max.bytes\",\"type\":\"alias\"},\"heap_used_in_bytes\":{\"path\":\"elasticsearch.cluster.stats.nodes.jvm.memory.heap.used.bytes\",\"type\":\"alias\"}}}}},\"count\":{\"properties\":{\"total\":{\"path\":\"elasticsearch.cluster.stats.nodes.count\",\"type\":\"alias\"}}}}}}},\"timestamp\":{\"path\":\"@timestamp\",\"type\":\"alias\"},\"job_stats\":{\"properties\":{\"job_id\":{\"path\":\"elasticsearch.ml.job.id\",\"type\":\"alias\"},\"forecasts_stats\":{\"properties\":{\"total\":{\"path\":\"elasticsearch.ml.job.forecasts_stats.total\",\"type\":\"alias\"}}}}}}}},\"composed_of\":[],\"version\":8000103,\"data_stream\":{\"hidden\":false,\"allow_custom_routing\":false}},\".slm-history\":{\"index_patterns\":[\".slm-history-5*\"],\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\"slm-history-ilm-policy\"},\"number_of_shards\":\"1\",\"auto_expand_replicas\":\"0-1\",\"number_of_replicas\":\"0\"}},\"mappings\":{\"dynamic\":false,\"properties\":{\"snapshot_name\":{\"type\":\"keyword\"},\"@timestamp\":{\"format\":\"epoch_millis\",\"type\":\"date\"},\"configuration\":{\"dynamic\":false,\"type\":\"object\",\"properties\":{\"indices\":{\"type\":\"keyword\"},\"include_global_state\":{\"type\":\"boolean\"},\"partial\":{\"type\":\"boolean\"}}},\"error_details\":{\"index\":false,\"type\":\"text\"},\"success\":{\"type\":\"boolean\"},\"repository\":{\"type\":\"keyword\"},\"operation\":{\"type\":\"keyword\"},\"policy\":{\"type\":\"keyword\"}}}},\"composed_of\":[],\"priority\":2147483647,\"version\":5,\"_meta\":{\"managed\":true,\"description\":\"index template for SLM history indices\"},\"data_stream\":{\"hidden\":true,\"allow_custom_routing\":false},\"allow_auto_create\":true},\".monitoring-logstash-mb\":{\"index_patterns\":[\".monitoring-logstash-8-*\"],\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\".monitoring-8-ilm-policy\"},\"mapping\":{\"total_fields\":{\"limit\":\"2000\"}}}},\"mappings\":{\"dynamic\":false,\"properties\":{\"logstash\":{\"properties\":{\"node\":{\"properties\":{\"jvm\":{\"properties\":{\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"stats\":{\"properties\":{\"jvm\":{\"properties\":{\"mem\":{\"properties\":{\"heap_max_in_bytes\":{\"type\":\"long\"},\"heap_used_in_bytes\":{\"type\":\"long\"}}},\"uptime_in_millis\":{\"type\":\"long\"}}},\"logstash\":{\"properties\":{\"uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"process\":{\"properties\":{\"cpu\":{\"properties\":{\"percent\":{\"type\":\"double\"}}}}},\"pipelines\":{\"type\":\"nested\",\"properties\":{\"vertices\":{\"type\":\"nested\",\"properties\":{\"events_out\":{\"type\":\"long\"},\"pipeline_ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"events_in\":{\"type\":\"long\"},\"queue_push_duration_in_millis\":{\"type\":\"long\"},\"duration_in_millis\":{\"type\":\"long\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"long_counters\":{\"type\":\"nested\",\"properties\":{\"name\":{\"type\":\"keyword\"},\"value\":{\"type\":\"long\"}}}}},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"type\":\"keyword\"},\"hash\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"queue\":{\"properties\":{\"events_count\":{\"type\":\"long\"},\"max_queue_size_in_bytes\":{\"type\":\"long\"},\"queue_size_in_bytes\":{\"type\":\"long\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"events\":{\"properties\":{\"filtered\":{\"type\":\"long\"},\"in\":{\"type\":\"long\"},\"queue_push_duration_in_millis\":{\"type\":\"long\"},\"duration_in_millis\":{\"type\":\"long\"},\"out\":{\"type\":\"long\"}}},\"reloads\":{\"properties\":{\"failures\":{\"type\":\"long\"},\"successes\":{\"type\":\"long\"}}}}},\"os\":{\"properties\":{\"cpu\":{\"properties\":{\"load_average\":{\"properties\":{\"5m\":{\"type\":\"half_float\"},\"15m\":{\"type\":\"half_float\"},\"1m\":{\"type\":\"half_float\"}}}}},\"cgroup\":{\"properties\":{\"cpu\":{\"properties\":{\"stat\":{\"properties\":{\"number_of_elapsed_periods\":{\"type\":\"long\"},\"number_of_times_throttled\":{\"type\":\"long\"},\"time_throttled_nanos\":{\"type\":\"long\"}}},\"cfs_quota_micros\":{\"type\":\"long\"}}},\"cpuacct\":{\"properties\":{\"usage_nanos\":{\"type\":\"long\"}}}}}}},\"events\":{\"properties\":{\"filtered\":{\"type\":\"long\"},\"in\":{\"type\":\"long\"},\"duration_in_millis\":{\"type\":\"long\"},\"out\":{\"type\":\"long\"}}},\"queue\":{\"properties\":{\"events_count\":{\"type\":\"long\"}}},\"timestamp\":{\"type\":\"date\"}}},\"host\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"properties\":{\"pipeline\":{\"properties\":{\"batch_size\":{\"type\":\"long\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"workers\":{\"type\":\"long\"},\"representation\":{\"properties\":{\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"hash\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"graph\":{\"properties\":{\"vertices\":{\"type\":\"object\"},\"edges\":{\"type\":\"object\"}}}}},\"hash\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"elasticsearch\":{\"properties\":{\"cluster\":{\"properties\":{\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}}}},\"logstash_state\":{\"properties\":{\"pipeline\":{\"properties\":{\"id\":{\"path\":\"logstash.node.state.pipeline.id\",\"type\":\"alias\"},\"hash\":{\"path\":\"logstash.node.state.pipeline.hash\",\"type\":\"alias\"}}}}},\"process\":{\"properties\":{\"pid\":{\"type\":\"long\"}}},\"@timestamp\":{\"type\":\"date\"},\"logstash_stats\":{\"properties\":{\"jvm\":{\"properties\":{\"mem\":{\"properties\":{\"heap_max_in_bytes\":{\"path\":\"logstash.node.stats.jvm.mem.heap_max_in_bytes\",\"type\":\"alias\"},\"heap_used_in_bytes\":{\"path\":\"logstash.node.stats.jvm.mem.heap_used_in_bytes\",\"type\":\"alias\"}}},\"uptime_in_millis\":{\"path\":\"logstash.node.stats.jvm.uptime_in_millis\",\"type\":\"alias\"}}},\"logstash\":{\"properties\":{\"uuid\":{\"path\":\"logstash.node.stats.logstash.uuid\",\"type\":\"alias\"},\"version\":{\"path\":\"logstash.node.stats.logstash.version\",\"type\":\"alias\"}}},\"process\":{\"properties\":{\"cpu\":{\"properties\":{\"percent\":{\"path\":\"logstash.node.stats.process.cpu.percent\",\"type\":\"alias\"}}}}},\"pipelines\":{\"type\":\"nested\"},\"os\":{\"properties\":{\"cpu\":{\"properties\":{\"load_average\":{\"properties\":{\"5m\":{\"path\":\"logstash.node.stats.os.cpu.load_average.5m\",\"type\":\"alias\"},\"15m\":{\"path\":\"logstash.node.stats.os.cpu.load_average.15m\",\"type\":\"alias\"},\"1m\":{\"path\":\"logstash.node.stats.os.cpu.load_average.1m\",\"type\":\"alias\"}}}}},\"cgroup\":{\"properties\":{\"cpu\":{\"properties\":{\"stat\":{\"properties\":{\"number_of_elapsed_periods\":{\"path\":\"logstash.node.stats.os.cgroup.cpu.stat.number_of_elapsed_periods\",\"type\":\"alias\"},\"number_of_times_throttled\":{\"path\":\"logstash.node.stats.os.cgroup.cpu.stat.number_of_times_throttled\",\"type\":\"alias\"},\"time_throttled_nanos\":{\"path\":\"logstash.node.stats.os.cgroup.cpu.stat.time_throttled_nanos\",\"type\":\"alias\"}}},\"cfs_quota_micros\":{\"path\":\"logstash.node.stats.os.cgroup.cpu.cfs_quota_micros\",\"type\":\"alias\"}}},\"cpuacct\":{\"properties\":{\"usage_nanos\":{\"path\":\"logstash.node.stats.os.cgroup.cpuacct.usage_nanos\",\"type\":\"alias\"}}}}}}},\"events\":{\"properties\":{\"in\":{\"path\":\"logstash.node.stats.events.in\",\"type\":\"alias\"},\"duration_in_millis\":{\"path\":\"logstash.node.stats.events.duration_in_millis\",\"type\":\"alias\"},\"out\":{\"path\":\"logstash.node.stats.events.out\",\"type\":\"alias\"}}},\"queue\":{\"properties\":{\"events_count\":{\"path\":\"logstash.node.stats.queue.events_count\",\"type\":\"alias\"}}},\"timestamp\":{\"path\":\"@timestamp\",\"type\":\"alias\"}}},\"ecs\":{\"properties\":{\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"cluster_uuid\":{\"path\":\"logstash.elasticsearch.cluster.id\",\"type\":\"alias\"},\"service\":{\"properties\":{\"hostname\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"host\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"metricset\":{\"properties\":{\"period\":{\"type\":\"long\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"event\":{\"properties\":{\"duration\":{\"type\":\"long\"},\"module\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"dataset\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"timestamp\":{\"path\":\"@timestamp\",\"type\":\"alias\"}}}},\"composed_of\":[],\"version\":8000103,\"data_stream\":{\"hidden\":false,\"allow_custom_routing\":false}},\".ml-anomalies-\":{\"index_patterns\":[\".ml-anomalies-*\"],\"template\":{\"settings\":{\"index\":{\"hidden\":\"true\",\"translog\":{\"durability\":\"async\"},\"auto_expand_replicas\":\"0-1\",\"query\":{\"default_field\":\"all_field_values\"}}},\"mappings\":{\"_meta\":{\"version\":\"8.6.2\"},\"dynamic_templates\":[{\"strings_as_keywords\":{\"mapping\":{\"type\":\"keyword\"},\"match\":\"*\"}}],\"properties\":{\"search_count\":{\"type\":\"long\"},\"bucket_count\":{\"type\":\"long\"},\"anomaly_score_explanation\":{\"properties\":{\"high_variance_penalty\":{\"type\":\"boolean\"},\"typical_value\":{\"type\":\"double\"},\"upper_confidence_bound\":{\"type\":\"double\"},\"lower_confidence_bound\":{\"type\":\"double\"},\"incomplete_bucket_penalty\":{\"type\":\"boolean\"},\"anomaly_type\":{\"type\":\"keyword\"},\"anomaly_characteristics_impact\":{\"type\":\"integer\"},\"multi_bucket_impact\":{\"type\":\"integer\"},\"anomaly_length\":{\"type\":\"integer\"},\"single_bucket_impact\":{\"type\":\"integer\"}}},\"terms\":{\"type\":\"text\"},\"record_score\":{\"type\":\"double\"},\"forecast_expiry_timestamp\":{\"type\":\"date\"},\"over_field_value\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"preferred_to_categories\":{\"type\":\"long\"},\"empty_bucket_count\":{\"type\":\"long\"},\"forecast_create_timestamp\":{\"type\":\"date\"},\"probability\":{\"type\":\"double\"},\"missing_field_count\":{\"type\":\"long\"},\"forecast_lower\":{\"type\":\"double\"},\"influencer_field_value\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"multi_bucket_impact\":{\"type\":\"double\"},\"forecast_progress\":{\"type\":\"double\"},\"max_matching_length\":{\"type\":\"long\"},\"latest_record_time_stamp\":{\"type\":\"date\"},\"regex\":{\"type\":\"keyword\"},\"examples\":{\"type\":\"text\"},\"snapshot_doc_count\":{\"type\":\"integer\"},\"average_bucket_processing_time_ms\":{\"type\":\"double\"},\"initial_anomaly_score\":{\"type\":\"double\"},\"initial_influencer_score\":{\"type\":\"double\"},\"partition_field_value\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"model_lower\":{\"type\":\"double\"},\"total_over_field_count\":{\"type\":\"long\"},\"retain\":{\"type\":\"boolean\"},\"forecast_upper\":{\"type\":\"double\"},\"latest_sparse_bucket_timestamp\":{\"type\":\"date\"},\"model_median\":{\"type\":\"double\"},\"category_id\":{\"type\":\"long\"},\"causes\":{\"type\":\"nested\",\"properties\":{\"actual\":{\"type\":\"double\"},\"partition_field_name\":{\"type\":\"keyword\"},\"partition_field_value\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"by_field_name\":{\"type\":\"keyword\"},\"probability\":{\"type\":\"double\"},\"geo_results\":{\"properties\":{\"actual_point\":{\"type\":\"geo_point\"},\"typical_point\":{\"type\":\"geo_point\"}}},\"field_name\":{\"type\":\"keyword\"},\"by_field_value\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"over_field_name\":{\"type\":\"keyword\"},\"correlated_by_field_value\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"function\":{\"type\":\"keyword\"},\"typical\":{\"type\":\"double\"},\"over_field_value\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"function_description\":{\"type\":\"keyword\"}}},\"all_field_values\":{\"analyzer\":\"whitespace\",\"type\":\"text\"},\"timestamp\":{\"type\":\"date\"},\"input_field_count\":{\"type\":\"long\"},\"model_bytes\":{\"type\":\"long\"},\"quantiles\":{\"type\":\"object\",\"enabled\":false},\"@timestamp\":{\"path\":\"timestamp\",\"type\":\"alias\"},\"function_description\":{\"type\":\"keyword\"},\"min_version\":{\"type\":\"keyword\"},\"raw_anomaly_score\":{\"type\":\"double\"},\"exponential_average_bucket_processing_time_ms\":{\"type\":\"double\"},\"invalid_date_count\":{\"type\":\"long\"},\"snapshot_id\":{\"type\":\"keyword\"},\"model_size_stats\":{\"properties\":{\"model_bytes\":{\"type\":\"long\"},\"result_type\":{\"type\":\"keyword\"},\"peak_model_bytes\":{\"type\":\"long\"},\"job_id\":{\"type\":\"keyword\"},\"total_over_field_count\":{\"type\":\"long\"},\"total_partition_field_count\":{\"type\":\"long\"},\"total_by_field_count\":{\"type\":\"long\"},\"assignment_memory_basis\":{\"type\":\"keyword\"},\"bucket_allocation_failures_count\":{\"type\":\"long\"},\"memory_status\":{\"type\":\"keyword\"},\"log_time\":{\"type\":\"date\"},\"timestamp\":{\"type\":\"date\"}}},\"total_search_time_ms\":{\"type\":\"double\"},\"latest_empty_bucket_timestamp\":{\"type\":\"date\"},\"anomaly_score\":{\"type\":\"double\"},\"over_field_name\":{\"type\":\"keyword\"},\"earliest_record_timestamp\":{\"type\":\"date\"},\"scheduled_events\":{\"type\":\"keyword\"},\"bucket_span\":{\"type\":\"long\"},\"maximum_bucket_processing_time_ms\":{\"type\":\"double\"},\"exponential_average_calculation_context\":{\"properties\":{\"incremental_metric_value_ms\":{\"type\":\"double\"},\"previous_exponential_average_ms\":{\"type\":\"double\"},\"latest_timestamp\":{\"type\":\"date\"}}},\"function\":{\"type\":\"keyword\"},\"influencers\":{\"type\":\"nested\",\"properties\":{\"influencer_field_name\":{\"type\":\"keyword\"},\"influencer_field_values\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"}}},\"input_record_count\":{\"type\":\"long\"},\"latest_result_time_stamp\":{\"type\":\"date\"},\"model_upper\":{\"type\":\"double\"},\"actual\":{\"type\":\"double\"},\"forecast_memory_bytes\":{\"type\":\"long\"},\"total_by_field_count\":{\"type\":\"long\"},\"geo_results\":{\"properties\":{\"actual_point\":{\"type\":\"geo_point\"},\"typical_point\":{\"type\":\"geo_point\"}}},\"detector_index\":{\"type\":\"integer\"},\"by_field_value\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"processed_record_count\":{\"type\":\"long\"},\"total_partition_field_count\":{\"type\":\"long\"},\"mlcategory\":{\"type\":\"keyword\"},\"assignment_memory_basis\":{\"type\":\"keyword\"},\"forecast_end_timestamp\":{\"type\":\"date\"},\"forecast_id\":{\"type\":\"keyword\"},\"partition_field_name\":{\"type\":\"keyword\"},\"by_field_name\":{\"type\":\"keyword\"},\"event_count\":{\"type\":\"long\"},\"description\":{\"type\":\"text\"},\"is_interim\":{\"type\":\"boolean\"},\"bucket_allocation_failures_count\":{\"type\":\"long\"},\"memory_status\":{\"type\":\"keyword\"},\"model_feature\":{\"type\":\"keyword\"},\"num_matches\":{\"type\":\"long\"},\"influencer_score\":{\"type\":\"double\"},\"out_of_order_timestamp_count\":{\"type\":\"long\"},\"result_type\":{\"type\":\"keyword\"},\"last_data_time\":{\"type\":\"date\"},\"latest_record_timestamp\":{\"type\":\"date\"},\"influencer_field_name\":{\"type\":\"keyword\"},\"forecast_messages\":{\"type\":\"keyword\"},\"sparse_bucket_count\":{\"type\":\"long\"},\"log_time\":{\"type\":\"date\"},\"field_name\":{\"type\":\"keyword\"},\"minimum_bucket_processing_time_ms\":{\"type\":\"double\"},\"bucket_influencers\":{\"type\":\"nested\",\"properties\":{\"anomaly_score\":{\"type\":\"double\"},\"initial_anomaly_score\":{\"type\":\"double\"},\"result_type\":{\"type\":\"keyword\"},\"raw_anomaly_score\":{\"type\":\"double\"},\"bucket_span\":{\"type\":\"long\"},\"job_id\":{\"type\":\"keyword\"},\"probability\":{\"type\":\"double\"},\"influencer_field_name\":{\"type\":\"keyword\"},\"is_interim\":{\"type\":\"boolean\"},\"timestamp\":{\"type\":\"date\"}}},\"processing_time_ms\":{\"type\":\"long\"},\"input_bytes\":{\"type\":\"long\"},\"initial_record_score\":{\"type\":\"double\"},\"job_id\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"processed_field_count\":{\"type\":\"long\"},\"forecast_status\":{\"type\":\"keyword\"},\"typical\":{\"type\":\"double\"},\"forecast_prediction\":{\"type\":\"double\"},\"forecast_start_timestamp\":{\"type\":\"date\"}}}},\"composed_of\":[],\"priority\":2147483647,\"version\":8060299,\"_meta\":{\"managed\":true,\"description\":\"index template for ML anomaly detection results indices\"}},\"metrics\":{\"index_patterns\":[\"metrics-*-*\"],\"composed_of\":[\"metrics-mappings\",\"data-streams-mappings\",\"metrics-settings\"],\"priority\":100,\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default metrics template installed by x-pack\"},\"data_stream\":{\"hidden\":false,\"allow_custom_routing\":false},\"allow_auto_create\":true},\".ml-notifications-000002\":{\"index_patterns\":[\".ml-notifications-000002\"],\"template\":{\"settings\":{\"index\":{\"hidden\":\"true\",\"number_of_shards\":\"1\",\"auto_expand_replicas\":\"0-1\"}},\"mappings\":{\"_meta\":{\"version\":\"8.6.2\"},\"dynamic\":\"false\",\"properties\":{\"job_type\":{\"type\":\"keyword\"},\"level\":{\"type\":\"keyword\"},\"job_id\":{\"type\":\"keyword\"},\"node_name\":{\"type\":\"keyword\"},\"message\":{\"type\":\"text\",\"fields\":{\"raw\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"cleared\":{\"type\":\"boolean\"},\"timestamp\":{\"type\":\"date\"}}}},\"composed_of\":[],\"priority\":2147483647,\"version\":8060299,\"_meta\":{\"managed\":true,\"description\":\"index template for ML notifications indices\"}},\".deprecation-indexing-template\":{\"index_patterns\":[\".logs-deprecation.*\"],\"composed_of\":[\".deprecation-indexing-mappings\",\".deprecation-indexing-settings\"],\"priority\":1000,\"version\":1,\"_meta\":{\"managed\":true,\"description\":\"default template for Stack deprecation logs index template installed by x-pack\"},\"data_stream\":{\"hidden\":true,\"allow_custom_routing\":false},\"allow_auto_create\":true},\"logs\":{\"index_patterns\":[\"logs-*-*\"],\"composed_of\":[\"logs-mappings\",\"data-streams-mappings\",\"logs-settings\"],\"priority\":100,\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default logs template installed by x-pack\"},\"data_stream\":{\"hidden\":false,\"allow_custom_routing\":false},\"allow_auto_create\":true},\".ml-stats\":{\"index_patterns\":[\".ml-stats-*\"],\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\"ml-size-based-ilm-policy\",\"rollover_alias\":\".ml-stats-write\"},\"hidden\":\"true\",\"number_of_shards\":\"1\",\"auto_expand_replicas\":\"0-1\"}},\"mappings\":{\"_meta\":{\"version\":\"8.6.2\"},\"dynamic\":false,\"properties\":{\"skipped_docs_count\":{\"type\":\"long\"},\"validation_loss\":{\"properties\":{\"fold_values\":{\"properties\":{\"fold\":{\"type\":\"integer\"},\"values\":{\"type\":\"double\"}}},\"loss_type\":{\"type\":\"keyword\"}}},\"cache_miss_count\":{\"type\":\"long\"},\"timing_stats\":{\"properties\":{\"iteration_time\":{\"type\":\"long\"},\"elapsed_time\":{\"type\":\"long\"}}},\"failure_count\":{\"type\":\"long\"},\"model_id\":{\"type\":\"keyword\"},\"type\":{\"type\":\"keyword\"},\"training_docs_count\":{\"type\":\"long\"},\"inference_count\":{\"type\":\"long\"},\"job_id\":{\"type\":\"keyword\"},\"missing_all_fields_count\":{\"type\":\"long\"},\"peak_usage_bytes\":{\"type\":\"long\"},\"iteration\":{\"type\":\"integer\"},\"hyperparameters\":{\"properties\":{\"max_attempts_to_add_tree\":{\"type\":\"integer\"},\"downsample_factor\":{\"type\":\"double\"},\"eta_growth_rate_per_tree\":{\"type\":\"double\"},\"soft_tree_depth_tolerance\":{\"type\":\"double\"},\"max_trees\":{\"type\":\"integer\"},\"lambda\":{\"type\":\"double\"},\"max_optimization_rounds_per_hyperparameter\":{\"type\":\"integer\"},\"eta\":{\"type\":\"double\"},\"soft_tree_depth_limit\":{\"type\":\"double\"},\"alpha\":{\"type\":\"double\"},\"class_assignment_objective\":{\"type\":\"keyword\"},\"feature_bag_fraction\":{\"type\":\"double\"},\"num_splits_per_feature\":{\"type\":\"integer\"},\"gamma\":{\"type\":\"double\"},\"num_folds\":{\"type\":\"integer\"}}},\"parameters\":{\"properties\":{\"compute_feature_influence\":{\"type\":\"boolean\"},\"feature_influence_threshold\":{\"type\":\"double\"},\"outlier_fraction\":{\"type\":\"double\"},\"method\":{\"type\":\"keyword\"},\"standardization_enabled\":{\"type\":\"boolean\"},\"n_neighbors\":{\"type\":\"integer\"}}},\"test_docs_count\":{\"type\":\"long\"},\"node_id\":{\"type\":\"keyword\"},\"timestamp\":{\"type\":\"date\"}}},\"aliases\":{}},\"composed_of\":[],\"priority\":2147483647,\"version\":8060299,\"_meta\":{\"managed\":true,\"description\":\"index template for ML stats indices\"}}}} index-graveyard: IndexGraveyard[[[index=[abacus_go/zV2aNzxVQxqMDQTD4A9TEQ], deleteDate=2023-03-28T03:56:57.282Z], [index=[access_platform_core/ZuN0_cUfSQKzXg_yG4Sv5g], deleteDate=2023-03-28T03:58:27.958Z], [index=[ad_bidder/6ihfVU6rRqOo4Gv4Djtetg], deleteDate=2023-03-28T03:59:11.628Z], [index=[apphealth/EthxrkUtTGCTP2mX-tS63Q], deleteDate=2023-03-28T03:59:36.596Z]]]\nnodes: \n {hostb9}{iKPGCkp9RVOKXOj20uOt4g}{cPRh9FqiSlGP9jClhm8Ugg}{hostb9}{127.0.0.1}{127.0.0.1:9300}{cdfhilmrstw}{ml.max_jvm_size=33285996544, ml.allocated_processors_double=16.0, xpack.installed=true, ml.machine_memory=135062048768, ml.allocated_processors=16}, local, master\nrouting_table (version 35):\n-- index [[abacus_go/cfvmY-Y_Su63-qSkPODDzg]]\n----shard_id [abacus_go][0]\n--------[abacus_go][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=AMhgJMlSTPOoKSsL7s3OKA], failed_attempts[0]\n--------[abacus_go][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=INDEX_CREATED], at[2023-03-28T03:57:31.683Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[web_wok/R1IULAXSQ4SC0pLFCU5oJA]]\n----shard_id [web_wok][0]\n--------[web_wok][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=JXruVMhdRVGvP1kQCbUNSQ], failed_attempts[0]\n--------[web_wok][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[access_platform_core/l4kdkOEQRaKfWGLpTZURCQ]]\n----shard_id [access_platform_core][0]\n--------[access_platform_core][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=6ajNSTQZSr6tP3YtmRaxgw], failed_attempts[0]\n--------[access_platform_core][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=INDEX_CREATED], at[2023-03-28T03:58:31.318Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[driverapp/cqbade6KQFqLEK-wPKlkBg]]\n----shard_id [driverapp][0]\n--------[driverapp][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=b5_Bhd3ITjOkl-xLF8H1jg], failed_attempts[0]\n--------[driverapp][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[k8s_kubelet/fQ9Iy7wlTKahsWLbs-oE_Q]]\n----shard_id [k8s_kubelet][0]\n--------[k8s_kubelet][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=Q4Xb11JATPaFZqmgTQHcVQ], failed_attempts[0]\n--------[k8s_kubelet][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.307Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[eats_store/hF-kN4XFRy--z_gUhYLwdA]]\n----shard_id [eats_store][0]\n--------[eats_store][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=WRNwRQ6gTNGfSNQok4iSyQ], failed_attempts[0]\n--------[eats_store][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[rider_product_cored/AdGiEBLBQbicsXDomsa3mw]]\n----shard_id [rider_product_cored][0]\n--------[rider_product_cored][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=GwRH_ZlFRtCV7m_n626z7g], failed_attempts[0]\n--------[rider_product_cored][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[finprod_rewards_eligibility/dJNRGPwrQS2bIPlpsXgUoQ]]\n----shard_id [finprod_rewards_eligibility][0]\n--------[finprod_rewards_eligibility][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=Hg4Dm9O8QEqXWKY7rH1X0A], failed_attempts[0]\n--------[finprod_rewards_eligibility][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[event_logs/DcomuHoGQzOjZq2bX67TFg]]\n----shard_id [event_logs][0]\n--------[event_logs][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=1uCl_LIDSBCWh6koVA0fjQ], failed_attempts[0]\n--------[event_logs][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[apphealth/rcPrX1aWRmeED9W5PU6A4g]]\n----shard_id [apphealth][0]\n--------[apphealth][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=Abip50RxQTmavqi4NTwSqQ], failed_attempts[0]\n--------[apphealth][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=INDEX_CREATED], at[2023-03-28T03:59:38.780Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[ad_bidder/tLHfw5r2QH-o7lG5tapKxw]]\n----shard_id [ad_bidder][0]\n--------[ad_bidder][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=YtEUwPgfQDGM0Cm_C4hXlw], failed_attempts[0]\n--------[ad_bidder][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=INDEX_CREATED], at[2023-03-28T03:59:13.333Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[invoice_gen/RPu5EzJVRfuqyI8MQ7iTjQ]]\n----shard_id [invoice_gen][0]\n--------[invoice_gen][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=onWbjhHuTD2C3AEayRJq1w], failed_attempts[0]\n--------[invoice_gen][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[fares_management/F1lH-QvyTkS5hnbDFwE2Xg]]\n----shard_id [fares_management][0]\n--------[fares_management][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=sOxfKX3nReynYN5HN88cuQ], failed_attempts[0]\n--------[fares_management][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[michelangelo_gateway/ZV2vZ8PmS0uysg-xzKO7QA]]\n----shard_id [michelangelo_gateway][0]\n--------[michelangelo_gateway][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=k5vDE54CSTSakhfgtGaTcA], failed_attempts[0]\n--------[michelangelo_gateway][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[.geoip_databases/5k1GHwKvQBKersi12DFh4A]]\n----shard_id [.geoip_databases][0]\n--------[.geoip_databases][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=UdvCKUGqTEmYFzZlnSu_tA], failed_attempts[0]\n\n-- index [[fares_intelligence/pSRJWxyXSqefhwLVhmT9nQ]]\n----shard_id [fares_intelligence][0]\n--------[fares_intelligence][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=-dD8ROsgS6qaYMoalciocA], failed_attempts[0]\n--------[fares_intelligence][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[fare_route/a4fjivWhQRee8JW7ZbXiXw]]\n----shard_id [fare_route][0]\n--------[fare_route][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=ltSLFgu4Rry0_0vXq28DbA], failed_attempts[0]\n--------[fare_route][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[k8s_apiserver/qQasynZgQ6e6AmD5ZWqh3w]]\n----shard_id [k8s_apiserver][0]\n--------[k8s_apiserver][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=Aaa0F0jiStCkteGDYR8yRQ], failed_attempts[0]\n--------[k8s_apiserver][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[fulfillment_compatibled/tMgrGBWeR3irgR5PyN8Oig]]\n----shard_id [fulfillment_compatibled][0]\n--------[fulfillment_compatibled][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=D_3C_3kMSNKa-XJk7pyoGg], failed_attempts[0]\n--------[fulfillment_compatibled][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\nrouting_nodes:\n-----node_id[iKPGCkp9RVOKXOj20uOt4g][V]\n--------[abacus_go][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=AMhgJMlSTPOoKSsL7s3OKA], failed_attempts[0]\n--------[web_wok][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=JXruVMhdRVGvP1kQCbUNSQ], failed_attempts[0]\n--------[access_platform_core][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=6ajNSTQZSr6tP3YtmRaxgw], failed_attempts[0]\n--------[driverapp][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=b5_Bhd3ITjOkl-xLF8H1jg], failed_attempts[0]\n--------[k8s_kubelet][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=Q4Xb11JATPaFZqmgTQHcVQ], failed_attempts[0]\n--------[eats_store][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=WRNwRQ6gTNGfSNQok4iSyQ], failed_attempts[0]\n--------[rider_product_cored][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=GwRH_ZlFRtCV7m_n626z7g], failed_attempts[0]\n--------[finprod_rewards_eligibility][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=Hg4Dm9O8QEqXWKY7rH1X0A], failed_attempts[0]\n--------[event_logs][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=1uCl_LIDSBCWh6koVA0fjQ], failed_attempts[0]\n--------[apphealth][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=Abip50RxQTmavqi4NTwSqQ], failed_attempts[0]\n--------[ad_bidder][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=YtEUwPgfQDGM0Cm_C4hXlw], failed_attempts[0]\n--------[invoice_gen][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=onWbjhHuTD2C3AEayRJq1w], failed_attempts[0]\n--------[fares_management][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=sOxfKX3nReynYN5HN88cuQ], failed_attempts[0]\n--------[michelangelo_gateway][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=k5vDE54CSTSakhfgtGaTcA], failed_attempts[0]\n--------[.geoip_databases][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=UdvCKUGqTEmYFzZlnSu_tA], failed_attempts[0]\n--------[fares_intelligence][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=-dD8ROsgS6qaYMoalciocA], failed_attempts[0]\n--------[fare_route][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=ltSLFgu4Rry0_0vXq28DbA], failed_attempts[0]\n--------[k8s_apiserver][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=Aaa0F0jiStCkteGDYR8yRQ], failed_attempts[0]\n--------[fulfillment_compatibled][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=D_3C_3kMSNKa-XJk7pyoGg], failed_attempts[0]\n---- unassigned\n--------[abacus_go][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=INDEX_CREATED], at[2023-03-28T03:57:31.683Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[web_wok][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[access_platform_core][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=INDEX_CREATED], at[2023-03-28T03:58:31.318Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[driverapp][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[k8s_kubelet][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.307Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[eats_store][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[rider_product_cored][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[finprod_rewards_eligibility][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[event_logs][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[apphealth][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=INDEX_CREATED], at[2023-03-28T03:59:38.780Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[ad_bidder][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=INDEX_CREATED], at[2023-03-28T03:59:13.333Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[invoice_gen][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[fares_management][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[michelangelo_gateway][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[fares_intelligence][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[fare_route][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[k8s_apiserver][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[fulfillment_compatibled][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\ncustoms:\n health: org.elasticsearch.health.metadata.HealthMetadata@66bc0723", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.service.MasterService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:02.058Z", "log.level":"DEBUG", "message":"publishing cluster state version [154]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.service.MasterService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:02.059Z", "log.level":"TRACE", "message":"handleClientValue: processing request for version [154] and term [6]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.coordination.CoordinationState","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:02.059Z", "log.level":"TRACE", "message":"setCurrentNodes: [{hostb9}{iKPGCkp9RVOKXOj20uOt4g}{cPRh9FqiSlGP9jClhm8Ugg}{hostb9}{127.0.0.1}{127.0.0.1:9300}{cdfhilmrstw}{ml.max_jvm_size=33285996544, ml.allocated_processors_double=16.0, xpack.installed=true, ml.machine_memory=135062048768, ml.allocated_processors=16}]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.coordination.LeaderChecker","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:02.064Z", "log.level":"TRACE", "message":"publishing PublishRequest{term=6, version=154, state=cluster uuid: bp-xkJD1S1iyBlj6I_JRHA [committed: true]\nversion: 154\nstate uuid: Xx549IAcSMCY30xGiugtFg\nfrom_diff: false\nmeta data version: 56\n coordination_metadata:\n term: 6\n last_committed_config: VotingConfiguration{iKPGCkp9RVOKXOj20uOt4g}\n last_accepted_config: VotingConfiguration{iKPGCkp9RVOKXOj20uOt4g}\n voting tombstones: []\n [k8s_kubelet/fQ9Iy7wlTKahsWLbs-oE_Q]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [Q4Xb11JATPaFZqmgTQHcVQ]\n [fares_intelligence/pSRJWxyXSqefhwLVhmT9nQ]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [-dD8ROsgS6qaYMoalciocA]\n [event_logs/DcomuHoGQzOjZq2bX67TFg]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [1uCl_LIDSBCWh6koVA0fjQ]\n [rider_product_cored/AdGiEBLBQbicsXDomsa3mw]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [GwRH_ZlFRtCV7m_n626z7g]\n [access_platform_core/l4kdkOEQRaKfWGLpTZURCQ]: v[6], mv[1], sv[2], av[1]\n 0: p_term [1], isa_ids [6ajNSTQZSr6tP3YtmRaxgw]\n [.geoip_databases/5k1GHwKvQBKersi12DFh4A]: v[6], mv[1], sv[1], av[1]\n 0: p_term [3], isa_ids [UdvCKUGqTEmYFzZlnSu_tA]\n [web_wok/R1IULAXSQ4SC0pLFCU5oJA]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [JXruVMhdRVGvP1kQCbUNSQ]\n [abacus_go/cfvmY-Y_Su63-qSkPODDzg]: v[6], mv[1], sv[2], av[1]\n 0: p_term [1], isa_ids [AMhgJMlSTPOoKSsL7s3OKA]\n [k8s_apiserver/qQasynZgQ6e6AmD5ZWqh3w]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [Aaa0F0jiStCkteGDYR8yRQ]\n [michelangelo_gateway/ZV2vZ8PmS0uysg-xzKO7QA]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [k5vDE54CSTSakhfgtGaTcA]\n [driverapp/cqbade6KQFqLEK-wPKlkBg]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [b5_Bhd3ITjOkl-xLF8H1jg]\n [apphealth/rcPrX1aWRmeED9W5PU6A4g]: v[4], mv[1], sv[1], av[1]\n 0: p_term [1], isa_ids [Abip50RxQTmavqi4NTwSqQ]\n [fares_management/F1lH-QvyTkS5hnbDFwE2Xg]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [sOxfKX3nReynYN5HN88cuQ]\n [finprod_rewards_eligibility/dJNRGPwrQS2bIPlpsXgUoQ]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [Hg4Dm9O8QEqXWKY7rH1X0A]\n [fare_route/a4fjivWhQRee8JW7ZbXiXw]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [ltSLFgu4Rry0_0vXq28DbA]\n [ad_bidder/tLHfw5r2QH-o7lG5tapKxw]: v[6], mv[1], sv[2], av[1]\n 0: p_term [1], isa_ids [YtEUwPgfQDGM0Cm_C4hXlw]\n [fulfillment_compatibled/tMgrGBWeR3irgR5PyN8Oig]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [D_3C_3kMSNKa-XJk7pyoGg]\n [eats_store/hF-kN4XFRy--z_gUhYLwdA]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [WRNwRQ6gTNGfSNQok4iSyQ]\n [invoice_gen/RPu5EzJVRfuqyI8MQ7iTjQ]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [onWbjhHuTD2C3AEayRJq1w]\nmetadata customs:\n licenses: LicensesMetadata{license={\"uid\":\"8e65ab16-3f3f-48cc-ab5f-e20097ceb81e\",\"type\":\"basic\",\"issue_date_in_millis\":1679968689560,\"max_nodes\":1000,\"max_resource_units\":null,\"issued_to\":\"elasticsearch\",\"issuer\":\"elasticsearch\",\"signature\":\"////+wAAAOBYLHp18OvjZrRTi0AmCZ6DOpkrd2cXJB/ffBB+AxaDsF+z7lkfdPbgYofMvUB7XbiD5j95mIfUcdlTJHVdk9NpPTFMKL5wC7BXn4CTS6kNox+Xi0+qSLpcgHzelOT63alJr1L5j/2pqffHnuMLG88Ye15eFhAb9H2A5MS3hX0HG9uTxyEidFCyu4gWV4qrZJDQKLpzC49MMnpP+ob2bNH/pA7dGVdWqEJpVNPe5/hAtGtCdVdUX6DvV0Q3dgwuejc0hRg/vl0YUtp45bQPBHDHECNKgxWmJCZrfstl/prRHA==\",\"start_date_in_millis\":-1}, trialVersion=null} component_template: {\"component_template\":{\"metrics-mappings\":{\"template\":{\"mappings\":{\"dynamic_templates\":[{\"long_metrics\":{\"mapping\":{\"index\":false,\"type\":\"long\"},\"match_mapping_type\":\"long\"}},{\"double_metrics\":{\"mapping\":{\"index\":false,\"type\":\"float\"},\"match_mapping_type\":\"double\"}}],\"properties\":{\"data_stream\":{\"properties\":{\"type\":{\"type\":\"constant_keyword\",\"value\":\"metrics\"}}}}}},\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default mappings for the metrics index template installed by x-pack\"}},\".deprecation-indexing-mappings\":{\"template\":{\"mappings\":{\"dynamic_templates\":[{\"strings_as_keyword\":{\"mapping\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"match_mapping_type\":\"string\"}}],\"date_detection\":false,\"properties\":{\"@timestamp\":{\"type\":\"date\"},\"ecs\":{\"properties\":{\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"data_stream\":{\"properties\":{\"namespace\":{\"type\":\"constant_keyword\"},\"type\":{\"type\":\"constant_keyword\",\"value\":\"logs\"},\"dataset\":{\"type\":\"constant_keyword\"}}},\"host\":{\"properties\":{\"ip\":{\"type\":\"ip\"}}},\"message\":{\"type\":\"text\"}}}},\"version\":1,\"_meta\":{\"managed\":true,\"description\":\"default mappings for Stack deprecation logs index template installed by x-pack\"}},\"synthetics-mappings\":{\"template\":{\"mappings\":{\"properties\":{\"data_stream\":{\"properties\":{\"type\":{\"type\":\"constant_keyword\",\"value\":\"synthetics\"}}}}}},\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default mappings for the synthetics index template installed by x-pack\"}},\"metrics-settings\":{\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\"metrics\"},\"codec\":\"best_compression\",\"query\":{\"default_field\":[\"message\"]}}}},\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default settings for the metrics index template installed by x-pack\"}},\"logs-settings\":{\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\"logs\"},\"codec\":\"best_compression\",\"query\":{\"default_field\":[\"message\"]}}}},\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default settings for the logs index template installed by x-pack\"}},\".deprecation-indexing-settings\":{\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\".deprecation-indexing-ilm-policy\"},\"codec\":\"best_compression\",\"hidden\":\"true\",\"auto_expand_replicas\":\"0-1\",\"query\":{\"default_field\":[\"message\"]}}}},\"version\":1,\"_meta\":{\"managed\":true,\"description\":\"default settings for Stack deprecation logs index template installed by x-pack\"}},\"synthetics-settings\":{\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\"synthetics\"},\"codec\":\"best_compression\"}}},\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default settings for the synthetics index template installed by x-pack\"}},\"logs-mappings\":{\"template\":{\"mappings\":{\"properties\":{\"data_stream\":{\"properties\":{\"type\":{\"type\":\"constant_keyword\",\"value\":\"logs\"}}}}}},\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default mappings for the logs index template installed by x-pack\"}},\"data-streams-mappings\":{\"template\":{\"mappings\":{\"dynamic_templates\":[{\"match_ip\":{\"mapping\":{\"type\":\"ip\"},\"match_mapping_type\":\"string\",\"match\":\"ip\"}},{\"match_message\":{\"mapping\":{\"type\":\"match_only_text\"},\"match_mapping_type\":\"string\",\"match\":\"message\"}},{\"strings_as_keyword\":{\"mapping\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"match_mapping_type\":\"string\"}}],\"date_detection\":false,\"properties\":{\"@timestamp\":{\"type\":\"date\"},\"ecs\":{\"properties\":{\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"data_stream\":{\"properties\":{\"namespace\":{\"type\":\"constant_keyword\"},\"dataset\":{\"type\":\"constant_keyword\"}}},\"host\":{\"type\":\"object\"}}}},\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"general mapping conventions for data streams\"}}}} persistent_tasks: {\"last_allocation_id\":6,\"tasks\":[{\"id\":\"health-node\",\"task\":{\"health-node\":{\"params\":{}}},\"allocation_id\":5,\"assignment\":{\"executor_node\":\"iKPGCkp9RVOKXOj20uOt4g\",\"explanation\":\"\"},\"allocation_id_on_last_status_update\":0},{\"id\":\"geoip-downloader\",\"task\":{\"geoip-downloader\":{\"params\":{},\"state\":{\"databases\":{\"GeoLite2-ASN.mmdb\":{\"last_update\":1679968651799,\"last_check\":1679968651799,\"first_chunk\":0,\"last_chunk\":4,\"md5\":\"33268a0d7cb3ba8d9d581bf1e5dc0980\"},\"GeoLite2-Country.mmdb\":{\"last_update\":1679968809323,\"last_check\":1679968809323,\"first_chunk\":0,\"last_chunk\":2,\"md5\":\"da6d05a7b044ea35910f20f756c057ad\"},\"GeoLite2-City.mmdb\":{\"last_update\":1679968693385,\"last_check\":1679968693385,\"first_chunk\":0,\"last_chunk\":33,\"md5\":\"2d91f3e42689a12c2f496199befdfccf\"}}}}},\"allocation_id\":6,\"assignment\":{\"executor_node\":\"iKPGCkp9RVOKXOj20uOt4g\",\"explanation\":\"\"},\"allocation_id_on_last_status_update\":0}]} index_lifecycle: {\n \"policies\" : {\n \"90-days-default\" : {\n \"policy\" : {\n \"phases\" : {\n \"warm\" : {\n \"min_age\" : \"2d\",\n \"actions\" : {\n \"shrink\" : {\n \"number_of_shards\" : 1\n },\n \"forcemerge\" : {\n \"max_num_segments\" : 1\n }\n }\n },\n \"cold\" : {\n \"min_age\" : \"30d\",\n \"actions\" : { }\n },\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"90d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"built-in ILM policy using the hot, warm, and cold phases with a retention of 90 days\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968678193,\n \"modified_date_string\" : \"2023-03-28T01:57:58.193Z\"\n },\n \"ilm-history-ilm-policy\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"90d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"default policy for the ILM history indices\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968685095,\n \"modified_date_string\" : \"2023-03-28T01:58:05.095Z\"\n },\n \"watch-history-ilm-policy-16\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"3d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"4d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"default policy for the watcher history indices\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968647327,\n \"modified_date_string\" : \"2023-03-28T01:57:27.327Z\"\n },\n \"30-days-default\" : {\n \"policy\" : {\n \"phases\" : {\n \"warm\" : {\n \"min_age\" : \"2d\",\n \"actions\" : {\n \"shrink\" : {\n \"number_of_shards\" : 1\n },\n \"forcemerge\" : {\n \"max_num_segments\" : 1\n }\n }\n },\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"30d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"built-in ILM policy using the hot and warm phases with a retention of 30 days\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968662496,\n \"modified_date_string\" : \"2023-03-28T01:57:42.496Z\"\n },\n \"synthetics\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"default policy for the synthetics index template installed by x-pack\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968682454,\n \"modified_date_string\" : \"2023-03-28T01:58:02.454Z\"\n },\n \".monitoring-8-ilm-policy\" : {\n \"policy\" : {\n \"phases\" : {\n \"warm\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"forcemerge\" : {\n \"max_num_segments\" : 1\n }\n }\n },\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"3d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"3d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"defaults\" : {\n \"delete_min_age\" : \"Using value of [3d] based on the monitoring plugin default\"\n },\n \"description\" : \"Index lifecycle policy generated for [monitoring-*-8] data streams\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968676492,\n \"modified_date_string\" : \"2023-03-28T01:57:56.492Z\"\n },\n \"slm-history-ilm-policy\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"90d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"default policy for the SLM history indices\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968644040,\n \"modified_date_string\" : \"2023-03-28T01:57:24.040Z\"\n },\n \".fleet-actions-results-ilm-policy\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_size\" : \"300gb\",\n \"max_age\" : \"30d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"90d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"default policy for fleet action results indices\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968686060,\n \"modified_date_string\" : \"2023-03-28T01:58:06.060Z\"\n },\n \"365-days-default\" : {\n \"policy\" : {\n \"phases\" : {\n \"warm\" : {\n \"min_age\" : \"2d\",\n \"actions\" : {\n \"shrink\" : {\n \"number_of_shards\" : 1\n },\n \"forcemerge\" : {\n \"max_num_segments\" : 1\n }\n }\n },\n \"cold\" : {\n \"min_age\" : \"30d\",\n \"actions\" : { }\n },\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"365d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"built-in ILM policy using the hot, warm, and cold phases with a retention of 365 days\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968675542,\n \"modified_date_string\" : \"2023-03-28T01:57:55.542Z\"\n },\n \"7-days-default\" : {\n \"policy\" : {\n \"phases\" : {\n \"warm\" : {\n \"min_age\" : \"2d\",\n \"actions\" : {\n \"shrink\" : {\n \"number_of_shards\" : 1\n },\n \"forcemerge\" : {\n \"max_num_segments\" : 1\n }\n }\n },\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"7d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"7d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"built-in ILM policy using the hot and warm phases with a retention of 7 days\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968672404,\n \"modified_date_string\" : \"2023-03-28T01:57:52.404Z\"\n },\n \".deprecation-indexing-ilm-policy\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"10gb\",\n \"max_age\" : \"30d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"30d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"ILM policy used for deprecation logs\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968670134,\n \"modified_date_string\" : \"2023-03-28T01:57:50.134Z\"\n },\n \"metrics\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"default policy for the metrics index template installed by x-pack\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968660707,\n \"modified_date_string\" : \"2023-03-28T01:57:40.707Z\"\n },\n \"ml-size-based-ilm-policy\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\"\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"default policy for machine learning state and stats indices\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968650499,\n \"modified_date_string\" : \"2023-03-28T01:57:30.499Z\"\n },\n \"180-days-default\" : {\n \"policy\" : {\n \"phases\" : {\n \"warm\" : {\n \"min_age\" : \"2d\",\n \"actions\" : {\n \"shrink\" : {\n \"number_of_shards\" : 1\n },\n \"forcemerge\" : {\n \"max_num_segments\" : 1\n }\n }\n },\n \"cold\" : {\n \"min_age\" : \"30d\",\n \"actions\" : { }\n },\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"180d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"built-in ILM policy using the hot, warm, and cold phases with a retention of 180 days\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968681386,\n \"modified_date_string\" : \"2023-03-28T01:58:01.386Z\"\n },\n \"logs\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"default policy for the logs index template installed by x-pack\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968679651,\n \"modified_date_string\" : \"2023-03-28T01:57:59.651Z\"\n }\n },\n \"operation_mode\" : \"RUNNING\"\n} index_template: {\"index_template\":{\".watch-history-16\":{\"index_patterns\":[\".watcher-history-16*\"],\"template\":{\"settings\":{\"index\":{\"format\":\"6\",\"lifecycle\":{\"name\":\"watch-history-ilm-policy-16\"},\"hidden\":\"true\",\"number_of_shards\":\"1\",\"auto_expand_replicas\":\"0-1\",\"number_of_replicas\":\"0\"}},\"mappings\":{\"_meta\":{\"watcher-history-version\":\"16\"},\"dynamic\":false,\"dynamic_templates\":[{\"disabled_payload_fields\":{\"match_pattern\":\"regex\",\"path_match\":\"result\\\\.(input(\\\\..+)*|(transform(\\\\..+)*)|(actions\\\\.transform(\\\\..+)*))\\\\.payload\",\"mapping\":{\"type\":\"object\",\"enabled\":false}}},{\"disabled_search_request_body_fields\":{\"match_pattern\":\"regex\",\"path_match\":\"result\\\\.(input(\\\\..+)*|(transform(\\\\..+)*)|(actions\\\\.transform(\\\\..+)*))\\\\.search\\\\.request\\\\.(body|template)\",\"mapping\":{\"type\":\"object\",\"enabled\":false}}},{\"disabled_exception_fields\":{\"match_pattern\":\"regex\",\"path_match\":\"result\\\\.(input(\\\\..+)*|(transform(\\\\..+)*)|(actions\\\\.transform(\\\\..+)*)|actions)\\\\.error\",\"mapping\":{\"type\":\"object\",\"enabled\":false}}},{\"disabled_jira_custom_fields\":{\"path_match\":\"result.actions.jira.fields.customfield_*\",\"mapping\":{\"type\":\"object\",\"enabled\":false}}}],\"properties\":{\"exception\":{\"type\":\"object\",\"enabled\":false},\"metadata\":{\"dynamic\":true,\"type\":\"object\"},\"trigger_event\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"schedule\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"scheduled_time\":{\"type\":\"date\"}}},\"triggered_time\":{\"type\":\"date\"},\"type\":{\"type\":\"keyword\"},\"manual\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"schedule\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"scheduled_time\":{\"type\":\"date\"}}}}}}},\"result\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"input\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"search\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"request\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"indices\":{\"type\":\"keyword\"},\"types\":{\"type\":\"keyword\"},\"search_type\":{\"type\":\"keyword\"}}}}},\"payload\":{\"type\":\"object\",\"enabled\":false},\"http\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"request\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"path\":{\"type\":\"keyword\"},\"host\":{\"type\":\"keyword\"}}}}},\"type\":{\"type\":\"keyword\"},\"status\":{\"type\":\"keyword\"}}},\"condition\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"compare\":{\"type\":\"object\",\"enabled\":false},\"array_compare\":{\"type\":\"object\",\"enabled\":false},\"type\":{\"type\":\"keyword\"},\"met\":{\"type\":\"boolean\"},\"script\":{\"type\":\"object\",\"enabled\":false},\"status\":{\"type\":\"keyword\"}}},\"transform\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"search\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"request\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"indices\":{\"type\":\"keyword\"},\"types\":{\"type\":\"keyword\"}}}}},\"type\":{\"type\":\"keyword\"}}},\"execution_duration\":{\"type\":\"long\"},\"actions\":{\"include_in_parent\":true,\"dynamic\":true,\"type\":\"nested\",\"properties\":{\"reason\":{\"type\":\"keyword\"},\"foreach\":{\"type\":\"object\",\"enabled\":false},\"webhook\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"request\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"path\":{\"type\":\"keyword\"},\"host\":{\"type\":\"keyword\"}}}}},\"number_of_actions_executed\":{\"type\":\"integer\"},\"slack\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"sent_messages\":{\"include_in_parent\":true,\"dynamic\":true,\"type\":\"nested\",\"properties\":{\"reason\":{\"type\":\"text\"},\"request\":{\"type\":\"object\",\"enabled\":false},\"response\":{\"type\":\"object\",\"enabled\":false},\"to\":{\"type\":\"keyword\"},\"message\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"attachments\":{\"include_in_parent\":true,\"dynamic\":true,\"type\":\"nested\",\"properties\":{\"color\":{\"type\":\"keyword\"},\"fields\":{\"properties\":{\"value\":{\"type\":\"text\"}}}}},\"icon\":{\"type\":\"keyword\"},\"from\":{\"type\":\"text\"},\"text\":{\"type\":\"text\"}}},\"status\":{\"type\":\"keyword\"}}},\"account\":{\"type\":\"keyword\"}}},\"index\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"response\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"index\":{\"type\":\"keyword\"},\"id\":{\"type\":\"keyword\"},\"type\":{\"type\":\"keyword\"}}}}},\"pagerduty\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"sent_event\":{\"include_in_parent\":true,\"dynamic\":true,\"type\":\"nested\",\"properties\":{\"reason\":{\"type\":\"text\"},\"request\":{\"type\":\"object\",\"enabled\":false},\"response\":{\"type\":\"object\",\"enabled\":false},\"event\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"client_url\":{\"type\":\"keyword\"},\"context\":{\"include_in_parent\":true,\"dynamic\":true,\"type\":\"nested\",\"properties\":{\"src\":{\"type\":\"keyword\"},\"alt\":{\"type\":\"text\"},\"href\":{\"type\":\"keyword\"},\"type\":{\"type\":\"keyword\"}}},\"client\":{\"type\":\"text\"},\"description\":{\"type\":\"text\"},\"attach_payload\":{\"type\":\"boolean\"},\"incident_key\":{\"type\":\"keyword\"},\"type\":{\"type\":\"keyword\"},\"account\":{\"type\":\"keyword\"}}}}},\"account\":{\"type\":\"keyword\"}}},\"id\":{\"type\":\"keyword\"},\"type\":{\"type\":\"keyword\"},\"email\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"message\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"cc\":{\"type\":\"keyword\"},\"bcc\":{\"type\":\"keyword\"},\"reply_to\":{\"type\":\"keyword\"},\"from\":{\"type\":\"keyword\"},\"id\":{\"type\":\"keyword\"},\"to\":{\"type\":\"keyword\"}}}}},\"status\":{\"type\":\"keyword\"},\"jira\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"result\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"self\":{\"type\":\"keyword\"},\"id\":{\"type\":\"keyword\"},\"key\":{\"type\":\"keyword\"}}},\"reason\":{\"type\":\"text\"},\"request\":{\"type\":\"object\",\"enabled\":false},\"response\":{\"type\":\"object\",\"enabled\":false},\"fields\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"summary\":{\"type\":\"text\"},\"issuetype\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"name\":{\"type\":\"keyword\"},\"id\":{\"type\":\"keyword\"}}},\"description\":{\"type\":\"text\"},\"project\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"id\":{\"type\":\"keyword\"},\"key\":{\"type\":\"keyword\"}}},\"labels\":{\"type\":\"text\"}}},\"account\":{\"type\":\"keyword\"}}}}},\"execution_time\":{\"type\":\"date\"}}},\"node\":{\"type\":\"keyword\"},\"input\":{\"type\":\"object\",\"enabled\":false},\"condition\":{\"type\":\"object\",\"enabled\":false},\"watch_id\":{\"type\":\"keyword\"},\"messages\":{\"type\":\"text\"},\"vars\":{\"type\":\"object\",\"enabled\":false},\"state\":{\"type\":\"keyword\"},\"user\":{\"type\":\"text\"},\"status\":{\"dynamic\":true,\"type\":\"object\",\"enabled\":false}}}},\"composed_of\":[],\"priority\":2147483647,\"version\":16,\"_meta\":{\"managed\":true,\"description\":\"index template for watcher history indices\"},\"data_stream\":{\"hidden\":true,\"allow_custom_routing\":false},\"allow_auto_create\":true},\".monitoring-beats-mb\":{\"index_patterns\":[\".monitoring-beats-8-*\"],\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\".monitoring-8-ilm-policy\"},\"mapping\":{\"total_fields\":{\"limit\":\"2000\"}}}},\"mappings\":{\"dynamic\":false,\"properties\":{\"@timestamp\":{\"type\":\"date\"},\"cluster_uuid\":{\"path\":\"beat.elasticsearch.cluster.id\",\"type\":\"alias\"},\"service\":{\"properties\":{\"node\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"environment\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"hostname\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"origin\":{\"properties\":{\"node\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"environment\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"target\":{\"properties\":{\"node\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"environment\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"beat\":{\"properties\":{\"elasticsearch\":{\"properties\":{\"cluster\":{\"properties\":{\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"stats\":{\"properties\":{\"handles\":{\"properties\":{\"limit\":{\"properties\":{\"hard\":{\"type\":\"long\"},\"soft\":{\"type\":\"long\"}}},\"open\":{\"type\":\"long\"}}},\"apm_server\":{\"properties\":{\"server\":{\"properties\":{\"request\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"response\":{\"properties\":{\"valid\":{\"properties\":{\"count\":{\"type\":\"long\"},\"accepted\":{\"type\":\"long\"},\"ok\":{\"type\":\"long\"}}},\"count\":{\"type\":\"long\"},\"errors\":{\"properties\":{\"ratelimit\":{\"type\":\"long\"},\"internal\":{\"type\":\"long\"},\"toolarge\":{\"type\":\"long\"},\"method\":{\"type\":\"long\"},\"unauthorized\":{\"type\":\"long\"},\"forbidden\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"},\"closed\":{\"type\":\"long\"},\"decode\":{\"type\":\"long\"},\"queue\":{\"type\":\"long\"},\"concurrency\":{\"type\":\"long\"},\"validate\":{\"type\":\"long\"}}}}},\"concurrent\":{\"properties\":{\"wait\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}}}},\"acm\":{\"properties\":{\"request\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"response\":{\"properties\":{\"valid\":{\"properties\":{\"notmodified\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"},\"accepted\":{\"type\":\"long\"},\"ok\":{\"type\":\"long\"}}},\"request\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"count\":{\"type\":\"long\"},\"unset\":{\"type\":\"long\"},\"errors\":{\"properties\":{\"ratelimit\":{\"type\":\"long\"},\"internal\":{\"type\":\"long\"},\"toolarge\":{\"type\":\"long\"},\"method\":{\"type\":\"long\"},\"unavailable\":{\"type\":\"long\"},\"forbidden\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"},\"decode\":{\"type\":\"long\"},\"unauthorized\":{\"type\":\"long\"},\"notfound\":{\"type\":\"long\"},\"closed\":{\"type\":\"long\"},\"invalidquery\":{\"type\":\"long\"},\"queue\":{\"type\":\"long\"},\"validate\":{\"type\":\"long\"}}}}}}},\"decoder\":{\"properties\":{\"deflate\":{\"properties\":{\"content-length\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"reader\":{\"properties\":{\"size\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"gzip\":{\"properties\":{\"content-length\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"uncompressed\":{\"properties\":{\"content-length\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"missing-content-length\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"processor\":{\"properties\":{\"metric\":{\"properties\":{\"transformations\":{\"type\":\"long\"},\"decoding\":{\"properties\":{\"count\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}},\"validation\":{\"properties\":{\"count\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}}}},\"sourcemap\":{\"properties\":{\"counter\":{\"type\":\"long\"},\"decoding\":{\"properties\":{\"count\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}},\"validation\":{\"properties\":{\"count\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}}}},\"error\":{\"properties\":{\"stacktraces\":{\"type\":\"long\"},\"spans\":{\"type\":\"long\"},\"frames\":{\"type\":\"long\"},\"transformations\":{\"type\":\"long\"},\"decoding\":{\"properties\":{\"count\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}},\"validation\":{\"properties\":{\"count\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}}}},\"transaction\":{\"properties\":{\"stacktraces\":{\"type\":\"long\"},\"spans\":{\"type\":\"long\"},\"frames\":{\"type\":\"long\"},\"transformations\":{\"type\":\"long\"},\"transactions\":{\"type\":\"long\"},\"decoding\":{\"properties\":{\"count\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}},\"validation\":{\"properties\":{\"count\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}}}},\"span\":{\"properties\":{\"transformations\":{\"type\":\"long\"}}}}}}},\"system\":{\"properties\":{\"load\":{\"properties\":{\"1\":{\"type\":\"double\"},\"15\":{\"type\":\"double\"},\"5\":{\"type\":\"double\"},\"norm\":{\"properties\":{\"1\":{\"type\":\"double\"},\"15\":{\"type\":\"double\"},\"5\":{\"type\":\"double\"}}}}},\"cpu\":{\"properties\":{\"cores\":{\"type\":\"long\"}}}}},\"beat\":{\"properties\":{\"host\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"cpu\":{\"properties\":{\"total\":{\"properties\":{\"ticks\":{\"type\":\"long\"},\"time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"value\":{\"type\":\"long\"}}},\"system\":{\"properties\":{\"ticks\":{\"type\":\"long\"},\"time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"user\":{\"properties\":{\"ticks\":{\"type\":\"long\"},\"time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}}}},\"runtime\":{\"properties\":{\"goroutines\":{\"type\":\"long\"}}},\"memstats\":{\"properties\":{\"memory\":{\"properties\":{\"total\":{\"type\":\"long\"},\"alloc\":{\"type\":\"long\"}}},\"rss\":{\"type\":\"long\"},\"gc_next\":{\"type\":\"long\"}}},\"libbeat\":{\"properties\":{\"output\":{\"properties\":{\"read\":{\"properties\":{\"bytes\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"write\":{\"properties\":{\"bytes\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}},\"events\":{\"properties\":{\"batches\":{\"type\":\"long\"},\"duplicates\":{\"type\":\"long\"},\"total\":{\"type\":\"long\"},\"dropped\":{\"type\":\"long\"},\"toomany\":{\"type\":\"long\"},\"active\":{\"type\":\"long\"},\"failed\":{\"type\":\"long\"},\"acked\":{\"type\":\"long\"}}}}},\"pipeline\":{\"properties\":{\"clients\":{\"type\":\"long\"},\"events\":{\"properties\":{\"total\":{\"type\":\"long\"},\"filtered\":{\"type\":\"long\"},\"dropped\":{\"type\":\"long\"},\"active\":{\"type\":\"long\"},\"failed\":{\"type\":\"long\"},\"published\":{\"type\":\"long\"},\"retry\":{\"type\":\"long\"}}},\"queue\":{\"properties\":{\"acked\":{\"type\":\"long\"}}}}},\"config\":{\"properties\":{\"running\":{\"type\":\"long\"},\"stops\":{\"type\":\"long\"},\"starts\":{\"type\":\"long\"},\"reloads\":{\"type\":\"long\"}}}}},\"cgroup\":{\"properties\":{\"memory\":{\"properties\":{\"mem\":{\"properties\":{\"usage\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"limit\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"cpu\":{\"properties\":{\"cfs\":{\"properties\":{\"period\":{\"properties\":{\"us\":{\"type\":\"long\"}}},\"quota\":{\"properties\":{\"us\":{\"type\":\"long\"}}}}},\"stats\":{\"properties\":{\"periods\":{\"type\":\"long\"},\"throttled\":{\"properties\":{\"ns\":{\"type\":\"long\"},\"periods\":{\"type\":\"long\"}}}}},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"cpuacct\":{\"properties\":{\"total\":{\"properties\":{\"ns\":{\"type\":\"long\"}}},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"info\":{\"properties\":{\"host\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"uptime\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"uptime\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"properties\":{\"output\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"cluster\":{\"properties\":{\"uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"input\":{\"properties\":{\"names\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"count\":{\"type\":\"long\"}}},\"management\":{\"properties\":{\"enabled\":{\"type\":\"boolean\"}}},\"service\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"module\":{\"properties\":{\"names\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"count\":{\"type\":\"long\"}}},\"beat\":{\"properties\":{\"host\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"host\":{\"properties\":{\"os\":{\"properties\":{\"kernel\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"platform\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"containerized\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"queue\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"host\":{\"properties\":{\"hostname\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"architecture\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"metricset\":{\"properties\":{\"period\":{\"type\":\"long\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"event\":{\"properties\":{\"reason\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"code\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"timezone\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"agent_id_status\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"duration\":{\"type\":\"long\"},\"reference\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ingested\":{\"type\":\"date\"},\"provider\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"action\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"end\":{\"type\":\"date\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"outcome\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"severity\":{\"type\":\"long\"},\"original\":{\"ignore_above\":1024,\"index\":false,\"type\":\"keyword\",\"doc_values\":false},\"risk_score\":{\"type\":\"float\"},\"created\":{\"type\":\"date\"},\"kind\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"module\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"start\":{\"type\":\"date\"},\"url\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"sequence\":{\"type\":\"long\"},\"risk_score_norm\":{\"type\":\"float\"},\"category\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"dataset\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"hash\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"beats_stats\":{\"properties\":{\"beat\":{\"properties\":{\"host\":{\"path\":\"beat.stats.beat.host\",\"type\":\"alias\"},\"name\":{\"path\":\"beat.stats.beat.name\",\"type\":\"alias\"},\"type\":{\"path\":\"beat.stats.beat.type\",\"type\":\"alias\"},\"uuid\":{\"path\":\"beat.stats.beat.uuid\",\"type\":\"alias\"},\"version\":{\"path\":\"beat.stats.beat.version\",\"type\":\"alias\"}}},\"metrics\":{\"properties\":{\"system\":{\"properties\":{\"load\":{\"properties\":{\"1\":{\"path\":\"beat.stats.system.load.1\",\"type\":\"alias\"},\"15\":{\"path\":\"beat.stats.system.load.15\",\"type\":\"alias\"},\"5\":{\"path\":\"beat.stats.system.load.5\",\"type\":\"alias\"},\"norm\":{\"properties\":{\"1\":{\"path\":\"beat.stats.system.load.norm.1\",\"type\":\"alias\"},\"15\":{\"path\":\"beat.stats.system.load.norm.15\",\"type\":\"alias\"},\"5\":{\"path\":\"beat.stats.system.load.norm.5\",\"type\":\"alias\"}}}}},\"cpu\":{\"properties\":{\"cores\":{\"path\":\"beat.stats.system.cpu.cores\",\"type\":\"alias\"}}}}},\"beat\":{\"properties\":{\"handles\":{\"properties\":{\"limit\":{\"properties\":{\"hard\":{\"path\":\"beat.stats.handles.limit.hard\",\"type\":\"alias\"},\"soft\":{\"path\":\"beat.stats.handles.limit.soft\",\"type\":\"alias\"}}},\"open\":{\"path\":\"beat.stats.handles.open\",\"type\":\"alias\"}}},\"cpu\":{\"properties\":{\"total\":{\"properties\":{\"ticks\":{\"path\":\"beat.stats.cpu.total.ticks\",\"type\":\"alias\"},\"time\":{\"properties\":{\"ms\":{\"path\":\"beat.stats.cpu.total.time.ms\",\"type\":\"alias\"}}},\"value\":{\"path\":\"beat.stats.cpu.total.value\",\"type\":\"alias\"}}},\"system\":{\"properties\":{\"ticks\":{\"path\":\"beat.stats.cpu.system.ticks\",\"type\":\"alias\"},\"time\":{\"properties\":{\"ms\":{\"path\":\"beat.stats.cpu.system.time.ms\",\"type\":\"alias\"}}}}},\"user\":{\"properties\":{\"ticks\":{\"path\":\"beat.stats.cpu.user.ticks\",\"type\":\"alias\"},\"time\":{\"properties\":{\"ms\":{\"path\":\"beat.stats.cpu.user.time.ms\",\"type\":\"alias\"}}}}}}},\"memstats\":{\"properties\":{\"rss\":{\"path\":\"beat.stats.memstats.rss\",\"type\":\"alias\"},\"gc_next\":{\"path\":\"beat.stats.memstats.gc_next\",\"type\":\"alias\"},\"memory_alloc\":{\"path\":\"beat.stats.memstats.memory.alloc\",\"type\":\"alias\"},\"memory_total\":{\"path\":\"beat.stats.memstats.memory.total\",\"type\":\"alias\"}}},\"cgroup\":{\"properties\":{\"memory\":{\"properties\":{\"id\":{\"path\":\"beat.stats.cgroup.memory.id\",\"type\":\"alias\"}}},\"mem\":{\"properties\":{\"usage\":{\"properties\":{\"bytes\":{\"path\":\"beat.stats.cgroup.memory.mem.usage.bytes\",\"type\":\"alias\"}}},\"limit\":{\"properties\":{\"bytes\":{\"path\":\"beat.stats.cgroup.memory.mem.limit.bytes\",\"type\":\"alias\"}}}}},\"cpu\":{\"properties\":{\"cfs\":{\"properties\":{\"period\":{\"properties\":{\"us\":{\"path\":\"beat.stats.cgroup.cpu.cfs.period.us\",\"type\":\"alias\"}}},\"quota\":{\"properties\":{\"us\":{\"path\":\"beat.stats.cgroup.cpu.cfs.quota.us\",\"type\":\"alias\"}}}}},\"stats\":{\"properties\":{\"periods\":{\"path\":\"beat.stats.cgroup.cpu.stats.periods\",\"type\":\"alias\"},\"throttled\":{\"properties\":{\"ns\":{\"path\":\"beat.stats.cgroup.cpu.stats.throttled.ns\",\"type\":\"alias\"},\"periods\":{\"path\":\"beat.stats.cgroup.cpu.stats.throttled.periods\",\"type\":\"alias\"}}}}},\"id\":{\"path\":\"beat.stats.cgroup.cpu.id\",\"type\":\"alias\"}}},\"cpuacct\":{\"properties\":{\"total\":{\"properties\":{\"ns\":{\"path\":\"beat.stats.cgroup.cpuacct.total.ns\",\"type\":\"alias\"}}},\"id\":{\"path\":\"beat.stats.cgroup.cpuacct.id\",\"type\":\"alias\"}}}}},\"info\":{\"properties\":{\"ephemeral_id\":{\"path\":\"beat.stats.info.ephemeral_id\",\"type\":\"alias\"},\"uptime\":{\"properties\":{\"ms\":{\"path\":\"beat.stats.info.uptime.ms\",\"type\":\"alias\"}}}}}}},\"libbeat\":{\"properties\":{\"output\":{\"properties\":{\"read\":{\"properties\":{\"bytes\":{\"path\":\"beat.stats.libbeat.output.read.bytes\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.libbeat.output.read.errors\",\"type\":\"alias\"}}},\"type\":{\"path\":\"beat.stats.libbeat.output.type\",\"type\":\"alias\"},\"write\":{\"properties\":{\"bytes\":{\"path\":\"beat.stats.libbeat.output.write.bytes\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.libbeat.output.write.errors\",\"type\":\"alias\"}}},\"events\":{\"properties\":{\"duplicated\":{\"path\":\"beat.stats.libbeat.output.events.duplicates\",\"type\":\"alias\"},\"batches\":{\"path\":\"beat.stats.libbeat.output.events.batches\",\"type\":\"alias\"},\"total\":{\"path\":\"beat.stats.libbeat.output.events.total\",\"type\":\"alias\"},\"dropped\":{\"path\":\"beat.stats.libbeat.output.events.dropped\",\"type\":\"alias\"},\"toomany\":{\"path\":\"beat.stats.libbeat.output.events.toomany\",\"type\":\"alias\"},\"active\":{\"path\":\"beat.stats.libbeat.output.events.active\",\"type\":\"alias\"},\"failed\":{\"path\":\"beat.stats.libbeat.output.events.failed\",\"type\":\"alias\"},\"acked\":{\"path\":\"beat.stats.libbeat.output.events.acked\",\"type\":\"alias\"}}}}},\"pipeline\":{\"properties\":{\"clients\":{\"path\":\"beat.stats.libbeat.pipeline.clients\",\"type\":\"alias\"},\"events\":{\"properties\":{\"total\":{\"path\":\"beat.stats.libbeat.pipeline.events.total\",\"type\":\"alias\"},\"filtered\":{\"path\":\"beat.stats.libbeat.pipeline.events.filtered\",\"type\":\"alias\"},\"dropped\":{\"path\":\"beat.stats.libbeat.pipeline.events.dropped\",\"type\":\"alias\"},\"active\":{\"path\":\"beat.stats.libbeat.pipeline.events.active\",\"type\":\"alias\"},\"failed\":{\"path\":\"beat.stats.libbeat.pipeline.events.failed\",\"type\":\"alias\"},\"published\":{\"path\":\"beat.stats.libbeat.pipeline.events.published\",\"type\":\"alias\"},\"retry\":{\"path\":\"beat.stats.libbeat.pipeline.events.retry\",\"type\":\"alias\"}}},\"queue\":{\"properties\":{\"acked\":{\"path\":\"beat.stats.libbeat.pipeline.queue.acked\",\"type\":\"alias\"}}}}},\"config\":{\"properties\":{\"module\":{\"properties\":{\"running\":{\"path\":\"beat.stats.libbeat.config.running\",\"type\":\"alias\"},\"stops\":{\"path\":\"beat.stats.libbeat.config.stops\",\"type\":\"alias\"},\"starts\":{\"path\":\"beat.stats.libbeat.config.starts\",\"type\":\"alias\"}}},\"reloads\":{\"path\":\"beat.stats.libbeat.config.reloads\",\"type\":\"alias\"}}}}},\"apm-server\":{\"properties\":{\"server\":{\"properties\":{\"request\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.server.request.count\",\"type\":\"alias\"}}},\"response\":{\"properties\":{\"valid\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.server.response.valid.count\",\"type\":\"alias\"},\"accepted\":{\"path\":\"beat.stats.apm_server.server.response.valid.accepted\",\"type\":\"alias\"},\"ok\":{\"path\":\"beat.stats.apm_server.server.response.valid.ok\",\"type\":\"alias\"}}},\"count\":{\"path\":\"beat.stats.apm_server.server.response.count\",\"type\":\"alias\"},\"errors\":{\"properties\":{\"ratelimit\":{\"path\":\"beat.stats.apm_server.server.response.errors.ratelimit\",\"type\":\"alias\"},\"internal\":{\"path\":\"beat.stats.apm_server.server.response.errors.internal\",\"type\":\"alias\"},\"toolarge\":{\"path\":\"beat.stats.apm_server.server.response.errors.toolarge\",\"type\":\"alias\"},\"method\":{\"path\":\"beat.stats.apm_server.server.response.errors.method\",\"type\":\"alias\"},\"unauthorized\":{\"path\":\"beat.stats.apm_server.server.response.errors.unauthorized\",\"type\":\"alias\"},\"forbidden\":{\"path\":\"beat.stats.apm_server.server.response.errors.forbidden\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.stats.apm_server.server.response.errors.count\",\"type\":\"alias\"},\"closed\":{\"path\":\"beat.stats.apm_server.server.response.errors.closed\",\"type\":\"alias\"},\"decode\":{\"path\":\"beat.stats.apm_server.server.response.errors.decode\",\"type\":\"alias\"},\"queue\":{\"path\":\"beat.stats.apm_server.server.response.errors.queue\",\"type\":\"alias\"},\"concurrency\":{\"path\":\"beat.stats.apm_server.server.response.errors.concurrency\",\"type\":\"alias\"},\"validate\":{\"path\":\"beat.stats.apm_server.server.response.errors.validate\",\"type\":\"alias\"}}}}},\"concurrent\":{\"properties\":{\"wait\":{\"properties\":{\"ms\":{\"path\":\"beat.stats.apm_server.server.concurrent.wait.ms\",\"type\":\"alias\"}}}}}}},\"sampling\":{\"properties\":{\"tail\":{\"properties\":{\"dynamic_service_groups\":{\"type\":\"long\"},\"storage\":{\"properties\":{\"lsm_size\":{\"type\":\"long\"},\"value_log_size\":{\"type\":\"long\"}}},\"events\":{\"properties\":{\"processed\":{\"type\":\"long\"},\"head_unsampled\":{\"type\":\"long\"},\"stored\":{\"type\":\"long\"},\"dropped\":{\"type\":\"long\"},\"sampled\":{\"type\":\"long\"}}}}},\"transactions_dropped\":{\"type\":\"long\"}}},\"acm\":{\"properties\":{\"request\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.acm.request.count\",\"type\":\"alias\"}}},\"response\":{\"properties\":{\"valid\":{\"properties\":{\"notmodified\":{\"path\":\"beat.stats.apm_server.acm.response.valid.notmodified\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.stats.apm_server.acm.response.valid.count\",\"type\":\"alias\"},\"accepted\":{\"path\":\"beat.stats.apm_server.acm.response.valid.accepted\",\"type\":\"alias\"},\"ok\":{\"path\":\"beat.stats.apm_server.acm.response.valid.ok\",\"type\":\"alias\"}}},\"request\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.acm.response.request.count\",\"type\":\"alias\"}}},\"count\":{\"path\":\"beat.stats.apm_server.acm.response.count\",\"type\":\"alias\"},\"unset\":{\"path\":\"beat.stats.apm_server.acm.response.unset\",\"type\":\"alias\"},\"errors\":{\"properties\":{\"ratelimit\":{\"path\":\"beat.stats.apm_server.acm.response.errors.ratelimit\",\"type\":\"alias\"},\"internal\":{\"path\":\"beat.stats.apm_server.acm.response.errors.internal\",\"type\":\"alias\"},\"toolarge\":{\"path\":\"beat.stats.apm_server.acm.response.errors.toolarge\",\"type\":\"alias\"},\"method\":{\"path\":\"beat.stats.apm_server.acm.response.errors.method\",\"type\":\"alias\"},\"unavailable\":{\"path\":\"beat.stats.apm_server.acm.response.errors.unavailable\",\"type\":\"alias\"},\"forbidden\":{\"path\":\"beat.stats.apm_server.acm.response.errors.forbidden\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.stats.apm_server.acm.response.errors.count\",\"type\":\"alias\"},\"decode\":{\"path\":\"beat.stats.apm_server.acm.response.errors.decode\",\"type\":\"alias\"},\"unauthorized\":{\"path\":\"beat.stats.apm_server.acm.response.errors.unauthorized\",\"type\":\"alias\"},\"notfound\":{\"path\":\"beat.stats.apm_server.acm.response.errors.notfound\",\"type\":\"alias\"},\"closed\":{\"path\":\"beat.stats.apm_server.acm.response.errors.closed\",\"type\":\"alias\"},\"invalidquery\":{\"path\":\"beat.stats.apm_server.acm.response.errors.invalidquery\",\"type\":\"alias\"},\"queue\":{\"path\":\"beat.stats.apm_server.acm.response.errors.queue\",\"type\":\"alias\"},\"validate\":{\"path\":\"beat.stats.apm_server.acm.response.errors.validate\",\"type\":\"alias\"}}}}}}},\"decoder\":{\"properties\":{\"deflate\":{\"properties\":{\"content-length\":{\"path\":\"beat.stats.apm_server.decoder.deflate.content-length\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.stats.apm_server.decoder.deflate.count\",\"type\":\"alias\"}}},\"reader\":{\"properties\":{\"size\":{\"path\":\"beat.stats.apm_server.decoder.reader.size\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.stats.apm_server.decoder.reader.count\",\"type\":\"alias\"}}},\"gzip\":{\"properties\":{\"content-length\":{\"path\":\"beat.stats.apm_server.decoder.gzip.content-length\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.stats.apm_server.decoder.gzip.count\",\"type\":\"alias\"}}},\"uncompressed\":{\"properties\":{\"content-length\":{\"path\":\"beat.stats.apm_server.decoder.uncompressed.content-length\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.stats.apm_server.decoder.uncompressed.count\",\"type\":\"alias\"}}},\"missing-content-length\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.decoder.missing-content-length.count\",\"type\":\"alias\"}}}}},\"processor\":{\"properties\":{\"metric\":{\"properties\":{\"transformations\":{\"path\":\"beat.stats.apm_server.processor.metric.transformations\",\"type\":\"alias\"},\"decoding\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.processor.metric.decoding.count\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.apm_server.processor.metric.decoding.errors\",\"type\":\"alias\"}}},\"validation\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.processor.metric.validation.count\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.apm_server.processor.metric.validation.errors\",\"type\":\"alias\"}}}}},\"sourcemap\":{\"properties\":{\"counter\":{\"path\":\"beat.stats.apm_server.processor.sourcemap.counter\",\"type\":\"alias\"},\"decoding\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.processor.sourcemap.decoding.count\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.apm_server.processor.sourcemap.decoding.errors\",\"type\":\"alias\"}}},\"validation\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.processor.sourcemap.validation.count\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.apm_server.processor.sourcemap.validation.errors\",\"type\":\"alias\"}}}}},\"error\":{\"properties\":{\"stacktraces\":{\"path\":\"beat.stats.apm_server.processor.error.stacktraces\",\"type\":\"alias\"},\"spans\":{\"path\":\"beat.stats.apm_server.processor.error.spans\",\"type\":\"alias\"},\"frames\":{\"path\":\"beat.stats.apm_server.processor.error.frames\",\"type\":\"alias\"},\"transformations\":{\"path\":\"beat.stats.apm_server.processor.error.transformations\",\"type\":\"alias\"},\"decoding\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.processor.error.decoding.count\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.apm_server.processor.error.decoding.errors\",\"type\":\"alias\"}}},\"validation\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.processor.error.validation.count\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.apm_server.processor.error.validation.errors\",\"type\":\"alias\"}}}}},\"transaction\":{\"properties\":{\"stacktraces\":{\"path\":\"beat.stats.apm_server.processor.transaction.stacktraces\",\"type\":\"alias\"},\"spans\":{\"path\":\"beat.stats.apm_server.processor.transaction.spans\",\"type\":\"alias\"},\"frames\":{\"path\":\"beat.stats.apm_server.processor.transaction.frames\",\"type\":\"alias\"},\"transformations\":{\"path\":\"beat.stats.apm_server.processor.transaction.transformations\",\"type\":\"alias\"},\"transactions\":{\"path\":\"beat.stats.apm_server.processor.transaction.transactions\",\"type\":\"alias\"},\"decoding\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.processor.transaction.decoding.count\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.apm_server.processor.transaction.decoding.errors\",\"type\":\"alias\"}}},\"validation\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.processor.transaction.validation.count\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.apm_server.processor.transaction.validation.errors\",\"type\":\"alias\"}}}}},\"span\":{\"properties\":{\"transformations\":{\"path\":\"beat.stats.apm_server.processor.span.transformations\",\"type\":\"alias\"}}}}}}}}},\"timestamp\":{\"path\":\"@timestamp\",\"type\":\"alias\"}}},\"beats_state\":{\"properties\":{\"beat\":{\"properties\":{\"host\":{\"path\":\"beat.state.beat.host\",\"type\":\"alias\"},\"name\":{\"path\":\"beat.state.beat.name\",\"type\":\"alias\"},\"type\":{\"path\":\"beat.state.beat.type\",\"type\":\"alias\"},\"uuid\":{\"path\":\"beat.state.beat.uuid\",\"type\":\"alias\"},\"version\":{\"path\":\"beat.state.beat.version\",\"type\":\"alias\"}}},\"state\":{\"properties\":{\"output\":{\"properties\":{\"name\":{\"path\":\"beat.state.output.name\",\"type\":\"alias\"}}},\"input\":{\"properties\":{\"names\":{\"path\":\"beat.state.input.names\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.state.input.count\",\"type\":\"alias\"}}},\"service\":{\"properties\":{\"name\":{\"path\":\"beat.state.service.name\",\"type\":\"alias\"},\"id\":{\"path\":\"beat.state.service.id\",\"type\":\"alias\"},\"version\":{\"path\":\"beat.state.service.version\",\"type\":\"alias\"}}},\"module\":{\"properties\":{\"names\":{\"path\":\"beat.state.module.names\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.state.module.count\",\"type\":\"alias\"}}},\"beat\":{\"properties\":{\"name\":{\"path\":\"beat.state.beat.name\",\"type\":\"alias\"}}},\"host\":{\"properties\":{\"hostname\":{\"path\":\"host.hostname\",\"type\":\"alias\"},\"os\":{\"properties\":{\"version\":{\"path\":\"beat.state.host.os.version\",\"type\":\"alias\"},\"platform\":{\"path\":\"beat.state.host.os.platform\",\"type\":\"alias\"}}},\"name\":{\"path\":\"host.name\",\"type\":\"alias\"},\"architecture\":{\"path\":\"host.architecture\",\"type\":\"alias\"}}}}},\"timestamp\":{\"path\":\"@timestamp\",\"type\":\"alias\"}}},\"timestamp\":{\"path\":\"@timestamp\",\"type\":\"alias\"}}}},\"composed_of\":[],\"version\":8000103,\"data_stream\":{\"hidden\":false,\"allow_custom_routing\":false}},\".monitoring-kibana-mb\":{\"index_patterns\":[\".monitoring-kibana-8-*\"],\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\".monitoring-8-ilm-policy\"},\"mapping\":{\"total_fields\":{\"limit\":\"2000\"}}}},\"mappings\":{\"dynamic\":false,\"properties\":{\"process\":{\"properties\":{\"pid\":{\"type\":\"long\"}}},\"@timestamp\":{\"type\":\"date\"},\"ecs\":{\"properties\":{\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"cluster_uuid\":{\"path\":\"kibana.elasticsearch.cluster.id\",\"type\":\"alias\"},\"service\":{\"properties\":{\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"host\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"metricset\":{\"properties\":{\"period\":{\"type\":\"long\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"kibana\":{\"properties\":{\"elasticsearch\":{\"properties\":{\"cluster\":{\"properties\":{\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"stats\":{\"properties\":{\"request\":{\"properties\":{\"total\":{\"type\":\"long\"},\"disconnects\":{\"type\":\"long\"}}},\"process\":{\"properties\":{\"memory\":{\"properties\":{\"resident_set_size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"heap\":{\"properties\":{\"total\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"used\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"size_limit\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"uptime\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}}}},\"event_loop_delay\":{\"properties\":{\"ms\":{\"scaling_factor\":1000,\"type\":\"scaled_float\"}}},\"uptime\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"os\":{\"properties\":{\"distroRelease\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"distro\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"memory\":{\"properties\":{\"used_in_bytes\":{\"type\":\"long\"},\"total_in_bytes\":{\"type\":\"long\"},\"free_in_bytes\":{\"type\":\"long\"}}},\"load\":{\"properties\":{\"5m\":{\"type\":\"half_float\"},\"15m\":{\"type\":\"half_float\"},\"1m\":{\"type\":\"half_float\"}}},\"platformRelease\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"platform\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"usage\":{\"properties\":{\"index\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"host\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"index\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"response_time\":{\"properties\":{\"avg\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"max\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"kibana\":{\"properties\":{\"status\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"concurrent_connections\":{\"type\":\"long\"},\"snapshot\":{\"type\":\"boolean\"},\"status\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"node_rules\":{\"properties\":{\"failures\":{\"type\":\"long\"},\"executions\":{\"type\":\"long\"},\"timeouts\":{\"type\":\"long\"}}},\"cluster_actions\":{\"properties\":{\"overdue\":{\"properties\":{\"delay\":{\"properties\":{\"p99\":{\"type\":\"float\"},\"p50\":{\"type\":\"float\"}}},\"count\":{\"type\":\"long\"}}}}},\"cluster_rules\":{\"properties\":{\"overdue\":{\"properties\":{\"delay\":{\"properties\":{\"p99\":{\"type\":\"float\"},\"p50\":{\"type\":\"float\"}}},\"count\":{\"type\":\"long\"}}}}},\"node_actions\":{\"properties\":{\"failures\":{\"type\":\"long\"},\"executions\":{\"type\":\"long\"},\"timeouts\":{\"type\":\"long\"}}},\"status\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"metrics\":{\"properties\":{\"requests\":{\"properties\":{\"total\":{\"type\":\"long\"},\"disconnects\":{\"type\":\"long\"}}},\"concurrent_connections\":{\"type\":\"long\"}}},\"status\":{\"properties\":{\"overall\":{\"properties\":{\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}}}}}},\"event\":{\"properties\":{\"duration\":{\"type\":\"long\"},\"module\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"dataset\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"kibana_stats\":{\"properties\":{\"process\":{\"properties\":{\"memory\":{\"properties\":{\"resident_set_size_in_bytes\":{\"path\":\"kibana.stats.process.memory.resident_set_size.bytes\",\"type\":\"alias\"},\"heap\":{\"properties\":{\"size_limit\":{\"path\":\"kibana.stats.process.memory.heap.size_limit.bytes\",\"type\":\"alias\"}}}}},\"event_loop_delay\":{\"path\":\"kibana.stats.process.event_loop_delay.ms\",\"type\":\"alias\"},\"uptime_in_millis\":{\"path\":\"kibana.stats.process.uptime.ms\",\"type\":\"alias\"}}},\"os\":{\"properties\":{\"memory\":{\"properties\":{\"free_in_bytes\":{\"path\":\"kibana.stats.os.memory.free_in_bytes\",\"type\":\"alias\"}}},\"load\":{\"properties\":{\"5m\":{\"path\":\"kibana.stats.os.load.5m\",\"type\":\"alias\"},\"15m\":{\"path\":\"kibana.stats.os.load.15m\",\"type\":\"alias\"},\"1m\":{\"path\":\"kibana.stats.os.load.1m\",\"type\":\"alias\"}}}}},\"response_times\":{\"properties\":{\"average\":{\"path\":\"kibana.stats.response_time.avg.ms\",\"type\":\"alias\"},\"max\":{\"path\":\"kibana.stats.response_time.max.ms\",\"type\":\"alias\"}}},\"requests\":{\"properties\":{\"total\":{\"path\":\"kibana.stats.request.total\",\"type\":\"alias\"},\"disconnects\":{\"path\":\"kibana.stats.request.disconnects\",\"type\":\"alias\"}}},\"kibana\":{\"properties\":{\"response_time\":{\"properties\":{\"max\":{\"path\":\"kibana.stats.response_time.max.ms\",\"type\":\"alias\"}}},\"uuid\":{\"path\":\"service.id\",\"type\":\"alias\"},\"version\":{\"path\":\"service.version\",\"type\":\"alias\"},\"status\":{\"path\":\"kibana.stats.status\",\"type\":\"alias\"}}},\"concurrent_connections\":{\"path\":\"kibana.stats.concurrent_connections\",\"type\":\"alias\"},\"timestamp\":{\"path\":\"@timestamp\",\"type\":\"alias\"}}},\"timestamp\":{\"path\":\"@timestamp\",\"type\":\"alias\"}}}},\"composed_of\":[],\"version\":8000103,\"data_stream\":{\"hidden\":false,\"allow_custom_routing\":false}},\".monitoring-ent-search-mb\":{\"index_patterns\":[\".monitoring-ent-search-8-*\"],\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\".monitoring-8-ilm-policy\"},\"mapping\":{\"total_fields\":{\"limit\":\"2000\"}}}},\"mappings\":{\"dynamic\":false,\"properties\":{\"enterprisesearch\":{\"properties\":{\"cluster_uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"stats\":{\"properties\":{\"connectors\":{\"properties\":{\"pool\":{\"properties\":{\"publish_worker_pool\":{\"properties\":{\"total_completed\":{\"type\":\"long\"},\"size\":{\"type\":\"long\"},\"idle\":{\"type\":\"long\"},\"busy\":{\"type\":\"long\"},\"queue_depth\":{\"type\":\"long\"},\"total_scheduled\":{\"type\":\"long\"}}},\"extract_worker_pool\":{\"properties\":{\"total_completed\":{\"type\":\"long\"},\"size\":{\"type\":\"long\"},\"idle\":{\"type\":\"long\"},\"busy\":{\"type\":\"long\"},\"queue_depth\":{\"type\":\"long\"},\"total_scheduled\":{\"type\":\"long\"}}},\"subextract_worker_pool\":{\"properties\":{\"total_completed\":{\"type\":\"long\"},\"size\":{\"type\":\"long\"},\"idle\":{\"type\":\"long\"},\"busy\":{\"type\":\"long\"},\"queue_depth\":{\"type\":\"long\"},\"total_scheduled\":{\"type\":\"long\"}}}}},\"job_store\":{\"properties\":{\"waiting\":{\"type\":\"long\"},\"job_types\":{\"properties\":{\"permissions\":{\"type\":\"long\"},\"incremental\":{\"type\":\"long\"},\"delete\":{\"type\":\"long\"},\"full\":{\"type\":\"long\"}}},\"working\":{\"type\":\"long\"}}}}},\"queues\":{\"properties\":{\"engine_destroyer\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"process_crawl\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"failed\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"mailer\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"http\":{\"properties\":{\"request_duration\":{\"properties\":{\"max\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"std_dev\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"mean\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"responses\":{\"properties\":{\"1xx\":{\"type\":\"long\"},\"2xx\":{\"type\":\"long\"},\"3xx\":{\"type\":\"long\"},\"4xx\":{\"type\":\"long\"},\"5xx\":{\"type\":\"long\"}}},\"connections\":{\"properties\":{\"current\":{\"type\":\"long\"},\"total\":{\"type\":\"long\"},\"max\":{\"type\":\"long\"}}},\"network\":{\"properties\":{\"received\":{\"properties\":{\"bytes_per_sec\":{\"type\":\"long\"},\"bytes\":{\"type\":\"long\"}}},\"sent\":{\"properties\":{\"bytes_per_sec\":{\"type\":\"long\"},\"bytes\":{\"type\":\"long\"}}}}}}},\"product_usage\":{\"properties\":{\"app_search\":{\"properties\":{\"total_engines\":{\"type\":\"long\"}}},\"workplace_search\":{\"properties\":{\"total_org_sources\":{\"type\":\"long\"},\"total_private_sources\":{\"type\":\"long\"}}}}}}},\"health\":{\"properties\":{\"jvm\":{\"properties\":{\"memory_usage\":{\"properties\":{\"heap_committed\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"heap_used\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"non_heap_committed\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"non_heap_init\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"heap_init\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"heap_max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"object_pending_finalization_count\":{\"type\":\"long\"}}},\"threads\":{\"properties\":{\"current\":{\"type\":\"long\"},\"total_started\":{\"type\":\"long\"},\"max\":{\"type\":\"long\"},\"daemon\":{\"type\":\"long\"}}},\"gc\":{\"properties\":{\"collection_count\":{\"type\":\"long\"},\"collection_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"process\":{\"properties\":{\"pid\":{\"type\":\"long\"},\"filebeat\":{\"properties\":{\"restart_count\":{\"type\":\"long\"},\"pid\":{\"type\":\"long\"},\"time_since_last_restart\":{\"properties\":{\"sec\":{\"type\":\"long\"}}}}},\"uptime\":{\"properties\":{\"sec\":{\"type\":\"long\"}}}}},\"crawler\":{\"properties\":{\"workers\":{\"properties\":{\"pool_size\":{\"type\":\"long\"},\"available\":{\"type\":\"long\"},\"active\":{\"type\":\"long\"}}}}},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"properties\":{\"number\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"build_hash\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}}}},\"agent\":{\"properties\":{\"id\":{\"type\":\"keyword\"},\"ephemeral_id\":{\"type\":\"keyword\"},\"type\":{\"type\":\"keyword\"},\"version\":{\"type\":\"keyword\"}}},\"process\":{\"properties\":{\"pid\":{\"type\":\"long\"}}},\"@timestamp\":{\"type\":\"date\"},\"ecs\":{\"properties\":{\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"service\":{\"properties\":{\"node\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"hostname\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"environment\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"origin\":{\"properties\":{\"node\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"environment\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"target\":{\"properties\":{\"node\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"environment\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"host\":{\"properties\":{\"hostname\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"architecture\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"metricset\":{\"properties\":{\"period\":{\"type\":\"long\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"event\":{\"properties\":{\"reason\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"code\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"timezone\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"agent_id_status\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"duration\":{\"type\":\"long\"},\"reference\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ingested\":{\"type\":\"date\"},\"provider\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"action\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"end\":{\"type\":\"date\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"outcome\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"severity\":{\"type\":\"long\"},\"original\":{\"ignore_above\":1024,\"index\":false,\"type\":\"keyword\",\"doc_values\":false},\"risk_score\":{\"type\":\"float\"},\"created\":{\"type\":\"date\"},\"kind\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"module\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"start\":{\"type\":\"date\"},\"url\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"sequence\":{\"type\":\"long\"},\"risk_score_norm\":{\"type\":\"float\"},\"category\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"dataset\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"hash\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}}},\"composed_of\":[],\"version\":8000103,\"_meta\":{\"managed\":true,\"description\":\"Template used by Enterprise Search Metricbeat module monitoring information for Stack Monitoring\"},\"data_stream\":{\"hidden\":false,\"allow_custom_routing\":false}},\"synthetics\":{\"index_patterns\":[\"synthetics-*-*\"],\"composed_of\":[\"synthetics-mappings\",\"data-streams-mappings\",\"synthetics-settings\"],\"priority\":100,\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default synthetics template installed by x-pack\"},\"data_stream\":{\"hidden\":false,\"allow_custom_routing\":false},\"allow_auto_create\":true},\"ilm-history\":{\"index_patterns\":[\"ilm-history-5*\"],\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\"ilm-history-ilm-policy\"},\"number_of_shards\":\"1\",\"auto_expand_replicas\":\"0-1\",\"number_of_replicas\":\"0\"}},\"mappings\":{\"dynamic\":false,\"properties\":{\"index_age\":{\"type\":\"long\"},\"@timestamp\":{\"format\":\"epoch_millis\",\"type\":\"date\"},\"error_details\":{\"type\":\"text\"},\"success\":{\"type\":\"boolean\"},\"index\":{\"type\":\"keyword\"},\"state\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"phase\":{\"type\":\"keyword\"},\"failed_step\":{\"type\":\"keyword\"},\"phase_definition\":{\"type\":\"text\"},\"action_time\":{\"format\":\"epoch_millis\",\"type\":\"date\"},\"phase_time\":{\"format\":\"epoch_millis\",\"type\":\"date\"},\"step_info\":{\"type\":\"text\"},\"action\":{\"type\":\"keyword\"},\"step\":{\"type\":\"keyword\"},\"is_auto-retryable_error\":{\"type\":\"keyword\"},\"creation_date\":{\"format\":\"epoch_millis\",\"type\":\"date\"},\"step_time\":{\"format\":\"epoch_millis\",\"type\":\"date\"}}},\"policy\":{\"type\":\"keyword\"}}}},\"composed_of\":[],\"priority\":2147483647,\"version\":5,\"_meta\":{\"managed\":true,\"description\":\"index template for ILM history indices\"},\"data_stream\":{\"hidden\":true,\"allow_custom_routing\":false},\"allow_auto_create\":true},\".ml-state\":{\"index_patterns\":[\".ml-state*\"],\"template\":{\"settings\":{\"index\":{\"hidden\":\"true\",\"lifecycle\":{\"name\":\"ml-size-based-ilm-policy\",\"rollover_alias\":\".ml-state-write\"},\"auto_expand_replicas\":\"0-1\"}},\"mappings\":{\"_meta\":{\"version\":\"8060299\"},\"enabled\":false},\"aliases\":{}},\"composed_of\":[],\"priority\":2147483647,\"version\":8060299,\"_meta\":{\"managed\":true,\"description\":\"index template for ML state indices\"}},\".monitoring-es-mb\":{\"index_patterns\":[\".monitoring-es-8-*\"],\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\".monitoring-8-ilm-policy\"},\"mapping\":{\"total_fields\":{\"limit\":\"2000\"}}}},\"mappings\":{\"dynamic\":false,\"properties\":{\"ccr_auto_follow_stats\":{\"properties\":{\"number_of_failed_remote_cluster_state_requests\":{\"path\":\"elasticsearch.ccr.auto_follow.failed.remote_cluster_state_requests.count\",\"type\":\"alias\"},\"follower\":{\"properties\":{\"failed_read_requests\":{\"path\":\"elasticsearch.ccr.requests.failed.read.count\",\"type\":\"alias\"}}},\"number_of_failed_follow_indices\":{\"path\":\"elasticsearch.ccr.auto_follow.failed.follow_indices.count\",\"type\":\"alias\"},\"number_of_successful_follow_indices\":{\"path\":\"elasticsearch.ccr.auto_follow.success.follow_indices.count\",\"type\":\"alias\"}}},\"ccr_stats\":{\"properties\":{\"write_buffer_size_in_bytes\":{\"path\":\"elasticsearch.ccr.write_buffer.size.bytes\",\"type\":\"alias\"},\"leader_global_checkpoint\":{\"path\":\"elasticsearch.ccr.leader.global_checkpoint\",\"type\":\"alias\"},\"follower_index\":{\"path\":\"elasticsearch.ccr.follower.index\",\"type\":\"alias\"},\"leader_max_seq_no\":{\"path\":\"elasticsearch.ccr.leader.max_seq_no\",\"type\":\"alias\"},\"last_requested_seq_no\":{\"path\":\"elasticsearch.ccr.last_requested_seq_no\",\"type\":\"alias\"},\"follower_settings_version\":{\"path\":\"elasticsearch.ccr.follower.settings_version\",\"type\":\"alias\"},\"successful_write_requests\":{\"path\":\"elasticsearch.ccr.requests.successful.write.count\",\"type\":\"alias\"},\"remote_cluster\":{\"path\":\"elasticsearch.ccr.remote_cluster\",\"type\":\"alias\"},\"outstanding_write_requests\":{\"path\":\"elasticsearch.ccr.requests.outstanding.write.count\",\"type\":\"alias\"},\"total_read_time_millis\":{\"path\":\"elasticsearch.ccr.total_time.read.ms\",\"type\":\"alias\"},\"outstanding_read_requests\":{\"path\":\"elasticsearch.ccr.requests.outstanding.read.count\",\"type\":\"alias\"},\"total_write_time_millis\":{\"path\":\"elasticsearch.ccr.total_time.write.ms\",\"type\":\"alias\"},\"failed_write_requests\":{\"path\":\"elasticsearch.ccr.requests.failed.write.count\",\"type\":\"alias\"},\"failed_read_requests\":{\"path\":\"elasticsearch.ccr.requests.failed.read.count\",\"type\":\"alias\"},\"bytes_read\":{\"path\":\"elasticsearch.ccr.bytes_read\",\"type\":\"alias\"},\"leader_index\":{\"path\":\"elasticsearch.ccr.leader.index\",\"type\":\"alias\"},\"follower_max_seq_no\":{\"path\":\"elasticsearch.ccr.follower.max_seq_no\",\"type\":\"alias\"},\"operations_written\":{\"path\":\"elasticsearch.ccr.follower.operations_written\",\"type\":\"alias\"},\"write_buffer_operation_count\":{\"path\":\"elasticsearch.ccr.write_buffer.operation.count\",\"type\":\"alias\"},\"successful_read_requests\":{\"path\":\"elasticsearch.ccr.requests.successful.read.count\",\"type\":\"alias\"},\"shard_id\":{\"path\":\"elasticsearch.ccr.follower.shard.number\",\"type\":\"alias\"},\"follower_mapping_version\":{\"path\":\"elasticsearch.ccr.follower.mapping_version\",\"type\":\"alias\"},\"follower_aliases_version\":{\"path\":\"elasticsearch.ccr.follower.aliases_version\",\"type\":\"alias\"},\"follower_global_checkpoint\":{\"path\":\"elasticsearch.ccr.follower.global_checkpoint\",\"type\":\"alias\"},\"total_read_remote_exec_time_millis\":{\"path\":\"elasticsearch.ccr.total_time.read.remote_exec.ms\",\"type\":\"alias\"},\"operations_read\":{\"path\":\"elasticsearch.ccr.follower.operations.read.count\",\"type\":\"alias\"}}},\"cluster_state\":{\"properties\":{\"nodes_hash\":{\"path\":\"elasticsearch.cluster.stats.state.nodes_hash\",\"type\":\"alias\"},\"master_node\":{\"path\":\"elasticsearch.cluster.stats.state.master_node\",\"type\":\"alias\"},\"state_uuid\":{\"path\":\"elasticsearch.cluster.stats.state.state_uuid\",\"type\":\"alias\"},\"version\":{\"path\":\"elasticsearch.cluster.stats.state.version\",\"type\":\"alias\"},\"status\":{\"path\":\"elasticsearch.cluster.stats.status\",\"type\":\"alias\"}}},\"indices_stats\":{\"properties\":{\"_all\":{\"properties\":{\"primaries\":{\"properties\":{\"indexing\":{\"properties\":{\"index_time_in_millis\":{\"path\":\"elasticsearch.index.summary.primaries.indexing.index.time.ms\",\"type\":\"alias\"},\"index_total\":{\"path\":\"elasticsearch.index.summary.primaries.indexing.index.count\",\"type\":\"alias\"}}}}},\"total\":{\"properties\":{\"search\":{\"properties\":{\"query_total\":{\"path\":\"elasticsearch.index.summary.total.search.query.count\",\"type\":\"alias\"},\"query_time_in_millis\":{\"path\":\"elasticsearch.index.summary.total.search.query.time.ms\",\"type\":\"alias\"}}},\"indexing\":{\"properties\":{\"index_total\":{\"path\":\"elasticsearch.index.summary.total.indexing.index.count\",\"type\":\"alias\"}}}}}}}}},\"node_stats\":{\"properties\":{\"jvm\":{\"properties\":{\"mem\":{\"properties\":{\"heap_used_percent\":{\"path\":\"elasticsearch.node.stats.jvm.mem.heap.used.pct\",\"type\":\"alias\"},\"heap_max_in_bytes\":{\"path\":\"elasticsearch.node.stats.jvm.mem.heap.max.bytes\",\"type\":\"alias\"},\"heap_used_in_bytes\":{\"path\":\"elasticsearch.node.stats.jvm.mem.heap.used.bytes\",\"type\":\"alias\"}}},\"gc\":{\"properties\":{\"collectors\":{\"properties\":{\"young\":{\"properties\":{\"collection_count\":{\"path\":\"elasticsearch.node.stats.jvm.gc.collectors.young.collection.count\",\"type\":\"alias\"},\"collection_time_in_millis\":{\"path\":\"elasticsearch.node.stats.jvm.gc.collectors.young.collection.ms\",\"type\":\"alias\"}}},\"old\":{\"properties\":{\"collection_count\":{\"path\":\"elasticsearch.node.stats.jvm.gc.collectors.old.collection.count\",\"type\":\"alias\"},\"collection_time_in_millis\":{\"path\":\"elasticsearch.node.stats.jvm.gc.collectors.old.collection.ms\",\"type\":\"alias\"}}}}}}}}},\"indices\":{\"properties\":{\"search\":{\"properties\":{\"query_total\":{\"path\":\"elasticsearch.node.stats.indices.search.query_total.count\",\"type\":\"alias\"},\"query_time_in_millis\":{\"path\":\"elasticsearch.node.stats.indices.search.query_time.ms\",\"type\":\"alias\"}}},\"query_cache\":{\"properties\":{\"memory_size_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.query_cache.memory.bytes\",\"type\":\"alias\"}}},\"docs\":{\"properties\":{\"count\":{\"path\":\"elasticsearch.node.stats.indices.docs.count\",\"type\":\"alias\"}}},\"indexing\":{\"properties\":{\"throttle_time_in_millis\":{\"path\":\"elasticsearch.node.stats.indices.indexing.throttle_time.ms\",\"type\":\"alias\"},\"index_time_in_millis\":{\"path\":\"elasticsearch.node.stats.indices.indexing.index_time.ms\",\"type\":\"alias\"},\"index_total\":{\"path\":\"elasticsearch.node.stats.indices.indexing.index_total.count\",\"type\":\"alias\"}}},\"fielddata\":{\"properties\":{\"memory_size_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.fielddata.memory.bytes\",\"type\":\"alias\"}}},\"store\":{\"properties\":{\"size\":{\"properties\":{\"bytes\":{\"path\":\"elasticsearch.node.stats.indices.store.size.bytes\",\"type\":\"alias\"}}},\"size_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.store.size.bytes\",\"type\":\"alias\"}}},\"request_cache\":{\"properties\":{\"memory_size_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.request_cache.memory.bytes\",\"type\":\"alias\"}}},\"segments\":{\"properties\":{\"version_map_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.version_map.memory.bytes\",\"type\":\"alias\"},\"norms_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.norms.memory.bytes\",\"type\":\"alias\"},\"count\":{\"path\":\"elasticsearch.node.stats.indices.segments.count\",\"type\":\"alias\"},\"term_vectors_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.term_vectors.memory.bytes\",\"type\":\"alias\"},\"points_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.points.memory.bytes\",\"type\":\"alias\"},\"index_writer_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.index_writer.memory.bytes\",\"type\":\"alias\"},\"memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.memory.bytes\",\"type\":\"alias\"},\"doc_values_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.doc_values.memory.bytes\",\"type\":\"alias\"},\"terms_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.terms.memory.bytes\",\"type\":\"alias\"},\"fixed_bit_set_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.fixed_bit_set.memory.bytes\",\"type\":\"alias\"},\"stored_fields_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.stored_fields.memory.bytes\",\"type\":\"alias\"}}}}},\"process\":{\"properties\":{\"cpu\":{\"properties\":{\"percent\":{\"path\":\"elasticsearch.node.stats.process.cpu.pct\",\"type\":\"alias\"}}}}},\"os\":{\"properties\":{\"cpu\":{\"properties\":{\"load_average\":{\"properties\":{\"1m\":{\"path\":\"elasticsearch.node.stats.os.cpu.load_avg.1m\",\"type\":\"alias\"}}}}},\"cgroup\":{\"properties\":{\"memory\":{\"properties\":{\"usage_in_bytes\":{\"path\":\"elasticsearch.node.stats.os.cgroup.memory.usage.bytes\",\"type\":\"alias\"},\"control_group\":{\"path\":\"elasticsearch.node.stats.os.cgroup.memory.control_group\",\"type\":\"alias\"},\"limit_in_bytes\":{\"path\":\"elasticsearch.node.stats.os.cgroup.memory.limit.bytes\",\"type\":\"alias\"}}},\"cpu\":{\"properties\":{\"stat\":{\"properties\":{\"number_of_elapsed_periods\":{\"path\":\"elasticsearch.node.stats.os.cgroup.cpu.stat.elapsed_periods.count\",\"type\":\"alias\"},\"number_of_times_throttled\":{\"path\":\"elasticsearch.node.stats.os.cgroup.cpu.stat.times_throttled.count\",\"type\":\"alias\"},\"time_throttled_nanos\":{\"path\":\"elasticsearch.node.stats.os.cgroup.cpu.stat.time_throttled.ns\",\"type\":\"alias\"}}},\"cfs_quota_micros\":{\"path\":\"elasticsearch.node.stats.os.cgroup.cpu.cfs.quota.us\",\"type\":\"alias\"}}},\"cpuacct\":{\"properties\":{\"usage_nanos\":{\"path\":\"elasticsearch.node.stats.os.cgroup.cpuacct.usage.ns\",\"type\":\"alias\"}}}}}}},\"thread_pool\":{\"properties\":{\"search\":{\"properties\":{\"rejected\":{\"path\":\"elasticsearch.node.stats.thread_pool.search.rejected.count\",\"type\":\"alias\"},\"queue\":{\"path\":\"elasticsearch.node.stats.thread_pool.search.queue.count\",\"type\":\"alias\"}}},\"get\":{\"properties\":{\"rejected\":{\"path\":\"elasticsearch.node.stats.thread_pool.get.rejected.count\",\"type\":\"alias\"},\"queue\":{\"path\":\"elasticsearch.node.stats.thread_pool.get.queue.count\",\"type\":\"alias\"}}},\"index\":{\"properties\":{\"rejected\":{\"path\":\"elasticsearch.node.stats.thread_pool.index.rejected.count\",\"type\":\"alias\"},\"queue\":{\"path\":\"elasticsearch.node.stats.thread_pool.index.queue.count\",\"type\":\"alias\"}}},\"bulk\":{\"properties\":{\"rejected\":{\"path\":\"elasticsearch.node.stats.thread_pool.bulk.rejected.count\",\"type\":\"alias\"},\"queue\":{\"path\":\"elasticsearch.node.stats.thread_pool.bulk.queue.count\",\"type\":\"alias\"}}},\"write\":{\"properties\":{\"rejected\":{\"path\":\"elasticsearch.node.stats.thread_pool.write.rejected.count\",\"type\":\"alias\"},\"queue\":{\"path\":\"elasticsearch.node.stats.thread_pool.write.queue.count\",\"type\":\"alias\"}}}}},\"fs\":{\"properties\":{\"summary\":{\"properties\":{\"total\":{\"properties\":{\"bytes\":{\"path\":\"elasticsearch.node.stats.fs.summary.total.bytes\",\"type\":\"alias\"}}},\"available\":{\"properties\":{\"bytes\":{\"path\":\"elasticsearch.node.stats.fs.summary.available.bytes\",\"type\":\"alias\"}}}}},\"io_stats\":{\"properties\":{\"total\":{\"properties\":{\"write_operations\":{\"path\":\"elasticsearch.node.stats.fs.io_stats.total.write.operations.count\",\"type\":\"alias\"},\"operations\":{\"path\":\"elasticsearch.node.stats.fs.io_stats.total.operations.count\",\"type\":\"alias\"},\"read_operations\":{\"path\":\"elasticsearch.node.stats.fs.io_stats.total.read.operations.count\",\"type\":\"alias\"}}}}},\"total\":{\"properties\":{\"total_in_bytes\":{\"path\":\"elasticsearch.node.stats.fs.summary.total.bytes\",\"type\":\"alias\"},\"available_in_bytes\":{\"path\":\"elasticsearch.node.stats.fs.summary.available.bytes\",\"type\":\"alias\"}}}}},\"node_id\":{\"path\":\"elasticsearch.node.id\",\"type\":\"alias\"}}},\"stack_stats\":{\"properties\":{\"xpack\":{\"properties\":{\"ccr\":{\"properties\":{\"available\":{\"path\":\"elasticsearch.cluster.stats.stack.xpack.ccr.available\",\"type\":\"alias\"},\"enabled\":{\"path\":\"elasticsearch.cluster.stats.stack.xpack.ccr.enabled\",\"type\":\"alias\"}}}}},\"apm\":{\"properties\":{\"found\":{\"path\":\"elasticsearch.cluster.stats.stack.apm.found\",\"type\":\"alias\"}}}}},\"license\":{\"properties\":{\"type\":{\"path\":\"elasticsearch.cluster.stats.license.type\",\"type\":\"alias\"},\"status\":{\"path\":\"elasticsearch.cluster.stats.license.status\",\"type\":\"alias\"}}},\"index_stats\":{\"properties\":{\"primaries\":{\"properties\":{\"docs\":{\"properties\":{\"count\":{\"path\":\"elasticsearch.index.primaries.docs.count\",\"type\":\"alias\"}}},\"indexing\":{\"properties\":{\"throttle_time_in_millis\":{\"path\":\"elasticsearch.index.primaries.indexing.throttle_time_in_millis\",\"type\":\"alias\"},\"index_time_in_millis\":{\"path\":\"elasticsearch.index.primaries.indexing.index_time_in_millis\",\"type\":\"alias\"},\"index_total\":{\"path\":\"elasticsearch.index.primaries.indexing.index_total\",\"type\":\"alias\"}}},\"refresh\":{\"properties\":{\"total_time_in_millis\":{\"path\":\"elasticsearch.index.primaries.refresh.total_time_in_millis\",\"type\":\"alias\"}}},\"store\":{\"properties\":{\"size_in_bytes\":{\"path\":\"elasticsearch.index.primaries.store.size_in_bytes\",\"type\":\"alias\"}}},\"merges\":{\"properties\":{\"total_size_in_bytes\":{\"path\":\"elasticsearch.index.primaries.merges.total_size_in_bytes\",\"type\":\"alias\"}}},\"segments\":{\"properties\":{\"count\":{\"path\":\"elasticsearch.index.primaries.segments.count\",\"type\":\"alias\"}}}}},\"total\":{\"properties\":{\"search\":{\"properties\":{\"query_total\":{\"path\":\"elasticsearch.index.total.search.query_total\",\"type\":\"alias\"},\"query_time_in_millis\":{\"path\":\"elasticsearch.index.total.search.query_time_in_millis\",\"type\":\"alias\"}}},\"query_cache\":{\"properties\":{\"memory_size_in_bytes\":{\"path\":\"elasticsearch.index.total.query_cache.memory_size_in_bytes\",\"type\":\"alias\"}}},\"fielddata\":{\"properties\":{\"memory_size_in_bytes\":{\"path\":\"elasticsearch.index.total.fielddata.memory_size_in_bytes\",\"type\":\"alias\"}}},\"indexing\":{\"properties\":{\"throttle_time_in_millis\":{\"path\":\"elasticsearch.index.total.indexing.throttle_time_in_millis\",\"type\":\"alias\"},\"index_time_in_millis\":{\"path\":\"elasticsearch.index.total.indexing.index_time_in_millis\",\"type\":\"alias\"},\"index_total\":{\"path\":\"elasticsearch.index.total.indexing.index_total\",\"type\":\"alias\"}}},\"refresh\":{\"properties\":{\"total_time_in_millis\":{\"path\":\"elasticsearch.index.total.refresh.total_time_in_millis\",\"type\":\"alias\"}}},\"request_cache\":{\"properties\":{\"memory_size_in_bytes\":{\"path\":\"elasticsearch.index.total.request_cache.memory_size_in_bytes\",\"type\":\"alias\"}}},\"store\":{\"properties\":{\"size_in_bytes\":{\"path\":\"elasticsearch.index.total.store.size_in_bytes\",\"type\":\"alias\"}}},\"merges\":{\"properties\":{\"total_size_in_bytes\":{\"path\":\"elasticsearch.index.total.merges.total_size_in_bytes\",\"type\":\"alias\"}}},\"segments\":{\"properties\":{\"version_map_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.version_map_memory_in_bytes\",\"type\":\"alias\"},\"norms_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.norms_memory_in_bytes\",\"type\":\"alias\"},\"count\":{\"path\":\"elasticsearch.index.total.segments.count\",\"type\":\"alias\"},\"term_vectors_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.term_vectors_memory_in_bytes\",\"type\":\"alias\"},\"points_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.points_memory_in_bytes\",\"type\":\"alias\"},\"index_writer_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.index_writer_memory_in_bytes\",\"type\":\"alias\"},\"memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.memory_in_bytes\",\"type\":\"alias\"},\"doc_values_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.doc_values_memory_in_bytes\",\"type\":\"alias\"},\"terms_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.terms_memory_in_bytes\",\"type\":\"alias\"},\"fixed_bit_set_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.fixed_bit_set_memory_in_bytes\",\"type\":\"alias\"},\"stored_fields_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.stored_fields_memory_in_bytes\",\"type\":\"alias\"}}}}},\"index\":{\"path\":\"elasticsearch.index.name\",\"type\":\"alias\"}}},\"@timestamp\":{\"type\":\"date\"},\"elasticsearch\":{\"properties\":{\"cluster\":{\"properties\":{\"stats\":{\"properties\":{\"license\":{\"properties\":{\"expiry_date_in_millis\":{\"type\":\"long\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"status\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"indices\":{\"properties\":{\"shards\":{\"properties\":{\"primaries\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"total\":{\"type\":\"long\"},\"docs\":{\"properties\":{\"total\":{\"type\":\"long\"}}},\"fielddata\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"store\":{\"properties\":{\"size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}}}},\"stack\":{\"properties\":{\"xpack\":{\"properties\":{\"ccr\":{\"properties\":{\"available\":{\"type\":\"boolean\"},\"enabled\":{\"type\":\"boolean\"}}}}},\"apm\":{\"properties\":{\"found\":{\"type\":\"boolean\"}}}}},\"nodes\":{\"properties\":{\"jvm\":{\"properties\":{\"memory\":{\"properties\":{\"heap\":{\"properties\":{\"max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"used\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}}}},\"max_uptime\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"data\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"},\"fs\":{\"properties\":{\"total\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"available\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"master\":{\"type\":\"long\"}}},\"state\":{\"properties\":{\"nodes_hash\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"master_node\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state_uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"status\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"properties\":{\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"node\":{\"properties\":{\"jvm\":{\"properties\":{\"memory\":{\"properties\":{\"heap\":{\"properties\":{\"init\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"nonheap\":{\"properties\":{\"init\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}}}},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"process\":{\"properties\":{\"mlockall\":{\"type\":\"boolean\"}}},\"stats\":{\"properties\":{\"jvm\":{\"properties\":{\"mem\":{\"properties\":{\"pools\":{\"properties\":{\"young\":{\"properties\":{\"max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"peak\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"peak_max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"used\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"old\":{\"properties\":{\"max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"peak\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"peak_max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"used\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"survivor\":{\"properties\":{\"max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"peak\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"peak_max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"used\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}}}},\"heap\":{\"properties\":{\"max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"used\":{\"properties\":{\"pct\":{\"type\":\"double\"},\"bytes\":{\"type\":\"long\"}}}}}}},\"gc\":{\"properties\":{\"collectors\":{\"properties\":{\"young\":{\"properties\":{\"collection\":{\"properties\":{\"ms\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}}}},\"old\":{\"properties\":{\"collection\":{\"properties\":{\"ms\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}}}}}}}}}},\"indices\":{\"properties\":{\"search\":{\"properties\":{\"query_total\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"query_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"query_cache\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"docs\":{\"properties\":{\"deleted\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"fielddata\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"indexing\":{\"properties\":{\"index_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"index_total\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"throttle_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"store\":{\"properties\":{\"size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"request_cache\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"bulk\":{\"properties\":{\"operations\":{\"properties\":{\"total\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"total_size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"avg_size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"total_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"avg_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"segments\":{\"properties\":{\"version_map\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"norms\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"terms\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"stored_fields\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"index_writer\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"count\":{\"type\":\"long\"},\"fixed_bit_set\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"term_vectors\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"doc_values\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"points\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}}}}}},\"process\":{\"properties\":{\"cpu\":{\"properties\":{\"pct\":{\"type\":\"double\"}}}}},\"os\":{\"properties\":{\"cpu\":{\"properties\":{\"load_avg\":{\"properties\":{\"1m\":{\"type\":\"half_float\"}}}}},\"cgroup\":{\"properties\":{\"memory\":{\"properties\":{\"control_group\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"usage\":{\"properties\":{\"bytes\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"limit\":{\"properties\":{\"bytes\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"cpu\":{\"properties\":{\"cfs\":{\"properties\":{\"quota\":{\"properties\":{\"us\":{\"type\":\"long\"}}}}},\"stat\":{\"properties\":{\"elapsed_periods\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"times_throttled\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"time_throttled\":{\"properties\":{\"ns\":{\"type\":\"long\"}}}}}}},\"cpuacct\":{\"properties\":{\"usage\":{\"properties\":{\"ns\":{\"type\":\"long\"}}}}}}}}},\"thread_pool\":{\"properties\":{\"search\":{\"properties\":{\"rejected\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"queue\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"get\":{\"properties\":{\"rejected\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"queue\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"index\":{\"properties\":{\"rejected\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"queue\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"bulk\":{\"properties\":{\"rejected\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"queue\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"write\":{\"properties\":{\"rejected\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"queue\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}}}},\"fs\":{\"properties\":{\"summary\":{\"properties\":{\"total\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"available\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"free\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"io_stats\":{\"properties\":{\"total\":{\"properties\":{\"operations\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"read\":{\"properties\":{\"operations\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"write\":{\"properties\":{\"operations\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}}}}}},\"total\":{\"properties\":{\"total_in_bytes\":{\"type\":\"long\"},\"available_in_bytes\":{\"type\":\"long\"}}}}}}},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"mlockall\":{\"type\":\"boolean\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"master\":{\"type\":\"boolean\"}}},\"ccr\":{\"properties\":{\"leader\":{\"properties\":{\"index\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"max_seq_no\":{\"type\":\"long\"},\"global_checkpoint\":{\"type\":\"long\"}}},\"follower\":{\"properties\":{\"operations\":{\"properties\":{\"read\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"time_since_last_read\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"index\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"settings_version\":{\"type\":\"long\"},\"shard\":{\"properties\":{\"number\":{\"type\":\"long\"}}},\"max_seq_no\":{\"type\":\"long\"},\"mapping_version\":{\"type\":\"long\"},\"aliases_version\":{\"type\":\"long\"},\"operations_written\":{\"type\":\"long\"},\"global_checkpoint\":{\"type\":\"long\"}}},\"remote_cluster\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"read_exceptions\":{\"type\":\"nested\"},\"shard_id\":{\"type\":\"long\"},\"bytes_read\":{\"type\":\"long\"},\"requests\":{\"properties\":{\"outstanding\":{\"properties\":{\"read\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"write\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"failed\":{\"properties\":{\"read\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"write\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"successful\":{\"properties\":{\"read\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"write\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}}}},\"auto_follow\":{\"properties\":{\"success\":{\"properties\":{\"follow_indices\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"failed\":{\"properties\":{\"follow_indices\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"remote_cluster_state_requests\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}}}},\"total_time\":{\"properties\":{\"read\":{\"properties\":{\"ms\":{\"type\":\"long\"},\"remote_exec\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"write\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"last_requested_seq_no\":{\"type\":\"long\"},\"write_buffer\":{\"properties\":{\"size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"operation\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}}}},\"pending_tasks\":{\"properties\":{\"time_in_queue\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"source\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"priority\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"insert_order\":{\"type\":\"long\"}}},\"enrich\":{\"properties\":{\"executed_searches\":{\"properties\":{\"total\":{\"type\":\"long\"}}},\"remote_requests\":{\"properties\":{\"current\":{\"type\":\"long\"},\"total\":{\"type\":\"long\"}}},\"executing_policy\":{\"properties\":{\"task\":{\"properties\":{\"task\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"parent_task_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"action\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"type\":\"long\"},\"time\":{\"properties\":{\"running\":{\"properties\":{\"nano\":{\"type\":\"long\"}}},\"start\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"cancellable\":{\"type\":\"boolean\"}}},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"queue\":{\"properties\":{\"size\":{\"type\":\"long\"}}}}},\"index\":{\"properties\":{\"summary\":{\"properties\":{\"primaries\":{\"properties\":{\"search\":{\"properties\":{\"query\":{\"properties\":{\"count\":{\"type\":\"long\"},\"time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}}}},\"docs\":{\"properties\":{\"deleted\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"indexing\":{\"properties\":{\"index\":{\"properties\":{\"count\":{\"type\":\"long\"},\"time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}}}},\"store\":{\"properties\":{\"size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"bulk\":{\"properties\":{\"operations\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"time\":{\"properties\":{\"avg\":{\"properties\":{\"ms\":{\"type\":\"long\"},\"bytes\":{\"type\":\"long\"}}},\"count\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}}}},\"segments\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"count\":{\"type\":\"long\"}}}}},\"total\":{\"properties\":{\"search\":{\"properties\":{\"query\":{\"properties\":{\"count\":{\"type\":\"long\"},\"time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}}}},\"docs\":{\"properties\":{\"deleted\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"indexing\":{\"properties\":{\"index\":{\"properties\":{\"count\":{\"type\":\"long\"},\"time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"is_throttled\":{\"type\":\"boolean\"},\"throttle_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"store\":{\"properties\":{\"size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"bulk\":{\"properties\":{\"operations\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"time\":{\"properties\":{\"avg\":{\"properties\":{\"ms\":{\"type\":\"long\"},\"bytes\":{\"type\":\"long\"}}}}}}},\"segments\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"count\":{\"type\":\"long\"}}}}}}},\"shards\":{\"properties\":{\"primaries\":{\"type\":\"long\"},\"total\":{\"type\":\"long\"}}},\"primaries\":{\"properties\":{\"search\":{\"properties\":{\"query_total\":{\"type\":\"long\"},\"query_time_in_millis\":{\"type\":\"long\"}}},\"query_cache\":{\"properties\":{\"miss_count\":{\"type\":\"long\"},\"memory_size_in_bytes\":{\"type\":\"long\"},\"hit_count\":{\"type\":\"long\"}}},\"docs\":{\"properties\":{\"deleted\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"indexing\":{\"properties\":{\"throttle_time_in_millis\":{\"type\":\"long\"},\"index_time_in_millis\":{\"type\":\"long\"},\"index_total\":{\"type\":\"long\"}}},\"refresh\":{\"properties\":{\"external_total_time_in_millis\":{\"type\":\"long\"},\"total_time_in_millis\":{\"type\":\"long\"}}},\"request_cache\":{\"properties\":{\"miss_count\":{\"type\":\"long\"},\"memory_size_in_bytes\":{\"type\":\"long\"},\"evictions\":{\"type\":\"long\"},\"hit_count\":{\"type\":\"long\"}}},\"store\":{\"properties\":{\"size_in_bytes\":{\"type\":\"long\"}}},\"segments\":{\"properties\":{\"version_map_memory_in_bytes\":{\"type\":\"long\"},\"norms_memory_in_bytes\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"},\"term_vectors_memory_in_bytes\":{\"type\":\"long\"},\"points_memory_in_bytes\":{\"type\":\"long\"},\"index_writer_memory_in_bytes\":{\"type\":\"long\"},\"memory_in_bytes\":{\"type\":\"long\"},\"terms_memory_in_bytes\":{\"type\":\"long\"},\"doc_values_memory_in_bytes\":{\"type\":\"long\"},\"stored_fields_memory_in_bytes\":{\"type\":\"long\"},\"fixed_bit_set_memory_in_bytes\":{\"type\":\"long\"}}},\"merges\":{\"properties\":{\"total_size_in_bytes\":{\"type\":\"long\"}}}}},\"total\":{\"properties\":{\"search\":{\"properties\":{\"query_total\":{\"type\":\"long\"},\"query_time_in_millis\":{\"type\":\"long\"}}},\"query_cache\":{\"properties\":{\"miss_count\":{\"type\":\"long\"},\"memory_size_in_bytes\":{\"type\":\"long\"},\"evictions\":{\"type\":\"long\"},\"hit_count\":{\"type\":\"long\"}}},\"docs\":{\"properties\":{\"deleted\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"fielddata\":{\"properties\":{\"memory_size_in_bytes\":{\"type\":\"long\"},\"evictions\":{\"type\":\"long\"}}},\"indexing\":{\"properties\":{\"throttle_time_in_millis\":{\"type\":\"long\"},\"index_time_in_millis\":{\"type\":\"long\"},\"index_total\":{\"type\":\"long\"}}},\"refresh\":{\"properties\":{\"external_total_time_in_millis\":{\"type\":\"long\"},\"total_time_in_millis\":{\"type\":\"long\"}}},\"store\":{\"properties\":{\"size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"size_in_bytes\":{\"type\":\"long\"}}},\"request_cache\":{\"properties\":{\"miss_count\":{\"type\":\"long\"},\"memory_size_in_bytes\":{\"type\":\"long\"},\"evictions\":{\"type\":\"long\"},\"hit_count\":{\"type\":\"long\"}}},\"bulk\":{\"properties\":{\"avg_time_in_millis\":{\"type\":\"long\"},\"avg_size_in_bytes\":{\"type\":\"long\"},\"total_time_in_millis\":{\"type\":\"long\"},\"total_operations\":{\"type\":\"long\"},\"total_size_in_bytes\":{\"type\":\"long\"}}},\"merges\":{\"properties\":{\"total_size_in_bytes\":{\"type\":\"long\"}}},\"segments\":{\"properties\":{\"version_map_memory_in_bytes\":{\"type\":\"long\"},\"norms_memory_in_bytes\":{\"type\":\"long\"},\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"count\":{\"type\":\"long\"},\"term_vectors_memory_in_bytes\":{\"type\":\"long\"},\"points_memory_in_bytes\":{\"type\":\"long\"},\"index_writer_memory_in_bytes\":{\"type\":\"long\"},\"memory_in_bytes\":{\"type\":\"long\"},\"terms_memory_in_bytes\":{\"type\":\"long\"},\"doc_values_memory_in_bytes\":{\"type\":\"long\"},\"stored_fields_memory_in_bytes\":{\"type\":\"long\"},\"fixed_bit_set_memory_in_bytes\":{\"type\":\"long\"}}}}},\"hidden\":{\"type\":\"boolean\"},\"created\":{\"type\":\"long\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"recovery\":{\"properties\":{\"stop_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"translog\":{\"properties\":{\"total_on_start\":{\"type\":\"long\"},\"total\":{\"type\":\"long\"},\"percent\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"verify_index\":{\"properties\":{\"total_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"check_index_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"index\":{\"properties\":{\"size\":{\"properties\":{\"total_in_bytes\":{\"type\":\"long\"},\"reused_in_bytes\":{\"type\":\"long\"},\"recovered_in_bytes\":{\"type\":\"long\"}}},\"files\":{\"properties\":{\"recovered\":{\"type\":\"long\"},\"total\":{\"type\":\"long\"},\"percent\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"reused\":{\"type\":\"long\"}}}}},\"source\":{\"properties\":{\"transport_address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"host\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"target\":{\"properties\":{\"transport_address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"host\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"start_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"stage\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"type\":\"long\"},\"total_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"primary\":{\"type\":\"boolean\"}}},\"uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"status\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"shard\":{\"properties\":{\"number\":{\"type\":\"long\"},\"source_node\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"relocating_node\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"primary\":{\"type\":\"boolean\"}}},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ml\":{\"properties\":{\"job\":{\"properties\":{\"data\":{\"properties\":{\"invalid_date\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"data_counts\":{\"properties\":{\"invalid_date_count\":{\"type\":\"long\"},\"processed_record_count\":{\"type\":\"long\"}}},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"forecasts_stats\":{\"properties\":{\"total\":{\"type\":\"long\"}}},\"model_size\":{\"properties\":{\"memory_status\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}}}}}},\"cluster_uuid\":{\"path\":\"elasticsearch.cluster.id\",\"type\":\"alias\"},\"ecs\":{\"properties\":{\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"service\":{\"properties\":{\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"index_recovery\":{\"properties\":{\"shards\":{\"properties\":{\"stop_time_in_millis\":{\"path\":\"elasticsearch.index.recovery.stop_time.ms\",\"type\":\"alias\"},\"start_time_in_millis\":{\"path\":\"elasticsearch.index.recovery.start_time.ms\",\"type\":\"alias\"}}}}},\"source_node\":{\"properties\":{\"name\":{\"path\":\"elasticsearch.node.name\",\"type\":\"alias\"},\"uuid\":{\"path\":\"elasticsearch.node.id\",\"type\":\"alias\"}}},\"host\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"metricset\":{\"properties\":{\"period\":{\"type\":\"long\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"shard\":{\"properties\":{\"node\":{\"path\":\"elasticsearch.node.id\",\"type\":\"alias\"},\"index\":{\"path\":\"elasticsearch.index.name\",\"type\":\"alias\"},\"state\":{\"path\":\"elasticsearch.shard.state\",\"type\":\"alias\"},\"shard\":{\"path\":\"elasticsearch.shard.number\",\"type\":\"alias\"},\"primary\":{\"path\":\"elasticsearch.shard.primary\",\"type\":\"alias\"}}},\"event\":{\"properties\":{\"duration\":{\"type\":\"long\"},\"module\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"dataset\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"cluster_stats\":{\"properties\":{\"indices\":{\"properties\":{\"shards\":{\"properties\":{\"total\":{\"path\":\"elasticsearch.cluster.stats.indices.shards.count\",\"type\":\"alias\"}}},\"count\":{\"path\":\"elasticsearch.cluster.stats.indices.total\",\"type\":\"alias\"}}},\"nodes\":{\"properties\":{\"jvm\":{\"properties\":{\"max_uptime_in_millis\":{\"path\":\"elasticsearch.cluster.stats.nodes.jvm.max_uptime.ms\",\"type\":\"alias\"},\"mem\":{\"properties\":{\"heap_max_in_bytes\":{\"path\":\"elasticsearch.cluster.stats.nodes.jvm.memory.heap.max.bytes\",\"type\":\"alias\"},\"heap_used_in_bytes\":{\"path\":\"elasticsearch.cluster.stats.nodes.jvm.memory.heap.used.bytes\",\"type\":\"alias\"}}}}},\"count\":{\"properties\":{\"total\":{\"path\":\"elasticsearch.cluster.stats.nodes.count\",\"type\":\"alias\"}}}}}}},\"timestamp\":{\"path\":\"@timestamp\",\"type\":\"alias\"},\"job_stats\":{\"properties\":{\"job_id\":{\"path\":\"elasticsearch.ml.job.id\",\"type\":\"alias\"},\"forecasts_stats\":{\"properties\":{\"total\":{\"path\":\"elasticsearch.ml.job.forecasts_stats.total\",\"type\":\"alias\"}}}}}}}},\"composed_of\":[],\"version\":8000103,\"data_stream\":{\"hidden\":false,\"allow_custom_routing\":false}},\".slm-history\":{\"index_patterns\":[\".slm-history-5*\"],\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\"slm-history-ilm-policy\"},\"number_of_shards\":\"1\",\"auto_expand_replicas\":\"0-1\",\"number_of_replicas\":\"0\"}},\"mappings\":{\"dynamic\":false,\"properties\":{\"snapshot_name\":{\"type\":\"keyword\"},\"@timestamp\":{\"format\":\"epoch_millis\",\"type\":\"date\"},\"configuration\":{\"dynamic\":false,\"type\":\"object\",\"properties\":{\"indices\":{\"type\":\"keyword\"},\"include_global_state\":{\"type\":\"boolean\"},\"partial\":{\"type\":\"boolean\"}}},\"error_details\":{\"index\":false,\"type\":\"text\"},\"success\":{\"type\":\"boolean\"},\"repository\":{\"type\":\"keyword\"},\"operation\":{\"type\":\"keyword\"},\"policy\":{\"type\":\"keyword\"}}}},\"composed_of\":[],\"priority\":2147483647,\"version\":5,\"_meta\":{\"managed\":true,\"description\":\"index template for SLM history indices\"},\"data_stream\":{\"hidden\":true,\"allow_custom_routing\":false},\"allow_auto_create\":true},\".monitoring-logstash-mb\":{\"index_patterns\":[\".monitoring-logstash-8-*\"],\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\".monitoring-8-ilm-policy\"},\"mapping\":{\"total_fields\":{\"limit\":\"2000\"}}}},\"mappings\":{\"dynamic\":false,\"properties\":{\"logstash\":{\"properties\":{\"node\":{\"properties\":{\"jvm\":{\"properties\":{\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"stats\":{\"properties\":{\"jvm\":{\"properties\":{\"mem\":{\"properties\":{\"heap_max_in_bytes\":{\"type\":\"long\"},\"heap_used_in_bytes\":{\"type\":\"long\"}}},\"uptime_in_millis\":{\"type\":\"long\"}}},\"logstash\":{\"properties\":{\"uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"process\":{\"properties\":{\"cpu\":{\"properties\":{\"percent\":{\"type\":\"double\"}}}}},\"pipelines\":{\"type\":\"nested\",\"properties\":{\"vertices\":{\"type\":\"nested\",\"properties\":{\"events_out\":{\"type\":\"long\"},\"pipeline_ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"events_in\":{\"type\":\"long\"},\"queue_push_duration_in_millis\":{\"type\":\"long\"},\"duration_in_millis\":{\"type\":\"long\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"long_counters\":{\"type\":\"nested\",\"properties\":{\"name\":{\"type\":\"keyword\"},\"value\":{\"type\":\"long\"}}}}},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"type\":\"keyword\"},\"hash\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"queue\":{\"properties\":{\"events_count\":{\"type\":\"long\"},\"max_queue_size_in_bytes\":{\"type\":\"long\"},\"queue_size_in_bytes\":{\"type\":\"long\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"events\":{\"properties\":{\"filtered\":{\"type\":\"long\"},\"in\":{\"type\":\"long\"},\"queue_push_duration_in_millis\":{\"type\":\"long\"},\"duration_in_millis\":{\"type\":\"long\"},\"out\":{\"type\":\"long\"}}},\"reloads\":{\"properties\":{\"failures\":{\"type\":\"long\"},\"successes\":{\"type\":\"long\"}}}}},\"os\":{\"properties\":{\"cpu\":{\"properties\":{\"load_average\":{\"properties\":{\"5m\":{\"type\":\"half_float\"},\"15m\":{\"type\":\"half_float\"},\"1m\":{\"type\":\"half_float\"}}}}},\"cgroup\":{\"properties\":{\"cpu\":{\"properties\":{\"stat\":{\"properties\":{\"number_of_elapsed_periods\":{\"type\":\"long\"},\"number_of_times_throttled\":{\"type\":\"long\"},\"time_throttled_nanos\":{\"type\":\"long\"}}},\"cfs_quota_micros\":{\"type\":\"long\"}}},\"cpuacct\":{\"properties\":{\"usage_nanos\":{\"type\":\"long\"}}}}}}},\"events\":{\"properties\":{\"filtered\":{\"type\":\"long\"},\"in\":{\"type\":\"long\"},\"duration_in_millis\":{\"type\":\"long\"},\"out\":{\"type\":\"long\"}}},\"queue\":{\"properties\":{\"events_count\":{\"type\":\"long\"}}},\"timestamp\":{\"type\":\"date\"}}},\"host\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"properties\":{\"pipeline\":{\"properties\":{\"batch_size\":{\"type\":\"long\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"workers\":{\"type\":\"long\"},\"representation\":{\"properties\":{\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"hash\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"graph\":{\"properties\":{\"vertices\":{\"type\":\"object\"},\"edges\":{\"type\":\"object\"}}}}},\"hash\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"elasticsearch\":{\"properties\":{\"cluster\":{\"properties\":{\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}}}},\"logstash_state\":{\"properties\":{\"pipeline\":{\"properties\":{\"id\":{\"path\":\"logstash.node.state.pipeline.id\",\"type\":\"alias\"},\"hash\":{\"path\":\"logstash.node.state.pipeline.hash\",\"type\":\"alias\"}}}}},\"process\":{\"properties\":{\"pid\":{\"type\":\"long\"}}},\"@timestamp\":{\"type\":\"date\"},\"logstash_stats\":{\"properties\":{\"jvm\":{\"properties\":{\"mem\":{\"properties\":{\"heap_max_in_bytes\":{\"path\":\"logstash.node.stats.jvm.mem.heap_max_in_bytes\",\"type\":\"alias\"},\"heap_used_in_bytes\":{\"path\":\"logstash.node.stats.jvm.mem.heap_used_in_bytes\",\"type\":\"alias\"}}},\"uptime_in_millis\":{\"path\":\"logstash.node.stats.jvm.uptime_in_millis\",\"type\":\"alias\"}}},\"logstash\":{\"properties\":{\"uuid\":{\"path\":\"logstash.node.stats.logstash.uuid\",\"type\":\"alias\"},\"version\":{\"path\":\"logstash.node.stats.logstash.version\",\"type\":\"alias\"}}},\"process\":{\"properties\":{\"cpu\":{\"properties\":{\"percent\":{\"path\":\"logstash.node.stats.process.cpu.percent\",\"type\":\"alias\"}}}}},\"pipelines\":{\"type\":\"nested\"},\"os\":{\"properties\":{\"cpu\":{\"properties\":{\"load_average\":{\"properties\":{\"5m\":{\"path\":\"logstash.node.stats.os.cpu.load_average.5m\",\"type\":\"alias\"},\"15m\":{\"path\":\"logstash.node.stats.os.cpu.load_average.15m\",\"type\":\"alias\"},\"1m\":{\"path\":\"logstash.node.stats.os.cpu.load_average.1m\",\"type\":\"alias\"}}}}},\"cgroup\":{\"properties\":{\"cpu\":{\"properties\":{\"stat\":{\"properties\":{\"number_of_elapsed_periods\":{\"path\":\"logstash.node.stats.os.cgroup.cpu.stat.number_of_elapsed_periods\",\"type\":\"alias\"},\"number_of_times_throttled\":{\"path\":\"logstash.node.stats.os.cgroup.cpu.stat.number_of_times_throttled\",\"type\":\"alias\"},\"time_throttled_nanos\":{\"path\":\"logstash.node.stats.os.cgroup.cpu.stat.time_throttled_nanos\",\"type\":\"alias\"}}},\"cfs_quota_micros\":{\"path\":\"logstash.node.stats.os.cgroup.cpu.cfs_quota_micros\",\"type\":\"alias\"}}},\"cpuacct\":{\"properties\":{\"usage_nanos\":{\"path\":\"logstash.node.stats.os.cgroup.cpuacct.usage_nanos\",\"type\":\"alias\"}}}}}}},\"events\":{\"properties\":{\"in\":{\"path\":\"logstash.node.stats.events.in\",\"type\":\"alias\"},\"duration_in_millis\":{\"path\":\"logstash.node.stats.events.duration_in_millis\",\"type\":\"alias\"},\"out\":{\"path\":\"logstash.node.stats.events.out\",\"type\":\"alias\"}}},\"queue\":{\"properties\":{\"events_count\":{\"path\":\"logstash.node.stats.queue.events_count\",\"type\":\"alias\"}}},\"timestamp\":{\"path\":\"@timestamp\",\"type\":\"alias\"}}},\"ecs\":{\"properties\":{\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"cluster_uuid\":{\"path\":\"logstash.elasticsearch.cluster.id\",\"type\":\"alias\"},\"service\":{\"properties\":{\"hostname\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"host\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"metricset\":{\"properties\":{\"period\":{\"type\":\"long\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"event\":{\"properties\":{\"duration\":{\"type\":\"long\"},\"module\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"dataset\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"timestamp\":{\"path\":\"@timestamp\",\"type\":\"alias\"}}}},\"composed_of\":[],\"version\":8000103,\"data_stream\":{\"hidden\":false,\"allow_custom_routing\":false}},\".ml-anomalies-\":{\"index_patterns\":[\".ml-anomalies-*\"],\"template\":{\"settings\":{\"index\":{\"hidden\":\"true\",\"translog\":{\"durability\":\"async\"},\"auto_expand_replicas\":\"0-1\",\"query\":{\"default_field\":\"all_field_values\"}}},\"mappings\":{\"_meta\":{\"version\":\"8.6.2\"},\"dynamic_templates\":[{\"strings_as_keywords\":{\"mapping\":{\"type\":\"keyword\"},\"match\":\"*\"}}],\"properties\":{\"search_count\":{\"type\":\"long\"},\"bucket_count\":{\"type\":\"long\"},\"anomaly_score_explanation\":{\"properties\":{\"high_variance_penalty\":{\"type\":\"boolean\"},\"typical_value\":{\"type\":\"double\"},\"upper_confidence_bound\":{\"type\":\"double\"},\"lower_confidence_bound\":{\"type\":\"double\"},\"incomplete_bucket_penalty\":{\"type\":\"boolean\"},\"anomaly_type\":{\"type\":\"keyword\"},\"anomaly_characteristics_impact\":{\"type\":\"integer\"},\"multi_bucket_impact\":{\"type\":\"integer\"},\"anomaly_length\":{\"type\":\"integer\"},\"single_bucket_impact\":{\"type\":\"integer\"}}},\"terms\":{\"type\":\"text\"},\"record_score\":{\"type\":\"double\"},\"forecast_expiry_timestamp\":{\"type\":\"date\"},\"over_field_value\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"preferred_to_categories\":{\"type\":\"long\"},\"empty_bucket_count\":{\"type\":\"long\"},\"forecast_create_timestamp\":{\"type\":\"date\"},\"probability\":{\"type\":\"double\"},\"missing_field_count\":{\"type\":\"long\"},\"forecast_lower\":{\"type\":\"double\"},\"influencer_field_value\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"multi_bucket_impact\":{\"type\":\"double\"},\"forecast_progress\":{\"type\":\"double\"},\"max_matching_length\":{\"type\":\"long\"},\"latest_record_time_stamp\":{\"type\":\"date\"},\"regex\":{\"type\":\"keyword\"},\"examples\":{\"type\":\"text\"},\"snapshot_doc_count\":{\"type\":\"integer\"},\"average_bucket_processing_time_ms\":{\"type\":\"double\"},\"initial_anomaly_score\":{\"type\":\"double\"},\"initial_influencer_score\":{\"type\":\"double\"},\"partition_field_value\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"model_lower\":{\"type\":\"double\"},\"total_over_field_count\":{\"type\":\"long\"},\"retain\":{\"type\":\"boolean\"},\"forecast_upper\":{\"type\":\"double\"},\"latest_sparse_bucket_timestamp\":{\"type\":\"date\"},\"model_median\":{\"type\":\"double\"},\"category_id\":{\"type\":\"long\"},\"causes\":{\"type\":\"nested\",\"properties\":{\"actual\":{\"type\":\"double\"},\"partition_field_name\":{\"type\":\"keyword\"},\"partition_field_value\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"by_field_name\":{\"type\":\"keyword\"},\"probability\":{\"type\":\"double\"},\"geo_results\":{\"properties\":{\"actual_point\":{\"type\":\"geo_point\"},\"typical_point\":{\"type\":\"geo_point\"}}},\"field_name\":{\"type\":\"keyword\"},\"by_field_value\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"over_field_name\":{\"type\":\"keyword\"},\"correlated_by_field_value\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"function\":{\"type\":\"keyword\"},\"typical\":{\"type\":\"double\"},\"over_field_value\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"function_description\":{\"type\":\"keyword\"}}},\"all_field_values\":{\"analyzer\":\"whitespace\",\"type\":\"text\"},\"timestamp\":{\"type\":\"date\"},\"input_field_count\":{\"type\":\"long\"},\"model_bytes\":{\"type\":\"long\"},\"quantiles\":{\"type\":\"object\",\"enabled\":false},\"@timestamp\":{\"path\":\"timestamp\",\"type\":\"alias\"},\"function_description\":{\"type\":\"keyword\"},\"min_version\":{\"type\":\"keyword\"},\"raw_anomaly_score\":{\"type\":\"double\"},\"exponential_average_bucket_processing_time_ms\":{\"type\":\"double\"},\"invalid_date_count\":{\"type\":\"long\"},\"snapshot_id\":{\"type\":\"keyword\"},\"model_size_stats\":{\"properties\":{\"model_bytes\":{\"type\":\"long\"},\"result_type\":{\"type\":\"keyword\"},\"peak_model_bytes\":{\"type\":\"long\"},\"job_id\":{\"type\":\"keyword\"},\"total_over_field_count\":{\"type\":\"long\"},\"total_partition_field_count\":{\"type\":\"long\"},\"total_by_field_count\":{\"type\":\"long\"},\"assignment_memory_basis\":{\"type\":\"keyword\"},\"bucket_allocation_failures_count\":{\"type\":\"long\"},\"memory_status\":{\"type\":\"keyword\"},\"log_time\":{\"type\":\"date\"},\"timestamp\":{\"type\":\"date\"}}},\"total_search_time_ms\":{\"type\":\"double\"},\"latest_empty_bucket_timestamp\":{\"type\":\"date\"},\"anomaly_score\":{\"type\":\"double\"},\"over_field_name\":{\"type\":\"keyword\"},\"earliest_record_timestamp\":{\"type\":\"date\"},\"scheduled_events\":{\"type\":\"keyword\"},\"bucket_span\":{\"type\":\"long\"},\"maximum_bucket_processing_time_ms\":{\"type\":\"double\"},\"exponential_average_calculation_context\":{\"properties\":{\"incremental_metric_value_ms\":{\"type\":\"double\"},\"previous_exponential_average_ms\":{\"type\":\"double\"},\"latest_timestamp\":{\"type\":\"date\"}}},\"function\":{\"type\":\"keyword\"},\"influencers\":{\"type\":\"nested\",\"properties\":{\"influencer_field_name\":{\"type\":\"keyword\"},\"influencer_field_values\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"}}},\"input_record_count\":{\"type\":\"long\"},\"latest_result_time_stamp\":{\"type\":\"date\"},\"model_upper\":{\"type\":\"double\"},\"actual\":{\"type\":\"double\"},\"forecast_memory_bytes\":{\"type\":\"long\"},\"total_by_field_count\":{\"type\":\"long\"},\"geo_results\":{\"properties\":{\"actual_point\":{\"type\":\"geo_point\"},\"typical_point\":{\"type\":\"geo_point\"}}},\"detector_index\":{\"type\":\"integer\"},\"by_field_value\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"processed_record_count\":{\"type\":\"long\"},\"total_partition_field_count\":{\"type\":\"long\"},\"mlcategory\":{\"type\":\"keyword\"},\"assignment_memory_basis\":{\"type\":\"keyword\"},\"forecast_end_timestamp\":{\"type\":\"date\"},\"forecast_id\":{\"type\":\"keyword\"},\"partition_field_name\":{\"type\":\"keyword\"},\"by_field_name\":{\"type\":\"keyword\"},\"event_count\":{\"type\":\"long\"},\"description\":{\"type\":\"text\"},\"is_interim\":{\"type\":\"boolean\"},\"bucket_allocation_failures_count\":{\"type\":\"long\"},\"memory_status\":{\"type\":\"keyword\"},\"model_feature\":{\"type\":\"keyword\"},\"num_matches\":{\"type\":\"long\"},\"influencer_score\":{\"type\":\"double\"},\"out_of_order_timestamp_count\":{\"type\":\"long\"},\"result_type\":{\"type\":\"keyword\"},\"last_data_time\":{\"type\":\"date\"},\"latest_record_timestamp\":{\"type\":\"date\"},\"influencer_field_name\":{\"type\":\"keyword\"},\"forecast_messages\":{\"type\":\"keyword\"},\"sparse_bucket_count\":{\"type\":\"long\"},\"log_time\":{\"type\":\"date\"},\"field_name\":{\"type\":\"keyword\"},\"minimum_bucket_processing_time_ms\":{\"type\":\"double\"},\"bucket_influencers\":{\"type\":\"nested\",\"properties\":{\"anomaly_score\":{\"type\":\"double\"},\"initial_anomaly_score\":{\"type\":\"double\"},\"result_type\":{\"type\":\"keyword\"},\"raw_anomaly_score\":{\"type\":\"double\"},\"bucket_span\":{\"type\":\"long\"},\"job_id\":{\"type\":\"keyword\"},\"probability\":{\"type\":\"double\"},\"influencer_field_name\":{\"type\":\"keyword\"},\"is_interim\":{\"type\":\"boolean\"},\"timestamp\":{\"type\":\"date\"}}},\"processing_time_ms\":{\"type\":\"long\"},\"input_bytes\":{\"type\":\"long\"},\"initial_record_score\":{\"type\":\"double\"},\"job_id\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"processed_field_count\":{\"type\":\"long\"},\"forecast_status\":{\"type\":\"keyword\"},\"typical\":{\"type\":\"double\"},\"forecast_prediction\":{\"type\":\"double\"},\"forecast_start_timestamp\":{\"type\":\"date\"}}}},\"composed_of\":[],\"priority\":2147483647,\"version\":8060299,\"_meta\":{\"managed\":true,\"description\":\"index template for ML anomaly detection results indices\"}},\"metrics\":{\"index_patterns\":[\"metrics-*-*\"],\"composed_of\":[\"metrics-mappings\",\"data-streams-mappings\",\"metrics-settings\"],\"priority\":100,\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default metrics template installed by x-pack\"},\"data_stream\":{\"hidden\":false,\"allow_custom_routing\":false},\"allow_auto_create\":true},\".ml-notifications-000002\":{\"index_patterns\":[\".ml-notifications-000002\"],\"template\":{\"settings\":{\"index\":{\"hidden\":\"true\",\"number_of_shards\":\"1\",\"auto_expand_replicas\":\"0-1\"}},\"mappings\":{\"_meta\":{\"version\":\"8.6.2\"},\"dynamic\":\"false\",\"properties\":{\"job_type\":{\"type\":\"keyword\"},\"level\":{\"type\":\"keyword\"},\"job_id\":{\"type\":\"keyword\"},\"node_name\":{\"type\":\"keyword\"},\"message\":{\"type\":\"text\",\"fields\":{\"raw\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"cleared\":{\"type\":\"boolean\"},\"timestamp\":{\"type\":\"date\"}}}},\"composed_of\":[],\"priority\":2147483647,\"version\":8060299,\"_meta\":{\"managed\":true,\"description\":\"index template for ML notifications indices\"}},\".deprecation-indexing-template\":{\"index_patterns\":[\".logs-deprecation.*\"],\"composed_of\":[\".deprecation-indexing-mappings\",\".deprecation-indexing-settings\"],\"priority\":1000,\"version\":1,\"_meta\":{\"managed\":true,\"description\":\"default template for Stack deprecation logs index template installed by x-pack\"},\"data_stream\":{\"hidden\":true,\"allow_custom_routing\":false},\"allow_auto_create\":true},\"logs\":{\"index_patterns\":[\"logs-*-*\"],\"composed_of\":[\"logs-mappings\",\"data-streams-mappings\",\"logs-settings\"],\"priority\":100,\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default logs template installed by x-pack\"},\"data_stream\":{\"hidden\":false,\"allow_custom_routing\":false},\"allow_auto_create\":true},\".ml-stats\":{\"index_patterns\":[\".ml-stats-*\"],\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\"ml-size-based-ilm-policy\",\"rollover_alias\":\".ml-stats-write\"},\"hidden\":\"true\",\"number_of_shards\":\"1\",\"auto_expand_replicas\":\"0-1\"}},\"mappings\":{\"_meta\":{\"version\":\"8.6.2\"},\"dynamic\":false,\"properties\":{\"skipped_docs_count\":{\"type\":\"long\"},\"validation_loss\":{\"properties\":{\"fold_values\":{\"properties\":{\"fold\":{\"type\":\"integer\"},\"values\":{\"type\":\"double\"}}},\"loss_type\":{\"type\":\"keyword\"}}},\"cache_miss_count\":{\"type\":\"long\"},\"timing_stats\":{\"properties\":{\"iteration_time\":{\"type\":\"long\"},\"elapsed_time\":{\"type\":\"long\"}}},\"failure_count\":{\"type\":\"long\"},\"model_id\":{\"type\":\"keyword\"},\"type\":{\"type\":\"keyword\"},\"training_docs_count\":{\"type\":\"long\"},\"inference_count\":{\"type\":\"long\"},\"job_id\":{\"type\":\"keyword\"},\"missing_all_fields_count\":{\"type\":\"long\"},\"peak_usage_bytes\":{\"type\":\"long\"},\"iteration\":{\"type\":\"integer\"},\"hyperparameters\":{\"properties\":{\"max_attempts_to_add_tree\":{\"type\":\"integer\"},\"downsample_factor\":{\"type\":\"double\"},\"eta_growth_rate_per_tree\":{\"type\":\"double\"},\"soft_tree_depth_tolerance\":{\"type\":\"double\"},\"max_trees\":{\"type\":\"integer\"},\"lambda\":{\"type\":\"double\"},\"max_optimization_rounds_per_hyperparameter\":{\"type\":\"integer\"},\"eta\":{\"type\":\"double\"},\"soft_tree_depth_limit\":{\"type\":\"double\"},\"alpha\":{\"type\":\"double\"},\"class_assignment_objective\":{\"type\":\"keyword\"},\"feature_bag_fraction\":{\"type\":\"double\"},\"num_splits_per_feature\":{\"type\":\"integer\"},\"gamma\":{\"type\":\"double\"},\"num_folds\":{\"type\":\"integer\"}}},\"parameters\":{\"properties\":{\"compute_feature_influence\":{\"type\":\"boolean\"},\"feature_influence_threshold\":{\"type\":\"double\"},\"outlier_fraction\":{\"type\":\"double\"},\"method\":{\"type\":\"keyword\"},\"standardization_enabled\":{\"type\":\"boolean\"},\"n_neighbors\":{\"type\":\"integer\"}}},\"test_docs_count\":{\"type\":\"long\"},\"node_id\":{\"type\":\"keyword\"},\"timestamp\":{\"type\":\"date\"}}},\"aliases\":{}},\"composed_of\":[],\"priority\":2147483647,\"version\":8060299,\"_meta\":{\"managed\":true,\"description\":\"index template for ML stats indices\"}}}} index-graveyard: IndexGraveyard[[[index=[abacus_go/zV2aNzxVQxqMDQTD4A9TEQ], deleteDate=2023-03-28T03:56:57.282Z], [index=[access_platform_core/ZuN0_cUfSQKzXg_yG4Sv5g], deleteDate=2023-03-28T03:58:27.958Z], [index=[ad_bidder/6ihfVU6rRqOo4Gv4Djtetg], deleteDate=2023-03-28T03:59:11.628Z], [index=[apphealth/EthxrkUtTGCTP2mX-tS63Q], deleteDate=2023-03-28T03:59:36.596Z]]]\nnodes: \n {hostb9}{iKPGCkp9RVOKXOj20uOt4g}{cPRh9FqiSlGP9jClhm8Ugg}{hostb9}{127.0.0.1}{127.0.0.1:9300}{cdfhilmrstw}{ml.max_jvm_size=33285996544, ml.allocated_processors_double=16.0, xpack.installed=true, ml.machine_memory=135062048768, ml.allocated_processors=16}, local, master\nrouting_table (version 35):\n-- index [[abacus_go/cfvmY-Y_Su63-qSkPODDzg]]\n----shard_id [abacus_go][0]\n--------[abacus_go][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=AMhgJMlSTPOoKSsL7s3OKA], failed_attempts[0]\n--------[abacus_go][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=INDEX_CREATED], at[2023-03-28T03:57:31.683Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[web_wok/R1IULAXSQ4SC0pLFCU5oJA]]\n----shard_id [web_wok][0]\n--------[web_wok][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=JXruVMhdRVGvP1kQCbUNSQ], failed_attempts[0]\n--------[web_wok][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[access_platform_core/l4kdkOEQRaKfWGLpTZURCQ]]\n----shard_id [access_platform_core][0]\n--------[access_platform_core][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=6ajNSTQZSr6tP3YtmRaxgw], failed_attempts[0]\n--------[access_platform_core][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=INDEX_CREATED], at[2023-03-28T03:58:31.318Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[driverapp/cqbade6KQFqLEK-wPKlkBg]]\n----shard_id [driverapp][0]\n--------[driverapp][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=b5_Bhd3ITjOkl-xLF8H1jg], failed_attempts[0]\n--------[driverapp][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[k8s_kubelet/fQ9Iy7wlTKahsWLbs-oE_Q]]\n----shard_id [k8s_kubelet][0]\n--------[k8s_kubelet][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=Q4Xb11JATPaFZqmgTQHcVQ], failed_attempts[0]\n--------[k8s_kubelet][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.307Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[eats_store/hF-kN4XFRy--z_gUhYLwdA]]\n----shard_id [eats_store][0]\n--------[eats_store][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=WRNwRQ6gTNGfSNQok4iSyQ], failed_attempts[0]\n--------[eats_store][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[rider_product_cored/AdGiEBLBQbicsXDomsa3mw]]\n----shard_id [rider_product_cored][0]\n--------[rider_product_cored][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=GwRH_ZlFRtCV7m_n626z7g], failed_attempts[0]\n--------[rider_product_cored][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[finprod_rewards_eligibility/dJNRGPwrQS2bIPlpsXgUoQ]]\n----shard_id [finprod_rewards_eligibility][0]\n--------[finprod_rewards_eligibility][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=Hg4Dm9O8QEqXWKY7rH1X0A], failed_attempts[0]\n--------[finprod_rewards_eligibility][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[event_logs/DcomuHoGQzOjZq2bX67TFg]]\n----shard_id [event_logs][0]\n--------[event_logs][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=1uCl_LIDSBCWh6koVA0fjQ], failed_attempts[0]\n--------[event_logs][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[apphealth/rcPrX1aWRmeED9W5PU6A4g]]\n----shard_id [apphealth][0]\n--------[apphealth][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=Abip50RxQTmavqi4NTwSqQ], failed_attempts[0]\n--------[apphealth][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=INDEX_CREATED], at[2023-03-28T03:59:38.780Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[ad_bidder/tLHfw5r2QH-o7lG5tapKxw]]\n----shard_id [ad_bidder][0]\n--------[ad_bidder][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=YtEUwPgfQDGM0Cm_C4hXlw], failed_attempts[0]\n--------[ad_bidder][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=INDEX_CREATED], at[2023-03-28T03:59:13.333Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[invoice_gen/RPu5EzJVRfuqyI8MQ7iTjQ]]\n----shard_id [invoice_gen][0]\n--------[invoice_gen][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=onWbjhHuTD2C3AEayRJq1w], failed_attempts[0]\n--------[invoice_gen][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[fares_management/F1lH-QvyTkS5hnbDFwE2Xg]]\n----shard_id [fares_management][0]\n--------[fares_management][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=sOxfKX3nReynYN5HN88cuQ], failed_attempts[0]\n--------[fares_management][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[michelangelo_gateway/ZV2vZ8PmS0uysg-xzKO7QA]]\n----shard_id [michelangelo_gateway][0]\n--------[michelangelo_gateway][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=k5vDE54CSTSakhfgtGaTcA], failed_attempts[0]\n--------[michelangelo_gateway][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[.geoip_databases/5k1GHwKvQBKersi12DFh4A]]\n----shard_id [.geoip_databases][0]\n--------[.geoip_databases][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=UdvCKUGqTEmYFzZlnSu_tA], failed_attempts[0]\n\n-- index [[fares_intelligence/pSRJWxyXSqefhwLVhmT9nQ]]\n----shard_id [fares_intelligence][0]\n--------[fares_intelligence][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=-dD8ROsgS6qaYMoalciocA], failed_attempts[0]\n--------[fares_intelligence][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[fare_route/a4fjivWhQRee8JW7ZbXiXw]]\n----shard_id [fare_route][0]\n--------[fare_route][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=ltSLFgu4Rry0_0vXq28DbA], failed_attempts[0]\n--------[fare_route][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[k8s_apiserver/qQasynZgQ6e6AmD5ZWqh3w]]\n----shard_id [k8s_apiserver][0]\n--------[k8s_apiserver][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=Aaa0F0jiStCkteGDYR8yRQ], failed_attempts[0]\n--------[k8s_apiserver][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[fulfillment_compatibled/tMgrGBWeR3irgR5PyN8Oig]]\n----shard_id [fulfillment_compatibled][0]\n--------[fulfillment_compatibled][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=D_3C_3kMSNKa-XJk7pyoGg], failed_attempts[0]\n--------[fulfillment_compatibled][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\nrouting_nodes:\n-----node_id[iKPGCkp9RVOKXOj20uOt4g][V]\n--------[abacus_go][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=AMhgJMlSTPOoKSsL7s3OKA], failed_attempts[0]\n--------[web_wok][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=JXruVMhdRVGvP1kQCbUNSQ], failed_attempts[0]\n--------[access_platform_core][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=6ajNSTQZSr6tP3YtmRaxgw], failed_attempts[0]\n--------[driverapp][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=b5_Bhd3ITjOkl-xLF8H1jg], failed_attempts[0]\n--------[k8s_kubelet][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=Q4Xb11JATPaFZqmgTQHcVQ], failed_attempts[0]\n--------[eats_store][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=WRNwRQ6gTNGfSNQok4iSyQ], failed_attempts[0]\n--------[rider_product_cored][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=GwRH_ZlFRtCV7m_n626z7g], failed_attempts[0]\n--------[finprod_rewards_eligibility][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=Hg4Dm9O8QEqXWKY7rH1X0A], failed_attempts[0]\n--------[event_logs][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=1uCl_LIDSBCWh6koVA0fjQ], failed_attempts[0]\n--------[apphealth][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=Abip50RxQTmavqi4NTwSqQ], failed_attempts[0]\n--------[ad_bidder][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=YtEUwPgfQDGM0Cm_C4hXlw], failed_attempts[0]\n--------[invoice_gen][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=onWbjhHuTD2C3AEayRJq1w], failed_attempts[0]\n--------[fares_management][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=sOxfKX3nReynYN5HN88cuQ], failed_attempts[0]\n--------[michelangelo_gateway][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=k5vDE54CSTSakhfgtGaTcA], failed_attempts[0]\n--------[.geoip_databases][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=UdvCKUGqTEmYFzZlnSu_tA], failed_attempts[0]\n--------[fares_intelligence][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=-dD8ROsgS6qaYMoalciocA], failed_attempts[0]\n--------[fare_route][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=ltSLFgu4Rry0_0vXq28DbA], failed_attempts[0]\n--------[k8s_apiserver][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=Aaa0F0jiStCkteGDYR8yRQ], failed_attempts[0]\n--------[fulfillment_compatibled][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=D_3C_3kMSNKa-XJk7pyoGg], failed_attempts[0]\n---- unassigned\n--------[abacus_go][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=INDEX_CREATED], at[2023-03-28T03:57:31.683Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[web_wok][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[access_platform_core][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=INDEX_CREATED], at[2023-03-28T03:58:31.318Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[driverapp][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[k8s_kubelet][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.307Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[eats_store][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[rider_product_cored][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[finprod_rewards_eligibility][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[event_logs][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[apphealth][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=INDEX_CREATED], at[2023-03-28T03:59:38.780Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[ad_bidder][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=INDEX_CREATED], at[2023-03-28T03:59:13.333Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[invoice_gen][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[fares_management][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[michelangelo_gateway][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[fares_intelligence][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[fare_route][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[k8s_apiserver][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[fulfillment_compatibled][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\ncustoms:\n health: org.elasticsearch.health.metadata.HealthMetadata@66bc0723} to [PublicationTarget{discoveryNode={hostb9}{iKPGCkp9RVOKXOj20uOt4g}{cPRh9FqiSlGP9jClhm8Ugg}{hostb9}{127.0.0.1}{127.0.0.1:9300}{cdfhilmrstw}{ml.max_jvm_size=33285996544, ml.allocated_processors_double=16.0, xpack.installed=true, ml.machine_memory=135062048768, ml.allocated_processors=16}, state=NOT_STARTED, ackIsPending=true}]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.coordination.Coordinator.CoordinatorPublication","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:02.067Z", "log.level":"TRACE", "message":"handling cluster state version [154] locally on [{hostb9}{iKPGCkp9RVOKXOj20uOt4g}{cPRh9FqiSlGP9jClhm8Ugg}{hostb9}{127.0.0.1}{127.0.0.1:9300}{cdfhilmrstw}{ml.max_jvm_size=33285996544, ml.allocated_processors_double=16.0, xpack.installed=true, ml.machine_memory=135062048768, ml.allocated_processors=16}]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.coordination.PublicationTransportHandler","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:02.074Z", "log.level":"TRACE", "message":"handlePublishRequest: handling [PublishRequest{term=6, version=154, state=cluster uuid: bp-xkJD1S1iyBlj6I_JRHA [committed: true]\nversion: 154\nstate uuid: Xx549IAcSMCY30xGiugtFg\nfrom_diff: false\nmeta data version: 56\n coordination_metadata:\n term: 6\n last_committed_config: VotingConfiguration{iKPGCkp9RVOKXOj20uOt4g}\n last_accepted_config: VotingConfiguration{iKPGCkp9RVOKXOj20uOt4g}\n voting tombstones: []\n [k8s_kubelet/fQ9Iy7wlTKahsWLbs-oE_Q]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [Q4Xb11JATPaFZqmgTQHcVQ]\n [fares_intelligence/pSRJWxyXSqefhwLVhmT9nQ]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [-dD8ROsgS6qaYMoalciocA]\n [event_logs/DcomuHoGQzOjZq2bX67TFg]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [1uCl_LIDSBCWh6koVA0fjQ]\n [rider_product_cored/AdGiEBLBQbicsXDomsa3mw]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [GwRH_ZlFRtCV7m_n626z7g]\n [access_platform_core/l4kdkOEQRaKfWGLpTZURCQ]: v[6], mv[1], sv[2], av[1]\n 0: p_term [1], isa_ids [6ajNSTQZSr6tP3YtmRaxgw]\n [.geoip_databases/5k1GHwKvQBKersi12DFh4A]: v[6], mv[1], sv[1], av[1]\n 0: p_term [3], isa_ids [UdvCKUGqTEmYFzZlnSu_tA]\n [web_wok/R1IULAXSQ4SC0pLFCU5oJA]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [JXruVMhdRVGvP1kQCbUNSQ]\n [abacus_go/cfvmY-Y_Su63-qSkPODDzg]: v[6], mv[1], sv[2], av[1]\n 0: p_term [1], isa_ids [AMhgJMlSTPOoKSsL7s3OKA]\n [k8s_apiserver/qQasynZgQ6e6AmD5ZWqh3w]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [Aaa0F0jiStCkteGDYR8yRQ]\n [michelangelo_gateway/ZV2vZ8PmS0uysg-xzKO7QA]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [k5vDE54CSTSakhfgtGaTcA]\n [driverapp/cqbade6KQFqLEK-wPKlkBg]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [b5_Bhd3ITjOkl-xLF8H1jg]\n [apphealth/rcPrX1aWRmeED9W5PU6A4g]: v[4], mv[1], sv[1], av[1]\n 0: p_term [1], isa_ids [Abip50RxQTmavqi4NTwSqQ]\n [fares_management/F1lH-QvyTkS5hnbDFwE2Xg]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [sOxfKX3nReynYN5HN88cuQ]\n [finprod_rewards_eligibility/dJNRGPwrQS2bIPlpsXgUoQ]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [Hg4Dm9O8QEqXWKY7rH1X0A]\n [fare_route/a4fjivWhQRee8JW7ZbXiXw]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [ltSLFgu4Rry0_0vXq28DbA]\n [ad_bidder/tLHfw5r2QH-o7lG5tapKxw]: v[6], mv[1], sv[2], av[1]\n 0: p_term [1], isa_ids [YtEUwPgfQDGM0Cm_C4hXlw]\n [fulfillment_compatibled/tMgrGBWeR3irgR5PyN8Oig]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [D_3C_3kMSNKa-XJk7pyoGg]\n [eats_store/hF-kN4XFRy--z_gUhYLwdA]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [WRNwRQ6gTNGfSNQok4iSyQ]\n [invoice_gen/RPu5EzJVRfuqyI8MQ7iTjQ]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [onWbjhHuTD2C3AEayRJq1w]\nmetadata customs:\n licenses: LicensesMetadata{license={\"uid\":\"8e65ab16-3f3f-48cc-ab5f-e20097ceb81e\",\"type\":\"basic\",\"issue_date_in_millis\":1679968689560,\"max_nodes\":1000,\"max_resource_units\":null,\"issued_to\":\"elasticsearch\",\"issuer\":\"elasticsearch\",\"signature\":\"////+wAAAOBYLHp18OvjZrRTi0AmCZ6DOpkrd2cXJB/ffBB+AxaDsF+z7lkfdPbgYofMvUB7XbiD5j95mIfUcdlTJHVdk9NpPTFMKL5wC7BXn4CTS6kNox+Xi0+qSLpcgHzelOT63alJr1L5j/2pqffHnuMLG88Ye15eFhAb9H2A5MS3hX0HG9uTxyEidFCyu4gWV4qrZJDQKLpzC49MMnpP+ob2bNH/pA7dGVdWqEJpVNPe5/hAtGtCdVdUX6DvV0Q3dgwuejc0hRg/vl0YUtp45bQPBHDHECNKgxWmJCZrfstl/prRHA==\",\"start_date_in_millis\":-1}, trialVersion=null} component_template: {\"component_template\":{\"metrics-mappings\":{\"template\":{\"mappings\":{\"dynamic_templates\":[{\"long_metrics\":{\"mapping\":{\"index\":false,\"type\":\"long\"},\"match_mapping_type\":\"long\"}},{\"double_metrics\":{\"mapping\":{\"index\":false,\"type\":\"float\"},\"match_mapping_type\":\"double\"}}],\"properties\":{\"data_stream\":{\"properties\":{\"type\":{\"type\":\"constant_keyword\",\"value\":\"metrics\"}}}}}},\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default mappings for the metrics index template installed by x-pack\"}},\".deprecation-indexing-mappings\":{\"template\":{\"mappings\":{\"dynamic_templates\":[{\"strings_as_keyword\":{\"mapping\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"match_mapping_type\":\"string\"}}],\"date_detection\":false,\"properties\":{\"@timestamp\":{\"type\":\"date\"},\"ecs\":{\"properties\":{\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"data_stream\":{\"properties\":{\"namespace\":{\"type\":\"constant_keyword\"},\"type\":{\"type\":\"constant_keyword\",\"value\":\"logs\"},\"dataset\":{\"type\":\"constant_keyword\"}}},\"host\":{\"properties\":{\"ip\":{\"type\":\"ip\"}}},\"message\":{\"type\":\"text\"}}}},\"version\":1,\"_meta\":{\"managed\":true,\"description\":\"default mappings for Stack deprecation logs index template installed by x-pack\"}},\"synthetics-mappings\":{\"template\":{\"mappings\":{\"properties\":{\"data_stream\":{\"properties\":{\"type\":{\"type\":\"constant_keyword\",\"value\":\"synthetics\"}}}}}},\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default mappings for the synthetics index template installed by x-pack\"}},\"metrics-settings\":{\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\"metrics\"},\"codec\":\"best_compression\",\"query\":{\"default_field\":[\"message\"]}}}},\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default settings for the metrics index template installed by x-pack\"}},\"logs-settings\":{\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\"logs\"},\"codec\":\"best_compression\",\"query\":{\"default_field\":[\"message\"]}}}},\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default settings for the logs index template installed by x-pack\"}},\".deprecation-indexing-settings\":{\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\".deprecation-indexing-ilm-policy\"},\"codec\":\"best_compression\",\"hidden\":\"true\",\"auto_expand_replicas\":\"0-1\",\"query\":{\"default_field\":[\"message\"]}}}},\"version\":1,\"_meta\":{\"managed\":true,\"description\":\"default settings for Stack deprecation logs index template installed by x-pack\"}},\"synthetics-settings\":{\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\"synthetics\"},\"codec\":\"best_compression\"}}},\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default settings for the synthetics index template installed by x-pack\"}},\"logs-mappings\":{\"template\":{\"mappings\":{\"properties\":{\"data_stream\":{\"properties\":{\"type\":{\"type\":\"constant_keyword\",\"value\":\"logs\"}}}}}},\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default mappings for the logs index template installed by x-pack\"}},\"data-streams-mappings\":{\"template\":{\"mappings\":{\"dynamic_templates\":[{\"match_ip\":{\"mapping\":{\"type\":\"ip\"},\"match_mapping_type\":\"string\",\"match\":\"ip\"}},{\"match_message\":{\"mapping\":{\"type\":\"match_only_text\"},\"match_mapping_type\":\"string\",\"match\":\"message\"}},{\"strings_as_keyword\":{\"mapping\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"match_mapping_type\":\"string\"}}],\"date_detection\":false,\"properties\":{\"@timestamp\":{\"type\":\"date\"},\"ecs\":{\"properties\":{\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"data_stream\":{\"properties\":{\"namespace\":{\"type\":\"constant_keyword\"},\"dataset\":{\"type\":\"constant_keyword\"}}},\"host\":{\"type\":\"object\"}}}},\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"general mapping conventions for data streams\"}}}} persistent_tasks: {\"last_allocation_id\":6,\"tasks\":[{\"id\":\"health-node\",\"task\":{\"health-node\":{\"params\":{}}},\"allocation_id\":5,\"assignment\":{\"executor_node\":\"iKPGCkp9RVOKXOj20uOt4g\",\"explanation\":\"\"},\"allocation_id_on_last_status_update\":0},{\"id\":\"geoip-downloader\",\"task\":{\"geoip-downloader\":{\"params\":{},\"state\":{\"databases\":{\"GeoLite2-ASN.mmdb\":{\"last_update\":1679968651799,\"last_check\":1679968651799,\"first_chunk\":0,\"last_chunk\":4,\"md5\":\"33268a0d7cb3ba8d9d581bf1e5dc0980\"},\"GeoLite2-Country.mmdb\":{\"last_update\":1679968809323,\"last_check\":1679968809323,\"first_chunk\":0,\"last_chunk\":2,\"md5\":\"da6d05a7b044ea35910f20f756c057ad\"},\"GeoLite2-City.mmdb\":{\"last_update\":1679968693385,\"last_check\":1679968693385,\"first_chunk\":0,\"last_chunk\":33,\"md5\":\"2d91f3e42689a12c2f496199befdfccf\"}}}}},\"allocation_id\":6,\"assignment\":{\"executor_node\":\"iKPGCkp9RVOKXOj20uOt4g\",\"explanation\":\"\"},\"allocation_id_on_last_status_update\":0}]} index_lifecycle: {\n \"policies\" : {\n \"90-days-default\" : {\n \"policy\" : {\n \"phases\" : {\n \"warm\" : {\n \"min_age\" : \"2d\",\n \"actions\" : {\n \"shrink\" : {\n \"number_of_shards\" : 1\n },\n \"forcemerge\" : {\n \"max_num_segments\" : 1\n }\n }\n },\n \"cold\" : {\n \"min_age\" : \"30d\",\n \"actions\" : { }\n },\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"90d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"built-in ILM policy using the hot, warm, and cold phases with a retention of 90 days\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968678193,\n \"modified_date_string\" : \"2023-03-28T01:57:58.193Z\"\n },\n \"ilm-history-ilm-policy\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"90d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"default policy for the ILM history indices\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968685095,\n \"modified_date_string\" : \"2023-03-28T01:58:05.095Z\"\n },\n \"watch-history-ilm-policy-16\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"3d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"4d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"default policy for the watcher history indices\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968647327,\n \"modified_date_string\" : \"2023-03-28T01:57:27.327Z\"\n },\n \"30-days-default\" : {\n \"policy\" : {\n \"phases\" : {\n \"warm\" : {\n \"min_age\" : \"2d\",\n \"actions\" : {\n \"shrink\" : {\n \"number_of_shards\" : 1\n },\n \"forcemerge\" : {\n \"max_num_segments\" : 1\n }\n }\n },\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"30d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"built-in ILM policy using the hot and warm phases with a retention of 30 days\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968662496,\n \"modified_date_string\" : \"2023-03-28T01:57:42.496Z\"\n },\n \"synthetics\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"default policy for the synthetics index template installed by x-pack\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968682454,\n \"modified_date_string\" : \"2023-03-28T01:58:02.454Z\"\n },\n \".monitoring-8-ilm-policy\" : {\n \"policy\" : {\n \"phases\" : {\n \"warm\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"forcemerge\" : {\n \"max_num_segments\" : 1\n }\n }\n },\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"3d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"3d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"defaults\" : {\n \"delete_min_age\" : \"Using value of [3d] based on the monitoring plugin default\"\n },\n \"description\" : \"Index lifecycle policy generated for [monitoring-*-8] data streams\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968676492,\n \"modified_date_string\" : \"2023-03-28T01:57:56.492Z\"\n },\n \"slm-history-ilm-policy\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"90d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"default policy for the SLM history indices\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968644040,\n \"modified_date_string\" : \"2023-03-28T01:57:24.040Z\"\n },\n \".fleet-actions-results-ilm-policy\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_size\" : \"300gb\",\n \"max_age\" : \"30d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"90d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"default policy for fleet action results indices\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968686060,\n \"modified_date_string\" : \"2023-03-28T01:58:06.060Z\"\n },\n \"365-days-default\" : {\n \"policy\" : {\n \"phases\" : {\n \"warm\" : {\n \"min_age\" : \"2d\",\n \"actions\" : {\n \"shrink\" : {\n \"number_of_shards\" : 1\n },\n \"forcemerge\" : {\n \"max_num_segments\" : 1\n }\n }\n },\n \"cold\" : {\n \"min_age\" : \"30d\",\n \"actions\" : { }\n },\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"365d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"built-in ILM policy using the hot, warm, and cold phases with a retention of 365 days\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968675542,\n \"modified_date_string\" : \"2023-03-28T01:57:55.542Z\"\n },\n \"7-days-default\" : {\n \"policy\" : {\n \"phases\" : {\n \"warm\" : {\n \"min_age\" : \"2d\",\n \"actions\" : {\n \"shrink\" : {\n \"number_of_shards\" : 1\n },\n \"forcemerge\" : {\n \"max_num_segments\" : 1\n }\n }\n },\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"7d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"7d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"built-in ILM policy using the hot and warm phases with a retention of 7 days\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968672404,\n \"modified_date_string\" : \"2023-03-28T01:57:52.404Z\"\n },\n \".deprecation-indexing-ilm-policy\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"10gb\",\n \"max_age\" : \"30d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"30d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"ILM policy used for deprecation logs\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968670134,\n \"modified_date_string\" : \"2023-03-28T01:57:50.134Z\"\n },\n \"metrics\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"default policy for the metrics index template installed by x-pack\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968660707,\n \"modified_date_string\" : \"2023-03-28T01:57:40.707Z\"\n },\n \"ml-size-based-ilm-policy\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\"\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"default policy for machine learning state and stats indices\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968650499,\n \"modified_date_string\" : \"2023-03-28T01:57:30.499Z\"\n },\n \"180-days-default\" : {\n \"policy\" : {\n \"phases\" : {\n \"warm\" : {\n \"min_age\" : \"2d\",\n \"actions\" : {\n \"shrink\" : {\n \"number_of_shards\" : 1\n },\n \"forcemerge\" : {\n \"max_num_segments\" : 1\n }\n }\n },\n \"cold\" : {\n \"min_age\" : \"30d\",\n \"actions\" : { }\n },\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"180d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"built-in ILM policy using the hot, warm, and cold phases with a retention of 180 days\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968681386,\n \"modified_date_string\" : \"2023-03-28T01:58:01.386Z\"\n },\n \"logs\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"default policy for the logs index template installed by x-pack\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968679651,\n \"modified_date_string\" : \"2023-03-28T01:57:59.651Z\"\n }\n },\n \"operation_mode\" : \"RUNNING\"\n} index_template: {\"index_template\":{\".watch-history-16\":{\"index_patterns\":[\".watcher-history-16*\"],\"template\":{\"settings\":{\"index\":{\"format\":\"6\",\"lifecycle\":{\"name\":\"watch-history-ilm-policy-16\"},\"hidden\":\"true\",\"number_of_shards\":\"1\",\"auto_expand_replicas\":\"0-1\",\"number_of_replicas\":\"0\"}},\"mappings\":{\"_meta\":{\"watcher-history-version\":\"16\"},\"dynamic\":false,\"dynamic_templates\":[{\"disabled_payload_fields\":{\"match_pattern\":\"regex\",\"path_match\":\"result\\\\.(input(\\\\..+)*|(transform(\\\\..+)*)|(actions\\\\.transform(\\\\..+)*))\\\\.payload\",\"mapping\":{\"type\":\"object\",\"enabled\":false}}},{\"disabled_search_request_body_fields\":{\"match_pattern\":\"regex\",\"path_match\":\"result\\\\.(input(\\\\..+)*|(transform(\\\\..+)*)|(actions\\\\.transform(\\\\..+)*))\\\\.search\\\\.request\\\\.(body|template)\",\"mapping\":{\"type\":\"object\",\"enabled\":false}}},{\"disabled_exception_fields\":{\"match_pattern\":\"regex\",\"path_match\":\"result\\\\.(input(\\\\..+)*|(transform(\\\\..+)*)|(actions\\\\.transform(\\\\..+)*)|actions)\\\\.error\",\"mapping\":{\"type\":\"object\",\"enabled\":false}}},{\"disabled_jira_custom_fields\":{\"path_match\":\"result.actions.jira.fields.customfield_*\",\"mapping\":{\"type\":\"object\",\"enabled\":false}}}],\"properties\":{\"exception\":{\"type\":\"object\",\"enabled\":false},\"metadata\":{\"dynamic\":true,\"type\":\"object\"},\"trigger_event\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"schedule\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"scheduled_time\":{\"type\":\"date\"}}},\"triggered_time\":{\"type\":\"date\"},\"type\":{\"type\":\"keyword\"},\"manual\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"schedule\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"scheduled_time\":{\"type\":\"date\"}}}}}}},\"result\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"input\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"search\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"request\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"indices\":{\"type\":\"keyword\"},\"types\":{\"type\":\"keyword\"},\"search_type\":{\"type\":\"keyword\"}}}}},\"payload\":{\"type\":\"object\",\"enabled\":false},\"http\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"request\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"path\":{\"type\":\"keyword\"},\"host\":{\"type\":\"keyword\"}}}}},\"type\":{\"type\":\"keyword\"},\"status\":{\"type\":\"keyword\"}}},\"condition\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"compare\":{\"type\":\"object\",\"enabled\":false},\"array_compare\":{\"type\":\"object\",\"enabled\":false},\"type\":{\"type\":\"keyword\"},\"met\":{\"type\":\"boolean\"},\"script\":{\"type\":\"object\",\"enabled\":false},\"status\":{\"type\":\"keyword\"}}},\"transform\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"search\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"request\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"indices\":{\"type\":\"keyword\"},\"types\":{\"type\":\"keyword\"}}}}},\"type\":{\"type\":\"keyword\"}}},\"execution_duration\":{\"type\":\"long\"},\"actions\":{\"include_in_parent\":true,\"dynamic\":true,\"type\":\"nested\",\"properties\":{\"reason\":{\"type\":\"keyword\"},\"foreach\":{\"type\":\"object\",\"enabled\":false},\"webhook\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"request\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"path\":{\"type\":\"keyword\"},\"host\":{\"type\":\"keyword\"}}}}},\"number_of_actions_executed\":{\"type\":\"integer\"},\"slack\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"sent_messages\":{\"include_in_parent\":true,\"dynamic\":true,\"type\":\"nested\",\"properties\":{\"reason\":{\"type\":\"text\"},\"request\":{\"type\":\"object\",\"enabled\":false},\"response\":{\"type\":\"object\",\"enabled\":false},\"to\":{\"type\":\"keyword\"},\"message\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"attachments\":{\"include_in_parent\":true,\"dynamic\":true,\"type\":\"nested\",\"properties\":{\"color\":{\"type\":\"keyword\"},\"fields\":{\"properties\":{\"value\":{\"type\":\"text\"}}}}},\"icon\":{\"type\":\"keyword\"},\"from\":{\"type\":\"text\"},\"text\":{\"type\":\"text\"}}},\"status\":{\"type\":\"keyword\"}}},\"account\":{\"type\":\"keyword\"}}},\"index\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"response\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"index\":{\"type\":\"keyword\"},\"id\":{\"type\":\"keyword\"},\"type\":{\"type\":\"keyword\"}}}}},\"pagerduty\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"sent_event\":{\"include_in_parent\":true,\"dynamic\":true,\"type\":\"nested\",\"properties\":{\"reason\":{\"type\":\"text\"},\"request\":{\"type\":\"object\",\"enabled\":false},\"response\":{\"type\":\"object\",\"enabled\":false},\"event\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"client_url\":{\"type\":\"keyword\"},\"context\":{\"include_in_parent\":true,\"dynamic\":true,\"type\":\"nested\",\"properties\":{\"src\":{\"type\":\"keyword\"},\"alt\":{\"type\":\"text\"},\"href\":{\"type\":\"keyword\"},\"type\":{\"type\":\"keyword\"}}},\"client\":{\"type\":\"text\"},\"description\":{\"type\":\"text\"},\"attach_payload\":{\"type\":\"boolean\"},\"incident_key\":{\"type\":\"keyword\"},\"type\":{\"type\":\"keyword\"},\"account\":{\"type\":\"keyword\"}}}}},\"account\":{\"type\":\"keyword\"}}},\"id\":{\"type\":\"keyword\"},\"type\":{\"type\":\"keyword\"},\"email\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"message\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"cc\":{\"type\":\"keyword\"},\"bcc\":{\"type\":\"keyword\"},\"reply_to\":{\"type\":\"keyword\"},\"from\":{\"type\":\"keyword\"},\"id\":{\"type\":\"keyword\"},\"to\":{\"type\":\"keyword\"}}}}},\"status\":{\"type\":\"keyword\"},\"jira\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"result\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"self\":{\"type\":\"keyword\"},\"id\":{\"type\":\"keyword\"},\"key\":{\"type\":\"keyword\"}}},\"reason\":{\"type\":\"text\"},\"request\":{\"type\":\"object\",\"enabled\":false},\"response\":{\"type\":\"object\",\"enabled\":false},\"fields\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"summary\":{\"type\":\"text\"},\"issuetype\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"name\":{\"type\":\"keyword\"},\"id\":{\"type\":\"keyword\"}}},\"description\":{\"type\":\"text\"},\"project\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"id\":{\"type\":\"keyword\"},\"key\":{\"type\":\"keyword\"}}},\"labels\":{\"type\":\"text\"}}},\"account\":{\"type\":\"keyword\"}}}}},\"execution_time\":{\"type\":\"date\"}}},\"node\":{\"type\":\"keyword\"},\"input\":{\"type\":\"object\",\"enabled\":false},\"condition\":{\"type\":\"object\",\"enabled\":false},\"watch_id\":{\"type\":\"keyword\"},\"messages\":{\"type\":\"text\"},\"vars\":{\"type\":\"object\",\"enabled\":false},\"state\":{\"type\":\"keyword\"},\"user\":{\"type\":\"text\"},\"status\":{\"dynamic\":true,\"type\":\"object\",\"enabled\":false}}}},\"composed_of\":[],\"priority\":2147483647,\"version\":16,\"_meta\":{\"managed\":true,\"description\":\"index template for watcher history indices\"},\"data_stream\":{\"hidden\":true,\"allow_custom_routing\":false},\"allow_auto_create\":true},\".monitoring-beats-mb\":{\"index_patterns\":[\".monitoring-beats-8-*\"],\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\".monitoring-8-ilm-policy\"},\"mapping\":{\"total_fields\":{\"limit\":\"2000\"}}}},\"mappings\":{\"dynamic\":false,\"properties\":{\"@timestamp\":{\"type\":\"date\"},\"cluster_uuid\":{\"path\":\"beat.elasticsearch.cluster.id\",\"type\":\"alias\"},\"service\":{\"properties\":{\"node\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"environment\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"hostname\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"origin\":{\"properties\":{\"node\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"environment\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"target\":{\"properties\":{\"node\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"environment\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"beat\":{\"properties\":{\"elasticsearch\":{\"properties\":{\"cluster\":{\"properties\":{\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"stats\":{\"properties\":{\"handles\":{\"properties\":{\"limit\":{\"properties\":{\"hard\":{\"type\":\"long\"},\"soft\":{\"type\":\"long\"}}},\"open\":{\"type\":\"long\"}}},\"apm_server\":{\"properties\":{\"server\":{\"properties\":{\"request\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"response\":{\"properties\":{\"valid\":{\"properties\":{\"count\":{\"type\":\"long\"},\"accepted\":{\"type\":\"long\"},\"ok\":{\"type\":\"long\"}}},\"count\":{\"type\":\"long\"},\"errors\":{\"properties\":{\"ratelimit\":{\"type\":\"long\"},\"internal\":{\"type\":\"long\"},\"toolarge\":{\"type\":\"long\"},\"method\":{\"type\":\"long\"},\"unauthorized\":{\"type\":\"long\"},\"forbidden\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"},\"closed\":{\"type\":\"long\"},\"decode\":{\"type\":\"long\"},\"queue\":{\"type\":\"long\"},\"concurrency\":{\"type\":\"long\"},\"validate\":{\"type\":\"long\"}}}}},\"concurrent\":{\"properties\":{\"wait\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}}}},\"acm\":{\"properties\":{\"request\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"response\":{\"properties\":{\"valid\":{\"properties\":{\"notmodified\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"},\"accepted\":{\"type\":\"long\"},\"ok\":{\"type\":\"long\"}}},\"request\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"count\":{\"type\":\"long\"},\"unset\":{\"type\":\"long\"},\"errors\":{\"properties\":{\"ratelimit\":{\"type\":\"long\"},\"internal\":{\"type\":\"long\"},\"toolarge\":{\"type\":\"long\"},\"method\":{\"type\":\"long\"},\"unavailable\":{\"type\":\"long\"},\"forbidden\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"},\"decode\":{\"type\":\"long\"},\"unauthorized\":{\"type\":\"long\"},\"notfound\":{\"type\":\"long\"},\"closed\":{\"type\":\"long\"},\"invalidquery\":{\"type\":\"long\"},\"queue\":{\"type\":\"long\"},\"validate\":{\"type\":\"long\"}}}}}}},\"decoder\":{\"properties\":{\"deflate\":{\"properties\":{\"content-length\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"reader\":{\"properties\":{\"size\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"gzip\":{\"properties\":{\"content-length\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"uncompressed\":{\"properties\":{\"content-length\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"missing-content-length\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"processor\":{\"properties\":{\"metric\":{\"properties\":{\"transformations\":{\"type\":\"long\"},\"decoding\":{\"properties\":{\"count\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}},\"validation\":{\"properties\":{\"count\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}}}},\"sourcemap\":{\"properties\":{\"counter\":{\"type\":\"long\"},\"decoding\":{\"properties\":{\"count\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}},\"validation\":{\"properties\":{\"count\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}}}},\"error\":{\"properties\":{\"stacktraces\":{\"type\":\"long\"},\"spans\":{\"type\":\"long\"},\"frames\":{\"type\":\"long\"},\"transformations\":{\"type\":\"long\"},\"decoding\":{\"properties\":{\"count\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}},\"validation\":{\"properties\":{\"count\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}}}},\"transaction\":{\"properties\":{\"stacktraces\":{\"type\":\"long\"},\"spans\":{\"type\":\"long\"},\"frames\":{\"type\":\"long\"},\"transformations\":{\"type\":\"long\"},\"transactions\":{\"type\":\"long\"},\"decoding\":{\"properties\":{\"count\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}},\"validation\":{\"properties\":{\"count\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}}}},\"span\":{\"properties\":{\"transformations\":{\"type\":\"long\"}}}}}}},\"system\":{\"properties\":{\"load\":{\"properties\":{\"1\":{\"type\":\"double\"},\"15\":{\"type\":\"double\"},\"5\":{\"type\":\"double\"},\"norm\":{\"properties\":{\"1\":{\"type\":\"double\"},\"15\":{\"type\":\"double\"},\"5\":{\"type\":\"double\"}}}}},\"cpu\":{\"properties\":{\"cores\":{\"type\":\"long\"}}}}},\"beat\":{\"properties\":{\"host\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"cpu\":{\"properties\":{\"total\":{\"properties\":{\"ticks\":{\"type\":\"long\"},\"time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"value\":{\"type\":\"long\"}}},\"system\":{\"properties\":{\"ticks\":{\"type\":\"long\"},\"time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"user\":{\"properties\":{\"ticks\":{\"type\":\"long\"},\"time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}}}},\"runtime\":{\"properties\":{\"goroutines\":{\"type\":\"long\"}}},\"memstats\":{\"properties\":{\"memory\":{\"properties\":{\"total\":{\"type\":\"long\"},\"alloc\":{\"type\":\"long\"}}},\"rss\":{\"type\":\"long\"},\"gc_next\":{\"type\":\"long\"}}},\"libbeat\":{\"properties\":{\"output\":{\"properties\":{\"read\":{\"properties\":{\"bytes\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"write\":{\"properties\":{\"bytes\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}},\"events\":{\"properties\":{\"batches\":{\"type\":\"long\"},\"duplicates\":{\"type\":\"long\"},\"total\":{\"type\":\"long\"},\"dropped\":{\"type\":\"long\"},\"toomany\":{\"type\":\"long\"},\"active\":{\"type\":\"long\"},\"failed\":{\"type\":\"long\"},\"acked\":{\"type\":\"long\"}}}}},\"pipeline\":{\"properties\":{\"clients\":{\"type\":\"long\"},\"events\":{\"properties\":{\"total\":{\"type\":\"long\"},\"filtered\":{\"type\":\"long\"},\"dropped\":{\"type\":\"long\"},\"active\":{\"type\":\"long\"},\"failed\":{\"type\":\"long\"},\"published\":{\"type\":\"long\"},\"retry\":{\"type\":\"long\"}}},\"queue\":{\"properties\":{\"acked\":{\"type\":\"long\"}}}}},\"config\":{\"properties\":{\"running\":{\"type\":\"long\"},\"stops\":{\"type\":\"long\"},\"starts\":{\"type\":\"long\"},\"reloads\":{\"type\":\"long\"}}}}},\"cgroup\":{\"properties\":{\"memory\":{\"properties\":{\"mem\":{\"properties\":{\"usage\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"limit\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"cpu\":{\"properties\":{\"cfs\":{\"properties\":{\"period\":{\"properties\":{\"us\":{\"type\":\"long\"}}},\"quota\":{\"properties\":{\"us\":{\"type\":\"long\"}}}}},\"stats\":{\"properties\":{\"periods\":{\"type\":\"long\"},\"throttled\":{\"properties\":{\"ns\":{\"type\":\"long\"},\"periods\":{\"type\":\"long\"}}}}},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"cpuacct\":{\"properties\":{\"total\":{\"properties\":{\"ns\":{\"type\":\"long\"}}},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"info\":{\"properties\":{\"host\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"uptime\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"uptime\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"properties\":{\"output\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"cluster\":{\"properties\":{\"uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"input\":{\"properties\":{\"names\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"count\":{\"type\":\"long\"}}},\"management\":{\"properties\":{\"enabled\":{\"type\":\"boolean\"}}},\"service\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"module\":{\"properties\":{\"names\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"count\":{\"type\":\"long\"}}},\"beat\":{\"properties\":{\"host\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"host\":{\"properties\":{\"os\":{\"properties\":{\"kernel\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"platform\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"containerized\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"queue\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"host\":{\"properties\":{\"hostname\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"architecture\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"metricset\":{\"properties\":{\"period\":{\"type\":\"long\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"event\":{\"properties\":{\"reason\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"code\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"timezone\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"agent_id_status\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"duration\":{\"type\":\"long\"},\"reference\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ingested\":{\"type\":\"date\"},\"provider\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"action\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"end\":{\"type\":\"date\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"outcome\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"severity\":{\"type\":\"long\"},\"original\":{\"ignore_above\":1024,\"index\":false,\"type\":\"keyword\",\"doc_values\":false},\"risk_score\":{\"type\":\"float\"},\"created\":{\"type\":\"date\"},\"kind\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"module\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"start\":{\"type\":\"date\"},\"url\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"sequence\":{\"type\":\"long\"},\"risk_score_norm\":{\"type\":\"float\"},\"category\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"dataset\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"hash\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"beats_stats\":{\"properties\":{\"beat\":{\"properties\":{\"host\":{\"path\":\"beat.stats.beat.host\",\"type\":\"alias\"},\"name\":{\"path\":\"beat.stats.beat.name\",\"type\":\"alias\"},\"type\":{\"path\":\"beat.stats.beat.type\",\"type\":\"alias\"},\"uuid\":{\"path\":\"beat.stats.beat.uuid\",\"type\":\"alias\"},\"version\":{\"path\":\"beat.stats.beat.version\",\"type\":\"alias\"}}},\"metrics\":{\"properties\":{\"system\":{\"properties\":{\"load\":{\"properties\":{\"1\":{\"path\":\"beat.stats.system.load.1\",\"type\":\"alias\"},\"15\":{\"path\":\"beat.stats.system.load.15\",\"type\":\"alias\"},\"5\":{\"path\":\"beat.stats.system.load.5\",\"type\":\"alias\"},\"norm\":{\"properties\":{\"1\":{\"path\":\"beat.stats.system.load.norm.1\",\"type\":\"alias\"},\"15\":{\"path\":\"beat.stats.system.load.norm.15\",\"type\":\"alias\"},\"5\":{\"path\":\"beat.stats.system.load.norm.5\",\"type\":\"alias\"}}}}},\"cpu\":{\"properties\":{\"cores\":{\"path\":\"beat.stats.system.cpu.cores\",\"type\":\"alias\"}}}}},\"beat\":{\"properties\":{\"handles\":{\"properties\":{\"limit\":{\"properties\":{\"hard\":{\"path\":\"beat.stats.handles.limit.hard\",\"type\":\"alias\"},\"soft\":{\"path\":\"beat.stats.handles.limit.soft\",\"type\":\"alias\"}}},\"open\":{\"path\":\"beat.stats.handles.open\",\"type\":\"alias\"}}},\"cpu\":{\"properties\":{\"total\":{\"properties\":{\"ticks\":{\"path\":\"beat.stats.cpu.total.ticks\",\"type\":\"alias\"},\"time\":{\"properties\":{\"ms\":{\"path\":\"beat.stats.cpu.total.time.ms\",\"type\":\"alias\"}}},\"value\":{\"path\":\"beat.stats.cpu.total.value\",\"type\":\"alias\"}}},\"system\":{\"properties\":{\"ticks\":{\"path\":\"beat.stats.cpu.system.ticks\",\"type\":\"alias\"},\"time\":{\"properties\":{\"ms\":{\"path\":\"beat.stats.cpu.system.time.ms\",\"type\":\"alias\"}}}}},\"user\":{\"properties\":{\"ticks\":{\"path\":\"beat.stats.cpu.user.ticks\",\"type\":\"alias\"},\"time\":{\"properties\":{\"ms\":{\"path\":\"beat.stats.cpu.user.time.ms\",\"type\":\"alias\"}}}}}}},\"memstats\":{\"properties\":{\"rss\":{\"path\":\"beat.stats.memstats.rss\",\"type\":\"alias\"},\"gc_next\":{\"path\":\"beat.stats.memstats.gc_next\",\"type\":\"alias\"},\"memory_alloc\":{\"path\":\"beat.stats.memstats.memory.alloc\",\"type\":\"alias\"},\"memory_total\":{\"path\":\"beat.stats.memstats.memory.total\",\"type\":\"alias\"}}},\"cgroup\":{\"properties\":{\"memory\":{\"properties\":{\"id\":{\"path\":\"beat.stats.cgroup.memory.id\",\"type\":\"alias\"}}},\"mem\":{\"properties\":{\"usage\":{\"properties\":{\"bytes\":{\"path\":\"beat.stats.cgroup.memory.mem.usage.bytes\",\"type\":\"alias\"}}},\"limit\":{\"properties\":{\"bytes\":{\"path\":\"beat.stats.cgroup.memory.mem.limit.bytes\",\"type\":\"alias\"}}}}},\"cpu\":{\"properties\":{\"cfs\":{\"properties\":{\"period\":{\"properties\":{\"us\":{\"path\":\"beat.stats.cgroup.cpu.cfs.period.us\",\"type\":\"alias\"}}},\"quota\":{\"properties\":{\"us\":{\"path\":\"beat.stats.cgroup.cpu.cfs.quota.us\",\"type\":\"alias\"}}}}},\"stats\":{\"properties\":{\"periods\":{\"path\":\"beat.stats.cgroup.cpu.stats.periods\",\"type\":\"alias\"},\"throttled\":{\"properties\":{\"ns\":{\"path\":\"beat.stats.cgroup.cpu.stats.throttled.ns\",\"type\":\"alias\"},\"periods\":{\"path\":\"beat.stats.cgroup.cpu.stats.throttled.periods\",\"type\":\"alias\"}}}}},\"id\":{\"path\":\"beat.stats.cgroup.cpu.id\",\"type\":\"alias\"}}},\"cpuacct\":{\"properties\":{\"total\":{\"properties\":{\"ns\":{\"path\":\"beat.stats.cgroup.cpuacct.total.ns\",\"type\":\"alias\"}}},\"id\":{\"path\":\"beat.stats.cgroup.cpuacct.id\",\"type\":\"alias\"}}}}},\"info\":{\"properties\":{\"ephemeral_id\":{\"path\":\"beat.stats.info.ephemeral_id\",\"type\":\"alias\"},\"uptime\":{\"properties\":{\"ms\":{\"path\":\"beat.stats.info.uptime.ms\",\"type\":\"alias\"}}}}}}},\"libbeat\":{\"properties\":{\"output\":{\"properties\":{\"read\":{\"properties\":{\"bytes\":{\"path\":\"beat.stats.libbeat.output.read.bytes\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.libbeat.output.read.errors\",\"type\":\"alias\"}}},\"type\":{\"path\":\"beat.stats.libbeat.output.type\",\"type\":\"alias\"},\"write\":{\"properties\":{\"bytes\":{\"path\":\"beat.stats.libbeat.output.write.bytes\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.libbeat.output.write.errors\",\"type\":\"alias\"}}},\"events\":{\"properties\":{\"duplicated\":{\"path\":\"beat.stats.libbeat.output.events.duplicates\",\"type\":\"alias\"},\"batches\":{\"path\":\"beat.stats.libbeat.output.events.batches\",\"type\":\"alias\"},\"total\":{\"path\":\"beat.stats.libbeat.output.events.total\",\"type\":\"alias\"},\"dropped\":{\"path\":\"beat.stats.libbeat.output.events.dropped\",\"type\":\"alias\"},\"toomany\":{\"path\":\"beat.stats.libbeat.output.events.toomany\",\"type\":\"alias\"},\"active\":{\"path\":\"beat.stats.libbeat.output.events.active\",\"type\":\"alias\"},\"failed\":{\"path\":\"beat.stats.libbeat.output.events.failed\",\"type\":\"alias\"},\"acked\":{\"path\":\"beat.stats.libbeat.output.events.acked\",\"type\":\"alias\"}}}}},\"pipeline\":{\"properties\":{\"clients\":{\"path\":\"beat.stats.libbeat.pipeline.clients\",\"type\":\"alias\"},\"events\":{\"properties\":{\"total\":{\"path\":\"beat.stats.libbeat.pipeline.events.total\",\"type\":\"alias\"},\"filtered\":{\"path\":\"beat.stats.libbeat.pipeline.events.filtered\",\"type\":\"alias\"},\"dropped\":{\"path\":\"beat.stats.libbeat.pipeline.events.dropped\",\"type\":\"alias\"},\"active\":{\"path\":\"beat.stats.libbeat.pipeline.events.active\",\"type\":\"alias\"},\"failed\":{\"path\":\"beat.stats.libbeat.pipeline.events.failed\",\"type\":\"alias\"},\"published\":{\"path\":\"beat.stats.libbeat.pipeline.events.published\",\"type\":\"alias\"},\"retry\":{\"path\":\"beat.stats.libbeat.pipeline.events.retry\",\"type\":\"alias\"}}},\"queue\":{\"properties\":{\"acked\":{\"path\":\"beat.stats.libbeat.pipeline.queue.acked\",\"type\":\"alias\"}}}}},\"config\":{\"properties\":{\"module\":{\"properties\":{\"running\":{\"path\":\"beat.stats.libbeat.config.running\",\"type\":\"alias\"},\"stops\":{\"path\":\"beat.stats.libbeat.config.stops\",\"type\":\"alias\"},\"starts\":{\"path\":\"beat.stats.libbeat.config.starts\",\"type\":\"alias\"}}},\"reloads\":{\"path\":\"beat.stats.libbeat.config.reloads\",\"type\":\"alias\"}}}}},\"apm-server\":{\"properties\":{\"server\":{\"properties\":{\"request\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.server.request.count\",\"type\":\"alias\"}}},\"response\":{\"properties\":{\"valid\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.server.response.valid.count\",\"type\":\"alias\"},\"accepted\":{\"path\":\"beat.stats.apm_server.server.response.valid.accepted\",\"type\":\"alias\"},\"ok\":{\"path\":\"beat.stats.apm_server.server.response.valid.ok\",\"type\":\"alias\"}}},\"count\":{\"path\":\"beat.stats.apm_server.server.response.count\",\"type\":\"alias\"},\"errors\":{\"properties\":{\"ratelimit\":{\"path\":\"beat.stats.apm_server.server.response.errors.ratelimit\",\"type\":\"alias\"},\"internal\":{\"path\":\"beat.stats.apm_server.server.response.errors.internal\",\"type\":\"alias\"},\"toolarge\":{\"path\":\"beat.stats.apm_server.server.response.errors.toolarge\",\"type\":\"alias\"},\"method\":{\"path\":\"beat.stats.apm_server.server.response.errors.method\",\"type\":\"alias\"},\"unauthorized\":{\"path\":\"beat.stats.apm_server.server.response.errors.unauthorized\",\"type\":\"alias\"},\"forbidden\":{\"path\":\"beat.stats.apm_server.server.response.errors.forbidden\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.stats.apm_server.server.response.errors.count\",\"type\":\"alias\"},\"closed\":{\"path\":\"beat.stats.apm_server.server.response.errors.closed\",\"type\":\"alias\"},\"decode\":{\"path\":\"beat.stats.apm_server.server.response.errors.decode\",\"type\":\"alias\"},\"queue\":{\"path\":\"beat.stats.apm_server.server.response.errors.queue\",\"type\":\"alias\"},\"concurrency\":{\"path\":\"beat.stats.apm_server.server.response.errors.concurrency\",\"type\":\"alias\"},\"validate\":{\"path\":\"beat.stats.apm_server.server.response.errors.validate\",\"type\":\"alias\"}}}}},\"concurrent\":{\"properties\":{\"wait\":{\"properties\":{\"ms\":{\"path\":\"beat.stats.apm_server.server.concurrent.wait.ms\",\"type\":\"alias\"}}}}}}},\"sampling\":{\"properties\":{\"tail\":{\"properties\":{\"dynamic_service_groups\":{\"type\":\"long\"},\"storage\":{\"properties\":{\"lsm_size\":{\"type\":\"long\"},\"value_log_size\":{\"type\":\"long\"}}},\"events\":{\"properties\":{\"processed\":{\"type\":\"long\"},\"head_unsampled\":{\"type\":\"long\"},\"stored\":{\"type\":\"long\"},\"dropped\":{\"type\":\"long\"},\"sampled\":{\"type\":\"long\"}}}}},\"transactions_dropped\":{\"type\":\"long\"}}},\"acm\":{\"properties\":{\"request\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.acm.request.count\",\"type\":\"alias\"}}},\"response\":{\"properties\":{\"valid\":{\"properties\":{\"notmodified\":{\"path\":\"beat.stats.apm_server.acm.response.valid.notmodified\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.stats.apm_server.acm.response.valid.count\",\"type\":\"alias\"},\"accepted\":{\"path\":\"beat.stats.apm_server.acm.response.valid.accepted\",\"type\":\"alias\"},\"ok\":{\"path\":\"beat.stats.apm_server.acm.response.valid.ok\",\"type\":\"alias\"}}},\"request\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.acm.response.request.count\",\"type\":\"alias\"}}},\"count\":{\"path\":\"beat.stats.apm_server.acm.response.count\",\"type\":\"alias\"},\"unset\":{\"path\":\"beat.stats.apm_server.acm.response.unset\",\"type\":\"alias\"},\"errors\":{\"properties\":{\"ratelimit\":{\"path\":\"beat.stats.apm_server.acm.response.errors.ratelimit\",\"type\":\"alias\"},\"internal\":{\"path\":\"beat.stats.apm_server.acm.response.errors.internal\",\"type\":\"alias\"},\"toolarge\":{\"path\":\"beat.stats.apm_server.acm.response.errors.toolarge\",\"type\":\"alias\"},\"method\":{\"path\":\"beat.stats.apm_server.acm.response.errors.method\",\"type\":\"alias\"},\"unavailable\":{\"path\":\"beat.stats.apm_server.acm.response.errors.unavailable\",\"type\":\"alias\"},\"forbidden\":{\"path\":\"beat.stats.apm_server.acm.response.errors.forbidden\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.stats.apm_server.acm.response.errors.count\",\"type\":\"alias\"},\"decode\":{\"path\":\"beat.stats.apm_server.acm.response.errors.decode\",\"type\":\"alias\"},\"unauthorized\":{\"path\":\"beat.stats.apm_server.acm.response.errors.unauthorized\",\"type\":\"alias\"},\"notfound\":{\"path\":\"beat.stats.apm_server.acm.response.errors.notfound\",\"type\":\"alias\"},\"closed\":{\"path\":\"beat.stats.apm_server.acm.response.errors.closed\",\"type\":\"alias\"},\"invalidquery\":{\"path\":\"beat.stats.apm_server.acm.response.errors.invalidquery\",\"type\":\"alias\"},\"queue\":{\"path\":\"beat.stats.apm_server.acm.response.errors.queue\",\"type\":\"alias\"},\"validate\":{\"path\":\"beat.stats.apm_server.acm.response.errors.validate\",\"type\":\"alias\"}}}}}}},\"decoder\":{\"properties\":{\"deflate\":{\"properties\":{\"content-length\":{\"path\":\"beat.stats.apm_server.decoder.deflate.content-length\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.stats.apm_server.decoder.deflate.count\",\"type\":\"alias\"}}},\"reader\":{\"properties\":{\"size\":{\"path\":\"beat.stats.apm_server.decoder.reader.size\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.stats.apm_server.decoder.reader.count\",\"type\":\"alias\"}}},\"gzip\":{\"properties\":{\"content-length\":{\"path\":\"beat.stats.apm_server.decoder.gzip.content-length\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.stats.apm_server.decoder.gzip.count\",\"type\":\"alias\"}}},\"uncompressed\":{\"properties\":{\"content-length\":{\"path\":\"beat.stats.apm_server.decoder.uncompressed.content-length\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.stats.apm_server.decoder.uncompressed.count\",\"type\":\"alias\"}}},\"missing-content-length\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.decoder.missing-content-length.count\",\"type\":\"alias\"}}}}},\"processor\":{\"properties\":{\"metric\":{\"properties\":{\"transformations\":{\"path\":\"beat.stats.apm_server.processor.metric.transformations\",\"type\":\"alias\"},\"decoding\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.processor.metric.decoding.count\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.apm_server.processor.metric.decoding.errors\",\"type\":\"alias\"}}},\"validation\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.processor.metric.validation.count\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.apm_server.processor.metric.validation.errors\",\"type\":\"alias\"}}}}},\"sourcemap\":{\"properties\":{\"counter\":{\"path\":\"beat.stats.apm_server.processor.sourcemap.counter\",\"type\":\"alias\"},\"decoding\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.processor.sourcemap.decoding.count\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.apm_server.processor.sourcemap.decoding.errors\",\"type\":\"alias\"}}},\"validation\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.processor.sourcemap.validation.count\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.apm_server.processor.sourcemap.validation.errors\",\"type\":\"alias\"}}}}},\"error\":{\"properties\":{\"stacktraces\":{\"path\":\"beat.stats.apm_server.processor.error.stacktraces\",\"type\":\"alias\"},\"spans\":{\"path\":\"beat.stats.apm_server.processor.error.spans\",\"type\":\"alias\"},\"frames\":{\"path\":\"beat.stats.apm_server.processor.error.frames\",\"type\":\"alias\"},\"transformations\":{\"path\":\"beat.stats.apm_server.processor.error.transformations\",\"type\":\"alias\"},\"decoding\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.processor.error.decoding.count\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.apm_server.processor.error.decoding.errors\",\"type\":\"alias\"}}},\"validation\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.processor.error.validation.count\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.apm_server.processor.error.validation.errors\",\"type\":\"alias\"}}}}},\"transaction\":{\"properties\":{\"stacktraces\":{\"path\":\"beat.stats.apm_server.processor.transaction.stacktraces\",\"type\":\"alias\"},\"spans\":{\"path\":\"beat.stats.apm_server.processor.transaction.spans\",\"type\":\"alias\"},\"frames\":{\"path\":\"beat.stats.apm_server.processor.transaction.frames\",\"type\":\"alias\"},\"transformations\":{\"path\":\"beat.stats.apm_server.processor.transaction.transformations\",\"type\":\"alias\"},\"transactions\":{\"path\":\"beat.stats.apm_server.processor.transaction.transactions\",\"type\":\"alias\"},\"decoding\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.processor.transaction.decoding.count\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.apm_server.processor.transaction.decoding.errors\",\"type\":\"alias\"}}},\"validation\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.processor.transaction.validation.count\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.apm_server.processor.transaction.validation.errors\",\"type\":\"alias\"}}}}},\"span\":{\"properties\":{\"transformations\":{\"path\":\"beat.stats.apm_server.processor.span.transformations\",\"type\":\"alias\"}}}}}}}}},\"timestamp\":{\"path\":\"@timestamp\",\"type\":\"alias\"}}},\"beats_state\":{\"properties\":{\"beat\":{\"properties\":{\"host\":{\"path\":\"beat.state.beat.host\",\"type\":\"alias\"},\"name\":{\"path\":\"beat.state.beat.name\",\"type\":\"alias\"},\"type\":{\"path\":\"beat.state.beat.type\",\"type\":\"alias\"},\"uuid\":{\"path\":\"beat.state.beat.uuid\",\"type\":\"alias\"},\"version\":{\"path\":\"beat.state.beat.version\",\"type\":\"alias\"}}},\"state\":{\"properties\":{\"output\":{\"properties\":{\"name\":{\"path\":\"beat.state.output.name\",\"type\":\"alias\"}}},\"input\":{\"properties\":{\"names\":{\"path\":\"beat.state.input.names\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.state.input.count\",\"type\":\"alias\"}}},\"service\":{\"properties\":{\"name\":{\"path\":\"beat.state.service.name\",\"type\":\"alias\"},\"id\":{\"path\":\"beat.state.service.id\",\"type\":\"alias\"},\"version\":{\"path\":\"beat.state.service.version\",\"type\":\"alias\"}}},\"module\":{\"properties\":{\"names\":{\"path\":\"beat.state.module.names\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.state.module.count\",\"type\":\"alias\"}}},\"beat\":{\"properties\":{\"name\":{\"path\":\"beat.state.beat.name\",\"type\":\"alias\"}}},\"host\":{\"properties\":{\"hostname\":{\"path\":\"host.hostname\",\"type\":\"alias\"},\"os\":{\"properties\":{\"version\":{\"path\":\"beat.state.host.os.version\",\"type\":\"alias\"},\"platform\":{\"path\":\"beat.state.host.os.platform\",\"type\":\"alias\"}}},\"name\":{\"path\":\"host.name\",\"type\":\"alias\"},\"architecture\":{\"path\":\"host.architecture\",\"type\":\"alias\"}}}}},\"timestamp\":{\"path\":\"@timestamp\",\"type\":\"alias\"}}},\"timestamp\":{\"path\":\"@timestamp\",\"type\":\"alias\"}}}},\"composed_of\":[],\"version\":8000103,\"data_stream\":{\"hidden\":false,\"allow_custom_routing\":false}},\".monitoring-kibana-mb\":{\"index_patterns\":[\".monitoring-kibana-8-*\"],\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\".monitoring-8-ilm-policy\"},\"mapping\":{\"total_fields\":{\"limit\":\"2000\"}}}},\"mappings\":{\"dynamic\":false,\"properties\":{\"process\":{\"properties\":{\"pid\":{\"type\":\"long\"}}},\"@timestamp\":{\"type\":\"date\"},\"ecs\":{\"properties\":{\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"cluster_uuid\":{\"path\":\"kibana.elasticsearch.cluster.id\",\"type\":\"alias\"},\"service\":{\"properties\":{\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"host\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"metricset\":{\"properties\":{\"period\":{\"type\":\"long\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"kibana\":{\"properties\":{\"elasticsearch\":{\"properties\":{\"cluster\":{\"properties\":{\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"stats\":{\"properties\":{\"request\":{\"properties\":{\"total\":{\"type\":\"long\"},\"disconnects\":{\"type\":\"long\"}}},\"process\":{\"properties\":{\"memory\":{\"properties\":{\"resident_set_size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"heap\":{\"properties\":{\"total\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"used\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"size_limit\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"uptime\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}}}},\"event_loop_delay\":{\"properties\":{\"ms\":{\"scaling_factor\":1000,\"type\":\"scaled_float\"}}},\"uptime\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"os\":{\"properties\":{\"distroRelease\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"distro\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"memory\":{\"properties\":{\"used_in_bytes\":{\"type\":\"long\"},\"total_in_bytes\":{\"type\":\"long\"},\"free_in_bytes\":{\"type\":\"long\"}}},\"load\":{\"properties\":{\"5m\":{\"type\":\"half_float\"},\"15m\":{\"type\":\"half_float\"},\"1m\":{\"type\":\"half_float\"}}},\"platformRelease\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"platform\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"usage\":{\"properties\":{\"index\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"host\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"index\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"response_time\":{\"properties\":{\"avg\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"max\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"kibana\":{\"properties\":{\"status\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"concurrent_connections\":{\"type\":\"long\"},\"snapshot\":{\"type\":\"boolean\"},\"status\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"node_rules\":{\"properties\":{\"failures\":{\"type\":\"long\"},\"executions\":{\"type\":\"long\"},\"timeouts\":{\"type\":\"long\"}}},\"cluster_actions\":{\"properties\":{\"overdue\":{\"properties\":{\"delay\":{\"properties\":{\"p99\":{\"type\":\"float\"},\"p50\":{\"type\":\"float\"}}},\"count\":{\"type\":\"long\"}}}}},\"cluster_rules\":{\"properties\":{\"overdue\":{\"properties\":{\"delay\":{\"properties\":{\"p99\":{\"type\":\"float\"},\"p50\":{\"type\":\"float\"}}},\"count\":{\"type\":\"long\"}}}}},\"node_actions\":{\"properties\":{\"failures\":{\"type\":\"long\"},\"executions\":{\"type\":\"long\"},\"timeouts\":{\"type\":\"long\"}}},\"status\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"metrics\":{\"properties\":{\"requests\":{\"properties\":{\"total\":{\"type\":\"long\"},\"disconnects\":{\"type\":\"long\"}}},\"concurrent_connections\":{\"type\":\"long\"}}},\"status\":{\"properties\":{\"overall\":{\"properties\":{\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}}}}}},\"event\":{\"properties\":{\"duration\":{\"type\":\"long\"},\"module\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"dataset\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"kibana_stats\":{\"properties\":{\"process\":{\"properties\":{\"memory\":{\"properties\":{\"resident_set_size_in_bytes\":{\"path\":\"kibana.stats.process.memory.resident_set_size.bytes\",\"type\":\"alias\"},\"heap\":{\"properties\":{\"size_limit\":{\"path\":\"kibana.stats.process.memory.heap.size_limit.bytes\",\"type\":\"alias\"}}}}},\"event_loop_delay\":{\"path\":\"kibana.stats.process.event_loop_delay.ms\",\"type\":\"alias\"},\"uptime_in_millis\":{\"path\":\"kibana.stats.process.uptime.ms\",\"type\":\"alias\"}}},\"os\":{\"properties\":{\"memory\":{\"properties\":{\"free_in_bytes\":{\"path\":\"kibana.stats.os.memory.free_in_bytes\",\"type\":\"alias\"}}},\"load\":{\"properties\":{\"5m\":{\"path\":\"kibana.stats.os.load.5m\",\"type\":\"alias\"},\"15m\":{\"path\":\"kibana.stats.os.load.15m\",\"type\":\"alias\"},\"1m\":{\"path\":\"kibana.stats.os.load.1m\",\"type\":\"alias\"}}}}},\"response_times\":{\"properties\":{\"average\":{\"path\":\"kibana.stats.response_time.avg.ms\",\"type\":\"alias\"},\"max\":{\"path\":\"kibana.stats.response_time.max.ms\",\"type\":\"alias\"}}},\"requests\":{\"properties\":{\"total\":{\"path\":\"kibana.stats.request.total\",\"type\":\"alias\"},\"disconnects\":{\"path\":\"kibana.stats.request.disconnects\",\"type\":\"alias\"}}},\"kibana\":{\"properties\":{\"response_time\":{\"properties\":{\"max\":{\"path\":\"kibana.stats.response_time.max.ms\",\"type\":\"alias\"}}},\"uuid\":{\"path\":\"service.id\",\"type\":\"alias\"},\"version\":{\"path\":\"service.version\",\"type\":\"alias\"},\"status\":{\"path\":\"kibana.stats.status\",\"type\":\"alias\"}}},\"concurrent_connections\":{\"path\":\"kibana.stats.concurrent_connections\",\"type\":\"alias\"},\"timestamp\":{\"path\":\"@timestamp\",\"type\":\"alias\"}}},\"timestamp\":{\"path\":\"@timestamp\",\"type\":\"alias\"}}}},\"composed_of\":[],\"version\":8000103,\"data_stream\":{\"hidden\":false,\"allow_custom_routing\":false}},\".monitoring-ent-search-mb\":{\"index_patterns\":[\".monitoring-ent-search-8-*\"],\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\".monitoring-8-ilm-policy\"},\"mapping\":{\"total_fields\":{\"limit\":\"2000\"}}}},\"mappings\":{\"dynamic\":false,\"properties\":{\"enterprisesearch\":{\"properties\":{\"cluster_uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"stats\":{\"properties\":{\"connectors\":{\"properties\":{\"pool\":{\"properties\":{\"publish_worker_pool\":{\"properties\":{\"total_completed\":{\"type\":\"long\"},\"size\":{\"type\":\"long\"},\"idle\":{\"type\":\"long\"},\"busy\":{\"type\":\"long\"},\"queue_depth\":{\"type\":\"long\"},\"total_scheduled\":{\"type\":\"long\"}}},\"extract_worker_pool\":{\"properties\":{\"total_completed\":{\"type\":\"long\"},\"size\":{\"type\":\"long\"},\"idle\":{\"type\":\"long\"},\"busy\":{\"type\":\"long\"},\"queue_depth\":{\"type\":\"long\"},\"total_scheduled\":{\"type\":\"long\"}}},\"subextract_worker_pool\":{\"properties\":{\"total_completed\":{\"type\":\"long\"},\"size\":{\"type\":\"long\"},\"idle\":{\"type\":\"long\"},\"busy\":{\"type\":\"long\"},\"queue_depth\":{\"type\":\"long\"},\"total_scheduled\":{\"type\":\"long\"}}}}},\"job_store\":{\"properties\":{\"waiting\":{\"type\":\"long\"},\"job_types\":{\"properties\":{\"permissions\":{\"type\":\"long\"},\"incremental\":{\"type\":\"long\"},\"delete\":{\"type\":\"long\"},\"full\":{\"type\":\"long\"}}},\"working\":{\"type\":\"long\"}}}}},\"queues\":{\"properties\":{\"engine_destroyer\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"process_crawl\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"failed\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"mailer\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"http\":{\"properties\":{\"request_duration\":{\"properties\":{\"max\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"std_dev\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"mean\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"responses\":{\"properties\":{\"1xx\":{\"type\":\"long\"},\"2xx\":{\"type\":\"long\"},\"3xx\":{\"type\":\"long\"},\"4xx\":{\"type\":\"long\"},\"5xx\":{\"type\":\"long\"}}},\"connections\":{\"properties\":{\"current\":{\"type\":\"long\"},\"total\":{\"type\":\"long\"},\"max\":{\"type\":\"long\"}}},\"network\":{\"properties\":{\"received\":{\"properties\":{\"bytes_per_sec\":{\"type\":\"long\"},\"bytes\":{\"type\":\"long\"}}},\"sent\":{\"properties\":{\"bytes_per_sec\":{\"type\":\"long\"},\"bytes\":{\"type\":\"long\"}}}}}}},\"product_usage\":{\"properties\":{\"app_search\":{\"properties\":{\"total_engines\":{\"type\":\"long\"}}},\"workplace_search\":{\"properties\":{\"total_org_sources\":{\"type\":\"long\"},\"total_private_sources\":{\"type\":\"long\"}}}}}}},\"health\":{\"properties\":{\"jvm\":{\"properties\":{\"memory_usage\":{\"properties\":{\"heap_committed\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"heap_used\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"non_heap_committed\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"non_heap_init\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"heap_init\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"heap_max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"object_pending_finalization_count\":{\"type\":\"long\"}}},\"threads\":{\"properties\":{\"current\":{\"type\":\"long\"},\"total_started\":{\"type\":\"long\"},\"max\":{\"type\":\"long\"},\"daemon\":{\"type\":\"long\"}}},\"gc\":{\"properties\":{\"collection_count\":{\"type\":\"long\"},\"collection_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"process\":{\"properties\":{\"pid\":{\"type\":\"long\"},\"filebeat\":{\"properties\":{\"restart_count\":{\"type\":\"long\"},\"pid\":{\"type\":\"long\"},\"time_since_last_restart\":{\"properties\":{\"sec\":{\"type\":\"long\"}}}}},\"uptime\":{\"properties\":{\"sec\":{\"type\":\"long\"}}}}},\"crawler\":{\"properties\":{\"workers\":{\"properties\":{\"pool_size\":{\"type\":\"long\"},\"available\":{\"type\":\"long\"},\"active\":{\"type\":\"long\"}}}}},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"properties\":{\"number\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"build_hash\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}}}},\"agent\":{\"properties\":{\"id\":{\"type\":\"keyword\"},\"ephemeral_id\":{\"type\":\"keyword\"},\"type\":{\"type\":\"keyword\"},\"version\":{\"type\":\"keyword\"}}},\"process\":{\"properties\":{\"pid\":{\"type\":\"long\"}}},\"@timestamp\":{\"type\":\"date\"},\"ecs\":{\"properties\":{\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"service\":{\"properties\":{\"node\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"hostname\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"environment\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"origin\":{\"properties\":{\"node\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"environment\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"target\":{\"properties\":{\"node\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"environment\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"host\":{\"properties\":{\"hostname\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"architecture\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"metricset\":{\"properties\":{\"period\":{\"type\":\"long\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"event\":{\"properties\":{\"reason\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"code\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"timezone\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"agent_id_status\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"duration\":{\"type\":\"long\"},\"reference\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ingested\":{\"type\":\"date\"},\"provider\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"action\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"end\":{\"type\":\"date\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"outcome\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"severity\":{\"type\":\"long\"},\"original\":{\"ignore_above\":1024,\"index\":false,\"type\":\"keyword\",\"doc_values\":false},\"risk_score\":{\"type\":\"float\"},\"created\":{\"type\":\"date\"},\"kind\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"module\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"start\":{\"type\":\"date\"},\"url\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"sequence\":{\"type\":\"long\"},\"risk_score_norm\":{\"type\":\"float\"},\"category\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"dataset\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"hash\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}}},\"composed_of\":[],\"version\":8000103,\"_meta\":{\"managed\":true,\"description\":\"Template used by Enterprise Search Metricbeat module monitoring information for Stack Monitoring\"},\"data_stream\":{\"hidden\":false,\"allow_custom_routing\":false}},\"synthetics\":{\"index_patterns\":[\"synthetics-*-*\"],\"composed_of\":[\"synthetics-mappings\",\"data-streams-mappings\",\"synthetics-settings\"],\"priority\":100,\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default synthetics template installed by x-pack\"},\"data_stream\":{\"hidden\":false,\"allow_custom_routing\":false},\"allow_auto_create\":true},\"ilm-history\":{\"index_patterns\":[\"ilm-history-5*\"],\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\"ilm-history-ilm-policy\"},\"number_of_shards\":\"1\",\"auto_expand_replicas\":\"0-1\",\"number_of_replicas\":\"0\"}},\"mappings\":{\"dynamic\":false,\"properties\":{\"index_age\":{\"type\":\"long\"},\"@timestamp\":{\"format\":\"epoch_millis\",\"type\":\"date\"},\"error_details\":{\"type\":\"text\"},\"success\":{\"type\":\"boolean\"},\"index\":{\"type\":\"keyword\"},\"state\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"phase\":{\"type\":\"keyword\"},\"failed_step\":{\"type\":\"keyword\"},\"phase_definition\":{\"type\":\"text\"},\"action_time\":{\"format\":\"epoch_millis\",\"type\":\"date\"},\"phase_time\":{\"format\":\"epoch_millis\",\"type\":\"date\"},\"step_info\":{\"type\":\"text\"},\"action\":{\"type\":\"keyword\"},\"step\":{\"type\":\"keyword\"},\"is_auto-retryable_error\":{\"type\":\"keyword\"},\"creation_date\":{\"format\":\"epoch_millis\",\"type\":\"date\"},\"step_time\":{\"format\":\"epoch_millis\",\"type\":\"date\"}}},\"policy\":{\"type\":\"keyword\"}}}},\"composed_of\":[],\"priority\":2147483647,\"version\":5,\"_meta\":{\"managed\":true,\"description\":\"index template for ILM history indices\"},\"data_stream\":{\"hidden\":true,\"allow_custom_routing\":false},\"allow_auto_create\":true},\".ml-state\":{\"index_patterns\":[\".ml-state*\"],\"template\":{\"settings\":{\"index\":{\"hidden\":\"true\",\"lifecycle\":{\"name\":\"ml-size-based-ilm-policy\",\"rollover_alias\":\".ml-state-write\"},\"auto_expand_replicas\":\"0-1\"}},\"mappings\":{\"_meta\":{\"version\":\"8060299\"},\"enabled\":false},\"aliases\":{}},\"composed_of\":[],\"priority\":2147483647,\"version\":8060299,\"_meta\":{\"managed\":true,\"description\":\"index template for ML state indices\"}},\".monitoring-es-mb\":{\"index_patterns\":[\".monitoring-es-8-*\"],\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\".monitoring-8-ilm-policy\"},\"mapping\":{\"total_fields\":{\"limit\":\"2000\"}}}},\"mappings\":{\"dynamic\":false,\"properties\":{\"ccr_auto_follow_stats\":{\"properties\":{\"number_of_failed_remote_cluster_state_requests\":{\"path\":\"elasticsearch.ccr.auto_follow.failed.remote_cluster_state_requests.count\",\"type\":\"alias\"},\"follower\":{\"properties\":{\"failed_read_requests\":{\"path\":\"elasticsearch.ccr.requests.failed.read.count\",\"type\":\"alias\"}}},\"number_of_failed_follow_indices\":{\"path\":\"elasticsearch.ccr.auto_follow.failed.follow_indices.count\",\"type\":\"alias\"},\"number_of_successful_follow_indices\":{\"path\":\"elasticsearch.ccr.auto_follow.success.follow_indices.count\",\"type\":\"alias\"}}},\"ccr_stats\":{\"properties\":{\"write_buffer_size_in_bytes\":{\"path\":\"elasticsearch.ccr.write_buffer.size.bytes\",\"type\":\"alias\"},\"leader_global_checkpoint\":{\"path\":\"elasticsearch.ccr.leader.global_checkpoint\",\"type\":\"alias\"},\"follower_index\":{\"path\":\"elasticsearch.ccr.follower.index\",\"type\":\"alias\"},\"leader_max_seq_no\":{\"path\":\"elasticsearch.ccr.leader.max_seq_no\",\"type\":\"alias\"},\"last_requested_seq_no\":{\"path\":\"elasticsearch.ccr.last_requested_seq_no\",\"type\":\"alias\"},\"follower_settings_version\":{\"path\":\"elasticsearch.ccr.follower.settings_version\",\"type\":\"alias\"},\"successful_write_requests\":{\"path\":\"elasticsearch.ccr.requests.successful.write.count\",\"type\":\"alias\"},\"remote_cluster\":{\"path\":\"elasticsearch.ccr.remote_cluster\",\"type\":\"alias\"},\"outstanding_write_requests\":{\"path\":\"elasticsearch.ccr.requests.outstanding.write.count\",\"type\":\"alias\"},\"total_read_time_millis\":{\"path\":\"elasticsearch.ccr.total_time.read.ms\",\"type\":\"alias\"},\"outstanding_read_requests\":{\"path\":\"elasticsearch.ccr.requests.outstanding.read.count\",\"type\":\"alias\"},\"total_write_time_millis\":{\"path\":\"elasticsearch.ccr.total_time.write.ms\",\"type\":\"alias\"},\"failed_write_requests\":{\"path\":\"elasticsearch.ccr.requests.failed.write.count\",\"type\":\"alias\"},\"failed_read_requests\":{\"path\":\"elasticsearch.ccr.requests.failed.read.count\",\"type\":\"alias\"},\"bytes_read\":{\"path\":\"elasticsearch.ccr.bytes_read\",\"type\":\"alias\"},\"leader_index\":{\"path\":\"elasticsearch.ccr.leader.index\",\"type\":\"alias\"},\"follower_max_seq_no\":{\"path\":\"elasticsearch.ccr.follower.max_seq_no\",\"type\":\"alias\"},\"operations_written\":{\"path\":\"elasticsearch.ccr.follower.operations_written\",\"type\":\"alias\"},\"write_buffer_operation_count\":{\"path\":\"elasticsearch.ccr.write_buffer.operation.count\",\"type\":\"alias\"},\"successful_read_requests\":{\"path\":\"elasticsearch.ccr.requests.successful.read.count\",\"type\":\"alias\"},\"shard_id\":{\"path\":\"elasticsearch.ccr.follower.shard.number\",\"type\":\"alias\"},\"follower_mapping_version\":{\"path\":\"elasticsearch.ccr.follower.mapping_version\",\"type\":\"alias\"},\"follower_aliases_version\":{\"path\":\"elasticsearch.ccr.follower.aliases_version\",\"type\":\"alias\"},\"follower_global_checkpoint\":{\"path\":\"elasticsearch.ccr.follower.global_checkpoint\",\"type\":\"alias\"},\"total_read_remote_exec_time_millis\":{\"path\":\"elasticsearch.ccr.total_time.read.remote_exec.ms\",\"type\":\"alias\"},\"operations_read\":{\"path\":\"elasticsearch.ccr.follower.operations.read.count\",\"type\":\"alias\"}}},\"cluster_state\":{\"properties\":{\"nodes_hash\":{\"path\":\"elasticsearch.cluster.stats.state.nodes_hash\",\"type\":\"alias\"},\"master_node\":{\"path\":\"elasticsearch.cluster.stats.state.master_node\",\"type\":\"alias\"},\"state_uuid\":{\"path\":\"elasticsearch.cluster.stats.state.state_uuid\",\"type\":\"alias\"},\"version\":{\"path\":\"elasticsearch.cluster.stats.state.version\",\"type\":\"alias\"},\"status\":{\"path\":\"elasticsearch.cluster.stats.status\",\"type\":\"alias\"}}},\"indices_stats\":{\"properties\":{\"_all\":{\"properties\":{\"primaries\":{\"properties\":{\"indexing\":{\"properties\":{\"index_time_in_millis\":{\"path\":\"elasticsearch.index.summary.primaries.indexing.index.time.ms\",\"type\":\"alias\"},\"index_total\":{\"path\":\"elasticsearch.index.summary.primaries.indexing.index.count\",\"type\":\"alias\"}}}}},\"total\":{\"properties\":{\"search\":{\"properties\":{\"query_total\":{\"path\":\"elasticsearch.index.summary.total.search.query.count\",\"type\":\"alias\"},\"query_time_in_millis\":{\"path\":\"elasticsearch.index.summary.total.search.query.time.ms\",\"type\":\"alias\"}}},\"indexing\":{\"properties\":{\"index_total\":{\"path\":\"elasticsearch.index.summary.total.indexing.index.count\",\"type\":\"alias\"}}}}}}}}},\"node_stats\":{\"properties\":{\"jvm\":{\"properties\":{\"mem\":{\"properties\":{\"heap_used_percent\":{\"path\":\"elasticsearch.node.stats.jvm.mem.heap.used.pct\",\"type\":\"alias\"},\"heap_max_in_bytes\":{\"path\":\"elasticsearch.node.stats.jvm.mem.heap.max.bytes\",\"type\":\"alias\"},\"heap_used_in_bytes\":{\"path\":\"elasticsearch.node.stats.jvm.mem.heap.used.bytes\",\"type\":\"alias\"}}},\"gc\":{\"properties\":{\"collectors\":{\"properties\":{\"young\":{\"properties\":{\"collection_count\":{\"path\":\"elasticsearch.node.stats.jvm.gc.collectors.young.collection.count\",\"type\":\"alias\"},\"collection_time_in_millis\":{\"path\":\"elasticsearch.node.stats.jvm.gc.collectors.young.collection.ms\",\"type\":\"alias\"}}},\"old\":{\"properties\":{\"collection_count\":{\"path\":\"elasticsearch.node.stats.jvm.gc.collectors.old.collection.count\",\"type\":\"alias\"},\"collection_time_in_millis\":{\"path\":\"elasticsearch.node.stats.jvm.gc.collectors.old.collection.ms\",\"type\":\"alias\"}}}}}}}}},\"indices\":{\"properties\":{\"search\":{\"properties\":{\"query_total\":{\"path\":\"elasticsearch.node.stats.indices.search.query_total.count\",\"type\":\"alias\"},\"query_time_in_millis\":{\"path\":\"elasticsearch.node.stats.indices.search.query_time.ms\",\"type\":\"alias\"}}},\"query_cache\":{\"properties\":{\"memory_size_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.query_cache.memory.bytes\",\"type\":\"alias\"}}},\"docs\":{\"properties\":{\"count\":{\"path\":\"elasticsearch.node.stats.indices.docs.count\",\"type\":\"alias\"}}},\"indexing\":{\"properties\":{\"throttle_time_in_millis\":{\"path\":\"elasticsearch.node.stats.indices.indexing.throttle_time.ms\",\"type\":\"alias\"},\"index_time_in_millis\":{\"path\":\"elasticsearch.node.stats.indices.indexing.index_time.ms\",\"type\":\"alias\"},\"index_total\":{\"path\":\"elasticsearch.node.stats.indices.indexing.index_total.count\",\"type\":\"alias\"}}},\"fielddata\":{\"properties\":{\"memory_size_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.fielddata.memory.bytes\",\"type\":\"alias\"}}},\"store\":{\"properties\":{\"size\":{\"properties\":{\"bytes\":{\"path\":\"elasticsearch.node.stats.indices.store.size.bytes\",\"type\":\"alias\"}}},\"size_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.store.size.bytes\",\"type\":\"alias\"}}},\"request_cache\":{\"properties\":{\"memory_size_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.request_cache.memory.bytes\",\"type\":\"alias\"}}},\"segments\":{\"properties\":{\"version_map_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.version_map.memory.bytes\",\"type\":\"alias\"},\"norms_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.norms.memory.bytes\",\"type\":\"alias\"},\"count\":{\"path\":\"elasticsearch.node.stats.indices.segments.count\",\"type\":\"alias\"},\"term_vectors_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.term_vectors.memory.bytes\",\"type\":\"alias\"},\"points_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.points.memory.bytes\",\"type\":\"alias\"},\"index_writer_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.index_writer.memory.bytes\",\"type\":\"alias\"},\"memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.memory.bytes\",\"type\":\"alias\"},\"doc_values_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.doc_values.memory.bytes\",\"type\":\"alias\"},\"terms_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.terms.memory.bytes\",\"type\":\"alias\"},\"fixed_bit_set_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.fixed_bit_set.memory.bytes\",\"type\":\"alias\"},\"stored_fields_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.stored_fields.memory.bytes\",\"type\":\"alias\"}}}}},\"process\":{\"properties\":{\"cpu\":{\"properties\":{\"percent\":{\"path\":\"elasticsearch.node.stats.process.cpu.pct\",\"type\":\"alias\"}}}}},\"os\":{\"properties\":{\"cpu\":{\"properties\":{\"load_average\":{\"properties\":{\"1m\":{\"path\":\"elasticsearch.node.stats.os.cpu.load_avg.1m\",\"type\":\"alias\"}}}}},\"cgroup\":{\"properties\":{\"memory\":{\"properties\":{\"usage_in_bytes\":{\"path\":\"elasticsearch.node.stats.os.cgroup.memory.usage.bytes\",\"type\":\"alias\"},\"control_group\":{\"path\":\"elasticsearch.node.stats.os.cgroup.memory.control_group\",\"type\":\"alias\"},\"limit_in_bytes\":{\"path\":\"elasticsearch.node.stats.os.cgroup.memory.limit.bytes\",\"type\":\"alias\"}}},\"cpu\":{\"properties\":{\"stat\":{\"properties\":{\"number_of_elapsed_periods\":{\"path\":\"elasticsearch.node.stats.os.cgroup.cpu.stat.elapsed_periods.count\",\"type\":\"alias\"},\"number_of_times_throttled\":{\"path\":\"elasticsearch.node.stats.os.cgroup.cpu.stat.times_throttled.count\",\"type\":\"alias\"},\"time_throttled_nanos\":{\"path\":\"elasticsearch.node.stats.os.cgroup.cpu.stat.time_throttled.ns\",\"type\":\"alias\"}}},\"cfs_quota_micros\":{\"path\":\"elasticsearch.node.stats.os.cgroup.cpu.cfs.quota.us\",\"type\":\"alias\"}}},\"cpuacct\":{\"properties\":{\"usage_nanos\":{\"path\":\"elasticsearch.node.stats.os.cgroup.cpuacct.usage.ns\",\"type\":\"alias\"}}}}}}},\"thread_pool\":{\"properties\":{\"search\":{\"properties\":{\"rejected\":{\"path\":\"elasticsearch.node.stats.thread_pool.search.rejected.count\",\"type\":\"alias\"},\"queue\":{\"path\":\"elasticsearch.node.stats.thread_pool.search.queue.count\",\"type\":\"alias\"}}},\"get\":{\"properties\":{\"rejected\":{\"path\":\"elasticsearch.node.stats.thread_pool.get.rejected.count\",\"type\":\"alias\"},\"queue\":{\"path\":\"elasticsearch.node.stats.thread_pool.get.queue.count\",\"type\":\"alias\"}}},\"index\":{\"properties\":{\"rejected\":{\"path\":\"elasticsearch.node.stats.thread_pool.index.rejected.count\",\"type\":\"alias\"},\"queue\":{\"path\":\"elasticsearch.node.stats.thread_pool.index.queue.count\",\"type\":\"alias\"}}},\"bulk\":{\"properties\":{\"rejected\":{\"path\":\"elasticsearch.node.stats.thread_pool.bulk.rejected.count\",\"type\":\"alias\"},\"queue\":{\"path\":\"elasticsearch.node.stats.thread_pool.bulk.queue.count\",\"type\":\"alias\"}}},\"write\":{\"properties\":{\"rejected\":{\"path\":\"elasticsearch.node.stats.thread_pool.write.rejected.count\",\"type\":\"alias\"},\"queue\":{\"path\":\"elasticsearch.node.stats.thread_pool.write.queue.count\",\"type\":\"alias\"}}}}},\"fs\":{\"properties\":{\"summary\":{\"properties\":{\"total\":{\"properties\":{\"bytes\":{\"path\":\"elasticsearch.node.stats.fs.summary.total.bytes\",\"type\":\"alias\"}}},\"available\":{\"properties\":{\"bytes\":{\"path\":\"elasticsearch.node.stats.fs.summary.available.bytes\",\"type\":\"alias\"}}}}},\"io_stats\":{\"properties\":{\"total\":{\"properties\":{\"write_operations\":{\"path\":\"elasticsearch.node.stats.fs.io_stats.total.write.operations.count\",\"type\":\"alias\"},\"operations\":{\"path\":\"elasticsearch.node.stats.fs.io_stats.total.operations.count\",\"type\":\"alias\"},\"read_operations\":{\"path\":\"elasticsearch.node.stats.fs.io_stats.total.read.operations.count\",\"type\":\"alias\"}}}}},\"total\":{\"properties\":{\"total_in_bytes\":{\"path\":\"elasticsearch.node.stats.fs.summary.total.bytes\",\"type\":\"alias\"},\"available_in_bytes\":{\"path\":\"elasticsearch.node.stats.fs.summary.available.bytes\",\"type\":\"alias\"}}}}},\"node_id\":{\"path\":\"elasticsearch.node.id\",\"type\":\"alias\"}}},\"stack_stats\":{\"properties\":{\"xpack\":{\"properties\":{\"ccr\":{\"properties\":{\"available\":{\"path\":\"elasticsearch.cluster.stats.stack.xpack.ccr.available\",\"type\":\"alias\"},\"enabled\":{\"path\":\"elasticsearch.cluster.stats.stack.xpack.ccr.enabled\",\"type\":\"alias\"}}}}},\"apm\":{\"properties\":{\"found\":{\"path\":\"elasticsearch.cluster.stats.stack.apm.found\",\"type\":\"alias\"}}}}},\"license\":{\"properties\":{\"type\":{\"path\":\"elasticsearch.cluster.stats.license.type\",\"type\":\"alias\"},\"status\":{\"path\":\"elasticsearch.cluster.stats.license.status\",\"type\":\"alias\"}}},\"index_stats\":{\"properties\":{\"primaries\":{\"properties\":{\"docs\":{\"properties\":{\"count\":{\"path\":\"elasticsearch.index.primaries.docs.count\",\"type\":\"alias\"}}},\"indexing\":{\"properties\":{\"throttle_time_in_millis\":{\"path\":\"elasticsearch.index.primaries.indexing.throttle_time_in_millis\",\"type\":\"alias\"},\"index_time_in_millis\":{\"path\":\"elasticsearch.index.primaries.indexing.index_time_in_millis\",\"type\":\"alias\"},\"index_total\":{\"path\":\"elasticsearch.index.primaries.indexing.index_total\",\"type\":\"alias\"}}},\"refresh\":{\"properties\":{\"total_time_in_millis\":{\"path\":\"elasticsearch.index.primaries.refresh.total_time_in_millis\",\"type\":\"alias\"}}},\"store\":{\"properties\":{\"size_in_bytes\":{\"path\":\"elasticsearch.index.primaries.store.size_in_bytes\",\"type\":\"alias\"}}},\"merges\":{\"properties\":{\"total_size_in_bytes\":{\"path\":\"elasticsearch.index.primaries.merges.total_size_in_bytes\",\"type\":\"alias\"}}},\"segments\":{\"properties\":{\"count\":{\"path\":\"elasticsearch.index.primaries.segments.count\",\"type\":\"alias\"}}}}},\"total\":{\"properties\":{\"search\":{\"properties\":{\"query_total\":{\"path\":\"elasticsearch.index.total.search.query_total\",\"type\":\"alias\"},\"query_time_in_millis\":{\"path\":\"elasticsearch.index.total.search.query_time_in_millis\",\"type\":\"alias\"}}},\"query_cache\":{\"properties\":{\"memory_size_in_bytes\":{\"path\":\"elasticsearch.index.total.query_cache.memory_size_in_bytes\",\"type\":\"alias\"}}},\"fielddata\":{\"properties\":{\"memory_size_in_bytes\":{\"path\":\"elasticsearch.index.total.fielddata.memory_size_in_bytes\",\"type\":\"alias\"}}},\"indexing\":{\"properties\":{\"throttle_time_in_millis\":{\"path\":\"elasticsearch.index.total.indexing.throttle_time_in_millis\",\"type\":\"alias\"},\"index_time_in_millis\":{\"path\":\"elasticsearch.index.total.indexing.index_time_in_millis\",\"type\":\"alias\"},\"index_total\":{\"path\":\"elasticsearch.index.total.indexing.index_total\",\"type\":\"alias\"}}},\"refresh\":{\"properties\":{\"total_time_in_millis\":{\"path\":\"elasticsearch.index.total.refresh.total_time_in_millis\",\"type\":\"alias\"}}},\"request_cache\":{\"properties\":{\"memory_size_in_bytes\":{\"path\":\"elasticsearch.index.total.request_cache.memory_size_in_bytes\",\"type\":\"alias\"}}},\"store\":{\"properties\":{\"size_in_bytes\":{\"path\":\"elasticsearch.index.total.store.size_in_bytes\",\"type\":\"alias\"}}},\"merges\":{\"properties\":{\"total_size_in_bytes\":{\"path\":\"elasticsearch.index.total.merges.total_size_in_bytes\",\"type\":\"alias\"}}},\"segments\":{\"properties\":{\"version_map_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.version_map_memory_in_bytes\",\"type\":\"alias\"},\"norms_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.norms_memory_in_bytes\",\"type\":\"alias\"},\"count\":{\"path\":\"elasticsearch.index.total.segments.count\",\"type\":\"alias\"},\"term_vectors_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.term_vectors_memory_in_bytes\",\"type\":\"alias\"},\"points_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.points_memory_in_bytes\",\"type\":\"alias\"},\"index_writer_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.index_writer_memory_in_bytes\",\"type\":\"alias\"},\"memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.memory_in_bytes\",\"type\":\"alias\"},\"doc_values_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.doc_values_memory_in_bytes\",\"type\":\"alias\"},\"terms_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.terms_memory_in_bytes\",\"type\":\"alias\"},\"fixed_bit_set_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.fixed_bit_set_memory_in_bytes\",\"type\":\"alias\"},\"stored_fields_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.stored_fields_memory_in_bytes\",\"type\":\"alias\"}}}}},\"index\":{\"path\":\"elasticsearch.index.name\",\"type\":\"alias\"}}},\"@timestamp\":{\"type\":\"date\"},\"elasticsearch\":{\"properties\":{\"cluster\":{\"properties\":{\"stats\":{\"properties\":{\"license\":{\"properties\":{\"expiry_date_in_millis\":{\"type\":\"long\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"status\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"indices\":{\"properties\":{\"shards\":{\"properties\":{\"primaries\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"total\":{\"type\":\"long\"},\"docs\":{\"properties\":{\"total\":{\"type\":\"long\"}}},\"fielddata\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"store\":{\"properties\":{\"size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}}}},\"stack\":{\"properties\":{\"xpack\":{\"properties\":{\"ccr\":{\"properties\":{\"available\":{\"type\":\"boolean\"},\"enabled\":{\"type\":\"boolean\"}}}}},\"apm\":{\"properties\":{\"found\":{\"type\":\"boolean\"}}}}},\"nodes\":{\"properties\":{\"jvm\":{\"properties\":{\"memory\":{\"properties\":{\"heap\":{\"properties\":{\"max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"used\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}}}},\"max_uptime\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"data\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"},\"fs\":{\"properties\":{\"total\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"available\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"master\":{\"type\":\"long\"}}},\"state\":{\"properties\":{\"nodes_hash\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"master_node\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state_uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"status\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"properties\":{\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"node\":{\"properties\":{\"jvm\":{\"properties\":{\"memory\":{\"properties\":{\"heap\":{\"properties\":{\"init\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"nonheap\":{\"properties\":{\"init\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}}}},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"process\":{\"properties\":{\"mlockall\":{\"type\":\"boolean\"}}},\"stats\":{\"properties\":{\"jvm\":{\"properties\":{\"mem\":{\"properties\":{\"pools\":{\"properties\":{\"young\":{\"properties\":{\"max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"peak\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"peak_max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"used\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"old\":{\"properties\":{\"max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"peak\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"peak_max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"used\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"survivor\":{\"properties\":{\"max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"peak\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"peak_max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"used\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}}}},\"heap\":{\"properties\":{\"max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"used\":{\"properties\":{\"pct\":{\"type\":\"double\"},\"bytes\":{\"type\":\"long\"}}}}}}},\"gc\":{\"properties\":{\"collectors\":{\"properties\":{\"young\":{\"properties\":{\"collection\":{\"properties\":{\"ms\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}}}},\"old\":{\"properties\":{\"collection\":{\"properties\":{\"ms\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}}}}}}}}}},\"indices\":{\"properties\":{\"search\":{\"properties\":{\"query_total\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"query_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"query_cache\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"docs\":{\"properties\":{\"deleted\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"fielddata\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"indexing\":{\"properties\":{\"index_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"index_total\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"throttle_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"store\":{\"properties\":{\"size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"request_cache\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"bulk\":{\"properties\":{\"operations\":{\"properties\":{\"total\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"total_size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"avg_size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"total_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"avg_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"segments\":{\"properties\":{\"version_map\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"norms\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"terms\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"stored_fields\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"index_writer\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"count\":{\"type\":\"long\"},\"fixed_bit_set\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"term_vectors\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"doc_values\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"points\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}}}}}},\"process\":{\"properties\":{\"cpu\":{\"properties\":{\"pct\":{\"type\":\"double\"}}}}},\"os\":{\"properties\":{\"cpu\":{\"properties\":{\"load_avg\":{\"properties\":{\"1m\":{\"type\":\"half_float\"}}}}},\"cgroup\":{\"properties\":{\"memory\":{\"properties\":{\"control_group\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"usage\":{\"properties\":{\"bytes\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"limit\":{\"properties\":{\"bytes\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"cpu\":{\"properties\":{\"cfs\":{\"properties\":{\"quota\":{\"properties\":{\"us\":{\"type\":\"long\"}}}}},\"stat\":{\"properties\":{\"elapsed_periods\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"times_throttled\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"time_throttled\":{\"properties\":{\"ns\":{\"type\":\"long\"}}}}}}},\"cpuacct\":{\"properties\":{\"usage\":{\"properties\":{\"ns\":{\"type\":\"long\"}}}}}}}}},\"thread_pool\":{\"properties\":{\"search\":{\"properties\":{\"rejected\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"queue\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"get\":{\"properties\":{\"rejected\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"queue\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"index\":{\"properties\":{\"rejected\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"queue\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"bulk\":{\"properties\":{\"rejected\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"queue\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"write\":{\"properties\":{\"rejected\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"queue\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}}}},\"fs\":{\"properties\":{\"summary\":{\"properties\":{\"total\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"available\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"free\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"io_stats\":{\"properties\":{\"total\":{\"properties\":{\"operations\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"read\":{\"properties\":{\"operations\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"write\":{\"properties\":{\"operations\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}}}}}},\"total\":{\"properties\":{\"total_in_bytes\":{\"type\":\"long\"},\"available_in_bytes\":{\"type\":\"long\"}}}}}}},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"mlockall\":{\"type\":\"boolean\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"master\":{\"type\":\"boolean\"}}},\"ccr\":{\"properties\":{\"leader\":{\"properties\":{\"index\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"max_seq_no\":{\"type\":\"long\"},\"global_checkpoint\":{\"type\":\"long\"}}},\"follower\":{\"properties\":{\"operations\":{\"properties\":{\"read\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"time_since_last_read\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"index\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"settings_version\":{\"type\":\"long\"},\"shard\":{\"properties\":{\"number\":{\"type\":\"long\"}}},\"max_seq_no\":{\"type\":\"long\"},\"mapping_version\":{\"type\":\"long\"},\"aliases_version\":{\"type\":\"long\"},\"operations_written\":{\"type\":\"long\"},\"global_checkpoint\":{\"type\":\"long\"}}},\"remote_cluster\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"read_exceptions\":{\"type\":\"nested\"},\"shard_id\":{\"type\":\"long\"},\"bytes_read\":{\"type\":\"long\"},\"requests\":{\"properties\":{\"outstanding\":{\"properties\":{\"read\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"write\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"failed\":{\"properties\":{\"read\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"write\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"successful\":{\"properties\":{\"read\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"write\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}}}},\"auto_follow\":{\"properties\":{\"success\":{\"properties\":{\"follow_indices\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"failed\":{\"properties\":{\"follow_indices\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"remote_cluster_state_requests\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}}}},\"total_time\":{\"properties\":{\"read\":{\"properties\":{\"ms\":{\"type\":\"long\"},\"remote_exec\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"write\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"last_requested_seq_no\":{\"type\":\"long\"},\"write_buffer\":{\"properties\":{\"size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"operation\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}}}},\"pending_tasks\":{\"properties\":{\"time_in_queue\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"source\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"priority\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"insert_order\":{\"type\":\"long\"}}},\"enrich\":{\"properties\":{\"executed_searches\":{\"properties\":{\"total\":{\"type\":\"long\"}}},\"remote_requests\":{\"properties\":{\"current\":{\"type\":\"long\"},\"total\":{\"type\":\"long\"}}},\"executing_policy\":{\"properties\":{\"task\":{\"properties\":{\"task\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"parent_task_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"action\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"type\":\"long\"},\"time\":{\"properties\":{\"running\":{\"properties\":{\"nano\":{\"type\":\"long\"}}},\"start\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"cancellable\":{\"type\":\"boolean\"}}},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"queue\":{\"properties\":{\"size\":{\"type\":\"long\"}}}}},\"index\":{\"properties\":{\"summary\":{\"properties\":{\"primaries\":{\"properties\":{\"search\":{\"properties\":{\"query\":{\"properties\":{\"count\":{\"type\":\"long\"},\"time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}}}},\"docs\":{\"properties\":{\"deleted\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"indexing\":{\"properties\":{\"index\":{\"properties\":{\"count\":{\"type\":\"long\"},\"time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}}}},\"store\":{\"properties\":{\"size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"bulk\":{\"properties\":{\"operations\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"time\":{\"properties\":{\"avg\":{\"properties\":{\"ms\":{\"type\":\"long\"},\"bytes\":{\"type\":\"long\"}}},\"count\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}}}},\"segments\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"count\":{\"type\":\"long\"}}}}},\"total\":{\"properties\":{\"search\":{\"properties\":{\"query\":{\"properties\":{\"count\":{\"type\":\"long\"},\"time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}}}},\"docs\":{\"properties\":{\"deleted\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"indexing\":{\"properties\":{\"index\":{\"properties\":{\"count\":{\"type\":\"long\"},\"time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"is_throttled\":{\"type\":\"boolean\"},\"throttle_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"store\":{\"properties\":{\"size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"bulk\":{\"properties\":{\"operations\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"time\":{\"properties\":{\"avg\":{\"properties\":{\"ms\":{\"type\":\"long\"},\"bytes\":{\"type\":\"long\"}}}}}}},\"segments\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"count\":{\"type\":\"long\"}}}}}}},\"shards\":{\"properties\":{\"primaries\":{\"type\":\"long\"},\"total\":{\"type\":\"long\"}}},\"primaries\":{\"properties\":{\"search\":{\"properties\":{\"query_total\":{\"type\":\"long\"},\"query_time_in_millis\":{\"type\":\"long\"}}},\"query_cache\":{\"properties\":{\"miss_count\":{\"type\":\"long\"},\"memory_size_in_bytes\":{\"type\":\"long\"},\"hit_count\":{\"type\":\"long\"}}},\"docs\":{\"properties\":{\"deleted\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"indexing\":{\"properties\":{\"throttle_time_in_millis\":{\"type\":\"long\"},\"index_time_in_millis\":{\"type\":\"long\"},\"index_total\":{\"type\":\"long\"}}},\"refresh\":{\"properties\":{\"external_total_time_in_millis\":{\"type\":\"long\"},\"total_time_in_millis\":{\"type\":\"long\"}}},\"request_cache\":{\"properties\":{\"miss_count\":{\"type\":\"long\"},\"memory_size_in_bytes\":{\"type\":\"long\"},\"evictions\":{\"type\":\"long\"},\"hit_count\":{\"type\":\"long\"}}},\"store\":{\"properties\":{\"size_in_bytes\":{\"type\":\"long\"}}},\"segments\":{\"properties\":{\"version_map_memory_in_bytes\":{\"type\":\"long\"},\"norms_memory_in_bytes\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"},\"term_vectors_memory_in_bytes\":{\"type\":\"long\"},\"points_memory_in_bytes\":{\"type\":\"long\"},\"index_writer_memory_in_bytes\":{\"type\":\"long\"},\"memory_in_bytes\":{\"type\":\"long\"},\"terms_memory_in_bytes\":{\"type\":\"long\"},\"doc_values_memory_in_bytes\":{\"type\":\"long\"},\"stored_fields_memory_in_bytes\":{\"type\":\"long\"},\"fixed_bit_set_memory_in_bytes\":{\"type\":\"long\"}}},\"merges\":{\"properties\":{\"total_size_in_bytes\":{\"type\":\"long\"}}}}},\"total\":{\"properties\":{\"search\":{\"properties\":{\"query_total\":{\"type\":\"long\"},\"query_time_in_millis\":{\"type\":\"long\"}}},\"query_cache\":{\"properties\":{\"miss_count\":{\"type\":\"long\"},\"memory_size_in_bytes\":{\"type\":\"long\"},\"evictions\":{\"type\":\"long\"},\"hit_count\":{\"type\":\"long\"}}},\"docs\":{\"properties\":{\"deleted\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"fielddata\":{\"properties\":{\"memory_size_in_bytes\":{\"type\":\"long\"},\"evictions\":{\"type\":\"long\"}}},\"indexing\":{\"properties\":{\"throttle_time_in_millis\":{\"type\":\"long\"},\"index_time_in_millis\":{\"type\":\"long\"},\"index_total\":{\"type\":\"long\"}}},\"refresh\":{\"properties\":{\"external_total_time_in_millis\":{\"type\":\"long\"},\"total_time_in_millis\":{\"type\":\"long\"}}},\"store\":{\"properties\":{\"size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"size_in_bytes\":{\"type\":\"long\"}}},\"request_cache\":{\"properties\":{\"miss_count\":{\"type\":\"long\"},\"memory_size_in_bytes\":{\"type\":\"long\"},\"evictions\":{\"type\":\"long\"},\"hit_count\":{\"type\":\"long\"}}},\"bulk\":{\"properties\":{\"avg_time_in_millis\":{\"type\":\"long\"},\"avg_size_in_bytes\":{\"type\":\"long\"},\"total_time_in_millis\":{\"type\":\"long\"},\"total_operations\":{\"type\":\"long\"},\"total_size_in_bytes\":{\"type\":\"long\"}}},\"merges\":{\"properties\":{\"total_size_in_bytes\":{\"type\":\"long\"}}},\"segments\":{\"properties\":{\"version_map_memory_in_bytes\":{\"type\":\"long\"},\"norms_memory_in_bytes\":{\"type\":\"long\"},\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"count\":{\"type\":\"long\"},\"term_vectors_memory_in_bytes\":{\"type\":\"long\"},\"points_memory_in_bytes\":{\"type\":\"long\"},\"index_writer_memory_in_bytes\":{\"type\":\"long\"},\"memory_in_bytes\":{\"type\":\"long\"},\"terms_memory_in_bytes\":{\"type\":\"long\"},\"doc_values_memory_in_bytes\":{\"type\":\"long\"},\"stored_fields_memory_in_bytes\":{\"type\":\"long\"},\"fixed_bit_set_memory_in_bytes\":{\"type\":\"long\"}}}}},\"hidden\":{\"type\":\"boolean\"},\"created\":{\"type\":\"long\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"recovery\":{\"properties\":{\"stop_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"translog\":{\"properties\":{\"total_on_start\":{\"type\":\"long\"},\"total\":{\"type\":\"long\"},\"percent\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"verify_index\":{\"properties\":{\"total_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"check_index_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"index\":{\"properties\":{\"size\":{\"properties\":{\"total_in_bytes\":{\"type\":\"long\"},\"reused_in_bytes\":{\"type\":\"long\"},\"recovered_in_bytes\":{\"type\":\"long\"}}},\"files\":{\"properties\":{\"recovered\":{\"type\":\"long\"},\"total\":{\"type\":\"long\"},\"percent\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"reused\":{\"type\":\"long\"}}}}},\"source\":{\"properties\":{\"transport_address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"host\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"target\":{\"properties\":{\"transport_address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"host\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"start_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"stage\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"type\":\"long\"},\"total_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"primary\":{\"type\":\"boolean\"}}},\"uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"status\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"shard\":{\"properties\":{\"number\":{\"type\":\"long\"},\"source_node\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"relocating_node\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"primary\":{\"type\":\"boolean\"}}},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ml\":{\"properties\":{\"job\":{\"properties\":{\"data\":{\"properties\":{\"invalid_date\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"data_counts\":{\"properties\":{\"invalid_date_count\":{\"type\":\"long\"},\"processed_record_count\":{\"type\":\"long\"}}},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"forecasts_stats\":{\"properties\":{\"total\":{\"type\":\"long\"}}},\"model_size\":{\"properties\":{\"memory_status\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}}}}}},\"cluster_uuid\":{\"path\":\"elasticsearch.cluster.id\",\"type\":\"alias\"},\"ecs\":{\"properties\":{\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"service\":{\"properties\":{\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"index_recovery\":{\"properties\":{\"shards\":{\"properties\":{\"stop_time_in_millis\":{\"path\":\"elasticsearch.index.recovery.stop_time.ms\",\"type\":\"alias\"},\"start_time_in_millis\":{\"path\":\"elasticsearch.index.recovery.start_time.ms\",\"type\":\"alias\"}}}}},\"source_node\":{\"properties\":{\"name\":{\"path\":\"elasticsearch.node.name\",\"type\":\"alias\"},\"uuid\":{\"path\":\"elasticsearch.node.id\",\"type\":\"alias\"}}},\"host\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"metricset\":{\"properties\":{\"period\":{\"type\":\"long\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"shard\":{\"properties\":{\"node\":{\"path\":\"elasticsearch.node.id\",\"type\":\"alias\"},\"index\":{\"path\":\"elasticsearch.index.name\",\"type\":\"alias\"},\"state\":{\"path\":\"elasticsearch.shard.state\",\"type\":\"alias\"},\"shard\":{\"path\":\"elasticsearch.shard.number\",\"type\":\"alias\"},\"primary\":{\"path\":\"elasticsearch.shard.primary\",\"type\":\"alias\"}}},\"event\":{\"properties\":{\"duration\":{\"type\":\"long\"},\"module\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"dataset\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"cluster_stats\":{\"properties\":{\"indices\":{\"properties\":{\"shards\":{\"properties\":{\"total\":{\"path\":\"elasticsearch.cluster.stats.indices.shards.count\",\"type\":\"alias\"}}},\"count\":{\"path\":\"elasticsearch.cluster.stats.indices.total\",\"type\":\"alias\"}}},\"nodes\":{\"properties\":{\"jvm\":{\"properties\":{\"max_uptime_in_millis\":{\"path\":\"elasticsearch.cluster.stats.nodes.jvm.max_uptime.ms\",\"type\":\"alias\"},\"mem\":{\"properties\":{\"heap_max_in_bytes\":{\"path\":\"elasticsearch.cluster.stats.nodes.jvm.memory.heap.max.bytes\",\"type\":\"alias\"},\"heap_used_in_bytes\":{\"path\":\"elasticsearch.cluster.stats.nodes.jvm.memory.heap.used.bytes\",\"type\":\"alias\"}}}}},\"count\":{\"properties\":{\"total\":{\"path\":\"elasticsearch.cluster.stats.nodes.count\",\"type\":\"alias\"}}}}}}},\"timestamp\":{\"path\":\"@timestamp\",\"type\":\"alias\"},\"job_stats\":{\"properties\":{\"job_id\":{\"path\":\"elasticsearch.ml.job.id\",\"type\":\"alias\"},\"forecasts_stats\":{\"properties\":{\"total\":{\"path\":\"elasticsearch.ml.job.forecasts_stats.total\",\"type\":\"alias\"}}}}}}}},\"composed_of\":[],\"version\":8000103,\"data_stream\":{\"hidden\":false,\"allow_custom_routing\":false}},\".slm-history\":{\"index_patterns\":[\".slm-history-5*\"],\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\"slm-history-ilm-policy\"},\"number_of_shards\":\"1\",\"auto_expand_replicas\":\"0-1\",\"number_of_replicas\":\"0\"}},\"mappings\":{\"dynamic\":false,\"properties\":{\"snapshot_name\":{\"type\":\"keyword\"},\"@timestamp\":{\"format\":\"epoch_millis\",\"type\":\"date\"},\"configuration\":{\"dynamic\":false,\"type\":\"object\",\"properties\":{\"indices\":{\"type\":\"keyword\"},\"include_global_state\":{\"type\":\"boolean\"},\"partial\":{\"type\":\"boolean\"}}},\"error_details\":{\"index\":false,\"type\":\"text\"},\"success\":{\"type\":\"boolean\"},\"repository\":{\"type\":\"keyword\"},\"operation\":{\"type\":\"keyword\"},\"policy\":{\"type\":\"keyword\"}}}},\"composed_of\":[],\"priority\":2147483647,\"version\":5,\"_meta\":{\"managed\":true,\"description\":\"index template for SLM history indices\"},\"data_stream\":{\"hidden\":true,\"allow_custom_routing\":false},\"allow_auto_create\":true},\".monitoring-logstash-mb\":{\"index_patterns\":[\".monitoring-logstash-8-*\"],\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\".monitoring-8-ilm-policy\"},\"mapping\":{\"total_fields\":{\"limit\":\"2000\"}}}},\"mappings\":{\"dynamic\":false,\"properties\":{\"logstash\":{\"properties\":{\"node\":{\"properties\":{\"jvm\":{\"properties\":{\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"stats\":{\"properties\":{\"jvm\":{\"properties\":{\"mem\":{\"properties\":{\"heap_max_in_bytes\":{\"type\":\"long\"},\"heap_used_in_bytes\":{\"type\":\"long\"}}},\"uptime_in_millis\":{\"type\":\"long\"}}},\"logstash\":{\"properties\":{\"uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"process\":{\"properties\":{\"cpu\":{\"properties\":{\"percent\":{\"type\":\"double\"}}}}},\"pipelines\":{\"type\":\"nested\",\"properties\":{\"vertices\":{\"type\":\"nested\",\"properties\":{\"events_out\":{\"type\":\"long\"},\"pipeline_ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"events_in\":{\"type\":\"long\"},\"queue_push_duration_in_millis\":{\"type\":\"long\"},\"duration_in_millis\":{\"type\":\"long\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"long_counters\":{\"type\":\"nested\",\"properties\":{\"name\":{\"type\":\"keyword\"},\"value\":{\"type\":\"long\"}}}}},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"type\":\"keyword\"},\"hash\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"queue\":{\"properties\":{\"events_count\":{\"type\":\"long\"},\"max_queue_size_in_bytes\":{\"type\":\"long\"},\"queue_size_in_bytes\":{\"type\":\"long\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"events\":{\"properties\":{\"filtered\":{\"type\":\"long\"},\"in\":{\"type\":\"long\"},\"queue_push_duration_in_millis\":{\"type\":\"long\"},\"duration_in_millis\":{\"type\":\"long\"},\"out\":{\"type\":\"long\"}}},\"reloads\":{\"properties\":{\"failures\":{\"type\":\"long\"},\"successes\":{\"type\":\"long\"}}}}},\"os\":{\"properties\":{\"cpu\":{\"properties\":{\"load_average\":{\"properties\":{\"5m\":{\"type\":\"half_float\"},\"15m\":{\"type\":\"half_float\"},\"1m\":{\"type\":\"half_float\"}}}}},\"cgroup\":{\"properties\":{\"cpu\":{\"properties\":{\"stat\":{\"properties\":{\"number_of_elapsed_periods\":{\"type\":\"long\"},\"number_of_times_throttled\":{\"type\":\"long\"},\"time_throttled_nanos\":{\"type\":\"long\"}}},\"cfs_quota_micros\":{\"type\":\"long\"}}},\"cpuacct\":{\"properties\":{\"usage_nanos\":{\"type\":\"long\"}}}}}}},\"events\":{\"properties\":{\"filtered\":{\"type\":\"long\"},\"in\":{\"type\":\"long\"},\"duration_in_millis\":{\"type\":\"long\"},\"out\":{\"type\":\"long\"}}},\"queue\":{\"properties\":{\"events_count\":{\"type\":\"long\"}}},\"timestamp\":{\"type\":\"date\"}}},\"host\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"properties\":{\"pipeline\":{\"properties\":{\"batch_size\":{\"type\":\"long\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"workers\":{\"type\":\"long\"},\"representation\":{\"properties\":{\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"hash\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"graph\":{\"properties\":{\"vertices\":{\"type\":\"object\"},\"edges\":{\"type\":\"object\"}}}}},\"hash\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"elasticsearch\":{\"properties\":{\"cluster\":{\"properties\":{\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}}}},\"logstash_state\":{\"properties\":{\"pipeline\":{\"properties\":{\"id\":{\"path\":\"logstash.node.state.pipeline.id\",\"type\":\"alias\"},\"hash\":{\"path\":\"logstash.node.state.pipeline.hash\",\"type\":\"alias\"}}}}},\"process\":{\"properties\":{\"pid\":{\"type\":\"long\"}}},\"@timestamp\":{\"type\":\"date\"},\"logstash_stats\":{\"properties\":{\"jvm\":{\"properties\":{\"mem\":{\"properties\":{\"heap_max_in_bytes\":{\"path\":\"logstash.node.stats.jvm.mem.heap_max_in_bytes\",\"type\":\"alias\"},\"heap_used_in_bytes\":{\"path\":\"logstash.node.stats.jvm.mem.heap_used_in_bytes\",\"type\":\"alias\"}}},\"uptime_in_millis\":{\"path\":\"logstash.node.stats.jvm.uptime_in_millis\",\"type\":\"alias\"}}},\"logstash\":{\"properties\":{\"uuid\":{\"path\":\"logstash.node.stats.logstash.uuid\",\"type\":\"alias\"},\"version\":{\"path\":\"logstash.node.stats.logstash.version\",\"type\":\"alias\"}}},\"process\":{\"properties\":{\"cpu\":{\"properties\":{\"percent\":{\"path\":\"logstash.node.stats.process.cpu.percent\",\"type\":\"alias\"}}}}},\"pipelines\":{\"type\":\"nested\"},\"os\":{\"properties\":{\"cpu\":{\"properties\":{\"load_average\":{\"properties\":{\"5m\":{\"path\":\"logstash.node.stats.os.cpu.load_average.5m\",\"type\":\"alias\"},\"15m\":{\"path\":\"logstash.node.stats.os.cpu.load_average.15m\",\"type\":\"alias\"},\"1m\":{\"path\":\"logstash.node.stats.os.cpu.load_average.1m\",\"type\":\"alias\"}}}}},\"cgroup\":{\"properties\":{\"cpu\":{\"properties\":{\"stat\":{\"properties\":{\"number_of_elapsed_periods\":{\"path\":\"logstash.node.stats.os.cgroup.cpu.stat.number_of_elapsed_periods\",\"type\":\"alias\"},\"number_of_times_throttled\":{\"path\":\"logstash.node.stats.os.cgroup.cpu.stat.number_of_times_throttled\",\"type\":\"alias\"},\"time_throttled_nanos\":{\"path\":\"logstash.node.stats.os.cgroup.cpu.stat.time_throttled_nanos\",\"type\":\"alias\"}}},\"cfs_quota_micros\":{\"path\":\"logstash.node.stats.os.cgroup.cpu.cfs_quota_micros\",\"type\":\"alias\"}}},\"cpuacct\":{\"properties\":{\"usage_nanos\":{\"path\":\"logstash.node.stats.os.cgroup.cpuacct.usage_nanos\",\"type\":\"alias\"}}}}}}},\"events\":{\"properties\":{\"in\":{\"path\":\"logstash.node.stats.events.in\",\"type\":\"alias\"},\"duration_in_millis\":{\"path\":\"logstash.node.stats.events.duration_in_millis\",\"type\":\"alias\"},\"out\":{\"path\":\"logstash.node.stats.events.out\",\"type\":\"alias\"}}},\"queue\":{\"properties\":{\"events_count\":{\"path\":\"logstash.node.stats.queue.events_count\",\"type\":\"alias\"}}},\"timestamp\":{\"path\":\"@timestamp\",\"type\":\"alias\"}}},\"ecs\":{\"properties\":{\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"cluster_uuid\":{\"path\":\"logstash.elasticsearch.cluster.id\",\"type\":\"alias\"},\"service\":{\"properties\":{\"hostname\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"host\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"metricset\":{\"properties\":{\"period\":{\"type\":\"long\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"event\":{\"properties\":{\"duration\":{\"type\":\"long\"},\"module\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"dataset\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"timestamp\":{\"path\":\"@timestamp\",\"type\":\"alias\"}}}},\"composed_of\":[],\"version\":8000103,\"data_stream\":{\"hidden\":false,\"allow_custom_routing\":false}},\".ml-anomalies-\":{\"index_patterns\":[\".ml-anomalies-*\"],\"template\":{\"settings\":{\"index\":{\"hidden\":\"true\",\"translog\":{\"durability\":\"async\"},\"auto_expand_replicas\":\"0-1\",\"query\":{\"default_field\":\"all_field_values\"}}},\"mappings\":{\"_meta\":{\"version\":\"8.6.2\"},\"dynamic_templates\":[{\"strings_as_keywords\":{\"mapping\":{\"type\":\"keyword\"},\"match\":\"*\"}}],\"properties\":{\"search_count\":{\"type\":\"long\"},\"bucket_count\":{\"type\":\"long\"},\"anomaly_score_explanation\":{\"properties\":{\"high_variance_penalty\":{\"type\":\"boolean\"},\"typical_value\":{\"type\":\"double\"},\"upper_confidence_bound\":{\"type\":\"double\"},\"lower_confidence_bound\":{\"type\":\"double\"},\"incomplete_bucket_penalty\":{\"type\":\"boolean\"},\"anomaly_type\":{\"type\":\"keyword\"},\"anomaly_characteristics_impact\":{\"type\":\"integer\"},\"multi_bucket_impact\":{\"type\":\"integer\"},\"anomaly_length\":{\"type\":\"integer\"},\"single_bucket_impact\":{\"type\":\"integer\"}}},\"terms\":{\"type\":\"text\"},\"record_score\":{\"type\":\"double\"},\"forecast_expiry_timestamp\":{\"type\":\"date\"},\"over_field_value\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"preferred_to_categories\":{\"type\":\"long\"},\"empty_bucket_count\":{\"type\":\"long\"},\"forecast_create_timestamp\":{\"type\":\"date\"},\"probability\":{\"type\":\"double\"},\"missing_field_count\":{\"type\":\"long\"},\"forecast_lower\":{\"type\":\"double\"},\"influencer_field_value\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"multi_bucket_impact\":{\"type\":\"double\"},\"forecast_progress\":{\"type\":\"double\"},\"max_matching_length\":{\"type\":\"long\"},\"latest_record_time_stamp\":{\"type\":\"date\"},\"regex\":{\"type\":\"keyword\"},\"examples\":{\"type\":\"text\"},\"snapshot_doc_count\":{\"type\":\"integer\"},\"average_bucket_processing_time_ms\":{\"type\":\"double\"},\"initial_anomaly_score\":{\"type\":\"double\"},\"initial_influencer_score\":{\"type\":\"double\"},\"partition_field_value\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"model_lower\":{\"type\":\"double\"},\"total_over_field_count\":{\"type\":\"long\"},\"retain\":{\"type\":\"boolean\"},\"forecast_upper\":{\"type\":\"double\"},\"latest_sparse_bucket_timestamp\":{\"type\":\"date\"},\"model_median\":{\"type\":\"double\"},\"category_id\":{\"type\":\"long\"},\"causes\":{\"type\":\"nested\",\"properties\":{\"actual\":{\"type\":\"double\"},\"partition_field_name\":{\"type\":\"keyword\"},\"partition_field_value\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"by_field_name\":{\"type\":\"keyword\"},\"probability\":{\"type\":\"double\"},\"geo_results\":{\"properties\":{\"actual_point\":{\"type\":\"geo_point\"},\"typical_point\":{\"type\":\"geo_point\"}}},\"field_name\":{\"type\":\"keyword\"},\"by_field_value\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"over_field_name\":{\"type\":\"keyword\"},\"correlated_by_field_value\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"function\":{\"type\":\"keyword\"},\"typical\":{\"type\":\"double\"},\"over_field_value\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"function_description\":{\"type\":\"keyword\"}}},\"all_field_values\":{\"analyzer\":\"whitespace\",\"type\":\"text\"},\"timestamp\":{\"type\":\"date\"},\"input_field_count\":{\"type\":\"long\"},\"model_bytes\":{\"type\":\"long\"},\"quantiles\":{\"type\":\"object\",\"enabled\":false},\"@timestamp\":{\"path\":\"timestamp\",\"type\":\"alias\"},\"function_description\":{\"type\":\"keyword\"},\"min_version\":{\"type\":\"keyword\"},\"raw_anomaly_score\":{\"type\":\"double\"},\"exponential_average_bucket_processing_time_ms\":{\"type\":\"double\"},\"invalid_date_count\":{\"type\":\"long\"},\"snapshot_id\":{\"type\":\"keyword\"},\"model_size_stats\":{\"properties\":{\"model_bytes\":{\"type\":\"long\"},\"result_type\":{\"type\":\"keyword\"},\"peak_model_bytes\":{\"type\":\"long\"},\"job_id\":{\"type\":\"keyword\"},\"total_over_field_count\":{\"type\":\"long\"},\"total_partition_field_count\":{\"type\":\"long\"},\"total_by_field_count\":{\"type\":\"long\"},\"assignment_memory_basis\":{\"type\":\"keyword\"},\"bucket_allocation_failures_count\":{\"type\":\"long\"},\"memory_status\":{\"type\":\"keyword\"},\"log_time\":{\"type\":\"date\"},\"timestamp\":{\"type\":\"date\"}}},\"total_search_time_ms\":{\"type\":\"double\"},\"latest_empty_bucket_timestamp\":{\"type\":\"date\"},\"anomaly_score\":{\"type\":\"double\"},\"over_field_name\":{\"type\":\"keyword\"},\"earliest_record_timestamp\":{\"type\":\"date\"},\"scheduled_events\":{\"type\":\"keyword\"},\"bucket_span\":{\"type\":\"long\"},\"maximum_bucket_processing_time_ms\":{\"type\":\"double\"},\"exponential_average_calculation_context\":{\"properties\":{\"incremental_metric_value_ms\":{\"type\":\"double\"},\"previous_exponential_average_ms\":{\"type\":\"double\"},\"latest_timestamp\":{\"type\":\"date\"}}},\"function\":{\"type\":\"keyword\"},\"influencers\":{\"type\":\"nested\",\"properties\":{\"influencer_field_name\":{\"type\":\"keyword\"},\"influencer_field_values\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"}}},\"input_record_count\":{\"type\":\"long\"},\"latest_result_time_stamp\":{\"type\":\"date\"},\"model_upper\":{\"type\":\"double\"},\"actual\":{\"type\":\"double\"},\"forecast_memory_bytes\":{\"type\":\"long\"},\"total_by_field_count\":{\"type\":\"long\"},\"geo_results\":{\"properties\":{\"actual_point\":{\"type\":\"geo_point\"},\"typical_point\":{\"type\":\"geo_point\"}}},\"detector_index\":{\"type\":\"integer\"},\"by_field_value\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"processed_record_count\":{\"type\":\"long\"},\"total_partition_field_count\":{\"type\":\"long\"},\"mlcategory\":{\"type\":\"keyword\"},\"assignment_memory_basis\":{\"type\":\"keyword\"},\"forecast_end_timestamp\":{\"type\":\"date\"},\"forecast_id\":{\"type\":\"keyword\"},\"partition_field_name\":{\"type\":\"keyword\"},\"by_field_name\":{\"type\":\"keyword\"},\"event_count\":{\"type\":\"long\"},\"description\":{\"type\":\"text\"},\"is_interim\":{\"type\":\"boolean\"},\"bucket_allocation_failures_count\":{\"type\":\"long\"},\"memory_status\":{\"type\":\"keyword\"},\"model_feature\":{\"type\":\"keyword\"},\"num_matches\":{\"type\":\"long\"},\"influencer_score\":{\"type\":\"double\"},\"out_of_order_timestamp_count\":{\"type\":\"long\"},\"result_type\":{\"type\":\"keyword\"},\"last_data_time\":{\"type\":\"date\"},\"latest_record_timestamp\":{\"type\":\"date\"},\"influencer_field_name\":{\"type\":\"keyword\"},\"forecast_messages\":{\"type\":\"keyword\"},\"sparse_bucket_count\":{\"type\":\"long\"},\"log_time\":{\"type\":\"date\"},\"field_name\":{\"type\":\"keyword\"},\"minimum_bucket_processing_time_ms\":{\"type\":\"double\"},\"bucket_influencers\":{\"type\":\"nested\",\"properties\":{\"anomaly_score\":{\"type\":\"double\"},\"initial_anomaly_score\":{\"type\":\"double\"},\"result_type\":{\"type\":\"keyword\"},\"raw_anomaly_score\":{\"type\":\"double\"},\"bucket_span\":{\"type\":\"long\"},\"job_id\":{\"type\":\"keyword\"},\"probability\":{\"type\":\"double\"},\"influencer_field_name\":{\"type\":\"keyword\"},\"is_interim\":{\"type\":\"boolean\"},\"timestamp\":{\"type\":\"date\"}}},\"processing_time_ms\":{\"type\":\"long\"},\"input_bytes\":{\"type\":\"long\"},\"initial_record_score\":{\"type\":\"double\"},\"job_id\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"processed_field_count\":{\"type\":\"long\"},\"forecast_status\":{\"type\":\"keyword\"},\"typical\":{\"type\":\"double\"},\"forecast_prediction\":{\"type\":\"double\"},\"forecast_start_timestamp\":{\"type\":\"date\"}}}},\"composed_of\":[],\"priority\":2147483647,\"version\":8060299,\"_meta\":{\"managed\":true,\"description\":\"index template for ML anomaly detection results indices\"}},\"metrics\":{\"index_patterns\":[\"metrics-*-*\"],\"composed_of\":[\"metrics-mappings\",\"data-streams-mappings\",\"metrics-settings\"],\"priority\":100,\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default metrics template installed by x-pack\"},\"data_stream\":{\"hidden\":false,\"allow_custom_routing\":false},\"allow_auto_create\":true},\".ml-notifications-000002\":{\"index_patterns\":[\".ml-notifications-000002\"],\"template\":{\"settings\":{\"index\":{\"hidden\":\"true\",\"number_of_shards\":\"1\",\"auto_expand_replicas\":\"0-1\"}},\"mappings\":{\"_meta\":{\"version\":\"8.6.2\"},\"dynamic\":\"false\",\"properties\":{\"job_type\":{\"type\":\"keyword\"},\"level\":{\"type\":\"keyword\"},\"job_id\":{\"type\":\"keyword\"},\"node_name\":{\"type\":\"keyword\"},\"message\":{\"type\":\"text\",\"fields\":{\"raw\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"cleared\":{\"type\":\"boolean\"},\"timestamp\":{\"type\":\"date\"}}}},\"composed_of\":[],\"priority\":2147483647,\"version\":8060299,\"_meta\":{\"managed\":true,\"description\":\"index template for ML notifications indices\"}},\".deprecation-indexing-template\":{\"index_patterns\":[\".logs-deprecation.*\"],\"composed_of\":[\".deprecation-indexing-mappings\",\".deprecation-indexing-settings\"],\"priority\":1000,\"version\":1,\"_meta\":{\"managed\":true,\"description\":\"default template for Stack deprecation logs index template installed by x-pack\"},\"data_stream\":{\"hidden\":true,\"allow_custom_routing\":false},\"allow_auto_create\":true},\"logs\":{\"index_patterns\":[\"logs-*-*\"],\"composed_of\":[\"logs-mappings\",\"data-streams-mappings\",\"logs-settings\"],\"priority\":100,\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default logs template installed by x-pack\"},\"data_stream\":{\"hidden\":false,\"allow_custom_routing\":false},\"allow_auto_create\":true},\".ml-stats\":{\"index_patterns\":[\".ml-stats-*\"],\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\"ml-size-based-ilm-policy\",\"rollover_alias\":\".ml-stats-write\"},\"hidden\":\"true\",\"number_of_shards\":\"1\",\"auto_expand_replicas\":\"0-1\"}},\"mappings\":{\"_meta\":{\"version\":\"8.6.2\"},\"dynamic\":false,\"properties\":{\"skipped_docs_count\":{\"type\":\"long\"},\"validation_loss\":{\"properties\":{\"fold_values\":{\"properties\":{\"fold\":{\"type\":\"integer\"},\"values\":{\"type\":\"double\"}}},\"loss_type\":{\"type\":\"keyword\"}}},\"cache_miss_count\":{\"type\":\"long\"},\"timing_stats\":{\"properties\":{\"iteration_time\":{\"type\":\"long\"},\"elapsed_time\":{\"type\":\"long\"}}},\"failure_count\":{\"type\":\"long\"},\"model_id\":{\"type\":\"keyword\"},\"type\":{\"type\":\"keyword\"},\"training_docs_count\":{\"type\":\"long\"},\"inference_count\":{\"type\":\"long\"},\"job_id\":{\"type\":\"keyword\"},\"missing_all_fields_count\":{\"type\":\"long\"},\"peak_usage_bytes\":{\"type\":\"long\"},\"iteration\":{\"type\":\"integer\"},\"hyperparameters\":{\"properties\":{\"max_attempts_to_add_tree\":{\"type\":\"integer\"},\"downsample_factor\":{\"type\":\"double\"},\"eta_growth_rate_per_tree\":{\"type\":\"double\"},\"soft_tree_depth_tolerance\":{\"type\":\"double\"},\"max_trees\":{\"type\":\"integer\"},\"lambda\":{\"type\":\"double\"},\"max_optimization_rounds_per_hyperparameter\":{\"type\":\"integer\"},\"eta\":{\"type\":\"double\"},\"soft_tree_depth_limit\":{\"type\":\"double\"},\"alpha\":{\"type\":\"double\"},\"class_assignment_objective\":{\"type\":\"keyword\"},\"feature_bag_fraction\":{\"type\":\"double\"},\"num_splits_per_feature\":{\"type\":\"integer\"},\"gamma\":{\"type\":\"double\"},\"num_folds\":{\"type\":\"integer\"}}},\"parameters\":{\"properties\":{\"compute_feature_influence\":{\"type\":\"boolean\"},\"feature_influence_threshold\":{\"type\":\"double\"},\"outlier_fraction\":{\"type\":\"double\"},\"method\":{\"type\":\"keyword\"},\"standardization_enabled\":{\"type\":\"boolean\"},\"n_neighbors\":{\"type\":\"integer\"}}},\"test_docs_count\":{\"type\":\"long\"},\"node_id\":{\"type\":\"keyword\"},\"timestamp\":{\"type\":\"date\"}}},\"aliases\":{}},\"composed_of\":[],\"priority\":2147483647,\"version\":8060299,\"_meta\":{\"managed\":true,\"description\":\"index template for ML stats indices\"}}}} index-graveyard: IndexGraveyard[[[index=[abacus_go/zV2aNzxVQxqMDQTD4A9TEQ], deleteDate=2023-03-28T03:56:57.282Z], [index=[access_platform_core/ZuN0_cUfSQKzXg_yG4Sv5g], deleteDate=2023-03-28T03:58:27.958Z], [index=[ad_bidder/6ihfVU6rRqOo4Gv4Djtetg], deleteDate=2023-03-28T03:59:11.628Z], [index=[apphealth/EthxrkUtTGCTP2mX-tS63Q], deleteDate=2023-03-28T03:59:36.596Z]]]\nnodes: \n {hostb9}{iKPGCkp9RVOKXOj20uOt4g}{cPRh9FqiSlGP9jClhm8Ugg}{hostb9}{127.0.0.1}{127.0.0.1:9300}{cdfhilmrstw}{ml.max_jvm_size=33285996544, ml.allocated_processors_double=16.0, xpack.installed=true, ml.machine_memory=135062048768, ml.allocated_processors=16}, local, master\nrouting_table (version 35):\n-- index [[abacus_go/cfvmY-Y_Su63-qSkPODDzg]]\n----shard_id [abacus_go][0]\n--------[abacus_go][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=AMhgJMlSTPOoKSsL7s3OKA], failed_attempts[0]\n--------[abacus_go][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=INDEX_CREATED], at[2023-03-28T03:57:31.683Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[web_wok/R1IULAXSQ4SC0pLFCU5oJA]]\n----shard_id [web_wok][0]\n--------[web_wok][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=JXruVMhdRVGvP1kQCbUNSQ], failed_attempts[0]\n--------[web_wok][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[access_platform_core/l4kdkOEQRaKfWGLpTZURCQ]]\n----shard_id [access_platform_core][0]\n--------[access_platform_core][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=6ajNSTQZSr6tP3YtmRaxgw], failed_attempts[0]\n--------[access_platform_core][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=INDEX_CREATED], at[2023-03-28T03:58:31.318Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[driverapp/cqbade6KQFqLEK-wPKlkBg]]\n----shard_id [driverapp][0]\n--------[driverapp][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=b5_Bhd3ITjOkl-xLF8H1jg], failed_attempts[0]\n--------[driverapp][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[k8s_kubelet/fQ9Iy7wlTKahsWLbs-oE_Q]]\n----shard_id [k8s_kubelet][0]\n--------[k8s_kubelet][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=Q4Xb11JATPaFZqmgTQHcVQ], failed_attempts[0]\n--------[k8s_kubelet][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.307Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[eats_store/hF-kN4XFRy--z_gUhYLwdA]]\n----shard_id [eats_store][0]\n--------[eats_store][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=WRNwRQ6gTNGfSNQok4iSyQ], failed_attempts[0]\n--------[eats_store][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[rider_product_cored/AdGiEBLBQbicsXDomsa3mw]]\n----shard_id [rider_product_cored][0]\n--------[rider_product_cored][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=GwRH_ZlFRtCV7m_n626z7g], failed_attempts[0]\n--------[rider_product_cored][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[finprod_rewards_eligibility/dJNRGPwrQS2bIPlpsXgUoQ]]\n----shard_id [finprod_rewards_eligibility][0]\n--------[finprod_rewards_eligibility][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=Hg4Dm9O8QEqXWKY7rH1X0A], failed_attempts[0]\n--------[finprod_rewards_eligibility][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[event_logs/DcomuHoGQzOjZq2bX67TFg]]\n----shard_id [event_logs][0]\n--------[event_logs][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=1uCl_LIDSBCWh6koVA0fjQ], failed_attempts[0]\n--------[event_logs][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[apphealth/rcPrX1aWRmeED9W5PU6A4g]]\n----shard_id [apphealth][0]\n--------[apphealth][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=Abip50RxQTmavqi4NTwSqQ], failed_attempts[0]\n--------[apphealth][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=INDEX_CREATED], at[2023-03-28T03:59:38.780Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[ad_bidder/tLHfw5r2QH-o7lG5tapKxw]]\n----shard_id [ad_bidder][0]\n--------[ad_bidder][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=YtEUwPgfQDGM0Cm_C4hXlw], failed_attempts[0]\n--------[ad_bidder][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=INDEX_CREATED], at[2023-03-28T03:59:13.333Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[invoice_gen/RPu5EzJVRfuqyI8MQ7iTjQ]]\n----shard_id [invoice_gen][0]\n--------[invoice_gen][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=onWbjhHuTD2C3AEayRJq1w], failed_attempts[0]\n--------[invoice_gen][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[fares_management/F1lH-QvyTkS5hnbDFwE2Xg]]\n----shard_id [fares_management][0]\n--------[fares_management][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=sOxfKX3nReynYN5HN88cuQ], failed_attempts[0]\n--------[fares_management][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[michelangelo_gateway/ZV2vZ8PmS0uysg-xzKO7QA]]\n----shard_id [michelangelo_gateway][0]\n--------[michelangelo_gateway][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=k5vDE54CSTSakhfgtGaTcA], failed_attempts[0]\n--------[michelangelo_gateway][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[.geoip_databases/5k1GHwKvQBKersi12DFh4A]]\n----shard_id [.geoip_databases][0]\n--------[.geoip_databases][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=UdvCKUGqTEmYFzZlnSu_tA], failed_attempts[0]\n\n-- index [[fares_intelligence/pSRJWxyXSqefhwLVhmT9nQ]]\n----shard_id [fares_intelligence][0]\n--------[fares_intelligence][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=-dD8ROsgS6qaYMoalciocA], failed_attempts[0]\n--------[fares_intelligence][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[fare_route/a4fjivWhQRee8JW7ZbXiXw]]\n----shard_id [fare_route][0]\n--------[fare_route][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=ltSLFgu4Rry0_0vXq28DbA], failed_attempts[0]\n--------[fare_route][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[k8s_apiserver/qQasynZgQ6e6AmD5ZWqh3w]]\n----shard_id [k8s_apiserver][0]\n--------[k8s_apiserver][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=Aaa0F0jiStCkteGDYR8yRQ], failed_attempts[0]\n--------[k8s_apiserver][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[fulfillment_compatibled/tMgrGBWeR3irgR5PyN8Oig]]\n----shard_id [fulfillment_compatibled][0]\n--------[fulfillment_compatibled][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=D_3C_3kMSNKa-XJk7pyoGg], failed_attempts[0]\n--------[fulfillment_compatibled][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\nrouting_nodes:\n-----node_id[iKPGCkp9RVOKXOj20uOt4g][V]\n--------[abacus_go][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=AMhgJMlSTPOoKSsL7s3OKA], failed_attempts[0]\n--------[web_wok][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=JXruVMhdRVGvP1kQCbUNSQ], failed_attempts[0]\n--------[access_platform_core][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=6ajNSTQZSr6tP3YtmRaxgw], failed_attempts[0]\n--------[driverapp][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=b5_Bhd3ITjOkl-xLF8H1jg], failed_attempts[0]\n--------[k8s_kubelet][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=Q4Xb11JATPaFZqmgTQHcVQ], failed_attempts[0]\n--------[eats_store][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=WRNwRQ6gTNGfSNQok4iSyQ], failed_attempts[0]\n--------[rider_product_cored][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=GwRH_ZlFRtCV7m_n626z7g], failed_attempts[0]\n--------[finprod_rewards_eligibility][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=Hg4Dm9O8QEqXWKY7rH1X0A], failed_attempts[0]\n--------[event_logs][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=1uCl_LIDSBCWh6koVA0fjQ], failed_attempts[0]\n--------[apphealth][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=Abip50RxQTmavqi4NTwSqQ], failed_attempts[0]\n--------[ad_bidder][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=YtEUwPgfQDGM0Cm_C4hXlw], failed_attempts[0]\n--------[invoice_gen][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=onWbjhHuTD2C3AEayRJq1w], failed_attempts[0]\n--------[fares_management][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=sOxfKX3nReynYN5HN88cuQ], failed_attempts[0]\n--------[michelangelo_gateway][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=k5vDE54CSTSakhfgtGaTcA], failed_attempts[0]\n--------[.geoip_databases][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=UdvCKUGqTEmYFzZlnSu_tA], failed_attempts[0]\n--------[fares_intelligence][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=-dD8ROsgS6qaYMoalciocA], failed_attempts[0]\n--------[fare_route][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=ltSLFgu4Rry0_0vXq28DbA], failed_attempts[0]\n--------[k8s_apiserver][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=Aaa0F0jiStCkteGDYR8yRQ], failed_attempts[0]\n--------[fulfillment_compatibled][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=D_3C_3kMSNKa-XJk7pyoGg], failed_attempts[0]\n---- unassigned\n--------[abacus_go][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=INDEX_CREATED], at[2023-03-28T03:57:31.683Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[web_wok][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[access_platform_core][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=INDEX_CREATED], at[2023-03-28T03:58:31.318Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[driverapp][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[k8s_kubelet][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.307Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[eats_store][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[rider_product_cored][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[finprod_rewards_eligibility][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[event_logs][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[apphealth][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=INDEX_CREATED], at[2023-03-28T03:59:38.780Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[ad_bidder][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=INDEX_CREATED], at[2023-03-28T03:59:13.333Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[invoice_gen][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[fares_management][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[michelangelo_gateway][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[fares_intelligence][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[fare_route][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[k8s_apiserver][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[fulfillment_compatibled][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\ncustoms:\n health: org.elasticsearch.health.metadata.HealthMetadata@66bc0723}] from [{hostb9}{iKPGCkp9RVOKXOj20uOt4g}{cPRh9FqiSlGP9jClhm8Ugg}{hostb9}{127.0.0.1}{127.0.0.1:9300}{cdfhilmrstw}{ml.max_jvm_size=33285996544, ml.allocated_processors_double=16.0, xpack.installed=true, ml.machine_memory=135062048768, ml.allocated_processors=16}]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][cluster_coordination][T#1]","log.logger":"org.elasticsearch.cluster.coordination.Coordinator","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:02.078Z", "log.level":"TRACE", "message":"handlePublishRequest: accepting publish request for version [154] and term [6]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][cluster_coordination][T#1]","log.logger":"org.elasticsearch.cluster.coordination.CoordinationState","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:02.078Z", "log.level":"TRACE", "message":"currentTerm [6] matches previous currentTerm, writing changes only", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][cluster_coordination][T#1]","log.logger":"org.elasticsearch.gateway.PersistedClusterStateService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:02.079Z", "log.level":"TRACE", "message":"no action required for mapping [FOi5Cox/V4KpXwFa2TtyrVRtJ6YqC9xfpc5SC5ChT0c=]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][cluster_coordination][T#1]","log.logger":"org.elasticsearch.gateway.PersistedClusterStateService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:02.079Z", "log.level":"TRACE", "message":"no action required for index [[k8s_kubelet/fQ9Iy7wlTKahsWLbs-oE_Q]]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][cluster_coordination][T#1]","log.logger":"org.elasticsearch.gateway.PersistedClusterStateService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:02.079Z", "log.level":"TRACE", "message":"no action required for index [[fares_intelligence/pSRJWxyXSqefhwLVhmT9nQ]]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][cluster_coordination][T#1]","log.logger":"org.elasticsearch.gateway.PersistedClusterStateService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:02.079Z", "log.level":"TRACE", "message":"no action required for index [[event_logs/DcomuHoGQzOjZq2bX67TFg]]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][cluster_coordination][T#1]","log.logger":"org.elasticsearch.gateway.PersistedClusterStateService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:02.079Z", "log.level":"TRACE", "message":"no action required for index [[rider_product_cored/AdGiEBLBQbicsXDomsa3mw]]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][cluster_coordination][T#1]","log.logger":"org.elasticsearch.gateway.PersistedClusterStateService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:02.079Z", "log.level":"TRACE", "message":"no action required for index [[access_platform_core/l4kdkOEQRaKfWGLpTZURCQ]]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][cluster_coordination][T#1]","log.logger":"org.elasticsearch.gateway.PersistedClusterStateService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:02.079Z", "log.level":"TRACE", "message":"no action required for index [[.geoip_databases/5k1GHwKvQBKersi12DFh4A]]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][cluster_coordination][T#1]","log.logger":"org.elasticsearch.gateway.PersistedClusterStateService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:02.080Z", "log.level":"TRACE", "message":"no action required for index [[web_wok/R1IULAXSQ4SC0pLFCU5oJA]]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][cluster_coordination][T#1]","log.logger":"org.elasticsearch.gateway.PersistedClusterStateService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:02.080Z", "log.level":"TRACE", "message":"no action required for index [[abacus_go/cfvmY-Y_Su63-qSkPODDzg]]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][cluster_coordination][T#1]","log.logger":"org.elasticsearch.gateway.PersistedClusterStateService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:02.080Z", "log.level":"TRACE", "message":"no action required for index [[k8s_apiserver/qQasynZgQ6e6AmD5ZWqh3w]]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][cluster_coordination][T#1]","log.logger":"org.elasticsearch.gateway.PersistedClusterStateService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:02.080Z", "log.level":"TRACE", "message":"no action required for index [[michelangelo_gateway/ZV2vZ8PmS0uysg-xzKO7QA]]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][cluster_coordination][T#1]","log.logger":"org.elasticsearch.gateway.PersistedClusterStateService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:02.080Z", "log.level":"TRACE", "message":"no action required for index [[driverapp/cqbade6KQFqLEK-wPKlkBg]]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][cluster_coordination][T#1]","log.logger":"org.elasticsearch.gateway.PersistedClusterStateService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:02.080Z", "log.level":"TRACE", "message":"updating metadata for [[apphealth/rcPrX1aWRmeED9W5PU6A4g]], changing version from [2] to [4]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][cluster_coordination][T#1]","log.logger":"org.elasticsearch.gateway.PersistedClusterStateService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:02.081Z", "log.level":"TRACE", "message":"removing metadata for [rcPrX1aWRmeED9W5PU6A4g]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][cluster_coordination][T#1]","log.logger":"org.elasticsearch.gateway.PersistedClusterStateService.MetadataIndexWriter","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":["[NIOFSDirectory@/var/lib/elasticsearch/_state lockFactory=org.apache.lucene.store.NativeFSLockFactory@e8d54fb]"]} +{"@timestamp":"2023-03-28T04:00:02.081Z", "log.level":"TRACE", "message":"updating metadata for [[apphealth/rcPrX1aWRmeED9W5PU6A4g]]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][cluster_coordination][T#1]","log.logger":"org.elasticsearch.gateway.PersistedClusterStateService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:02.082Z", "log.level":"TRACE", "message":"no action required for index [[fares_management/F1lH-QvyTkS5hnbDFwE2Xg]]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][cluster_coordination][T#1]","log.logger":"org.elasticsearch.gateway.PersistedClusterStateService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:02.083Z", "log.level":"TRACE", "message":"no action required for index [[finprod_rewards_eligibility/dJNRGPwrQS2bIPlpsXgUoQ]]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][cluster_coordination][T#1]","log.logger":"org.elasticsearch.gateway.PersistedClusterStateService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:02.083Z", "log.level":"TRACE", "message":"no action required for index [[fare_route/a4fjivWhQRee8JW7ZbXiXw]]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][cluster_coordination][T#1]","log.logger":"org.elasticsearch.gateway.PersistedClusterStateService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:02.083Z", "log.level":"TRACE", "message":"no action required for index [[ad_bidder/tLHfw5r2QH-o7lG5tapKxw]]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][cluster_coordination][T#1]","log.logger":"org.elasticsearch.gateway.PersistedClusterStateService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:02.083Z", "log.level":"TRACE", "message":"no action required for index [[fulfillment_compatibled/tMgrGBWeR3irgR5PyN8Oig]]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][cluster_coordination][T#1]","log.logger":"org.elasticsearch.gateway.PersistedClusterStateService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:02.083Z", "log.level":"TRACE", "message":"no action required for index [[eats_store/hF-kN4XFRy--z_gUhYLwdA]]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][cluster_coordination][T#1]","log.logger":"org.elasticsearch.gateway.PersistedClusterStateService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:02.083Z", "log.level":"TRACE", "message":"no action required for index [[invoice_gen/RPu5EzJVRfuqyI8MQ7iTjQ]]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][cluster_coordination][T#1]","log.logger":"org.elasticsearch.gateway.PersistedClusterStateService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:02.083Z", "log.level":"TRACE", "message":"flushing", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][cluster_coordination][T#1]","log.logger":"org.elasticsearch.gateway.PersistedClusterStateService.MetadataIndexWriter","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":["[NIOFSDirectory@/var/lib/elasticsearch/_state lockFactory=org.apache.lucene.store.NativeFSLockFactory@e8d54fb]"]} +{"@timestamp":"2023-03-28T04:00:02.119Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#6]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [abacus_go]"]} +{"@timestamp":"2023-03-28T04:00:02.119Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#8]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [event_logs]"]} +{"@timestamp":"2023-03-28T04:00:02.119Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#2]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [fulfillment_compatibled]"]} +{"@timestamp":"2023-03-28T04:00:02.119Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#5]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [k8s_kubelet]"]} +{"@timestamp":"2023-03-28T04:00:02.120Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#7]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [driverapp]"]} +{"@timestamp":"2023-03-28T04:00:02.394Z", "log.level":"TRACE", "message":"checking jobs [2023-03-28T04:00:02.394359200Z]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"ticker-schedule-trigger-engine","log.logger":"org.elasticsearch.xpack.watcher.trigger.schedule.engine.TickerScheduleTriggerEngine","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:02.443Z", "log.level":"TRACE", "message":"total indexing heap bytes used [0b] vs indices.memory.index_buffer_size [3gb], currently writing bytes [0b]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][scheduler][T#1]","log.logger":"org.elasticsearch.indices.IndexingMemoryController","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} +{"@timestamp":"2023-03-28T04:00:02.892Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#1]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [.geoip_databases]"]} +{"@timestamp":"2023-03-28T04:00:02.892Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#4]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [finprod_rewards_eligibility]"]} +{"@timestamp":"2023-03-28T04:00:02.893Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#3]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [rider_product_cored]"]} +{"@timestamp":"2023-03-28T04:00:02.893Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#6]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [fares_management]"]} +{"@timestamp":"2023-03-28T04:00:02.893Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#8]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [michelangelo_gateway]"]} +{"@timestamp":"2023-03-28T04:00:02.893Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#2]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [k8s_apiserver]"]} +{"@timestamp":"2023-03-28T04:00:02.894Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#5]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [access_platform_core]"]} +{"@timestamp":"2023-03-28T04:00:02.894Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#7]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [web_wok]"]} diff --git a/tests/test_ir/test_json/spark-event-logs.json b/tests/test_ir/test_json/spark-event-logs.json new file mode 100644 index 00000000..a9d33a68 --- /dev/null +++ b/tests/test_ir/test_json/spark-event-logs.json @@ -0,0 +1,332 @@ +{"Event":"SparkListenerLogStart","Spark Version":"3.1.2"} +{"Event":"SparkListenerResourceProfileAdded","Resource Profile Id":0,"Executor Resource Requests":{"cores":{"Resource Name":"cores","Amount":1,"Discovery Script":"","Vendor":""},"memory":{"Resource Name":"memory","Amount":4096,"Discovery Script":"","Vendor":""},"offHeap":{"Resource Name":"offHeap","Amount":0,"Discovery Script":"","Vendor":""}},"Task Resource Requests":{"cpus":{"Resource Name":"cpus","Amount":1.0}}} +{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"driver","Host":"hostb7.dsrg.utoronto.ca","Port":41823},"Maximum Memory":2101975449,"Timestamp":1633789164234,"Maximum Onheap Memory":2101975449,"Maximum Offheap Memory":0} +{"Event":"SparkListenerEnvironmentUpdate","JVM Information":{"Java Home":"/usr/lib/jvm/java-8-openjdk-amd64/jre","Java Version":"1.8.0_162 (Oracle Corporation)","Scala Version":"version 2.12.10"},"Spark Properties":{"spark.driver.host":"hostb7.dsrg.utoronto.ca","spark.eventLog.enabled":"true","spark.driver.port":"44011","spark.jars":"file:/home/usera112/projects/log-compressor/deployments/HiBench/sparkbench/assembly/target/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.app.name":"ScalaJoin","spark.scheduler.mode":"FIFO","spark.driver.memory":"4g","spark.submit.pyFiles":"","spark.default.parallelism":"8","spark.app.startTime":"1633789163045","spark.executor.id":"driver","spark.app.initial.jar.urls":"spark://hostb7.dsrg.utoronto.ca:44011/jars/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.submit.deployMode":"client","spark.master":"spark://hostb7:7077","spark.executor.memory":"4g","spark.eventLog.dir":"hdfs://hostb7:8020/directory","spark.app.id":"app-20211009101924-2782","spark.sql.shuffle.partitions":"8"},"Hadoop Properties":{"hadoop.service.shutdown.timeout":"30s","yarn.resourcemanager.amlauncher.thread-count":"50","yarn.sharedcache.enabled":"false","fs.s3a.connection.maximum":"15","yarn.nodemanager.numa-awareness.numactl.cmd":"/usr/bin/numactl","fs.s3a.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","yarn.app.mapreduce.am.scheduler.heartbeat.interval-ms":"1000","yarn.timeline-service.timeline-client.number-of-async-entities-to-merge":"10","hadoop.security.kms.client.timeout":"60","hadoop.http.authentication.kerberos.principal":"HTTP/_HOST@LOCALHOST","mapreduce.jobhistory.loadedjob.tasks.max":"-1","mapreduce.framework.name":"yarn","yarn.sharedcache.uploader.server.thread-count":"50","yarn.nodemanager.linux-container-executor.nonsecure-mode.user-pattern":"^[_.A-Za-z0-9][-@_.A-Za-z0-9]{0,255}?[$]?$","tfile.fs.output.buffer.size":"262144","yarn.app.mapreduce.am.job.task.listener.thread-count":"30","hadoop.security.groups.cache.background.reload.threads":"3","yarn.resourcemanager.webapp.cross-origin.enabled":"false","fs.AbstractFileSystem.ftp.impl":"org.apache.hadoop.fs.ftp.FtpFs","hadoop.registry.secure":"false","hadoop.shell.safely.delete.limit.num.files":"100","dfs.bytes-per-checksum":"512","mapreduce.job.acl-view-job":" ","fs.s3a.s3guard.ddb.background.sleep":"25ms","fs.s3a.retry.limit":"${fs.s3a.attempts.maximum}","mapreduce.jobhistory.loadedjobs.cache.size":"5","fs.s3a.s3guard.ddb.table.create":"false","yarn.nodemanager.amrmproxy.enabled":"false","yarn.timeline-service.entity-group-fs-store.with-user-dir":"false","mapreduce.input.fileinputformat.split.minsize":"0","yarn.resourcemanager.container.liveness-monitor.interval-ms":"600000","yarn.resourcemanager.client.thread-count":"50","io.seqfile.compress.blocksize":"1000000","yarn.sharedcache.checksum.algo.impl":"org.apache.hadoop.yarn.sharedcache.ChecksumSHA256Impl","yarn.nodemanager.amrmproxy.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.nodemanager.amrmproxy.DefaultRequestInterceptor","dfs.replication":"3","yarn.timeline-service.entity-group-fs-store.leveldb-cache-read-cache-size":"10485760","mapreduce.reduce.shuffle.fetch.retry.interval-ms":"1000","mapreduce.task.profile.maps":"0-2","yarn.scheduler.include-port-in-node-name":"false","yarn.nodemanager.admin-env":"MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX","yarn.resourcemanager.node-removal-untracked.timeout-ms":"60000","mapreduce.am.max-attempts":"2","hadoop.security.kms.client.failover.sleep.base.millis":"100","mapreduce.jobhistory.webapp.https.address":"0.0.0.0:19890","yarn.node-labels.fs-store.impl.class":"org.apache.hadoop.yarn.nodelabels.FileSystemNodeLabelsStore","yarn.nodemanager.collector-service.address":"${yarn.nodemanager.hostname}:8048","fs.trash.checkpoint.interval":"0","mapreduce.job.map.output.collector.class":"org.apache.hadoop.mapred.MapTask$MapOutputBuffer","yarn.resourcemanager.node-ip-cache.expiry-interval-secs":"-1","hadoop.http.authentication.signature.secret.file":"*********(redacted)","hadoop.jetty.logs.serve.aliases":"true","yarn.resourcemanager.placement-constraints.handler":"disabled","yarn.timeline-service.handler-thread-count":"10","yarn.resourcemanager.max-completed-applications":"1000","yarn.resourcemanager.system-metrics-publisher.enabled":"false","yarn.resourcemanager.placement-constraints.algorithm.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm.DefaultPlacementAlgorithm","yarn.sharedcache.webapp.address":"0.0.0.0:8788","yarn.resourcemanager.delegation.token.renew-interval":"*********(redacted)","yarn.sharedcache.nm.uploader.replication.factor":"10","hadoop.security.groups.negative-cache.secs":"30","yarn.app.mapreduce.task.container.log.backups":"0","mapreduce.reduce.skip.proc-count.auto-incr":"true","hadoop.security.group.mapping.ldap.posix.attr.gid.name":"gidNumber","rpc.engine.org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolPB":"org.apache.hadoop.ipc.ProtobufRpcEngine","ipc.client.fallback-to-simple-auth-allowed":"false","yarn.nodemanager.resource.memory.enforced":"true","yarn.client.failover-proxy-provider":"org.apache.hadoop.yarn.client.ConfiguredRMFailoverProxyProvider","yarn.timeline-service.http-authentication.simple.anonymous.allowed":"true","ha.health-monitor.check-interval.ms":"1000","yarn.acl.reservation-enable":"false","yarn.resourcemanager.store.class":"org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore","yarn.app.mapreduce.am.hard-kill-timeout-ms":"10000","fs.s3a.etag.checksum.enabled":"false","yarn.nodemanager.container-metrics.enable":"true","yarn.timeline-service.client.fd-clean-interval-secs":"60","yarn.resourcemanager.nodemanagers.heartbeat-interval-ms":"1000","hadoop.common.configuration.version":"3.0.0","fs.s3a.s3guard.ddb.table.capacity.read":"500","yarn.nodemanager.remote-app-log-dir-suffix":"logs","yarn.nodemanager.windows-container.cpu-limit.enabled":"false","yarn.nodemanager.runtime.linux.docker.privileged-containers.allowed":"false","file.blocksize":"67108864","hadoop.registry.zk.retry.ceiling.ms":"60000","yarn.scheduler.configuration.leveldb-store.path":"${hadoop.tmp.dir}/yarn/system/confstore","yarn.sharedcache.store.in-memory.initial-delay-mins":"10","mapreduce.jobhistory.principal":"jhs/_HOST@REALM.TLD","mapreduce.map.skip.proc-count.auto-incr":"true","fs.s3a.committer.name":"file","mapreduce.task.profile.reduces":"0-2","hadoop.zk.num-retries":"1000","yarn.webapp.xfs-filter.enabled":"true","seq.io.sort.mb":"100","yarn.scheduler.configuration.max.version":"100","yarn.timeline-service.webapp.https.address":"${yarn.timeline-service.hostname}:8190","yarn.resourcemanager.scheduler.address":"${yarn.resourcemanager.hostname}:8030","yarn.node-labels.enabled":"false","yarn.resourcemanager.webapp.ui-actions.enabled":"true","mapreduce.task.timeout":"600000","yarn.sharedcache.client-server.thread-count":"50","hadoop.security.groups.shell.command.timeout":"0s","hadoop.security.crypto.cipher.suite":"AES/CTR/NoPadding","yarn.nodemanager.elastic-memory-control.oom-handler":"org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.DefaultOOMHandler","yarn.resourcemanager.connect.max-wait.ms":"900000","fs.defaultFS":"hdfs://hostb7:8020","yarn.minicluster.use-rpc":"false","fs.har.impl.disable.cache":"true","yarn.webapp.ui2.enable":"false","io.compression.codec.bzip2.library":"system-native","yarn.nodemanager.distributed-scheduling.enabled":"false","mapreduce.shuffle.connection-keep-alive.timeout":"5","yarn.resourcemanager.webapp.https.address":"${yarn.resourcemanager.hostname}:8090","mapreduce.jobhistory.address":"0.0.0.0:10020","yarn.resourcemanager.nm-tokens.master-key-rolling-interval-secs":"*********(redacted)","yarn.is.minicluster":"false","yarn.nodemanager.address":"${yarn.nodemanager.hostname}:0","fs.abfss.impl":"org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","fs.AbstractFileSystem.s3a.impl":"org.apache.hadoop.fs.s3a.S3A","mapreduce.task.combine.progress.records":"10000","yarn.resourcemanager.epoch.range":"0","yarn.resourcemanager.am.max-attempts":"2","yarn.nodemanager.linux-container-executor.cgroups.hierarchy":"/hadoop-yarn","fs.AbstractFileSystem.wasbs.impl":"org.apache.hadoop.fs.azure.Wasbs","yarn.timeline-service.entity-group-fs-store.cache-store-class":"org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore","fs.ftp.transfer.mode":"BLOCK_TRANSFER_MODE","ipc.server.log.slow.rpc":"false","yarn.resourcemanager.node-labels.provider.fetch-interval-ms":"1800000","yarn.router.webapp.https.address":"0.0.0.0:8091","yarn.nodemanager.webapp.cross-origin.enabled":"false","fs.wasb.impl":"org.apache.hadoop.fs.azure.NativeAzureFileSystem","yarn.resourcemanager.auto-update.containers":"false","yarn.app.mapreduce.am.job.committer.cancel-timeout":"60000","yarn.scheduler.configuration.zk-store.parent-path":"/confstore","yarn.nodemanager.default-container-executor.log-dirs.permissions":"710","yarn.app.attempt.diagnostics.limit.kc":"64","ftp.bytes-per-checksum":"512","yarn.nodemanager.resource.memory-mb":"-1","fs.AbstractFileSystem.abfs.impl":"org.apache.hadoop.fs.azurebfs.Abfs","yarn.timeline-service.writer.flush-interval-seconds":"60","fs.s3a.fast.upload.active.blocks":"4","hadoop.security.credential.clear-text-fallback":"true","yarn.nodemanager.collector-service.thread-count":"5","fs.azure.secure.mode":"false","mapreduce.jobhistory.joblist.cache.size":"20000","fs.ftp.host":"0.0.0.0","yarn.resourcemanager.fs.state-store.num-retries":"0","yarn.resourcemanager.nodemanager-connect-retries":"10","yarn.nodemanager.log-aggregation.num-log-files-per-app":"30","hadoop.security.kms.client.encrypted.key.cache.low-watermark":"0.3f","fs.s3a.committer.magic.enabled":"false","yarn.timeline-service.client.max-retries":"30","dfs.ha.fencing.ssh.connect-timeout":"30000","yarn.log-aggregation-enable":"false","yarn.system-metrics-publisher.enabled":"false","mapreduce.reduce.markreset.buffer.percent":"0.0","fs.AbstractFileSystem.viewfs.impl":"org.apache.hadoop.fs.viewfs.ViewFs","mapreduce.task.io.sort.factor":"10","yarn.nodemanager.amrmproxy.client.thread-count":"25","ha.failover-controller.new-active.rpc-timeout.ms":"60000","yarn.nodemanager.container-localizer.java.opts":"-Xmx256m","mapreduce.jobhistory.datestring.cache.size":"200000","mapreduce.job.acl-modify-job":" ","yarn.nodemanager.windows-container.memory-limit.enabled":"false","yarn.timeline-service.webapp.address":"${yarn.timeline-service.hostname}:8188","yarn.app.mapreduce.am.job.committer.commit-window":"10000","yarn.nodemanager.container-manager.thread-count":"20","yarn.minicluster.fixed.ports":"false","hadoop.tags.system":"YARN,HDFS,NAMENODE,DATANODE,REQUIRED,SECURITY,KERBEROS,PERFORMANCE,CLIENT\n ,SERVER,DEBUG,DEPRECATED,COMMON,OPTIONAL","yarn.cluster.max-application-priority":"0","yarn.timeline-service.ttl-enable":"true","mapreduce.jobhistory.recovery.store.fs.uri":"${hadoop.tmp.dir}/mapred/history/recoverystore","hadoop.caller.context.signature.max.size":"40","yarn.client.load.resource-types.from-server":"false","ha.zookeeper.session-timeout.ms":"10000","tfile.io.chunk.size":"1048576","fs.s3a.s3guard.ddb.table.capacity.write":"100","mapreduce.job.speculative.slowtaskthreshold":"1.0","io.serializations":"org.apache.hadoop.io.serializer.WritableSerialization, org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization, org.apache.hadoop.io.serializer.avro.AvroReflectSerialization","hadoop.security.kms.client.failover.sleep.max.millis":"2000","hadoop.security.group.mapping.ldap.directory.search.timeout":"10000","yarn.scheduler.configuration.store.max-logs":"1000","yarn.nodemanager.node-attributes.provider.fetch-interval-ms":"600000","fs.swift.impl":"org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem","yarn.nodemanager.local-cache.max-files-per-directory":"8192","hadoop.http.cross-origin.enabled":"false","hadoop.zk.acl":"world:anyone:rwcda","mapreduce.map.sort.spill.percent":"0.80","yarn.timeline-service.entity-group-fs-store.scan-interval-seconds":"60","yarn.node-attribute.fs-store.impl.class":"org.apache.hadoop.yarn.server.resourcemanager.nodelabels.FileSystemNodeAttributeStore","fs.s3a.retry.interval":"500ms","yarn.timeline-service.client.best-effort":"false","yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled":"*********(redacted)","hadoop.security.group.mapping.ldap.posix.attr.uid.name":"uidNumber","fs.AbstractFileSystem.swebhdfs.impl":"org.apache.hadoop.fs.SWebHdfs","yarn.nodemanager.elastic-memory-control.timeout-sec":"5","mapreduce.ifile.readahead":"true","yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms":"300000","yarn.timeline-service.reader.webapp.address":"${yarn.timeline-service.webapp.address}","yarn.resourcemanager.placement-constraints.algorithm.pool-size":"1","yarn.timeline-service.hbase.coprocessor.jar.hdfs.location":"/hbase/coprocessor/hadoop-yarn-server-timelineservice.jar","hadoop.security.kms.client.encrypted.key.cache.num.refill.threads":"2","yarn.resourcemanager.scheduler.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler","yarn.app.mapreduce.am.command-opts":"-Xmx1024m","mapreduce.cluster.local.dir":"${hadoop.tmp.dir}/mapred/local","io.mapfile.bloom.error.rate":"0.005","fs.client.resolve.topology.enabled":"false","yarn.nodemanager.runtime.linux.allowed-runtimes":"default","yarn.sharedcache.store.class":"org.apache.hadoop.yarn.server.sharedcachemanager.store.InMemorySCMStore","ha.failover-controller.graceful-fence.rpc-timeout.ms":"5000","ftp.replication":"3","hadoop.security.uid.cache.secs":"14400","mapreduce.job.maxtaskfailures.per.tracker":"3","fs.s3a.metadatastore.impl":"org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore","io.skip.checksum.errors":"false","yarn.app.mapreduce.client-am.ipc.max-retries-on-timeouts":"3","yarn.timeline-service.webapp.xfs-filter.xframe-options":"SAMEORIGIN","fs.s3a.connection.timeout":"200000","mapreduce.job.max.split.locations":"15","yarn.resourcemanager.nm-container-queuing.max-queue-length":"15","hadoop.registry.zk.session.timeout.ms":"60000","yarn.federation.cache-ttl.secs":"300","mapreduce.jvm.system-properties-to-log":"os.name,os.version,java.home,java.runtime.version,java.vendor,java.version,java.vm.name,java.class.path,java.io.tmpdir,user.dir,user.name","yarn.resourcemanager.opportunistic-container-allocation.nodes-used":"10","yarn.timeline-service.entity-group-fs-store.active-dir":"/tmp/entity-file-history/active","mapreduce.shuffle.transfer.buffer.size":"131072","yarn.timeline-service.client.retry-interval-ms":"1000","yarn.http.policy":"HTTP_ONLY","fs.s3a.socket.send.buffer":"8192","fs.AbstractFileSystem.abfss.impl":"org.apache.hadoop.fs.azurebfs.Abfss","yarn.sharedcache.uploader.server.address":"0.0.0.0:8046","yarn.resourcemanager.delegation-token.max-conf-size-bytes":"*********(redacted)","hadoop.http.authentication.token.validity":"*********(redacted)","mapreduce.shuffle.max.connections":"0","yarn.minicluster.yarn.nodemanager.resource.memory-mb":"4096","mapreduce.job.emit-timeline-data":"false","yarn.nodemanager.resource.system-reserved-memory-mb":"-1","hadoop.kerberos.min.seconds.before.relogin":"60","mapreduce.jobhistory.move.thread-count":"3","yarn.resourcemanager.admin.client.thread-count":"1","yarn.dispatcher.drain-events.timeout":"300000","fs.s3a.buffer.dir":"${hadoop.tmp.dir}/s3a","hadoop.ssl.enabled.protocols":"TLSv1,SSLv2Hello,TLSv1.1,TLSv1.2","mapreduce.jobhistory.admin.address":"0.0.0.0:10033","yarn.log-aggregation-status.time-out.ms":"600000","fs.s3a.assumed.role.sts.endpoint.region":"us-west-1","mapreduce.shuffle.port":"13562","yarn.resourcemanager.max-log-aggregation-diagnostics-in-memory":"10","yarn.nodemanager.health-checker.interval-ms":"600000","yarn.router.clientrm.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.clientrm.DefaultClientRequestInterceptor","yarn.resourcemanager.zk-appid-node.split-index":"0","ftp.blocksize":"67108864","yarn.nodemanager.runtime.linux.sandbox-mode.local-dirs.permissions":"read","yarn.router.rmadmin.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.rmadmin.DefaultRMAdminRequestInterceptor","yarn.nodemanager.log-container-debug-info.enabled":"true","yarn.client.max-cached-nodemanagers-proxies":"0","yarn.nodemanager.linux-container-executor.cgroups.delete-delay-ms":"20","yarn.nodemanager.delete.debug-delay-sec":"0","yarn.nodemanager.pmem-check-enabled":"true","yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage":"90.0","mapreduce.app-submission.cross-platform":"false","yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms":"10000","yarn.nodemanager.container-retry-minimum-interval-ms":"1000","hadoop.security.groups.cache.secs":"300","yarn.federation.enabled":"false","fs.azure.local.sas.key.mode":"false","ipc.maximum.data.length":"67108864","mapreduce.shuffle.max.threads":"0","yarn.router.pipeline.cache-max-size":"25","yarn.resourcemanager.nm-container-queuing.load-comparator":"QUEUE_LENGTH","hadoop.security.authorization":"false","mapreduce.job.complete.cancel.delegation.tokens":"*********(redacted)","fs.s3a.paging.maximum":"5000","nfs.exports.allowed.hosts":"* rw","yarn.nodemanager.amrmproxy.ha.enable":"false","mapreduce.jobhistory.http.policy":"HTTP_ONLY","yarn.sharedcache.store.in-memory.check-period-mins":"720","hadoop.security.group.mapping.ldap.ssl":"false","yarn.client.application-client-protocol.poll-interval-ms":"200","yarn.scheduler.configuration.leveldb-store.compaction-interval-secs":"86400","yarn.timeline-service.writer.class":"org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl","ha.zookeeper.parent-znode":"/hadoop-ha","yarn.nodemanager.log-aggregation.policy.class":"org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.AllContainerLogAggregationPolicy","mapreduce.reduce.shuffle.merge.percent":"0.66","hadoop.security.group.mapping.ldap.search.filter.group":"(objectClass=group)","yarn.resourcemanager.placement-constraints.scheduler.pool-size":"1","yarn.nodemanager.resourcemanager.minimum.version":"NONE","mapreduce.job.speculative.speculative-cap-running-tasks":"0.1","yarn.admin.acl":"*","yarn.nodemanager.recovery.supervised":"false","yarn.sharedcache.admin.thread-count":"1","yarn.resourcemanager.ha.automatic-failover.enabled":"true","mapreduce.reduce.skip.maxgroups":"0","mapreduce.reduce.shuffle.connect.timeout":"180000","yarn.resourcemanager.address":"${yarn.resourcemanager.hostname}:8032","ipc.client.ping":"true","mapreduce.task.local-fs.write-limit.bytes":"-1","fs.adl.oauth2.access.token.provider.type":"*********(redacted)","mapreduce.shuffle.ssl.file.buffer.size":"65536","yarn.resourcemanager.ha.automatic-failover.embedded":"true","yarn.nodemanager.resource-plugins.gpu.docker-plugin":"nvidia-docker-v1","hadoop.ssl.enabled":"false","fs.s3a.multipart.purge":"false","yarn.scheduler.configuration.store.class":"file","yarn.resourcemanager.nm-container-queuing.queue-limit-stdev":"1.0f","mapreduce.job.end-notification.max.attempts":"5","mapreduce.output.fileoutputformat.compress.codec":"org.apache.hadoop.io.compress.DefaultCodec","yarn.nodemanager.container-monitor.procfs-tree.smaps-based-rss.enabled":"false","ipc.client.bind.wildcard.addr":"false","yarn.resourcemanager.webapp.rest-csrf.enabled":"false","ha.health-monitor.connect-retry-interval.ms":"1000","yarn.nodemanager.keytab":"/etc/krb5.keytab","mapreduce.jobhistory.keytab":"/etc/security/keytab/jhs.service.keytab","fs.s3a.threads.max":"10","mapreduce.reduce.shuffle.input.buffer.percent":"0.70","yarn.nodemanager.runtime.linux.docker.allowed-container-networks":"host,none,bridge","yarn.nodemanager.node-labels.resync-interval-ms":"120000","hadoop.tmp.dir":"/tmp/hadoop-${user.name}","mapreduce.job.maps":"2","mapreduce.jobhistory.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.job.end-notification.max.retry.interval":"5000","yarn.log-aggregation.retain-check-interval-seconds":"-1","yarn.resourcemanager.resource-tracker.client.thread-count":"50","yarn.rm.system-metrics-publisher.emit-container-events":"false","yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size":"10000","yarn.resourcemanager.ha.automatic-failover.zk-base-path":"/yarn-leader-election","io.seqfile.local.dir":"${hadoop.tmp.dir}/io/local","fs.s3a.s3guard.ddb.throttle.retry.interval":"100ms","fs.AbstractFileSystem.wasb.impl":"org.apache.hadoop.fs.azure.Wasb","mapreduce.client.submit.file.replication":"10","mapreduce.jobhistory.minicluster.fixed.ports":"false","fs.s3a.multipart.threshold":"2147483647","yarn.resourcemanager.webapp.xfs-filter.xframe-options":"SAMEORIGIN","mapreduce.jobhistory.done-dir":"${yarn.app.mapreduce.am.staging-dir}/history/done","ipc.client.idlethreshold":"4000","yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage":"false","mapreduce.reduce.input.buffer.percent":"0.0","yarn.nodemanager.runtime.linux.docker.userremapping-gid-threshold":"1","yarn.nodemanager.webapp.rest-csrf.enabled":"false","fs.ftp.host.port":"21","ipc.ping.interval":"60000","yarn.resourcemanager.history-writer.multi-threaded-dispatcher.pool-size":"10","yarn.resourcemanager.admin.address":"${yarn.resourcemanager.hostname}:8033","file.client-write-packet-size":"65536","ipc.client.kill.max":"10","mapreduce.reduce.speculative":"true","hadoop.security.key.default.bitlength":"128","mapreduce.job.reducer.unconditional-preempt.delay.sec":"300","yarn.nodemanager.disk-health-checker.interval-ms":"120000","yarn.nodemanager.log.deletion-threads-count":"4","yarn.webapp.filter-entity-list-by-user":"false","ipc.client.connection.maxidletime":"10000","mapreduce.task.io.sort.mb":"100","yarn.nodemanager.localizer.client.thread-count":"5","io.erasurecode.codec.rs.rawcoders":"rs_native,rs_java","io.erasurecode.codec.rs-legacy.rawcoders":"rs-legacy_java","yarn.sharedcache.admin.address":"0.0.0.0:8047","yarn.resourcemanager.placement-constraints.algorithm.iterator":"SERIAL","yarn.nodemanager.localizer.cache.cleanup.interval-ms":"600000","hadoop.security.crypto.codec.classes.aes.ctr.nopadding":"org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec, org.apache.hadoop.crypto.JceAesCtrCryptoCodec","mapreduce.job.cache.limit.max-resources-mb":"0","fs.s3a.connection.ssl.enabled":"true","yarn.nodemanager.process-kill-wait.ms":"5000","mapreduce.job.hdfs-servers":"${fs.defaultFS}","hadoop.workaround.non.threadsafe.getpwuid":"true","fs.df.interval":"60000","fs.s3a.multiobjectdelete.enable":"true","yarn.sharedcache.cleaner.resource-sleep-ms":"0","yarn.nodemanager.disk-health-checker.min-healthy-disks":"0.25","hadoop.shell.missing.defaultFs.warning":"false","io.file.buffer.size":"65536","hadoop.security.group.mapping.ldap.search.attr.member":"member","hadoop.security.random.device.file.path":"/dev/urandom","hadoop.security.sensitive-config-keys":"*********(redacted)","fs.s3a.s3guard.ddb.max.retries":"9","hadoop.rpc.socket.factory.class.default":"org.apache.hadoop.net.StandardSocketFactory","yarn.intermediate-data-encryption.enable":"false","yarn.resourcemanager.connect.retry-interval.ms":"30000","yarn.nodemanager.container.stderr.pattern":"{*stderr*,*STDERR*}","yarn.scheduler.minimum-allocation-mb":"1024","yarn.app.mapreduce.am.staging-dir":"/tmp/hadoop-yarn/staging","mapreduce.reduce.shuffle.read.timeout":"180000","hadoop.http.cross-origin.max-age":"1800","io.erasurecode.codec.xor.rawcoders":"xor_native,xor_java","fs.s3a.connection.establish.timeout":"5000","mapreduce.job.running.map.limit":"0","yarn.minicluster.control-resource-monitoring":"false","hadoop.ssl.require.client.cert":"false","hadoop.kerberos.kinit.command":"kinit","yarn.federation.state-store.class":"org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore","mapreduce.reduce.log.level":"INFO","hadoop.security.dns.log-slow-lookups.threshold.ms":"1000","mapreduce.job.ubertask.enable":"false","adl.http.timeout":"-1","yarn.resourcemanager.placement-constraints.retry-attempts":"3","hadoop.caller.context.enabled":"false","yarn.nodemanager.vmem-pmem-ratio":"2.1","hadoop.rpc.protection":"authentication","ha.health-monitor.rpc-timeout.ms":"45000","yarn.nodemanager.remote-app-log-dir":"/tmp/logs","hadoop.zk.timeout-ms":"10000","fs.s3a.s3guard.cli.prune.age":"86400000","yarn.nodemanager.resource.pcores-vcores-multiplier":"1.0","yarn.nodemanager.runtime.linux.sandbox-mode":"disabled","yarn.app.mapreduce.am.containerlauncher.threadpool-initial-size":"10","fs.s3a.committer.threads":"8","hadoop.zk.retry-interval-ms":"1000","hadoop.security.crypto.buffer.size":"8192","yarn.nodemanager.node-labels.provider.fetch-interval-ms":"600000","mapreduce.jobhistory.recovery.store.leveldb.path":"${hadoop.tmp.dir}/mapred/history/recoverystore","yarn.client.failover-retries-on-socket-timeouts":"0","yarn.nodemanager.resource.memory.enabled":"false","fs.azure.authorization.caching.enable":"true","hadoop.security.instrumentation.requires.admin":"false","yarn.nodemanager.delete.thread-count":"4","mapreduce.job.finish-when-all-reducers-done":"true","hadoop.registry.jaas.context":"Client","yarn.timeline-service.leveldb-timeline-store.path":"${hadoop.tmp.dir}/yarn/timeline","io.map.index.interval":"128","yarn.resourcemanager.nm-container-queuing.max-queue-wait-time-ms":"100","fs.abfs.impl":"org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","mapreduce.job.counters.max":"120","mapreduce.jobhistory.webapp.rest-csrf.enabled":"false","yarn.timeline-service.store-class":"org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore","mapreduce.jobhistory.move.interval-ms":"180000","yarn.nodemanager.localizer.fetch.thread-count":"4","yarn.resourcemanager.scheduler.client.thread-count":"50","hadoop.ssl.hostname.verifier":"DEFAULT","yarn.timeline-service.leveldb-state-store.path":"${hadoop.tmp.dir}/yarn/timeline","mapreduce.job.classloader":"false","mapreduce.task.profile.map.params":"${mapreduce.task.profile.params}","ipc.client.connect.timeout":"20000","hadoop.security.auth_to_local.mechanism":"hadoop","yarn.timeline-service.app-collector.linger-period.ms":"60000","yarn.nm.liveness-monitor.expiry-interval-ms":"600000","yarn.resourcemanager.reservation-system.planfollower.time-step":"1000","yarn.nodemanager.runtime.linux.docker.enable-userremapping.allowed":"true","yarn.webapp.api-service.enable":"false","yarn.nodemanager.recovery.enabled":"false","mapreduce.job.end-notification.retry.interval":"1000","fs.du.interval":"600000","fs.ftp.impl":"org.apache.hadoop.fs.ftp.FTPFileSystem","yarn.nodemanager.container.stderr.tail.bytes":"4096","hadoop.security.group.mapping.ldap.read.timeout.ms":"60000","hadoop.security.groups.cache.warn.after.ms":"5000","file.bytes-per-checksum":"512","mapreduce.outputcommitter.factory.scheme.s3a":"org.apache.hadoop.fs.s3a.commit.S3ACommitterFactory","hadoop.security.groups.cache.background.reload":"false","yarn.nodemanager.container-monitor.enabled":"true","yarn.nodemanager.elastic-memory-control.enabled":"false","net.topology.script.number.args":"100","mapreduce.task.merge.progress.records":"10000","yarn.nodemanager.localizer.address":"${yarn.nodemanager.hostname}:8040","yarn.timeline-service.keytab":"/etc/krb5.keytab","mapreduce.reduce.shuffle.fetch.retry.timeout-ms":"30000","yarn.resourcemanager.rm.container-allocation.expiry-interval-ms":"600000","mapreduce.fileoutputcommitter.algorithm.version":"1","yarn.resourcemanager.work-preserving-recovery.enabled":"true","mapreduce.map.skip.maxrecords":"0","yarn.sharedcache.root-dir":"/sharedcache","fs.s3a.retry.throttle.limit":"${fs.s3a.attempts.maximum}","hadoop.http.authentication.type":"simple","mapreduce.job.cache.limit.max-resources":"0","mapreduce.task.userlog.limit.kb":"0","yarn.resourcemanager.scheduler.monitor.enable":"false","ipc.client.connect.max.retries":"10","hadoop.registry.zk.retry.times":"5","yarn.nodemanager.resource-monitor.interval-ms":"3000","yarn.nodemanager.resource-plugins.gpu.allowed-gpu-devices":"auto","mapreduce.job.sharedcache.mode":"disabled","yarn.nodemanager.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.shuffle.listen.queue.size":"128","yarn.scheduler.configuration.mutation.acl-policy.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy","mapreduce.map.cpu.vcores":"1","yarn.log-aggregation.file-formats":"TFile","yarn.timeline-service.client.fd-retain-secs":"300","hadoop.user.group.static.mapping.overrides":"dr.who=;","fs.azure.sas.expiry.period":"90d","mapreduce.jobhistory.recovery.store.class":"org.apache.hadoop.mapreduce.v2.hs.HistoryServerFileSystemStateStoreService","yarn.resourcemanager.fail-fast":"${yarn.fail-fast}","yarn.resourcemanager.proxy-user-privileges.enabled":"false","yarn.router.webapp.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.webapp.DefaultRequestInterceptorREST","yarn.nodemanager.resource.memory.cgroups.soft-limit-percentage":"90.0","mapreduce.job.reducer.preempt.delay.sec":"0","hadoop.util.hash.type":"murmur","yarn.nodemanager.disk-validator":"basic","yarn.app.mapreduce.client.job.max-retries":"3","mapreduce.reduce.shuffle.retry-delay.max.ms":"60000","hadoop.security.group.mapping.ldap.connection.timeout.ms":"60000","mapreduce.task.profile.params":"-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,verbose=n,file=%s","yarn.app.mapreduce.shuffle.log.backups":"0","yarn.nodemanager.container-diagnostics-maximum-size":"10000","hadoop.registry.zk.retry.interval.ms":"1000","yarn.nodemanager.linux-container-executor.cgroups.delete-timeout-ms":"1000","fs.AbstractFileSystem.file.impl":"org.apache.hadoop.fs.local.LocalFs","yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds":"-1","mapreduce.jobhistory.cleaner.interval-ms":"86400000","hadoop.registry.zk.quorum":"localhost:2181","mapreduce.output.fileoutputformat.compress":"false","yarn.resourcemanager.am-rm-tokens.master-key-rolling-interval-secs":"*********(redacted)","fs.s3a.assumed.role.session.duration":"30m","hadoop.security.group.mapping.ldap.conversion.rule":"none","hadoop.ssl.server.conf":"ssl-server.xml","fs.s3a.retry.throttle.interval":"1000ms","seq.io.sort.factor":"100","yarn.sharedcache.cleaner.initial-delay-mins":"10","mapreduce.client.completion.pollinterval":"5000","hadoop.ssl.keystores.factory.class":"org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory","yarn.app.mapreduce.am.resource.cpu-vcores":"1","yarn.timeline-service.enabled":"false","yarn.nodemanager.runtime.linux.docker.capabilities":"CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE","yarn.acl.enable":"false","yarn.timeline-service.entity-group-fs-store.done-dir":"/tmp/entity-file-history/done/","mapreduce.task.profile":"false","yarn.resourcemanager.fs.state-store.uri":"${hadoop.tmp.dir}/yarn/system/rmstore","mapreduce.jobhistory.always-scan-user-dir":"false","yarn.nodemanager.opportunistic-containers-use-pause-for-preemption":"false","yarn.nodemanager.linux-container-executor.nonsecure-mode.local-user":"nobody","yarn.timeline-service.reader.class":"org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineReaderImpl","yarn.resourcemanager.configuration.provider-class":"org.apache.hadoop.yarn.LocalConfigurationProvider","yarn.nodemanager.runtime.linux.docker.userremapping-uid-threshold":"1","yarn.resourcemanager.configuration.file-system-based-store":"/yarn/conf","mapreduce.job.cache.limit.max-single-resource-mb":"0","yarn.nodemanager.runtime.linux.docker.stop.grace-period":"10","yarn.resourcemanager.resource-profiles.source-file":"resource-profiles.json","yarn.nodemanager.resource.percentage-physical-cpu-limit":"100","mapreduce.jobhistory.client.thread-count":"10","tfile.fs.input.buffer.size":"262144","mapreduce.client.progressmonitor.pollinterval":"1000","yarn.nodemanager.log-dirs":"${yarn.log.dir}/userlogs","fs.automatic.close":"true","yarn.nodemanager.hostname":"0.0.0.0","yarn.nodemanager.resource.memory.cgroups.swappiness":"0","ftp.stream-buffer-size":"4096","yarn.fail-fast":"false","yarn.timeline-service.app-aggregation-interval-secs":"15","hadoop.security.group.mapping.ldap.search.filter.user":"(&(objectClass=user)(sAMAccountName={0}))","yarn.nodemanager.container-localizer.log.level":"INFO","yarn.timeline-service.address":"${yarn.timeline-service.hostname}:10200","mapreduce.job.ubertask.maxmaps":"9","fs.s3a.threads.keepalivetime":"60","mapreduce.jobhistory.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","mapreduce.task.files.preserve.failedtasks":"false","yarn.app.mapreduce.client.job.retry-interval":"2000","ha.failover-controller.graceful-fence.connection.retries":"1","yarn.resourcemanager.delegation.token.max-lifetime":"*********(redacted)","yarn.timeline-service.client.drain-entities.timeout.ms":"2000","yarn.nodemanager.resource-plugins.fpga.vendor-plugin.class":"org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.IntelFpgaOpenclPlugin","yarn.timeline-service.entity-group-fs-store.summary-store":"org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore","mapreduce.reduce.cpu.vcores":"1","mapreduce.job.encrypted-intermediate-data.buffer.kb":"128","fs.client.resolve.remote.symlinks":"true","yarn.nodemanager.webapp.https.address":"0.0.0.0:8044","hadoop.http.cross-origin.allowed-origins":"*","mapreduce.job.encrypted-intermediate-data":"false","yarn.timeline-service.entity-group-fs-store.retain-seconds":"604800","yarn.resourcemanager.metrics.runtime.buckets":"60,300,1440","yarn.timeline-service.generic-application-history.max-applications":"10000","yarn.nodemanager.local-dirs":"${hadoop.tmp.dir}/nm-local-dir","mapreduce.shuffle.connection-keep-alive.enable":"false","yarn.node-labels.configuration-type":"centralized","fs.s3a.path.style.access":"false","yarn.nodemanager.aux-services.mapreduce_shuffle.class":"org.apache.hadoop.mapred.ShuffleHandler","yarn.sharedcache.store.in-memory.staleness-period-mins":"10080","fs.adl.impl":"org.apache.hadoop.fs.adl.AdlFileSystem","yarn.resourcemanager.nodemanager.minimum.version":"NONE","mapreduce.jobhistory.webapp.xfs-filter.xframe-options":"SAMEORIGIN","yarn.app.mapreduce.am.staging-dir.erasurecoding.enabled":"false","net.topology.impl":"org.apache.hadoop.net.NetworkTopology","io.map.index.skip":"0","yarn.timeline-service.reader.webapp.https.address":"${yarn.timeline-service.webapp.https.address}","fs.ftp.data.connection.mode":"ACTIVE_LOCAL_DATA_CONNECTION_MODE","mapreduce.job.local-fs.single-disk-limit.check.kill-limit-exceed":"true","yarn.scheduler.maximum-allocation-vcores":"4","hadoop.http.cross-origin.allowed-headers":"X-Requested-With,Content-Type,Accept,Origin","yarn.nodemanager.log-aggregation.compression-type":"none","yarn.timeline-service.version":"1.0f","yarn.ipc.rpc.class":"org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC","mapreduce.reduce.maxattempts":"4","hadoop.security.dns.log-slow-lookups.enabled":"false","mapreduce.job.committer.setup.cleanup.needed":"true","mapreduce.job.running.reduce.limit":"0","ipc.maximum.response.length":"134217728","yarn.resourcemanager.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","mapreduce.job.token.tracking.ids.enabled":"*********(redacted)","hadoop.caller.context.max.size":"128","yarn.nodemanager.runtime.linux.docker.host-pid-namespace.allowed":"false","yarn.nodemanager.runtime.linux.docker.delayed-removal.allowed":"false","hadoop.registry.system.acls":"sasl:yarn@, sasl:mapred@, sasl:hdfs@","yarn.nodemanager.recovery.dir":"${hadoop.tmp.dir}/yarn-nm-recovery","fs.s3a.fast.upload.buffer":"disk","mapreduce.jobhistory.intermediate-done-dir":"${yarn.app.mapreduce.am.staging-dir}/history/done_intermediate","yarn.app.mapreduce.shuffle.log.separate":"true","fs.s3a.max.total.tasks":"5","fs.s3a.readahead.range":"64K","hadoop.http.authentication.simple.anonymous.allowed":"true","fs.s3a.attempts.maximum":"20","hadoop.registry.zk.connection.timeout.ms":"15000","yarn.resourcemanager.delegation-token-renewer.thread-count":"*********(redacted)","yarn.nodemanager.health-checker.script.timeout-ms":"1200000","yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size":"10000","yarn.resourcemanager.resource-profiles.enabled":"false","yarn.timeline-service.hbase-schema.prefix":"prod.","fs.azure.authorization":"false","mapreduce.map.log.level":"INFO","yarn.resourcemanager.decommissioning-nodes-watcher.poll-interval-secs":"20","mapreduce.output.fileoutputformat.compress.type":"RECORD","yarn.resourcemanager.leveldb-state-store.path":"${hadoop.tmp.dir}/yarn/system/rmstore","yarn.timeline-service.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.ifile.readahead.bytes":"4194304","yarn.sharedcache.app-checker.class":"org.apache.hadoop.yarn.server.sharedcachemanager.RemoteAppChecker","yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users":"true","yarn.nodemanager.resource.detect-hardware-capabilities":"false","mapreduce.cluster.acls.enabled":"false","mapreduce.job.speculative.retry-after-no-speculate":"1000","hadoop.security.group.mapping.ldap.search.group.hierarchy.levels":"0","yarn.resourcemanager.fs.state-store.retry-interval-ms":"1000","file.stream-buffer-size":"4096","yarn.resourcemanager.application-timeouts.monitor.interval-ms":"3000","mapreduce.map.output.compress.codec":"org.apache.hadoop.io.compress.DefaultCodec","mapreduce.map.speculative":"true","mapreduce.job.speculative.retry-after-speculate":"15000","yarn.nodemanager.linux-container-executor.cgroups.mount":"false","yarn.app.mapreduce.am.container.log.backups":"0","yarn.app.mapreduce.am.log.level":"INFO","mapreduce.job.reduce.slowstart.completedmaps":"0.05","yarn.timeline-service.http-authentication.type":"simple","hadoop.security.group.mapping.ldap.search.attr.group.name":"cn","yarn.nodemanager.resource-plugins.fpga.allowed-fpga-devices":"0,1","yarn.timeline-service.client.internal-timers-ttl-secs":"420","hadoop.http.logs.enabled":"true","fs.s3a.block.size":"32M","yarn.sharedcache.client-server.address":"0.0.0.0:8045","yarn.nodemanager.logaggregation.threadpool-size-max":"100","yarn.resourcemanager.hostname":"hostb7","yarn.resourcemanager.delegation.key.update-interval":"86400000","mapreduce.reduce.shuffle.fetch.retry.enabled":"${yarn.nodemanager.recovery.enabled}","mapreduce.map.memory.mb":"-1","mapreduce.task.skip.start.attempts":"2","fs.AbstractFileSystem.hdfs.impl":"org.apache.hadoop.fs.Hdfs","yarn.nodemanager.disk-health-checker.enable":"true","ipc.client.tcpnodelay":"true","ipc.client.rpc-timeout.ms":"0","yarn.nodemanager.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","ipc.client.low-latency":"false","mapreduce.input.lineinputformat.linespermap":"1","yarn.router.interceptor.user.threadpool-size":"5","ipc.client.connect.max.retries.on.timeouts":"45","yarn.timeline-service.leveldb-timeline-store.read-cache-size":"104857600","fs.AbstractFileSystem.har.impl":"org.apache.hadoop.fs.HarFs","mapreduce.job.split.metainfo.maxsize":"10000000","yarn.am.liveness-monitor.expiry-interval-ms":"600000","yarn.resourcemanager.container-tokens.master-key-rolling-interval-secs":"*********(redacted)","yarn.timeline-service.entity-group-fs-store.app-cache-size":"10","fs.s3a.socket.recv.buffer":"8192","yarn.resourcemanager.resource-tracker.address":"${yarn.resourcemanager.hostname}:8031","yarn.nodemanager.node-labels.provider.fetch-timeout-ms":"1200000","mapreduce.job.heap.memory-mb.ratio":"0.8","yarn.resourcemanager.leveldb-state-store.compaction-interval-secs":"3600","yarn.resourcemanager.webapp.rest-csrf.custom-header":"X-XSRF-Header","yarn.scheduler.configuration.fs.path":"file://${hadoop.tmp.dir}/yarn/system/schedconf","mapreduce.client.output.filter":"FAILED","hadoop.http.filter.initializers":"org.apache.hadoop.http.lib.StaticUserWebFilter","mapreduce.reduce.memory.mb":"-1","yarn.timeline-service.hostname":"0.0.0.0","file.replication":"1","yarn.nodemanager.container-metrics.unregister-delay-ms":"10000","yarn.nodemanager.container-metrics.period-ms":"-1","mapreduce.fileoutputcommitter.task.cleanup.enabled":"false","yarn.nodemanager.log.retain-seconds":"10800","yarn.timeline-service.entity-group-fs-store.cleaner-interval-seconds":"3600","yarn.resourcemanager.keytab":"/etc/krb5.keytab","hadoop.security.group.mapping.providers.combined":"true","mapreduce.reduce.merge.inmem.threshold":"1000","yarn.timeline-service.recovery.enabled":"false","fs.azure.saskey.usecontainersaskeyforallaccess":"true","yarn.sharedcache.nm.uploader.thread-count":"20","yarn.resourcemanager.nodemanager-graceful-decommission-timeout-secs":"3600","mapreduce.shuffle.ssl.enabled":"false","yarn.timeline-service.hbase.coprocessor.app-final-value-retention-milliseconds":"259200000","fs.s3a.committer.staging.abort.pending.uploads":"true","yarn.nodemanager.opportunistic-containers-max-queue-length":"0","yarn.resourcemanager.state-store.max-completed-applications":"${yarn.resourcemanager.max-completed-applications}","mapreduce.job.speculative.minimum-allowed-tasks":"10","yarn.log-aggregation.retain-seconds":"-1","yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb":"0","mapreduce.jobhistory.max-age-ms":"604800000","hadoop.http.cross-origin.allowed-methods":"GET,POST,HEAD","yarn.resourcemanager.opportunistic-container-allocation.enabled":"false","mapreduce.jobhistory.webapp.address":"0.0.0.0:19888","hadoop.system.tags":"YARN,HDFS,NAMENODE,DATANODE,REQUIRED,SECURITY,KERBEROS,PERFORMANCE,CLIENT\n ,SERVER,DEBUG,DEPRECATED,COMMON,OPTIONAL","yarn.log-aggregation.file-controller.TFile.class":"org.apache.hadoop.yarn.logaggregation.filecontroller.tfile.LogAggregationTFileController","yarn.client.nodemanager-connect.max-wait-ms":"180000","yarn.resourcemanager.webapp.address":"${yarn.resourcemanager.hostname}:8088","mapreduce.jobhistory.recovery.enable":"false","mapreduce.reduce.shuffle.parallelcopies":"5","fs.AbstractFileSystem.webhdfs.impl":"org.apache.hadoop.fs.WebHdfs","fs.trash.interval":"0","yarn.app.mapreduce.client.max-retries":"3","hadoop.security.authentication":"simple","mapreduce.task.profile.reduce.params":"${mapreduce.task.profile.params}","yarn.app.mapreduce.am.resource.mb":"1536","mapreduce.input.fileinputformat.list-status.num-threads":"1","yarn.nodemanager.container-executor.class":"org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor","io.mapfile.bloom.size":"1048576","yarn.timeline-service.ttl-ms":"604800000","yarn.resourcemanager.nm-container-queuing.min-queue-length":"5","yarn.nodemanager.resource.cpu-vcores":"-1","mapreduce.job.reduces":"1","fs.s3a.multipart.size":"100M","yarn.scheduler.minimum-allocation-vcores":"1","mapreduce.job.speculative.speculative-cap-total-tasks":"0.01","hadoop.ssl.client.conf":"ssl-client.xml","mapreduce.job.queuename":"default","mapreduce.job.encrypted-intermediate-data-key-size-bits":"128","fs.s3a.metadatastore.authoritative":"false","yarn.nodemanager.webapp.xfs-filter.xframe-options":"SAMEORIGIN","ha.health-monitor.sleep-after-disconnect.ms":"1000","yarn.app.mapreduce.shuffle.log.limit.kb":"0","hadoop.security.group.mapping":"org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback","yarn.client.application-client-protocol.poll-timeout-ms":"-1","mapreduce.jobhistory.jhist.format":"binary","yarn.resourcemanager.ha.enabled":"false","hadoop.http.staticuser.user":"dr.who","mapreduce.task.exit.timeout.check-interval-ms":"20000","mapreduce.jobhistory.intermediate-user-done-dir.permissions":"770","mapreduce.task.exit.timeout":"60000","yarn.nodemanager.linux-container-executor.resources-handler.class":"org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler","mapreduce.reduce.shuffle.memory.limit.percent":"0.25","yarn.resourcemanager.reservation-system.enable":"false","mapreduce.map.output.compress":"false","ha.zookeeper.acl":"world:anyone:rwcda","ipc.server.max.connections":"0","yarn.nodemanager.aux-services":"mapreduce_shuffle","yarn.nodemanager.runtime.linux.docker.default-container-network":"host","yarn.router.webapp.address":"0.0.0.0:8089","yarn.scheduler.maximum-allocation-mb":"8192","yarn.resourcemanager.scheduler.monitor.policies":"org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy","yarn.sharedcache.cleaner.period-mins":"1440","yarn.nodemanager.resource-plugins.gpu.docker-plugin.nvidia-docker-v1.endpoint":"http://localhost:3476/v1.0/docker/cli","yarn.app.mapreduce.am.container.log.limit.kb":"0","ipc.client.connect.retry.interval":"1000","yarn.timeline-service.http-cross-origin.enabled":"false","fs.wasbs.impl":"org.apache.hadoop.fs.azure.NativeAzureFileSystem$Secure","yarn.federation.subcluster-resolver.class":"org.apache.hadoop.yarn.server.federation.resolver.DefaultSubClusterResolverImpl","yarn.resourcemanager.zk-state-store.parent-path":"/rmstore","mapreduce.jobhistory.cleaner.enable":"true","yarn.timeline-service.client.fd-flush-interval-secs":"10","hadoop.security.kms.client.encrypted.key.cache.expiry":"43200000","yarn.client.nodemanager-client-async.thread-pool-max-size":"500","mapreduce.map.maxattempts":"4","yarn.resourcemanager.nm-container-queuing.sorting-nodes-interval-ms":"1000","fs.s3a.committer.staging.tmp.path":"tmp/staging","yarn.nodemanager.sleep-delay-before-sigkill.ms":"250","yarn.resourcemanager.nm-container-queuing.min-queue-wait-time-ms":"10","mapreduce.job.end-notification.retry.attempts":"0","yarn.nodemanager.resource.count-logical-processors-as-cores":"false","hadoop.registry.zk.root":"/registry","adl.feature.ownerandgroup.enableupn":"false","yarn.resourcemanager.zk-max-znode-size.bytes":"1048576","mapreduce.job.reduce.shuffle.consumer.plugin.class":"org.apache.hadoop.mapreduce.task.reduce.Shuffle","yarn.resourcemanager.delayed.delegation-token.removal-interval-ms":"*********(redacted)","yarn.nodemanager.localizer.cache.target-size-mb":"10240","fs.s3a.committer.staging.conflict-mode":"fail","mapreduce.client.libjars.wildcard":"true","fs.s3a.committer.staging.unique-filenames":"true","yarn.nodemanager.node-attributes.provider.fetch-timeout-ms":"1200000","fs.s3a.list.version":"2","ftp.client-write-packet-size":"65536","fs.AbstractFileSystem.adl.impl":"org.apache.hadoop.fs.adl.Adl","hadoop.security.key.default.cipher":"AES/CTR/NoPadding","yarn.client.failover-retries":"0","fs.s3a.multipart.purge.age":"86400","mapreduce.job.local-fs.single-disk-limit.check.interval-ms":"5000","net.topology.node.switch.mapping.impl":"org.apache.hadoop.net.ScriptBasedMapping","yarn.nodemanager.amrmproxy.address":"0.0.0.0:8049","ipc.server.listen.queue.size":"128","map.sort.class":"org.apache.hadoop.util.QuickSort","fs.viewfs.rename.strategy":"SAME_MOUNTPOINT","hadoop.security.kms.client.authentication.retry-count":"1","fs.permissions.umask-mode":"022","fs.s3a.assumed.role.credentials.provider":"org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider","yarn.nodemanager.vmem-check-enabled":"true","yarn.nodemanager.numa-awareness.enabled":"false","yarn.nodemanager.recovery.compaction-interval-secs":"3600","yarn.app.mapreduce.client-am.ipc.max-retries":"3","yarn.federation.registry.base-dir":"yarnfederation/","mapreduce.job.max.map":"-1","mapreduce.job.local-fs.single-disk-limit.bytes":"-1","mapreduce.job.ubertask.maxreduces":"1","hadoop.security.kms.client.encrypted.key.cache.size":"500","hadoop.security.java.secure.random.algorithm":"SHA1PRNG","ha.failover-controller.cli-check.rpc-timeout.ms":"20000","mapreduce.jobhistory.jobname.limit":"50","mapreduce.application.classpath":"$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*:$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*","yarn.client.nodemanager-connect.retry-interval-ms":"10000","yarn.timeline-service.state-store-class":"org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore","yarn.nodemanager.env-whitelist":"JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,PATH,LANG,TZ,HADOOP_MAPRED_HOME","yarn.sharedcache.nested-level":"3","yarn.timeline-service.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","fs.azure.user.agent.prefix":"unknown","yarn.resourcemanager.zk-delegation-token-node.split-index":"*********(redacted)","yarn.nodemanager.numa-awareness.read-topology":"false","yarn.nodemanager.webapp.address":"${yarn.nodemanager.hostname}:8042","rpc.metrics.quantile.enable":"false","yarn.registry.class":"org.apache.hadoop.registry.client.impl.FSRegistryOperationsService","mapreduce.jobhistory.admin.acl":"*","yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size":"10","yarn.scheduler.queue-placement-rules":"user-group","hadoop.http.authentication.kerberos.keytab":"${user.home}/hadoop.keytab","yarn.resourcemanager.recovery.enabled":"false","yarn.timeline-service.webapp.rest-csrf.enabled":"false"},"System Properties":{"java.io.tmpdir":"/tmp","line.separator":"\n","path.separator":":","sun.management.compiler":"HotSpot 64-Bit Tiered Compilers","SPARK_SUBMIT":"true","sun.cpu.endian":"little","java.specification.version":"1.8","java.vm.specification.name":"Java Virtual Machine Specification","java.vendor":"Oracle Corporation","java.vm.specification.version":"1.8","user.home":"/home/usera112","file.encoding.pkg":"sun.io","sun.nio.ch.bugLevel":"","sun.arch.data.model":"64","sun.boot.library.path":"/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64","user.dir":"/home/usera112/projects/log-compressor/deployments/HiBench","java.library.path":"/usr/java/packages/lib/amd64:/usr/lib/x86_64-linux-gnu/jni:/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:/usr/lib/jni:/lib:/usr/lib","sun.cpu.isalist":"","os.arch":"amd64","java.vm.version":"25.162-b12","jetty.git.hash":"b881a572662e1943a14ae12e7e1207989f218b74","java.endorsed.dirs":"/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/endorsed","java.runtime.version":"1.8.0_162-8u162-b12-1-b12","java.vm.info":"mixed mode","java.ext.dirs":"/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/ext:/usr/java/packages/lib/ext","java.runtime.name":"OpenJDK Runtime Environment","file.separator":"/","java.class.version":"52.0","java.specification.name":"Java Platform API Specification","sun.boot.class.path":"/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/resources.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/rt.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/sunrsasign.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/jsse.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/jce.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/charsets.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/jfr.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/classes","file.encoding":"UTF-8","user.timezone":"America/Toronto","java.specification.vendor":"Oracle Corporation","sun.java.launcher":"SUN_STANDARD","os.version":"4.15.0-29-generic","sun.os.patch.level":"unknown","java.vm.specification.vendor":"Oracle Corporation","user.country":"US","sun.jnu.encoding":"UTF-8","user.language":"en","java.vendor.url":"http://java.oracle.com/","java.awt.printerjob":"sun.print.PSPrinterJob","java.awt.graphicsenv":"sun.awt.X11GraphicsEnvironment","awt.toolkit":"sun.awt.X11.XToolkit","os.name":"Linux","java.vm.vendor":"Oracle Corporation","java.vendor.url.bug":"http://bugreport.sun.com/bugreport/","user.name":"usera112","java.vm.name":"OpenJDK 64-Bit Server VM","sun.java.command":"org.apache.spark.deploy.SparkSubmit --master spark://hostb7:7077 --properties-file /home/usera112/projects/log-compressor/deployments/HiBench/report/join/spark/conf/sparkbench/spark.conf --class com.intel.hibench.sparkbench.sql.ScalaSparkSQLBench /home/usera112/projects/log-compressor/deployments/HiBench/sparkbench/assembly/target/sparkbench-assembly-8.0-SNAPSHOT-dist.jar ScalaJoin /home/usera112/projects/log-compressor/deployments/HiBench/report/join/spark/conf/../rankings_uservisits_join.hive","java.home":"/usr/lib/jvm/java-8-openjdk-amd64/jre","java.version":"1.8.0_162","sun.io.unicode.encoding":"UnicodeLittle"},"Classpath Entries":{"/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jackson-mapper-asl-1.9.13.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-yarn-api-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hk2-locator-2.6.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/nimbus-jose-jwt-4.41.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/htrace-core4-4.1.0-incubating.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/cats-kernel_2.12-2.0.0-M4.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/httpclient-4.5.6.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-collections-3.2.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-beeline-2.3.7.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/json-smart-2.3.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/curator-recipes-2.13.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/stream-2.9.6.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-graphx_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jline-2.14.6.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-admissionregistration-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/guava-14.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-sql_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/parquet-common-1.10.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/chill-java-0.9.5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-beanutils-1.9.4.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kryo-shaded-4.0.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hk2-api-2.6.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-io-2.5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/datanucleus-rdbms-4.1.19.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/metrics-graphite-4.1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/breeze_2.12-1.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jakarta.xml.bind-api-2.3.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jakarta.activation-api-1.2.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-lang3-3.10.jar":"System Classpath","spark://hostb7.dsrg.utoronto.ca:44011/jars/sparkbench-assembly-8.0-SNAPSHOT-dist.jar":"Added By User","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-mesos_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/avro-mapred-1.8.2-hadoop2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jsr305-3.0.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/HikariCP-2.5.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/ehcache-3.3.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-cli-1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-core-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/joda-time-2.10.5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/javassist-3.25.0-GA.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerby-config-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/token-provider-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/leveldbjni-all-1.8.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-sketch_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-mllib-local_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-apps-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/json4s-ast_2.12-3.7.0-M5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/json-1.8.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/datanucleus-core-4.1.17.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spire-macros_2.12-0.17.0-M1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/parquet-jackson-1.10.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-yarn-server-web-proxy-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-math3-3.4.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-vector-code-gen-2.3.7.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-network-shuffle_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/RoaringBitmap-0.9.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-mapreduce-client-jobclient-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/super-csv-2.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/pyrolite-4.30.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-yarn-common-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/slf4j-log4j12-1.7.30.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/macro-compat_2.12-1.1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-discovery-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-hive_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-metastore-2.3.7.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-streaming_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jackson-databind-2.10.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/snakeyaml-1.24.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-lang-2.6.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/libfb303-0.9.3.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/curator-framework-2.13.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerby-xdr-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/xz-1.5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-shims-common-2.3.7.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/mesos-1.4.0-shaded-protobuf.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerb-crypto-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-llap-common-2.3.7.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jersey-server-2.30.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/minlog-1.3.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jackson-module-scala_2.12-2.10.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/okhttp-2.7.5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/transaction-api-1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-exec-2.3.7-core.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-tags_2.12-3.1.2-tests.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/shims-0.9.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-jdbc-2.3.7.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/paranamer-2.8.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/woodstox-core-5.0.3.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/generex-1.0.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jackson-annotations-2.10.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spire-platform_2.12-0.17.0-M1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/chill_2.12-0.9.5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/JLargeArrays-1.5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-net-3.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-service-rpc-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/py4j-0.10.9.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/okhttp-3.12.12.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-crypto-1.1.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-core_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/zjsonpatch-0.3.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/guice-servlet-4.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/derby-10.12.1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-pool-1.5.4.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/parquet-encoding-1.10.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/scala-xml_2.12-1.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jersey-client-2.30.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerby-pkix-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerb-common-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-annotations-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-shims-2.3.7.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-compiler-3.0.16.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-kvstore_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-repl_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jcip-annotations-1.0-1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/avro-1.8.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/osgi-resource-locator-1.0.3.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/javolution-5.5.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/re2j-1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/orc-core-1.5.12.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/javax.jdo-3.2.0-m3.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/bonecp-0.8.0.RELEASE.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jakarta.inject-2.6.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/objenesis-2.6.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jersey-hk2-2.30.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spire-util_2.12-0.17.0-M1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/datanucleus-api-jdo-4.2.4.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-settings-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/aopalliance-1.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-kubernetes_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/httpcore-4.4.12.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/janino-3.0.16.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/scala-reflect-2.12.10.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/aopalliance-repackaged-2.6.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/libthrift-0.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/json4s-jackson_2.12-3.7.0-M5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-policy-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jackson-module-paranamer-2.10.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jaxb-runtime-2.3.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/orc-mapreduce-1.5.12.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-storageclass-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jpam-1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/gson-2.2.4.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/automaton-1.11-8.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/curator-client-2.13.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-codec-1.10.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-httpclient-3.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/flatbuffers-java-1.9.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-apiextensions-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/zookeeper-3.4.14.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/netty-all-4.1.51.Final.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-yarn-client-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jersey-container-servlet-2.30.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jackson-module-jaxb-annotations-2.10.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jakarta.ws.rs-api-2.1.6.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/xbean-asm7-shaded-4.15.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-auth-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-batch-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/arrow-vector-2.0.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jakarta.servlet-api-4.0.3.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-unsafe_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-client-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-cli-2.3.7.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-mllib_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/json4s-core_2.12-3.7.0-M5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/scala-parser-combinators_2.12-1.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/arrow-memory-core-2.0.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jersey-common-2.30.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerby-util-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jdo-api-3.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-metrics-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-storage-api-2.7.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jackson-jaxrs-base-2.9.5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/ivy-2.4.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-extensions-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-networking-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-events-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hk2-utils-2.6.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-common-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-rbac-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/protobuf-java-2.5.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/scala-compiler-2.12.10.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/lz4-java-1.7.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/threeten-extra-1.5.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/stax-api-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-client-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/parquet-hadoop-1.10.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/breeze-macros_2.12-1.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/ST4-4.0.4.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-serde-2.3.7.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/hadoop-3.3.1/etc/hadoop/":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/snappy-java-1.1.8.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/core-1.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerb-identity-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/orc-shims-1.5.12.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-configuration2-2.1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jaxb-api-2.2.11.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/arrow-format-2.0.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jsp-api-2.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/activation-1.1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jakarta.annotation-api-1.3.5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/oro-2.0.8.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/shapeless_2.12-2.3.3.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jackson-core-2.10.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-daemon-1.0.13.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-launcher_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/opencsv-2.3.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-tags_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-text-1.6.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/arrow-memory-netty-2.0.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerb-util-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerby-asn1-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jackson-core-asl-1.9.13.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-coordination-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jackson-dataformat-yaml-2.10.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerb-admin-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/avro-ipc-1.8.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-mapreduce-client-common-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/antlr4-runtime-4.8-1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jackson-jaxrs-json-provider-2.9.5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-common-2.3.7.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/arpack_combined_all-0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/geronimo-jcache_1.0_spec-1.0-alpha-1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/machinist_2.12-0.6.8.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/aircompressor-0.10.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/univocity-parsers-2.9.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerb-server-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-dbcp-1.4.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/stax2-api-3.1.4.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/istack-commons-runtime-3.0.8.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-yarn-server-common-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerb-simplekdc-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/antlr-runtime-3.5.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/JTransforms-3.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/guice-4.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-yarn_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/slf4j-api-1.7.30.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jackson-datatype-jsr310-2.11.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/parquet-format-2.4.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-yarn-registry-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/velocity-1.5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/javax.inject-1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-scheduling-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-autoscaling-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerb-client-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spire_2.12-0.17.0-M1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/log4j-1.2.17.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-network-common_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/logging-interceptor-3.12.12.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/dnsjava-2.1.7.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/audience-annotations-0.5.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/metrics-jmx-4.1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jta-1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-shims-scheduler-2.3.7.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/json4s-scalap_2.12-3.7.0-M5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/conf/":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/metrics-json-4.1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jersey-container-servlet-core-2.30.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jodd-core-3.5.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/compress-lzf-1.0.3.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/metrics-jvm-4.1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/scala-collection-compat_2.12-2.1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/parquet-column-1.10.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/metrics-core-4.1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/scala-library-2.12.10.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-common-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-hive-thriftserver_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/zstd-jni-1.4.8-1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jakarta.validation-api-2.0.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerb-core-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/okio-1.14.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-logging-1.1.3.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jcl-over-slf4j-1.7.30.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jersey-media-jaxb-2.30.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/algebra_2.12-2.0.0-M2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-compress-1.20.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-hdfs-client-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-shims-0.23-2.3.7.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jul-to-slf4j-1.7.30.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-certificates-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-mapreduce-client-core-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-catalyst_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/accessors-smart-1.2.jar":"System Classpath"}} +{"Event":"SparkListenerApplicationStart","App Name":"ScalaJoin","App ID":"app-20211009101924-2782","Timestamp":1633789163045,"User":"usera112"} +{"Event":"SparkListenerExecutorAdded","Timestamp":1633789166140,"Executor ID":"5","Executor Info":{"Host":"10.1.0.19","Total Cores":16,"Log Urls":{"stdout":"http://10.1.0.19:8081/logPage/?appId=app-20211009101924-2782&executorId=5&logType=stdout","stderr":"http://10.1.0.19:8081/logPage/?appId=app-20211009101924-2782&executorId=5&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0}} +{"Event":"SparkListenerExecutorAdded","Timestamp":1633789166162,"Executor ID":"10","Executor Info":{"Host":"10.1.0.16","Total Cores":16,"Log Urls":{"stdout":"http://10.1.0.16:8081/logPage/?appId=app-20211009101924-2782&executorId=10&logType=stdout","stderr":"http://10.1.0.16:8081/logPage/?appId=app-20211009101924-2782&executorId=10&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0}} +{"Event":"SparkListenerExecutorAdded","Timestamp":1633789166196,"Executor ID":"11","Executor Info":{"Host":"10.1.0.13","Total Cores":16,"Log Urls":{"stdout":"http://10.1.0.13:8081/logPage/?appId=app-20211009101924-2782&executorId=11&logType=stdout","stderr":"http://10.1.0.13:8081/logPage/?appId=app-20211009101924-2782&executorId=11&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0}} +{"Event":"SparkListenerExecutorAdded","Timestamp":1633789166205,"Executor ID":"1","Executor Info":{"Host":"10.1.0.17","Total Cores":16,"Log Urls":{"stdout":"http://10.1.0.17:8081/logPage/?appId=app-20211009101924-2782&executorId=1&logType=stdout","stderr":"http://10.1.0.17:8081/logPage/?appId=app-20211009101924-2782&executorId=1&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0}} +{"Event":"SparkListenerExecutorAdded","Timestamp":1633789166218,"Executor ID":"4","Executor Info":{"Host":"10.1.0.8","Total Cores":16,"Log Urls":{"stdout":"http://10.1.0.8:8081/logPage/?appId=app-20211009101924-2782&executorId=4&logType=stdout","stderr":"http://10.1.0.8:8081/logPage/?appId=app-20211009101924-2782&executorId=4&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0}} +{"Event":"SparkListenerExecutorAdded","Timestamp":1633789166224,"Executor ID":"6","Executor Info":{"Host":"10.1.0.18","Total Cores":16,"Log Urls":{"stdout":"http://10.1.0.18:8081/logPage/?appId=app-20211009101924-2782&executorId=6&logType=stdout","stderr":"http://10.1.0.18:8081/logPage/?appId=app-20211009101924-2782&executorId=6&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0}} +{"Event":"SparkListenerExecutorAdded","Timestamp":1633789166231,"Executor ID":"0","Executor Info":{"Host":"10.1.0.12","Total Cores":16,"Log Urls":{"stdout":"http://10.1.0.12:8081/logPage/?appId=app-20211009101924-2782&executorId=0&logType=stdout","stderr":"http://10.1.0.12:8081/logPage/?appId=app-20211009101924-2782&executorId=0&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0}} +{"Event":"SparkListenerExecutorAdded","Timestamp":1633789166233,"Executor ID":"2","Executor Info":{"Host":"10.1.0.10","Total Cores":16,"Log Urls":{"stdout":"http://10.1.0.10:8081/logPage/?appId=app-20211009101924-2782&executorId=2&logType=stdout","stderr":"http://10.1.0.10:8081/logPage/?appId=app-20211009101924-2782&executorId=2&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0}} +{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"5","Host":"10.1.0.19","Port":33541},"Maximum Memory":2101975449,"Timestamp":1633789166236,"Maximum Onheap Memory":2101975449,"Maximum Offheap Memory":0} +{"Event":"SparkListenerExecutorAdded","Timestamp":1633789166249,"Executor ID":"8","Executor Info":{"Host":"10.1.0.11","Total Cores":16,"Log Urls":{"stdout":"http://10.1.0.11:8081/logPage/?appId=app-20211009101924-2782&executorId=8&logType=stdout","stderr":"http://10.1.0.11:8081/logPage/?appId=app-20211009101924-2782&executorId=8&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0}} +{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"10","Host":"10.1.0.16","Port":32829},"Maximum Memory":2101975449,"Timestamp":1633789166253,"Maximum Onheap Memory":2101975449,"Maximum Offheap Memory":0} +{"Event":"SparkListenerExecutorAdded","Timestamp":1633789166258,"Executor ID":"9","Executor Info":{"Host":"10.1.0.14","Total Cores":16,"Log Urls":{"stdout":"http://10.1.0.14:8081/logPage/?appId=app-20211009101924-2782&executorId=9&logType=stdout","stderr":"http://10.1.0.14:8081/logPage/?appId=app-20211009101924-2782&executorId=9&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0}} +{"Event":"SparkListenerExecutorAdded","Timestamp":1633789166270,"Executor ID":"3","Executor Info":{"Host":"10.1.0.15","Total Cores":16,"Log Urls":{"stdout":"http://10.1.0.15:8081/logPage/?appId=app-20211009101924-2782&executorId=3&logType=stdout","stderr":"http://10.1.0.15:8081/logPage/?appId=app-20211009101924-2782&executorId=3&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0}} +{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"11","Host":"10.1.0.13","Port":37477},"Maximum Memory":2101975449,"Timestamp":1633789166302,"Maximum Onheap Memory":2101975449,"Maximum Offheap Memory":0} +{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"1","Host":"10.1.0.17","Port":36603},"Maximum Memory":2101975449,"Timestamp":1633789166304,"Maximum Onheap Memory":2101975449,"Maximum Offheap Memory":0} +{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"4","Host":"10.1.0.8","Port":42035},"Maximum Memory":2101975449,"Timestamp":1633789166315,"Maximum Onheap Memory":2101975449,"Maximum Offheap Memory":0} +{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"6","Host":"10.1.0.18","Port":42233},"Maximum Memory":2101975449,"Timestamp":1633789166325,"Maximum Onheap Memory":2101975449,"Maximum Offheap Memory":0} +{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"0","Host":"10.1.0.12","Port":44023},"Maximum Memory":2101975449,"Timestamp":1633789166327,"Maximum Onheap Memory":2101975449,"Maximum Offheap Memory":0} +{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"2","Host":"10.1.0.10","Port":34715},"Maximum Memory":2101975449,"Timestamp":1633789166331,"Maximum Onheap Memory":2101975449,"Maximum Offheap Memory":0} +{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"8","Host":"10.1.0.11","Port":41045},"Maximum Memory":2101975449,"Timestamp":1633789166342,"Maximum Onheap Memory":2101975449,"Maximum Offheap Memory":0} +{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"9","Host":"10.1.0.14","Port":35127},"Maximum Memory":2101975449,"Timestamp":1633789166357,"Maximum Onheap Memory":2101975449,"Maximum Offheap Memory":0} +{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"3","Host":"10.1.0.15","Port":34131},"Maximum Memory":2101975449,"Timestamp":1633789166373,"Maximum Onheap Memory":2101975449,"Maximum Offheap Memory":0} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":0,"description":"sql at ScalaSparkSQLBench.scala:43","details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nSetCatalogAndNamespace (1)\n\n\n(1) SetCatalogAndNamespace\nArguments: org.apache.spark.sql.connector.catalog.CatalogManager@5a6f6cac, spark_catalog, ArrayBuffer(DEFAULT)\n\n","sparkPlanInfo":{"nodeName":"SetCatalogAndNamespace","simpleString":"SetCatalogAndNamespace org.apache.spark.sql.connector.catalog.CatalogManager@5a6f6cac, spark_catalog, ArrayBuffer(DEFAULT)","children":[],"metadata":{},"metrics":[]},"time":1633789168029} +{"Event":"SparkListenerExecutorAdded","Timestamp":1633789171242,"Executor ID":"7","Executor Info":{"Host":"10.1.0.9","Total Cores":16,"Log Urls":{"stdout":"http://10.1.0.9:8081/logPage/?appId=app-20211009101924-2782&executorId=7&logType=stdout","stderr":"http://10.1.0.9:8081/logPage/?appId=app-20211009101924-2782&executorId=7&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0}} +{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"7","Host":"10.1.0.9","Port":45863},"Maximum Memory":2101975449,"Timestamp":1633789171347,"Maximum Onheap Memory":2101975449,"Maximum Offheap Memory":0} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":0,"time":1633789190680} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":1,"description":"sql at ScalaSparkSQLBench.scala:43","details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nExecute SetCommand (1)\n +- SetCommand (2)\n\n\n(1) Execute SetCommand\nOutput [2]: [key#8, value#9]\n\n(2) SetCommand\nArguments: (hive.input.format,Some(org.apache.hadoop.hive.ql.io.HiveInputFormat))\n\n","sparkPlanInfo":{"nodeName":"Execute SetCommand","simpleString":"Execute SetCommand","children":[],"metadata":{},"metrics":[]},"time":1633789190733} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":1,"time":1633789190738} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":2,"description":"sql at ScalaSparkSQLBench.scala:43","details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nExecute SetCommand (1)\n +- SetCommand (2)\n\n\n(1) Execute SetCommand\nOutput [2]: [key#18, value#19]\n\n(2) SetCommand\nArguments: (mapreduce.job.maps,Some(8))\n\n","sparkPlanInfo":{"nodeName":"Execute SetCommand","simpleString":"Execute SetCommand","children":[],"metadata":{},"metrics":[]},"time":1633789190754} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":2,"time":1633789190754} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":3,"description":"sql at ScalaSparkSQLBench.scala:43","details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nExecute SetCommand (1)\n +- SetCommand (2)\n\n\n(1) Execute SetCommand\nOutput [2]: [key#28, value#29]\n\n(2) SetCommand\nArguments: (mapreduce.job.reduces,Some(8))\n\n","sparkPlanInfo":{"nodeName":"Execute SetCommand","simpleString":"Execute SetCommand","children":[],"metadata":{},"metrics":[]},"time":1633789190767} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":3,"time":1633789190767} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":4,"description":"sql at ScalaSparkSQLBench.scala:43","details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nExecute SetCommand (1)\n +- SetCommand (2)\n\n\n(1) Execute SetCommand\nOutput [2]: [key#38, value#39]\n\n(2) SetCommand\nArguments: (hive.stats.autogather,Some(false))\n\n","sparkPlanInfo":{"nodeName":"Execute SetCommand","simpleString":"Execute SetCommand","children":[],"metadata":{},"metrics":[]},"time":1633789190779} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":4,"time":1633789190780} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":5,"description":"sql at ScalaSparkSQLBench.scala:43","details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nLocalTableScan (1)\n\n\n(1) LocalTableScan\nOutput: []\nArguments: \n\n","sparkPlanInfo":{"nodeName":"LocalTableScan","simpleString":"LocalTableScan ","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":0,"metricType":"sum"}]},"time":1633789190834} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":5,"time":1633789190835} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":6,"description":"sql at ScalaSparkSQLBench.scala:43","details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nExecute CreateTableCommand (1)\n +- CreateTableCommand (2)\n\n\n(1) Execute CreateTableCommand\nOutput: []\n\n(2) CreateTableCommand\nArguments: `default`.`rankings`, org.apache.hadoop.hive.serde2.OpenCSVSerde, false\n\n","sparkPlanInfo":{"nodeName":"Execute CreateTableCommand","simpleString":"Execute CreateTableCommand","children":[],"metadata":{},"metrics":[]},"time":1633789190897} +{"Event":"org.apache.spark.sql.catalyst.catalog.CreateTablePreEvent","database":"default","name":"rankings"} +{"Event":"org.apache.spark.sql.catalyst.catalog.CreateTableEvent","database":"default","name":"rankings"} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":6,"time":1633789191681} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":7,"description":"sql at ScalaSparkSQLBench.scala:43","details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nLocalTableScan (1)\n\n\n(1) LocalTableScan\nOutput: []\nArguments: \n\n","sparkPlanInfo":{"nodeName":"LocalTableScan","simpleString":"LocalTableScan ","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1,"metricType":"sum"}]},"time":1633789191704} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":7,"time":1633789191704} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":8,"description":"sql at ScalaSparkSQLBench.scala:43","details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nExecute CreateTableCommand (1)\n +- CreateTableCommand (2)\n\n\n(1) Execute CreateTableCommand\nOutput: []\n\n(2) CreateTableCommand\nArguments: `default`.`uservisits_copy`, org.apache.hadoop.hive.serde2.OpenCSVSerde, false\n\n","sparkPlanInfo":{"nodeName":"Execute CreateTableCommand","simpleString":"Execute CreateTableCommand","children":[],"metadata":{},"metrics":[]},"time":1633789191720} +{"Event":"org.apache.spark.sql.catalyst.catalog.CreateTablePreEvent","database":"default","name":"uservisits_copy"} +{"Event":"org.apache.spark.sql.catalyst.catalog.CreateTableEvent","database":"default","name":"uservisits_copy"} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":8,"time":1633789191791} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":9,"description":"sql at ScalaSparkSQLBench.scala:43","details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nLocalTableScan (1)\n\n\n(1) LocalTableScan\nOutput: []\nArguments: \n\n","sparkPlanInfo":{"nodeName":"LocalTableScan","simpleString":"LocalTableScan ","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2,"metricType":"sum"}]},"time":1633789191803} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":9,"time":1633789191803} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":10,"description":"sql at ScalaSparkSQLBench.scala:43","details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nExecute CreateTableCommand (1)\n +- CreateTableCommand (2)\n\n\n(1) Execute CreateTableCommand\nOutput: []\n\n(2) CreateTableCommand\nArguments: `default`.`rankings_uservisits_join`, org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, false\n\n","sparkPlanInfo":{"nodeName":"Execute CreateTableCommand","simpleString":"Execute CreateTableCommand","children":[],"metadata":{},"metrics":[]},"time":1633789191813} +{"Event":"org.apache.spark.sql.catalyst.catalog.CreateTablePreEvent","database":"default","name":"rankings_uservisits_join"} +{"Event":"org.apache.spark.sql.catalyst.catalog.CreateTableEvent","database":"default","name":"rankings_uservisits_join"} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":10,"time":1633789191990} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":11,"description":"sql at ScalaSparkSQLBench.scala:43","details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nExecute InsertIntoHiveTable (15)\n+- * Project (14)\n +- * Sort (13)\n +- Exchange (12)\n +- * HashAggregate (11)\n +- Exchange (10)\n +- * HashAggregate (9)\n +- * Project (8)\n +- * BroadcastHashJoin Inner BuildLeft (7)\n :- BroadcastExchange (3)\n : +- * Filter (2)\n : +- Scan hive default.rankings (1)\n +- * Project (6)\n +- * Filter (5)\n +- Scan hive default.uservisits_copy (4)\n\n\n(1) Scan hive default.rankings\nOutput [2]: [pageurl#41, pagerank#42]\nArguments: [pageurl#41, pagerank#42], HiveTableRelation [`default`.`rankings`, org.apache.hadoop.hive.serde2.OpenCSVSerde, Data Cols: [pageurl#41, pagerank#42, avgduration#43], Partition Cols: []]\n\n(2) Filter [codegen id : 1]\nInput [2]: [pageurl#41, pagerank#42]\nCondition : isnotnull(pageURL#41)\n\n(3) BroadcastExchange\nInput [2]: [pageurl#41, pagerank#42]\nArguments: HashedRelationBroadcastMode(List(input[0, string, false]),false), [id=#88]\n\n(4) Scan hive default.uservisits_copy\nOutput [4]: [adrevenue#47, desturl#45, sourceip#44, visitdate#46]\nArguments: [adRevenue#47, destURL#45, sourceIP#44, visitDate#46], HiveTableRelation [`default`.`uservisits_copy`, org.apache.hadoop.hive.serde2.OpenCSVSerde, Data Cols: [sourceip#44, desturl#45, visitdate#46, adrevenue#47, useragent#48, countrycode#49, languagecode#..., Partition Cols: []]\n\n(5) Filter\nInput [4]: [adrevenue#47, desturl#45, sourceip#44, visitdate#46]\nCondition : (((isnotnull(visitDate#46) AND (datediff(cast(visitDate#46 as date), 10592) >= 0)) AND (datediff(cast(visitDate#46 as date), 10957) <= 0)) AND isnotnull(destURL#45))\n\n(6) Project\nOutput [3]: [sourceIP#44, destURL#45, adRevenue#47]\nInput [4]: [adrevenue#47, desturl#45, sourceip#44, visitdate#46]\n\n(7) BroadcastHashJoin [codegen id : 2]\nLeft keys [1]: [pageURL#41]\nRight keys [1]: [destURL#45]\nJoin condition: None\n\n(8) Project [codegen id : 2]\nOutput [3]: [pagerank#42, sourceIP#44, adRevenue#47]\nInput [5]: [pageurl#41, pagerank#42, sourceIP#44, destURL#45, adRevenue#47]\n\n(9) HashAggregate [codegen id : 2]\nInput [3]: [pagerank#42, sourceIP#44, adRevenue#47]\nKeys [1]: [sourceIP#44]\nFunctions [2]: [partial_avg(cast(pageRank#42 as double)), partial_sum(cast(adRevenue#47 as double))]\nAggregate Attributes [3]: [sum#62, count#63L, sum#64]\nResults [4]: [sourceIP#44, sum#65, count#66L, sum#67]\n\n(10) Exchange\nInput [4]: [sourceIP#44, sum#65, count#66L, sum#67]\nArguments: hashpartitioning(sourceIP#44, 8), ENSURE_REQUIREMENTS, [id=#97]\n\n(11) HashAggregate [codegen id : 3]\nInput [4]: [sourceIP#44, sum#65, count#66L, sum#67]\nKeys [1]: [sourceIP#44]\nFunctions [2]: [avg(cast(pageRank#42 as double)), sum(cast(adRevenue#47 as double))]\nAggregate Attributes [2]: [avg(cast(pageRank#42 as double))#53, sum(cast(adRevenue#47 as double))#54]\nResults [3]: [sourceIP#44, avg(cast(pageRank#42 as double))#53 AS avg(CAST(pageRank AS DOUBLE))#55, sum(cast(adRevenue#47 as double))#54 AS totalRevenue#40]\n\n(12) Exchange\nInput [3]: [sourceIP#44, avg(CAST(pageRank AS DOUBLE))#55, totalRevenue#40]\nArguments: rangepartitioning(totalRevenue#40 DESC NULLS LAST, 8), ENSURE_REQUIREMENTS, [id=#101]\n\n(13) Sort [codegen id : 4]\nInput [3]: [sourceIP#44, avg(CAST(pageRank AS DOUBLE))#55, totalRevenue#40]\nArguments: [totalRevenue#40 DESC NULLS LAST], true, 0\n\n(14) Project [codegen id : 4]\nOutput [3]: [ansi_cast(sourceIP#44 as string) AS sourceIP#60, ansi_cast(avg(CAST(pageRank AS DOUBLE))#55 as double) AS avgPageRank#61, totalRevenue#40]\nInput [3]: [sourceIP#44, avg(CAST(pageRank AS DOUBLE))#55, totalRevenue#40]\n\n(15) Execute InsertIntoHiveTable\nInput [3]: [sourceIP#60, avgPageRank#61, totalRevenue#40]\nArguments: `default`.`rankings_uservisits_join`, org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, true, false, [sourceIP, avgPageRank, totalRevenue]\n\n","sparkPlanInfo":{"nodeName":"Execute InsertIntoHiveTable","simpleString":"Execute InsertIntoHiveTable `default`.`rankings_uservisits_join`, org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, true, false, [sourceIP, avgPageRank, totalRevenue]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"Project","simpleString":"Project [ansi_cast(sourceIP#44 as string) AS sourceIP#60, ansi_cast(avg(CAST(pageRank AS DOUBLE))#55 as double) AS avgPageRank#61, totalRevenue#40]","children":[{"nodeName":"Sort","simpleString":"Sort [totalRevenue#40 DESC NULLS LAST], true, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Exchange","simpleString":"Exchange rangepartitioning(totalRevenue#40 DESC NULLS LAST, 8), ENSURE_REQUIREMENTS, [id=#101]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[sourceIP#44], functions=[avg(cast(pageRank#42 as double)), sum(cast(adRevenue#47 as double))])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(sourceIP#44, 8), ENSURE_REQUIREMENTS, [id=#97]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[sourceIP#44], functions=[partial_avg(cast(pageRank#42 as double)), partial_sum(cast(adRevenue#47 as double))])","children":[{"nodeName":"Project","simpleString":"Project [pagerank#42, sourceIP#44, adRevenue#47]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [pageURL#41], [destURL#45], Inner, BuildLeft, false","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, false]),false), [id=#88]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Filter","simpleString":"Filter isnotnull(pageURL#41)","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan hive default.rankings","simpleString":"Scan hive default.rankings [pageurl#41, pagerank#42], HiveTableRelation [`default`.`rankings`, org.apache.hadoop.hive.serde2.OpenCSVSerde, Data Cols: [pageurl#41, pagerank#42, avgduration#43], Partition Cols: []]","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":75,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":74,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":73,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":72,"metricType":"timing"},{"name":"time to build","accumulatorId":71,"metricType":"timing"},{"name":"time to collect","accumulatorId":70,"metricType":"timing"},{"name":"number of output rows","accumulatorId":69,"metricType":"sum"},{"name":"data size","accumulatorId":68,"metricType":"size"}]}],"metadata":{},"metrics":[]},{"nodeName":"Project","simpleString":"Project [sourceIP#44, destURL#45, adRevenue#47]","children":[{"nodeName":"Filter","simpleString":"Filter (((isnotnull(visitDate#46) AND (datediff(cast(visitDate#46 as date), 10592) >= 0)) AND (datediff(cast(visitDate#46 as date), 10957) <= 0)) AND isnotnull(destURL#45))","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan hive default.uservisits_copy","simpleString":"Scan hive default.uservisits_copy [adRevenue#47, destURL#45, sourceIP#44, visitDate#46], HiveTableRelation [`default`.`uservisits_copy`, org.apache.hadoop.hive.serde2.OpenCSVSerde, Data Cols: [sourceip#44, desturl#45, visitdate#46, adrevenue#47, useragent#48, countrycode#49, languagecode#..., Partition Cols: []]","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":77,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":76,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":67,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":64,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":65,"metricType":"timing"},{"name":"peak memory","accumulatorId":63,"metricType":"size"},{"name":"number of output rows","accumulatorId":62,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":66,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":61,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":34,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":35,"metricType":"nsTiming"},{"name":"records read","accumulatorId":32,"metricType":"sum"},{"name":"local bytes read","accumulatorId":30,"metricType":"size"},{"name":"fetch wait time","accumulatorId":31,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":28,"metricType":"size"},{"name":"local blocks read","accumulatorId":27,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":26,"metricType":"sum"},{"name":"data size","accumulatorId":25,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":29,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":33,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":58,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":59,"metricType":"timing"},{"name":"peak memory","accumulatorId":57,"metricType":"size"},{"name":"number of output rows","accumulatorId":56,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":60,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":55,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":45,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":46,"metricType":"nsTiming"},{"name":"records read","accumulatorId":43,"metricType":"sum"},{"name":"local bytes read","accumulatorId":41,"metricType":"size"},{"name":"fetch wait time","accumulatorId":42,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":39,"metricType":"size"},{"name":"local blocks read","accumulatorId":38,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":37,"metricType":"sum"},{"name":"data size","accumulatorId":36,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":40,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":44,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":52,"metricType":"timing"},{"name":"peak memory","accumulatorId":53,"metricType":"size"},{"name":"spill size","accumulatorId":54,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":51,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"number of written files","accumulatorId":47,"metricType":"sum"},{"name":"written output","accumulatorId":48,"metricType":"size"},{"name":"number of output rows","accumulatorId":49,"metricType":"sum"},{"name":"number of dynamic part","accumulatorId":50,"metricType":"sum"}]},"time":1633789193359} +{"Event":"SparkListenerJobStart","Job ID":0,"Submission Time":1633789194271,"Stage Infos":[{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Number of Tasks":8,"RDD Info":[{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"2\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"Scan hive default.rankings\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"Scan hive default.rankings\"}","Callsite":"","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"HadoopRDD","Scope":"{\"id\":\"7\",\"name\":\"Scan hive default.rankings\"}","Callsite":"","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"Scan hive default.rankings\"}","Callsite":"","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:185)\njava.util.concurrent.FutureTask.run(FutureTask.java:266)\njava.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\njava.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[0],"Properties":{"spark.sql.warehouse.dir":"file:/home/usera112/projects/log-compressor/deployments/HiBench/spark-warehouse","spark.driver.host":"hostb7.dsrg.utoronto.ca","spark.eventLog.enabled":"true","spark.job.interruptOnCancel":"true","spark.driver.port":"44011","spark.jars":"file:/home/usera112/projects/log-compressor/deployments/HiBench/sparkbench/assembly/target/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.app.name":"ScalaJoin","spark.rdd.scope":"{\"id\":\"0\",\"name\":\"WholeStageCodegen (4)\"}","spark.driver.memory":"4g","spark.rdd.scope.noOverride":"true","spark.submit.pyFiles":"","spark.default.parallelism":"8","spark.job.description":"broadcast exchange (runId d47509dc-011f-41d9-9b59-eb259b8dacbe)","spark.app.startTime":"1633789163045","spark.executor.id":"driver","spark.jobGroup.id":"d47509dc-011f-41d9-9b59-eb259b8dacbe","spark.app.initial.jar.urls":"spark://hostb7.dsrg.utoronto.ca:44011/jars/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.submit.deployMode":"client","spark.master":"spark://hostb7:7077","spark.executor.memory":"4g","spark.eventLog.dir":"hdfs://hostb7:8020/directory","spark.sql.execution.id":"11","spark.sql.catalogImplementation":"hive","spark.app.id":"app-20211009101924-2782","spark.sql.shuffle.partitions":"8"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Number of Tasks":8,"RDD Info":[{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"2\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"Scan hive default.rankings\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"Scan hive default.rankings\"}","Callsite":"","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"HadoopRDD","Scope":"{\"id\":\"7\",\"name\":\"Scan hive default.rankings\"}","Callsite":"","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"Scan hive default.rankings\"}","Callsite":"","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:185)\njava.util.concurrent.FutureTask.run(FutureTask.java:266)\njava.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\njava.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1633789194289,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.sql.warehouse.dir":"file:/home/usera112/projects/log-compressor/deployments/HiBench/spark-warehouse","spark.driver.host":"hostb7.dsrg.utoronto.ca","spark.eventLog.enabled":"true","spark.job.interruptOnCancel":"true","spark.driver.port":"44011","spark.jars":"file:/home/usera112/projects/log-compressor/deployments/HiBench/sparkbench/assembly/target/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.app.name":"ScalaJoin","spark.rdd.scope":"{\"id\":\"0\",\"name\":\"WholeStageCodegen (4)\"}","spark.driver.memory":"4g","spark.rdd.scope.noOverride":"true","spark.submit.pyFiles":"","spark.default.parallelism":"8","spark.job.description":"broadcast exchange (runId d47509dc-011f-41d9-9b59-eb259b8dacbe)","spark.app.startTime":"1633789163045","spark.executor.id":"driver","spark.jobGroup.id":"d47509dc-011f-41d9-9b59-eb259b8dacbe","spark.app.initial.jar.urls":"spark://hostb7.dsrg.utoronto.ca:44011/jars/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.submit.deployMode":"client","spark.master":"spark://hostb7:7077","spark.executor.memory":"4g","spark.eventLog.dir":"hdfs://hostb7:8020/directory","spark.sql.execution.id":"11","spark.sql.catalogImplementation":"hive","spark.app.id":"app-20211009101924-2782","spark.sql.shuffle.partitions":"8"}} +{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Launch Time":1633789194341,"Executor ID":"5","Host":"10.1.0.19","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":1,"Index":1,"Attempt":0,"Launch Time":1633789194357,"Executor ID":"7","Host":"10.1.0.9","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":2,"Index":2,"Attempt":0,"Launch Time":1633789194357,"Executor ID":"1","Host":"10.1.0.17","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":3,"Index":3,"Attempt":0,"Launch Time":1633789194358,"Executor ID":"11","Host":"10.1.0.13","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":4,"Index":4,"Attempt":0,"Launch Time":1633789194358,"Executor ID":"9","Host":"10.1.0.14","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":5,"Index":5,"Attempt":0,"Launch Time":1633789194358,"Executor ID":"4","Host":"10.1.0.8","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":6,"Index":6,"Attempt":0,"Launch Time":1633789194359,"Executor ID":"10","Host":"10.1.0.16","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":7,"Index":7,"Attempt":0,"Launch Time":1633789194360,"Executor ID":"2","Host":"10.1.0.10","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskGettingResult","Task Info":{"Task ID":2,"Index":2,"Attempt":0,"Launch Time":1633789194357,"Executor ID":"1","Host":"10.1.0.17","Locality":"ANY","Speculative":false,"Getting Result Time":1633789196623,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskGettingResult","Task Info":{"Task ID":6,"Index":6,"Attempt":0,"Launch Time":1633789194359,"Executor ID":"10","Host":"10.1.0.16","Locality":"ANY","Speculative":false,"Getting Result Time":1633789196651,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskGettingResult","Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Launch Time":1633789194341,"Executor ID":"5","Host":"10.1.0.19","Locality":"ANY","Speculative":false,"Getting Result Time":1633789196662,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":2,"Index":2,"Attempt":0,"Launch Time":1633789194357,"Executor ID":"1","Host":"10.1.0.17","Locality":"ANY","Speculative":false,"Getting Result Time":1633789196623,"Finish Time":1633789196670,"Failed":false,"Killed":false,"Accumulables":[{"ID":73,"Name":"duration","Update":"202","Value":"202","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"number of output rows","Update":"15000","Value":"15000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":75,"Name":"number of output rows","Update":"15000","Value":"15000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":78,"Name":"internal.metrics.executorDeserializeTime","Update":671,"Value":671,"Internal":true,"Count Failed Values":true},{"ID":79,"Name":"internal.metrics.executorDeserializeCpuTime","Update":602637939,"Value":602637939,"Internal":true,"Count Failed Values":true},{"ID":80,"Name":"internal.metrics.executorRunTime","Update":1532,"Value":1532,"Internal":true,"Count Failed Values":true},{"ID":81,"Name":"internal.metrics.executorCpuTime","Update":1384098580,"Value":1384098580,"Internal":true,"Count Failed Values":true},{"ID":82,"Name":"internal.metrics.resultSize","Update":1064422,"Value":1064422,"Internal":true,"Count Failed Values":true},{"ID":83,"Name":"internal.metrics.jvmGCTime","Update":146,"Value":146,"Internal":true,"Count Failed Values":true},{"ID":84,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":99,"Name":"internal.metrics.input.bytesRead","Update":654511,"Value":654511,"Internal":true,"Count Failed Values":true},{"ID":100,"Name":"internal.metrics.input.recordsRead","Update":15000,"Value":15000,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":671,"Executor Deserialize CPU Time":602637939,"Executor Run Time":1532,"Executor CPU Time":1384098580,"Peak Execution Memory":0,"Result Size":1064422,"JVM GC Time":146,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":654511,"Records Read":15000},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":6,"Index":6,"Attempt":0,"Launch Time":1633789194359,"Executor ID":"10","Host":"10.1.0.16","Locality":"ANY","Speculative":false,"Getting Result Time":1633789196651,"Finish Time":1633789196675,"Failed":false,"Killed":false,"Accumulables":[{"ID":73,"Name":"duration","Update":"191","Value":"393","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"number of output rows","Update":"15000","Value":"30000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":75,"Name":"number of output rows","Update":"15000","Value":"30000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":78,"Name":"internal.metrics.executorDeserializeTime","Update":694,"Value":1365,"Internal":true,"Count Failed Values":true},{"ID":79,"Name":"internal.metrics.executorDeserializeCpuTime","Update":623857712,"Value":1226495651,"Internal":true,"Count Failed Values":true},{"ID":80,"Name":"internal.metrics.executorRunTime","Update":1542,"Value":3074,"Internal":true,"Count Failed Values":true},{"ID":81,"Name":"internal.metrics.executorCpuTime","Update":1387426225,"Value":2771524805,"Internal":true,"Count Failed Values":true},{"ID":82,"Name":"internal.metrics.resultSize","Update":1067500,"Value":2131922,"Internal":true,"Count Failed Values":true},{"ID":83,"Name":"internal.metrics.jvmGCTime","Update":147,"Value":293,"Internal":true,"Count Failed Values":true},{"ID":84,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":99,"Name":"internal.metrics.input.bytesRead","Update":655811,"Value":1310322,"Internal":true,"Count Failed Values":true},{"ID":100,"Name":"internal.metrics.input.recordsRead","Update":15000,"Value":30000,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":694,"Executor Deserialize CPU Time":623857712,"Executor Run Time":1542,"Executor CPU Time":1387426225,"Peak Execution Memory":0,"Result Size":1067500,"JVM GC Time":147,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":655811,"Records Read":15000},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskGettingResult","Task Info":{"Task ID":4,"Index":4,"Attempt":0,"Launch Time":1633789194358,"Executor ID":"9","Host":"10.1.0.14","Locality":"ANY","Speculative":false,"Getting Result Time":1633789196690,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Launch Time":1633789194341,"Executor ID":"5","Host":"10.1.0.19","Locality":"ANY","Speculative":false,"Getting Result Time":1633789196662,"Finish Time":1633789196691,"Failed":false,"Killed":false,"Accumulables":[{"ID":73,"Name":"duration","Update":"179","Value":"572","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"number of output rows","Update":"15000","Value":"45000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":75,"Name":"number of output rows","Update":"15000","Value":"45000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":78,"Name":"internal.metrics.executorDeserializeTime","Update":684,"Value":2049,"Internal":true,"Count Failed Values":true},{"ID":79,"Name":"internal.metrics.executorDeserializeCpuTime","Update":607279304,"Value":1833774955,"Internal":true,"Count Failed Values":true},{"ID":80,"Name":"internal.metrics.executorRunTime","Update":1570,"Value":4644,"Internal":true,"Count Failed Values":true},{"ID":81,"Name":"internal.metrics.executorCpuTime","Update":1419605504,"Value":4191130309,"Internal":true,"Count Failed Values":true},{"ID":82,"Name":"internal.metrics.resultSize","Update":1063975,"Value":3195897,"Internal":true,"Count Failed Values":true},{"ID":83,"Name":"internal.metrics.jvmGCTime","Update":136,"Value":429,"Internal":true,"Count Failed Values":true},{"ID":84,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":99,"Name":"internal.metrics.input.bytesRead","Update":652408,"Value":1962730,"Internal":true,"Count Failed Values":true},{"ID":100,"Name":"internal.metrics.input.recordsRead","Update":15000,"Value":45000,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":24095592,"JVMOffHeapMemory":49273008,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":9994,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":2,"MinorGCTime":18,"MajorGCCount":2,"MajorGCTime":48},"Task Metrics":{"Executor Deserialize Time":684,"Executor Deserialize CPU Time":607279304,"Executor Run Time":1570,"Executor CPU Time":1419605504,"Peak Execution Memory":0,"Result Size":1063975,"JVM GC Time":136,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":652408,"Records Read":15000},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":4,"Index":4,"Attempt":0,"Launch Time":1633789194358,"Executor ID":"9","Host":"10.1.0.14","Locality":"ANY","Speculative":false,"Getting Result Time":1633789196690,"Finish Time":1633789196716,"Failed":false,"Killed":false,"Accumulables":[{"ID":73,"Name":"duration","Update":"199","Value":"771","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"number of output rows","Update":"15000","Value":"60000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":75,"Name":"number of output rows","Update":"15000","Value":"60000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":78,"Name":"internal.metrics.executorDeserializeTime","Update":696,"Value":2745,"Internal":true,"Count Failed Values":true},{"ID":79,"Name":"internal.metrics.executorDeserializeCpuTime","Update":616746885,"Value":2450521840,"Internal":true,"Count Failed Values":true},{"ID":80,"Name":"internal.metrics.executorRunTime","Update":1583,"Value":6227,"Internal":true,"Count Failed Values":true},{"ID":81,"Name":"internal.metrics.executorCpuTime","Update":1433872773,"Value":5625003082,"Internal":true,"Count Failed Values":true},{"ID":82,"Name":"internal.metrics.resultSize","Update":1062242,"Value":4258139,"Internal":true,"Count Failed Values":true},{"ID":83,"Name":"internal.metrics.jvmGCTime","Update":145,"Value":574,"Internal":true,"Count Failed Values":true},{"ID":84,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":6,"Internal":true,"Count Failed Values":true},{"ID":99,"Name":"internal.metrics.input.bytesRead","Update":653437,"Value":2616167,"Internal":true,"Count Failed Values":true},{"ID":100,"Name":"internal.metrics.input.recordsRead","Update":15000,"Value":60000,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":237641824,"JVMOffHeapMemory":65001768,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":497850,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":497850,"OffHeapUnifiedMemory":0,"DirectPoolMemory":10117,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":2,"MinorGCTime":18,"MajorGCCount":2,"MajorGCTime":46},"Task Metrics":{"Executor Deserialize Time":696,"Executor Deserialize CPU Time":616746885,"Executor Run Time":1583,"Executor CPU Time":1433872773,"Peak Execution Memory":0,"Result Size":1062242,"JVM GC Time":145,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":653437,"Records Read":15000},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskGettingResult","Task Info":{"Task ID":3,"Index":3,"Attempt":0,"Launch Time":1633789194358,"Executor ID":"11","Host":"10.1.0.13","Locality":"ANY","Speculative":false,"Getting Result Time":1633789196752,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":3,"Index":3,"Attempt":0,"Launch Time":1633789194358,"Executor ID":"11","Host":"10.1.0.13","Locality":"ANY","Speculative":false,"Getting Result Time":1633789196752,"Finish Time":1633789196763,"Failed":false,"Killed":false,"Accumulables":[{"ID":73,"Name":"duration","Update":"186","Value":"957","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"number of output rows","Update":"15000","Value":"75000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":75,"Name":"number of output rows","Update":"15000","Value":"75000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":78,"Name":"internal.metrics.executorDeserializeTime","Update":700,"Value":3445,"Internal":true,"Count Failed Values":true},{"ID":79,"Name":"internal.metrics.executorDeserializeCpuTime","Update":618580776,"Value":3069102616,"Internal":true,"Count Failed Values":true},{"ID":80,"Name":"internal.metrics.executorRunTime","Update":1644,"Value":7871,"Internal":true,"Count Failed Values":true},{"ID":81,"Name":"internal.metrics.executorCpuTime","Update":1494597853,"Value":7119600935,"Internal":true,"Count Failed Values":true},{"ID":82,"Name":"internal.metrics.resultSize","Update":1057603,"Value":5315742,"Internal":true,"Count Failed Values":true},{"ID":83,"Name":"internal.metrics.jvmGCTime","Update":148,"Value":722,"Internal":true,"Count Failed Values":true},{"ID":99,"Name":"internal.metrics.input.bytesRead","Update":650282,"Value":3266449,"Internal":true,"Count Failed Values":true},{"ID":100,"Name":"internal.metrics.input.recordsRead","Update":15000,"Value":75000,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":285876616,"JVMOffHeapMemory":65073704,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":497850,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":497850,"OffHeapUnifiedMemory":0,"DirectPoolMemory":39200,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":2,"MinorGCTime":19,"MajorGCCount":2,"MajorGCTime":47},"Task Metrics":{"Executor Deserialize Time":700,"Executor Deserialize CPU Time":618580776,"Executor Run Time":1644,"Executor CPU Time":1494597853,"Peak Execution Memory":0,"Result Size":1057603,"JVM GC Time":148,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":650282,"Records Read":15000},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskGettingResult","Task Info":{"Task ID":7,"Index":7,"Attempt":0,"Launch Time":1633789194360,"Executor ID":"2","Host":"10.1.0.10","Locality":"ANY","Speculative":false,"Getting Result Time":1633789196773,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":7,"Index":7,"Attempt":0,"Launch Time":1633789194360,"Executor ID":"2","Host":"10.1.0.10","Locality":"ANY","Speculative":false,"Getting Result Time":1633789196773,"Finish Time":1633789196794,"Failed":false,"Killed":false,"Accumulables":[{"ID":73,"Name":"duration","Update":"200","Value":"1157","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"number of output rows","Update":"15000","Value":"90000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":75,"Name":"number of output rows","Update":"15000","Value":"90000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":78,"Name":"internal.metrics.executorDeserializeTime","Update":720,"Value":4165,"Internal":true,"Count Failed Values":true},{"ID":79,"Name":"internal.metrics.executorDeserializeCpuTime","Update":635126187,"Value":3704228803,"Internal":true,"Count Failed Values":true},{"ID":80,"Name":"internal.metrics.executorRunTime","Update":1630,"Value":9501,"Internal":true,"Count Failed Values":true},{"ID":81,"Name":"internal.metrics.executorCpuTime","Update":1486285416,"Value":8605886351,"Internal":true,"Count Failed Values":true},{"ID":82,"Name":"internal.metrics.resultSize","Update":1058522,"Value":6374264,"Internal":true,"Count Failed Values":true},{"ID":83,"Name":"internal.metrics.jvmGCTime","Update":144,"Value":866,"Internal":true,"Count Failed Values":true},{"ID":84,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":99,"Name":"internal.metrics.input.bytesRead","Update":650945,"Value":3917394,"Internal":true,"Count Failed Values":true},{"ID":100,"Name":"internal.metrics.input.recordsRead","Update":15000,"Value":90000,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":720,"Executor Deserialize CPU Time":635126187,"Executor Run Time":1630,"Executor CPU Time":1486285416,"Peak Execution Memory":0,"Result Size":1058522,"JVM GC Time":144,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":650945,"Records Read":15000},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskGettingResult","Task Info":{"Task ID":5,"Index":5,"Attempt":0,"Launch Time":1633789194358,"Executor ID":"4","Host":"10.1.0.8","Locality":"ANY","Speculative":false,"Getting Result Time":1633789196823,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":5,"Index":5,"Attempt":0,"Launch Time":1633789194358,"Executor ID":"4","Host":"10.1.0.8","Locality":"ANY","Speculative":false,"Getting Result Time":1633789196823,"Finish Time":1633789196844,"Failed":false,"Killed":false,"Accumulables":[{"ID":73,"Name":"duration","Update":"186","Value":"1343","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"number of output rows","Update":"15000","Value":"105000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":75,"Name":"number of output rows","Update":"15000","Value":"105000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":78,"Name":"internal.metrics.executorDeserializeTime","Update":744,"Value":4909,"Internal":true,"Count Failed Values":true},{"ID":79,"Name":"internal.metrics.executorDeserializeCpuTime","Update":668702485,"Value":4372931288,"Internal":true,"Count Failed Values":true},{"ID":80,"Name":"internal.metrics.executorRunTime","Update":1666,"Value":11167,"Internal":true,"Count Failed Values":true},{"ID":81,"Name":"internal.metrics.executorCpuTime","Update":1516220264,"Value":10122106615,"Internal":true,"Count Failed Values":true},{"ID":82,"Name":"internal.metrics.resultSize","Update":1059139,"Value":7433403,"Internal":true,"Count Failed Values":true},{"ID":83,"Name":"internal.metrics.jvmGCTime","Update":129,"Value":995,"Internal":true,"Count Failed Values":true},{"ID":84,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":8,"Internal":true,"Count Failed Values":true},{"ID":99,"Name":"internal.metrics.input.bytesRead","Update":651025,"Value":4568419,"Internal":true,"Count Failed Values":true},{"ID":100,"Name":"internal.metrics.input.recordsRead","Update":15000,"Value":105000,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":744,"Executor Deserialize CPU Time":668702485,"Executor Run Time":1666,"Executor CPU Time":1516220264,"Peak Execution Memory":0,"Result Size":1059139,"JVM GC Time":129,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":651025,"Records Read":15000},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskGettingResult","Task Info":{"Task ID":1,"Index":1,"Attempt":0,"Launch Time":1633789194357,"Executor ID":"7","Host":"10.1.0.9","Locality":"ANY","Speculative":false,"Getting Result Time":1633789196876,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":1,"Index":1,"Attempt":0,"Launch Time":1633789194357,"Executor ID":"7","Host":"10.1.0.9","Locality":"ANY","Speculative":false,"Getting Result Time":1633789196876,"Finish Time":1633789196886,"Failed":false,"Killed":false,"Accumulables":[{"ID":73,"Name":"duration","Update":"191","Value":"1534","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"number of output rows","Update":"15000","Value":"120000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":75,"Name":"number of output rows","Update":"15000","Value":"120000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":78,"Name":"internal.metrics.executorDeserializeTime","Update":711,"Value":5620,"Internal":true,"Count Failed Values":true},{"ID":79,"Name":"internal.metrics.executorDeserializeCpuTime","Update":627471483,"Value":5000402771,"Internal":true,"Count Failed Values":true},{"ID":80,"Name":"internal.metrics.executorRunTime","Update":1751,"Value":12918,"Internal":true,"Count Failed Values":true},{"ID":81,"Name":"internal.metrics.executorCpuTime","Update":1587465036,"Value":11709571651,"Internal":true,"Count Failed Values":true},{"ID":82,"Name":"internal.metrics.resultSize","Update":1060240,"Value":8493643,"Internal":true,"Count Failed Values":true},{"ID":83,"Name":"internal.metrics.jvmGCTime","Update":144,"Value":1139,"Internal":true,"Count Failed Values":true},{"ID":84,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":9,"Internal":true,"Count Failed Values":true},{"ID":99,"Name":"internal.metrics.input.bytesRead","Update":651866,"Value":5220285,"Internal":true,"Count Failed Values":true},{"ID":100,"Name":"internal.metrics.input.recordsRead","Update":15000,"Value":120000,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":711,"Executor Deserialize CPU Time":627471483,"Executor Run Time":1751,"Executor CPU Time":1587465036,"Peak Execution Memory":0,"Result Size":1060240,"JVM GC Time":144,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":651866,"Records Read":15000},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Number of Tasks":8,"RDD Info":[{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"2\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"Scan hive default.rankings\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"Scan hive default.rankings\"}","Callsite":"","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"HadoopRDD","Scope":"{\"id\":\"7\",\"name\":\"Scan hive default.rankings\"}","Callsite":"","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"Scan hive default.rankings\"}","Callsite":"","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:185)\njava.util.concurrent.FutureTask.run(FutureTask.java:266)\njava.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\njava.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1633789194289,"Completion Time":1633789196889,"Accumulables":[{"ID":73,"Name":"duration","Value":"1534","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"number of output rows","Value":"120000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":75,"Name":"number of output rows","Value":"120000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":78,"Name":"internal.metrics.executorDeserializeTime","Value":5620,"Internal":true,"Count Failed Values":true},{"ID":79,"Name":"internal.metrics.executorDeserializeCpuTime","Value":5000402771,"Internal":true,"Count Failed Values":true},{"ID":80,"Name":"internal.metrics.executorRunTime","Value":12918,"Internal":true,"Count Failed Values":true},{"ID":81,"Name":"internal.metrics.executorCpuTime","Value":11709571651,"Internal":true,"Count Failed Values":true},{"ID":82,"Name":"internal.metrics.resultSize","Value":8493643,"Internal":true,"Count Failed Values":true},{"ID":83,"Name":"internal.metrics.jvmGCTime","Value":1139,"Internal":true,"Count Failed Values":true},{"ID":84,"Name":"internal.metrics.resultSerializationTime","Value":9,"Internal":true,"Count Failed Values":true},{"ID":99,"Name":"internal.metrics.input.bytesRead","Value":5220285,"Internal":true,"Count Failed Values":true},{"ID":100,"Name":"internal.metrics.input.recordsRead","Value":120000,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":0,"Completion Time":1633789196894,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":11,"accumUpdates":[[72,118],[71,158],[70,3343],[69,120000],[68,29360128]]} +{"Event":"SparkListenerJobStart","Job ID":1,"Submission Time":1633789197481,"Stage Infos":[{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"sql at ScalaSparkSQLBench.scala:43","Number of Tasks":8,"RDD Info":[{"RDD ID":11,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"12\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[10],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":10,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"13\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[9],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":6,"Name":"HadoopRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":2,"Stage Attempt ID":0,"Stage Name":"sql at ScalaSparkSQLBench.scala:43","Number of Tasks":8,"RDD Info":[{"RDD ID":16,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[15],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":12,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"12\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[11],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":15,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[14],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":14,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[1],"Details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[1,2],"Properties":{"spark.sql.warehouse.dir":"file:/home/usera112/projects/log-compressor/deployments/HiBench/spark-warehouse","spark.driver.host":"hostb7.dsrg.utoronto.ca","spark.eventLog.enabled":"true","spark.driver.port":"44011","__fetch_continuous_blocks_in_batch_enabled":"true","spark.jars":"file:/home/usera112/projects/log-compressor/deployments/HiBench/sparkbench/assembly/target/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.app.name":"ScalaJoin","spark.rdd.scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","spark.driver.memory":"4g","spark.rdd.scope.noOverride":"true","spark.submit.pyFiles":"","spark.default.parallelism":"8","spark.app.startTime":"1633789163045","spark.executor.id":"driver","spark.app.initial.jar.urls":"spark://hostb7.dsrg.utoronto.ca:44011/jars/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.submit.deployMode":"client","spark.master":"spark://hostb7:7077","spark.executor.memory":"4g","spark.eventLog.dir":"hdfs://hostb7:8020/directory","spark.sql.execution.id":"11","spark.sql.catalogImplementation":"hive","spark.app.id":"app-20211009101924-2782","spark.sql.shuffle.partitions":"8"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"sql at ScalaSparkSQLBench.scala:43","Number of Tasks":8,"RDD Info":[{"RDD ID":11,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"12\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[10],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":10,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"13\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[9],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":6,"Name":"HadoopRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1633789197487,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.sql.warehouse.dir":"file:/home/usera112/projects/log-compressor/deployments/HiBench/spark-warehouse","spark.driver.host":"hostb7.dsrg.utoronto.ca","spark.eventLog.enabled":"true","spark.driver.port":"44011","__fetch_continuous_blocks_in_batch_enabled":"true","spark.jars":"file:/home/usera112/projects/log-compressor/deployments/HiBench/sparkbench/assembly/target/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.app.name":"ScalaJoin","spark.rdd.scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","spark.driver.memory":"4g","spark.rdd.scope.noOverride":"true","spark.submit.pyFiles":"","spark.default.parallelism":"8","spark.app.startTime":"1633789163045","spark.executor.id":"driver","spark.app.initial.jar.urls":"spark://hostb7.dsrg.utoronto.ca:44011/jars/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.submit.deployMode":"client","spark.master":"spark://hostb7:7077","spark.executor.memory":"4g","spark.eventLog.dir":"hdfs://hostb7:8020/directory","spark.sql.execution.id":"11","spark.sql.catalogImplementation":"hive","spark.app.id":"app-20211009101924-2782","spark.sql.shuffle.partitions":"8"}} +{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":8,"Index":0,"Attempt":0,"Launch Time":1633789197498,"Executor ID":"9","Host":"10.1.0.14","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":9,"Index":1,"Attempt":0,"Launch Time":1633789197499,"Executor ID":"1","Host":"10.1.0.17","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":10,"Index":2,"Attempt":0,"Launch Time":1633789197499,"Executor ID":"11","Host":"10.1.0.13","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":11,"Index":3,"Attempt":0,"Launch Time":1633789197500,"Executor ID":"8","Host":"10.1.0.11","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":12,"Index":4,"Attempt":0,"Launch Time":1633789197501,"Executor ID":"6","Host":"10.1.0.18","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":13,"Index":5,"Attempt":0,"Launch Time":1633789197501,"Executor ID":"2","Host":"10.1.0.10","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":14,"Index":6,"Attempt":0,"Launch Time":1633789197502,"Executor ID":"10","Host":"10.1.0.16","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":15,"Index":7,"Attempt":0,"Launch Time":1633789197502,"Executor ID":"4","Host":"10.1.0.8","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":9,"Index":1,"Attempt":0,"Launch Time":1633789197499,"Executor ID":"1","Host":"10.1.0.17","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":1633789199211,"Failed":false,"Killed":false,"Accumulables":[{"ID":25,"Name":"data size","Update":"165920","Value":"165920","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"shuffle bytes written","Update":"83472","Value":"83472","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"shuffle records written","Update":"2963","Value":"2963","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":35,"Name":"shuffle write time","Update":"8415526","Value":"8415526","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":61,"Name":"duration","Update":"1280","Value":"1280","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":62,"Name":"number of output rows","Update":"2963","Value":"2963","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":63,"Name":"peak memory","Update":"4456448","Value":"4456448","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":65,"Name":"time in aggregation build","Update":"972","Value":"972","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":66,"Name":"avg hash probe bucket list iters","Update":"10","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":67,"Name":"number of output rows","Update":"2963","Value":"2963","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":76,"Name":"number of output rows","Update":"2963","Value":"2963","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":77,"Name":"number of output rows","Update":"125061","Value":"125061","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":103,"Name":"internal.metrics.executorDeserializeTime","Update":196,"Value":196,"Internal":true,"Count Failed Values":true},{"ID":104,"Name":"internal.metrics.executorDeserializeCpuTime","Update":189573155,"Value":189573155,"Internal":true,"Count Failed Values":true},{"ID":105,"Name":"internal.metrics.executorRunTime","Update":1499,"Value":1499,"Internal":true,"Count Failed Values":true},{"ID":106,"Name":"internal.metrics.executorCpuTime","Update":1390424789,"Value":1390424789,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.resultSize","Update":2904,"Value":2904,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.jvmGCTime","Update":36,"Value":36,"Internal":true,"Count Failed Values":true},{"ID":112,"Name":"internal.metrics.peakExecutionMemory","Update":33816576,"Value":33816576,"Internal":true,"Count Failed Values":true},{"ID":121,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":83472,"Value":83472,"Internal":true,"Count Failed Values":true},{"ID":122,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":2963,"Value":2963,"Internal":true,"Count Failed Values":true},{"ID":123,"Name":"internal.metrics.shuffle.write.writeTime","Update":8415526,"Value":8415526,"Internal":true,"Count Failed Values":true},{"ID":124,"Name":"internal.metrics.input.bytesRead","Update":23125676,"Value":23125676,"Internal":true,"Count Failed Values":true},{"ID":125,"Name":"internal.metrics.input.recordsRead","Update":125061,"Value":125061,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":196,"Executor Deserialize CPU Time":189573155,"Executor Run Time":1499,"Executor CPU Time":1390424789,"Peak Execution Memory":33816576,"Result Size":2904,"JVM GC Time":36,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":83472,"Shuffle Write Time":8415526,"Shuffle Records Written":2963},"Input Metrics":{"Bytes Read":23125676,"Records Read":125061},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":14,"Index":6,"Attempt":0,"Launch Time":1633789197502,"Executor ID":"10","Host":"10.1.0.16","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":1633789199224,"Failed":false,"Killed":false,"Accumulables":[{"ID":25,"Name":"data size","Update":"169456","Value":"335376","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"shuffle bytes written","Update":"85405","Value":"168877","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"shuffle records written","Update":"3026","Value":"5989","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":35,"Name":"shuffle write time","Update":"8115759","Value":"16531285","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":61,"Name":"duration","Update":"1329","Value":"2609","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":62,"Name":"number of output rows","Update":"3026","Value":"5989","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":63,"Name":"peak memory","Update":"4456448","Value":"8912896","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":65,"Name":"time in aggregation build","Update":"1001","Value":"1973","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":66,"Name":"avg hash probe bucket list iters","Update":"10","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":67,"Name":"number of output rows","Update":"3026","Value":"5989","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":76,"Name":"number of output rows","Update":"3026","Value":"5989","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":77,"Name":"number of output rows","Update":"125342","Value":"250403","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":103,"Name":"internal.metrics.executorDeserializeTime","Update":186,"Value":382,"Internal":true,"Count Failed Values":true},{"ID":104,"Name":"internal.metrics.executorDeserializeCpuTime","Update":179340004,"Value":368913159,"Internal":true,"Count Failed Values":true},{"ID":105,"Name":"internal.metrics.executorRunTime","Update":1524,"Value":3023,"Internal":true,"Count Failed Values":true},{"ID":106,"Name":"internal.metrics.executorCpuTime","Update":1413829348,"Value":2804254137,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.resultSize","Update":2905,"Value":5809,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.jvmGCTime","Update":35,"Value":71,"Internal":true,"Count Failed Values":true},{"ID":112,"Name":"internal.metrics.peakExecutionMemory","Update":33816576,"Value":67633152,"Internal":true,"Count Failed Values":true},{"ID":121,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":85405,"Value":168877,"Internal":true,"Count Failed Values":true},{"ID":122,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":3026,"Value":5989,"Internal":true,"Count Failed Values":true},{"ID":123,"Name":"internal.metrics.shuffle.write.writeTime","Update":8115759,"Value":16531285,"Internal":true,"Count Failed Values":true},{"ID":124,"Name":"internal.metrics.input.bytesRead","Update":23247031,"Value":46372707,"Internal":true,"Count Failed Values":true},{"ID":125,"Name":"internal.metrics.input.recordsRead","Update":125342,"Value":250403,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":488649352,"JVMOffHeapMemory":84231984,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":11504640,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":11504640,"OffHeapUnifiedMemory":0,"DirectPoolMemory":5312351,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":3,"MinorGCTime":28,"MajorGCCount":3,"MajorGCTime":149},"Task Metrics":{"Executor Deserialize Time":186,"Executor Deserialize CPU Time":179340004,"Executor Run Time":1524,"Executor CPU Time":1413829348,"Peak Execution Memory":33816576,"Result Size":2905,"JVM GC Time":35,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":85405,"Shuffle Write Time":8115759,"Shuffle Records Written":3026},"Input Metrics":{"Bytes Read":23247031,"Records Read":125342},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":10,"Index":2,"Attempt":0,"Launch Time":1633789197499,"Executor ID":"11","Host":"10.1.0.13","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":1633789199230,"Failed":false,"Killed":false,"Accumulables":[{"ID":25,"Name":"data size","Update":"165872","Value":"501248","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"shuffle bytes written","Update":"83597","Value":"252474","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"shuffle records written","Update":"2962","Value":"8951","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":35,"Name":"shuffle write time","Update":"8919410","Value":"25450695","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":61,"Name":"duration","Update":"1323","Value":"3932","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":62,"Name":"number of output rows","Update":"2962","Value":"8951","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":63,"Name":"peak memory","Update":"4456448","Value":"13369344","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":65,"Name":"time in aggregation build","Update":"1001","Value":"2974","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":66,"Name":"avg hash probe bucket list iters","Update":"10","Value":"30","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":67,"Name":"number of output rows","Update":"2962","Value":"8951","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":76,"Name":"number of output rows","Update":"2962","Value":"8951","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":77,"Name":"number of output rows","Update":"125003","Value":"375406","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":103,"Name":"internal.metrics.executorDeserializeTime","Update":186,"Value":568,"Internal":true,"Count Failed Values":true},{"ID":104,"Name":"internal.metrics.executorDeserializeCpuTime","Update":179512290,"Value":548425449,"Internal":true,"Count Failed Values":true},{"ID":105,"Name":"internal.metrics.executorRunTime","Update":1530,"Value":4553,"Internal":true,"Count Failed Values":true},{"ID":106,"Name":"internal.metrics.executorCpuTime","Update":1423658983,"Value":4227913120,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.resultSize","Update":2905,"Value":8714,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.jvmGCTime","Update":37,"Value":108,"Internal":true,"Count Failed Values":true},{"ID":112,"Name":"internal.metrics.peakExecutionMemory","Update":33816576,"Value":101449728,"Internal":true,"Count Failed Values":true},{"ID":121,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":83597,"Value":252474,"Internal":true,"Count Failed Values":true},{"ID":122,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":2962,"Value":8951,"Internal":true,"Count Failed Values":true},{"ID":123,"Name":"internal.metrics.shuffle.write.writeTime","Update":8919410,"Value":25450695,"Internal":true,"Count Failed Values":true},{"ID":124,"Name":"internal.metrics.input.bytesRead","Update":23143498,"Value":69516205,"Internal":true,"Count Failed Values":true},{"ID":125,"Name":"internal.metrics.input.recordsRead","Update":125003,"Value":375406,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":186,"Executor Deserialize CPU Time":179512290,"Executor Run Time":1530,"Executor CPU Time":1423658983,"Peak Execution Memory":33816576,"Result Size":2905,"JVM GC Time":37,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":83597,"Shuffle Write Time":8919410,"Shuffle Records Written":2962},"Input Metrics":{"Bytes Read":23143498,"Records Read":125003},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":13,"Index":5,"Attempt":0,"Launch Time":1633789197501,"Executor ID":"2","Host":"10.1.0.10","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":1633789199236,"Failed":false,"Killed":false,"Accumulables":[{"ID":25,"Name":"data size","Update":"170848","Value":"672096","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"shuffle bytes written","Update":"85986","Value":"338460","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"shuffle records written","Update":"3051","Value":"12002","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":35,"Name":"shuffle write time","Update":"8490760","Value":"33941455","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":61,"Name":"duration","Update":"1328","Value":"5260","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":62,"Name":"number of output rows","Update":"3051","Value":"12002","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":63,"Name":"peak memory","Update":"4456448","Value":"17825792","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":65,"Name":"time in aggregation build","Update":"1012","Value":"3986","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":66,"Name":"avg hash probe bucket list iters","Update":"10","Value":"40","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":67,"Name":"number of output rows","Update":"3051","Value":"12002","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":76,"Name":"number of output rows","Update":"3051","Value":"12002","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":77,"Name":"number of output rows","Update":"124533","Value":"499939","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":103,"Name":"internal.metrics.executorDeserializeTime","Update":188,"Value":756,"Internal":true,"Count Failed Values":true},{"ID":104,"Name":"internal.metrics.executorDeserializeCpuTime","Update":181300121,"Value":729725570,"Internal":true,"Count Failed Values":true},{"ID":105,"Name":"internal.metrics.executorRunTime","Update":1533,"Value":6086,"Internal":true,"Count Failed Values":true},{"ID":106,"Name":"internal.metrics.executorCpuTime","Update":1426032738,"Value":5653945858,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.resultSize","Update":2904,"Value":11618,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.jvmGCTime","Update":38,"Value":146,"Internal":true,"Count Failed Values":true},{"ID":112,"Name":"internal.metrics.peakExecutionMemory","Update":33816576,"Value":135266304,"Internal":true,"Count Failed Values":true},{"ID":121,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":85986,"Value":338460,"Internal":true,"Count Failed Values":true},{"ID":122,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":3051,"Value":12002,"Internal":true,"Count Failed Values":true},{"ID":123,"Name":"internal.metrics.shuffle.write.writeTime","Update":8490760,"Value":33941455,"Internal":true,"Count Failed Values":true},{"ID":124,"Name":"internal.metrics.input.bytesRead","Update":23014586,"Value":92530791,"Internal":true,"Count Failed Values":true},{"ID":125,"Name":"internal.metrics.input.recordsRead","Update":124533,"Value":499939,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":188,"Executor Deserialize CPU Time":181300121,"Executor Run Time":1533,"Executor CPU Time":1426032738,"Peak Execution Memory":33816576,"Result Size":2904,"JVM GC Time":38,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":85986,"Shuffle Write Time":8490760,"Shuffle Records Written":3051},"Input Metrics":{"Bytes Read":23014586,"Records Read":124533},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":15,"Index":7,"Attempt":0,"Launch Time":1633789197502,"Executor ID":"4","Host":"10.1.0.8","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":1633789199295,"Failed":false,"Killed":false,"Accumulables":[{"ID":25,"Name":"data size","Update":"167832","Value":"839928","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"shuffle bytes written","Update":"84637","Value":"423097","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"shuffle records written","Update":"2997","Value":"14999","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":35,"Name":"shuffle write time","Update":"8947328","Value":"42888783","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":61,"Name":"duration","Update":"1382","Value":"6642","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":62,"Name":"number of output rows","Update":"2997","Value":"14999","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":63,"Name":"peak memory","Update":"4456448","Value":"22282240","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":65,"Name":"time in aggregation build","Update":"1042","Value":"5028","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":66,"Name":"avg hash probe bucket list iters","Update":"10","Value":"50","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":67,"Name":"number of output rows","Update":"2997","Value":"14999","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":76,"Name":"number of output rows","Update":"2997","Value":"14999","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":77,"Name":"number of output rows","Update":"125488","Value":"625427","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":103,"Name":"internal.metrics.executorDeserializeTime","Update":195,"Value":951,"Internal":true,"Count Failed Values":true},{"ID":104,"Name":"internal.metrics.executorDeserializeCpuTime","Update":188608128,"Value":918333698,"Internal":true,"Count Failed Values":true},{"ID":105,"Name":"internal.metrics.executorRunTime","Update":1585,"Value":7671,"Internal":true,"Count Failed Values":true},{"ID":106,"Name":"internal.metrics.executorCpuTime","Update":1465371554,"Value":7119317412,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.resultSize","Update":2903,"Value":14521,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.jvmGCTime","Update":35,"Value":181,"Internal":true,"Count Failed Values":true},{"ID":112,"Name":"internal.metrics.peakExecutionMemory","Update":33816576,"Value":169082880,"Internal":true,"Count Failed Values":true},{"ID":121,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":84637,"Value":423097,"Internal":true,"Count Failed Values":true},{"ID":122,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":2997,"Value":14999,"Internal":true,"Count Failed Values":true},{"ID":123,"Name":"internal.metrics.shuffle.write.writeTime","Update":8947328,"Value":42888783,"Internal":true,"Count Failed Values":true},{"ID":124,"Name":"internal.metrics.input.bytesRead","Update":23178313,"Value":115709104,"Internal":true,"Count Failed Values":true},{"ID":125,"Name":"internal.metrics.input.recordsRead","Update":125488,"Value":625427,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":195,"Executor Deserialize CPU Time":188608128,"Executor Run Time":1585,"Executor CPU Time":1465371554,"Peak Execution Memory":33816576,"Result Size":2903,"JVM GC Time":35,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":84637,"Shuffle Write Time":8947328,"Shuffle Records Written":2997},"Input Metrics":{"Bytes Read":23178313,"Records Read":125488},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":8,"Index":0,"Attempt":0,"Launch Time":1633789197498,"Executor ID":"9","Host":"10.1.0.14","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":1633789199320,"Failed":false,"Killed":false,"Accumulables":[{"ID":25,"Name":"data size","Update":"158088","Value":"998016","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"shuffle bytes written","Update":"79832","Value":"502929","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"shuffle records written","Update":"2823","Value":"17822","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":35,"Name":"shuffle write time","Update":"9288526","Value":"52177309","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":61,"Name":"duration","Update":"1408","Value":"8050","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":62,"Name":"number of output rows","Update":"2823","Value":"17822","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":63,"Name":"peak memory","Update":"262144","Value":"22544384","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":65,"Name":"time in aggregation build","Update":"1070","Value":"6098","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":67,"Name":"number of output rows","Update":"2823","Value":"17822","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":76,"Name":"number of output rows","Update":"2823","Value":"17822","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":77,"Name":"number of output rows","Update":"124858","Value":"750285","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":103,"Name":"internal.metrics.executorDeserializeTime","Update":196,"Value":1147,"Internal":true,"Count Failed Values":true},{"ID":104,"Name":"internal.metrics.executorDeserializeCpuTime","Update":188541690,"Value":1106875388,"Internal":true,"Count Failed Values":true},{"ID":105,"Name":"internal.metrics.executorRunTime","Update":1610,"Value":9281,"Internal":true,"Count Failed Values":true},{"ID":106,"Name":"internal.metrics.executorCpuTime","Update":1487640781,"Value":8606958193,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.resultSize","Update":2904,"Value":17425,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.jvmGCTime","Update":35,"Value":216,"Internal":true,"Count Failed Values":true},{"ID":112,"Name":"internal.metrics.peakExecutionMemory","Update":29622272,"Value":198705152,"Internal":true,"Count Failed Values":true},{"ID":121,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":79832,"Value":502929,"Internal":true,"Count Failed Values":true},{"ID":122,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":2823,"Value":17822,"Internal":true,"Count Failed Values":true},{"ID":123,"Name":"internal.metrics.shuffle.write.writeTime","Update":9288526,"Value":52177309,"Internal":true,"Count Failed Values":true},{"ID":124,"Name":"internal.metrics.input.bytesRead","Update":23122020,"Value":138831124,"Internal":true,"Count Failed Values":true},{"ID":125,"Name":"internal.metrics.input.recordsRead","Update":124858,"Value":750285,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":196,"Executor Deserialize CPU Time":188541690,"Executor Run Time":1610,"Executor CPU Time":1487640781,"Peak Execution Memory":29622272,"Result Size":2904,"JVM GC Time":35,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":79832,"Shuffle Write Time":9288526,"Shuffle Records Written":2823},"Input Metrics":{"Bytes Read":23122020,"Records Read":124858},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":12,"Index":4,"Attempt":0,"Launch Time":1633789197501,"Executor ID":"6","Host":"10.1.0.18","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":1633789201142,"Failed":false,"Killed":false,"Accumulables":[{"ID":25,"Name":"data size","Update":"161944","Value":"1159960","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"shuffle bytes written","Update":"81819","Value":"584748","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"shuffle records written","Update":"2892","Value":"20714","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":35,"Name":"shuffle write time","Update":"9220560","Value":"61397869","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":61,"Name":"duration","Update":"1422","Value":"9472","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":62,"Name":"number of output rows","Update":"2892","Value":"20714","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":63,"Name":"peak memory","Update":"4456448","Value":"27000832","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":65,"Name":"time in aggregation build","Update":"1080","Value":"7178","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":66,"Name":"avg hash probe bucket list iters","Update":"10","Value":"60","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":67,"Name":"number of output rows","Update":"2892","Value":"20714","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":76,"Name":"number of output rows","Update":"2892","Value":"20714","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":77,"Name":"number of output rows","Update":"124603","Value":"874888","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":103,"Name":"internal.metrics.executorDeserializeTime","Update":856,"Value":2003,"Internal":true,"Count Failed Values":true},{"ID":104,"Name":"internal.metrics.executorDeserializeCpuTime","Update":792100442,"Value":1898975830,"Internal":true,"Count Failed Values":true},{"ID":105,"Name":"internal.metrics.executorRunTime","Update":2745,"Value":12026,"Internal":true,"Count Failed Values":true},{"ID":106,"Name":"internal.metrics.executorCpuTime","Update":2515745749,"Value":11122703942,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.resultSize","Update":2904,"Value":20329,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.jvmGCTime","Update":163,"Value":379,"Internal":true,"Count Failed Values":true},{"ID":112,"Name":"internal.metrics.peakExecutionMemory","Update":33816576,"Value":232521728,"Internal":true,"Count Failed Values":true},{"ID":121,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":81819,"Value":584748,"Internal":true,"Count Failed Values":true},{"ID":122,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":2892,"Value":20714,"Internal":true,"Count Failed Values":true},{"ID":123,"Name":"internal.metrics.shuffle.write.writeTime","Update":9220560,"Value":61397869,"Internal":true,"Count Failed Values":true},{"ID":124,"Name":"internal.metrics.input.bytesRead","Update":23064712,"Value":161895836,"Internal":true,"Count Failed Values":true},{"ID":125,"Name":"internal.metrics.input.recordsRead","Update":124603,"Value":874888,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":856,"Executor Deserialize CPU Time":792100442,"Executor Run Time":2745,"Executor CPU Time":2515745749,"Peak Execution Memory":33816576,"Result Size":2904,"JVM GC Time":163,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":81819,"Shuffle Write Time":9220560,"Shuffle Records Written":2892},"Input Metrics":{"Bytes Read":23064712,"Records Read":124603},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":11,"Index":3,"Attempt":0,"Launch Time":1633789197500,"Executor ID":"8","Host":"10.1.0.11","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":1633789201242,"Failed":false,"Killed":false,"Accumulables":[{"ID":25,"Name":"data size","Update":"161504","Value":"1321464","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"shuffle bytes written","Update":"81474","Value":"666222","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"shuffle records written","Update":"2884","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":35,"Name":"shuffle write time","Update":"9090562","Value":"70488431","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":61,"Name":"duration","Update":"1491","Value":"10963","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":62,"Name":"number of output rows","Update":"2884","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":63,"Name":"peak memory","Update":"4456448","Value":"31457280","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":65,"Name":"time in aggregation build","Update":"1137","Value":"8315","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":66,"Name":"avg hash probe bucket list iters","Update":"10","Value":"70","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":67,"Name":"number of output rows","Update":"2884","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":76,"Name":"number of output rows","Update":"2884","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":77,"Name":"number of output rows","Update":"125112","Value":"1000000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":103,"Name":"internal.metrics.executorDeserializeTime","Update":875,"Value":2878,"Internal":true,"Count Failed Values":true},{"ID":104,"Name":"internal.metrics.executorDeserializeCpuTime","Update":805744939,"Value":2704720769,"Internal":true,"Count Failed Values":true},{"ID":105,"Name":"internal.metrics.executorRunTime","Update":2827,"Value":14853,"Internal":true,"Count Failed Values":true},{"ID":106,"Name":"internal.metrics.executorCpuTime","Update":2598933099,"Value":13721637041,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.resultSize","Update":2904,"Value":23233,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.jvmGCTime","Update":169,"Value":548,"Internal":true,"Count Failed Values":true},{"ID":112,"Name":"internal.metrics.peakExecutionMemory","Update":33816576,"Value":266338304,"Internal":true,"Count Failed Values":true},{"ID":121,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":81474,"Value":666222,"Internal":true,"Count Failed Values":true},{"ID":122,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":2884,"Value":23598,"Internal":true,"Count Failed Values":true},{"ID":123,"Name":"internal.metrics.shuffle.write.writeTime","Update":9090562,"Value":70488431,"Internal":true,"Count Failed Values":true},{"ID":124,"Name":"internal.metrics.input.bytesRead","Update":23103793,"Value":184999629,"Internal":true,"Count Failed Values":true},{"ID":125,"Name":"internal.metrics.input.recordsRead","Update":125112,"Value":1000000,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":875,"Executor Deserialize CPU Time":805744939,"Executor Run Time":2827,"Executor CPU Time":2598933099,"Peak Execution Memory":33816576,"Result Size":2904,"JVM GC Time":169,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":81474,"Shuffle Write Time":9090562,"Shuffle Records Written":2884},"Input Metrics":{"Bytes Read":23103793,"Records Read":125112},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"sql at ScalaSparkSQLBench.scala:43","Number of Tasks":8,"RDD Info":[{"RDD ID":11,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"12\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[10],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":10,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"13\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[9],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":6,"Name":"HadoopRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1633789197487,"Completion Time":1633789201243,"Accumulables":[{"ID":25,"Name":"data size","Value":"1321464","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"shuffle bytes written","Value":"666222","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"shuffle records written","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":35,"Name":"shuffle write time","Value":"70488431","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":61,"Name":"duration","Value":"10963","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":62,"Name":"number of output rows","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":63,"Name":"peak memory","Value":"31457280","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":65,"Name":"time in aggregation build","Value":"8315","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":66,"Name":"avg hash probe bucket list iters","Value":"70","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":67,"Name":"number of output rows","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":76,"Name":"number of output rows","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":77,"Name":"number of output rows","Value":"1000000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":103,"Name":"internal.metrics.executorDeserializeTime","Value":2878,"Internal":true,"Count Failed Values":true},{"ID":104,"Name":"internal.metrics.executorDeserializeCpuTime","Value":2704720769,"Internal":true,"Count Failed Values":true},{"ID":105,"Name":"internal.metrics.executorRunTime","Value":14853,"Internal":true,"Count Failed Values":true},{"ID":106,"Name":"internal.metrics.executorCpuTime","Value":13721637041,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.resultSize","Value":23233,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.jvmGCTime","Value":548,"Internal":true,"Count Failed Values":true},{"ID":112,"Name":"internal.metrics.peakExecutionMemory","Value":266338304,"Internal":true,"Count Failed Values":true},{"ID":121,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":666222,"Internal":true,"Count Failed Values":true},{"ID":122,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":23598,"Internal":true,"Count Failed Values":true},{"ID":123,"Name":"internal.metrics.shuffle.write.writeTime","Value":70488431,"Internal":true,"Count Failed Values":true},{"ID":124,"Name":"internal.metrics.input.bytesRead","Value":184999629,"Internal":true,"Count Failed Values":true},{"ID":125,"Name":"internal.metrics.input.recordsRead","Value":1000000,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":2,"Stage Attempt ID":0,"Stage Name":"sql at ScalaSparkSQLBench.scala:43","Number of Tasks":8,"RDD Info":[{"RDD ID":16,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[15],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":12,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"12\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[11],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":15,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[14],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":14,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[1],"Details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1633789201258,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.sql.warehouse.dir":"file:/home/usera112/projects/log-compressor/deployments/HiBench/spark-warehouse","spark.driver.host":"hostb7.dsrg.utoronto.ca","spark.eventLog.enabled":"true","spark.driver.port":"44011","__fetch_continuous_blocks_in_batch_enabled":"true","spark.jars":"file:/home/usera112/projects/log-compressor/deployments/HiBench/sparkbench/assembly/target/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.app.name":"ScalaJoin","spark.rdd.scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","spark.driver.memory":"4g","spark.rdd.scope.noOverride":"true","spark.submit.pyFiles":"","spark.default.parallelism":"8","spark.app.startTime":"1633789163045","spark.executor.id":"driver","spark.app.initial.jar.urls":"spark://hostb7.dsrg.utoronto.ca:44011/jars/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.submit.deployMode":"client","spark.master":"spark://hostb7:7077","spark.executor.memory":"4g","spark.eventLog.dir":"hdfs://hostb7:8020/directory","spark.sql.execution.id":"11","spark.sql.catalogImplementation":"hive","spark.app.id":"app-20211009101924-2782","spark.sql.shuffle.partitions":"8"}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":16,"Index":0,"Attempt":0,"Launch Time":1633789201269,"Executor ID":"10","Host":"10.1.0.16","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":17,"Index":1,"Attempt":0,"Launch Time":1633789201271,"Executor ID":"5","Host":"10.1.0.19","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":18,"Index":2,"Attempt":0,"Launch Time":1633789201271,"Executor ID":"4","Host":"10.1.0.8","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":19,"Index":3,"Attempt":0,"Launch Time":1633789201271,"Executor ID":"3","Host":"10.1.0.15","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":20,"Index":4,"Attempt":0,"Launch Time":1633789201272,"Executor ID":"2","Host":"10.1.0.10","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":21,"Index":5,"Attempt":0,"Launch Time":1633789201272,"Executor ID":"9","Host":"10.1.0.14","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":22,"Index":6,"Attempt":0,"Launch Time":1633789201273,"Executor ID":"1","Host":"10.1.0.17","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":23,"Index":7,"Attempt":0,"Launch Time":1633789201273,"Executor ID":"11","Host":"10.1.0.13","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":16,"Index":0,"Attempt":0,"Launch Time":1633789201269,"Executor ID":"10","Host":"10.1.0.16","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633789201495,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"7","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":27,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"71772","Value":"71772","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"local bytes read","Update":"10445","Value":"10445","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2905","Value":"2905","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"39","Value":"39","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2905","Value":"2905","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"4456448","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"16","Value":"16","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":128,"Name":"internal.metrics.executorDeserializeTime","Update":45,"Value":45,"Internal":true,"Count Failed Values":true},{"ID":129,"Name":"internal.metrics.executorDeserializeCpuTime","Update":40194173,"Value":40194173,"Internal":true,"Count Failed Values":true},{"ID":130,"Name":"internal.metrics.executorRunTime","Update":163,"Value":163,"Internal":true,"Count Failed Values":true},{"ID":131,"Name":"internal.metrics.executorCpuTime","Update":127454999,"Value":127454999,"Internal":true,"Count Failed Values":true},{"ID":132,"Name":"internal.metrics.resultSize","Update":14169,"Value":14169,"Internal":true,"Count Failed Values":true},{"ID":134,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":137,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":4456448,"Internal":true,"Count Failed Values":true},{"ID":139,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":140,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":141,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":71772,"Value":71772,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":10445,"Value":10445,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2905,"Value":2905,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":45,"Executor Deserialize CPU Time":40194173,"Executor Run Time":163,"Executor CPU Time":127454999,"Peak Execution Memory":4456448,"Result Size":14169,"JVM GC Time":0,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":71772,"Remote Bytes Read To Disk":0,"Local Bytes Read":10445,"Total Records Read":2905},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":18,"Index":2,"Attempt":0,"Launch Time":1633789201271,"Executor ID":"4","Host":"10.1.0.8","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633789201502,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"7","Value":"14","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":27,"Name":"local blocks read","Update":"1","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"75167","Value":"146939","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"local bytes read","Update":"10737","Value":"21182","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"3049","Value":"5954","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"40","Value":"79","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"3049","Value":"5954","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"8912896","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"16","Value":"32","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"22","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":128,"Name":"internal.metrics.executorDeserializeTime","Update":46,"Value":91,"Internal":true,"Count Failed Values":true},{"ID":129,"Name":"internal.metrics.executorDeserializeCpuTime","Update":39995158,"Value":80189331,"Internal":true,"Count Failed Values":true},{"ID":130,"Name":"internal.metrics.executorRunTime","Update":171,"Value":334,"Internal":true,"Count Failed Values":true},{"ID":131,"Name":"internal.metrics.executorCpuTime","Update":134762848,"Value":262217847,"Internal":true,"Count Failed Values":true},{"ID":132,"Name":"internal.metrics.resultSize","Update":14169,"Value":28338,"Internal":true,"Count Failed Values":true},{"ID":134,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":137,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":8912896,"Internal":true,"Count Failed Values":true},{"ID":139,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":14,"Internal":true,"Count Failed Values":true},{"ID":140,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":141,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":75167,"Value":146939,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":10737,"Value":21182,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.shuffle.read.recordsRead","Update":3049,"Value":5954,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":46,"Executor Deserialize CPU Time":39995158,"Executor Run Time":171,"Executor CPU Time":134762848,"Peak Execution Memory":4456448,"Result Size":14169,"JVM GC Time":0,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":75167,"Remote Bytes Read To Disk":0,"Local Bytes Read":10737,"Total Records Read":3049},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":22,"Index":6,"Attempt":0,"Launch Time":1633789201273,"Executor ID":"1","Host":"10.1.0.17","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633789201505,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"7","Value":"21","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":27,"Name":"local blocks read","Update":"1","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"71833","Value":"218772","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"local bytes read","Update":"10308","Value":"31490","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2910","Value":"8864","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"43","Value":"122","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2910","Value":"8864","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"13369344","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"18","Value":"50","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"33","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":128,"Name":"internal.metrics.executorDeserializeTime","Update":46,"Value":137,"Internal":true,"Count Failed Values":true},{"ID":129,"Name":"internal.metrics.executorDeserializeCpuTime","Update":40980094,"Value":121169425,"Internal":true,"Count Failed Values":true},{"ID":130,"Name":"internal.metrics.executorRunTime","Update":172,"Value":506,"Internal":true,"Count Failed Values":true},{"ID":131,"Name":"internal.metrics.executorCpuTime","Update":133541679,"Value":395759526,"Internal":true,"Count Failed Values":true},{"ID":132,"Name":"internal.metrics.resultSize","Update":14169,"Value":42507,"Internal":true,"Count Failed Values":true},{"ID":134,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":137,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":13369344,"Internal":true,"Count Failed Values":true},{"ID":139,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":21,"Internal":true,"Count Failed Values":true},{"ID":140,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":141,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":71833,"Value":218772,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":10308,"Value":31490,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2910,"Value":8864,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":46,"Executor Deserialize CPU Time":40980094,"Executor Run Time":172,"Executor CPU Time":133541679,"Peak Execution Memory":4456448,"Result Size":14169,"JVM GC Time":0,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":71833,"Remote Bytes Read To Disk":0,"Local Bytes Read":10308,"Total Records Read":2910},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":21,"Index":5,"Attempt":0,"Launch Time":1633789201272,"Executor ID":"9","Host":"10.1.0.14","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633789201505,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"7","Value":"28","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":27,"Name":"local blocks read","Update":"1","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"73503","Value":"292275","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"local bytes read","Update":"9706","Value":"41196","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2948","Value":"11812","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"42","Value":"164","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2948","Value":"11812","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"17825792","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"16","Value":"66","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"44","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":128,"Name":"internal.metrics.executorDeserializeTime","Update":48,"Value":185,"Internal":true,"Count Failed Values":true},{"ID":129,"Name":"internal.metrics.executorDeserializeCpuTime","Update":41014065,"Value":162183490,"Internal":true,"Count Failed Values":true},{"ID":130,"Name":"internal.metrics.executorRunTime","Update":170,"Value":676,"Internal":true,"Count Failed Values":true},{"ID":131,"Name":"internal.metrics.executorCpuTime","Update":134801322,"Value":530560848,"Internal":true,"Count Failed Values":true},{"ID":132,"Name":"internal.metrics.resultSize","Update":14169,"Value":56676,"Internal":true,"Count Failed Values":true},{"ID":134,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":137,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":17825792,"Internal":true,"Count Failed Values":true},{"ID":139,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":28,"Internal":true,"Count Failed Values":true},{"ID":140,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":141,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":73503,"Value":292275,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":9706,"Value":41196,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2948,"Value":11812,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":48,"Executor Deserialize CPU Time":41014065,"Executor Run Time":170,"Executor CPU Time":134801322,"Peak Execution Memory":4456448,"Result Size":14169,"JVM GC Time":0,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":73503,"Remote Bytes Read To Disk":0,"Local Bytes Read":9706,"Total Records Read":2948},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":20,"Index":4,"Attempt":0,"Launch Time":1633789201272,"Executor ID":"2","Host":"10.1.0.10","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633789201506,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"7","Value":"35","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":27,"Name":"local blocks read","Update":"1","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"71983","Value":"364258","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"local bytes read","Update":"11005","Value":"52201","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2937","Value":"14749","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"41","Value":"205","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2937","Value":"14749","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"22282240","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"17","Value":"83","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"55","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":128,"Name":"internal.metrics.executorDeserializeTime","Update":47,"Value":232,"Internal":true,"Count Failed Values":true},{"ID":129,"Name":"internal.metrics.executorDeserializeCpuTime","Update":41987190,"Value":204170680,"Internal":true,"Count Failed Values":true},{"ID":130,"Name":"internal.metrics.executorRunTime","Update":173,"Value":849,"Internal":true,"Count Failed Values":true},{"ID":131,"Name":"internal.metrics.executorCpuTime","Update":132010463,"Value":662571311,"Internal":true,"Count Failed Values":true},{"ID":132,"Name":"internal.metrics.resultSize","Update":14169,"Value":70845,"Internal":true,"Count Failed Values":true},{"ID":134,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":137,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":22282240,"Internal":true,"Count Failed Values":true},{"ID":139,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":35,"Internal":true,"Count Failed Values":true},{"ID":140,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":141,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":71983,"Value":364258,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":11005,"Value":52201,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2937,"Value":14749,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":47,"Executor Deserialize CPU Time":41987190,"Executor Run Time":173,"Executor CPU Time":132010463,"Peak Execution Memory":4456448,"Result Size":14169,"JVM GC Time":0,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":71983,"Remote Bytes Read To Disk":0,"Local Bytes Read":11005,"Total Records Read":2937},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":23,"Index":7,"Attempt":0,"Launch Time":1633789201273,"Executor ID":"11","Host":"10.1.0.13","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633789201506,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"7","Value":"42","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":27,"Name":"local blocks read","Update":"1","Value":"6","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"70073","Value":"434331","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"local bytes read","Update":"10623","Value":"62824","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2856","Value":"17605","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"45","Value":"250","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2856","Value":"17605","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"26738688","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"17","Value":"100","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"66","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":128,"Name":"internal.metrics.executorDeserializeTime","Update":49,"Value":281,"Internal":true,"Count Failed Values":true},{"ID":129,"Name":"internal.metrics.executorDeserializeCpuTime","Update":42961717,"Value":247132397,"Internal":true,"Count Failed Values":true},{"ID":130,"Name":"internal.metrics.executorRunTime","Update":171,"Value":1020,"Internal":true,"Count Failed Values":true},{"ID":131,"Name":"internal.metrics.executorCpuTime","Update":135033817,"Value":797605128,"Internal":true,"Count Failed Values":true},{"ID":132,"Name":"internal.metrics.resultSize","Update":14169,"Value":85014,"Internal":true,"Count Failed Values":true},{"ID":134,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":6,"Internal":true,"Count Failed Values":true},{"ID":137,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":26738688,"Internal":true,"Count Failed Values":true},{"ID":139,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":42,"Internal":true,"Count Failed Values":true},{"ID":140,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":6,"Internal":true,"Count Failed Values":true},{"ID":141,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":70073,"Value":434331,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":10623,"Value":62824,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2856,"Value":17605,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":49,"Executor Deserialize CPU Time":42961717,"Executor Run Time":171,"Executor CPU Time":135033817,"Peak Execution Memory":4456448,"Result Size":14169,"JVM GC Time":0,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":70073,"Remote Bytes Read To Disk":0,"Local Bytes Read":10623,"Total Records Read":2856},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":17,"Index":1,"Attempt":0,"Launch Time":1633789201271,"Executor ID":"5","Host":"10.1.0.19","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633789201776,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"8","Value":"50","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"84612","Value":"518943","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2999","Value":"20604","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"127","Value":"377","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2999","Value":"20604","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"31195136","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"35","Value":"135","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"77","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":128,"Name":"internal.metrics.executorDeserializeTime","Update":218,"Value":499,"Internal":true,"Count Failed Values":true},{"ID":129,"Name":"internal.metrics.executorDeserializeCpuTime","Update":208729295,"Value":455861692,"Internal":true,"Count Failed Values":true},{"ID":130,"Name":"internal.metrics.executorRunTime","Update":270,"Value":1290,"Internal":true,"Count Failed Values":true},{"ID":131,"Name":"internal.metrics.executorCpuTime","Update":253112853,"Value":1050717981,"Internal":true,"Count Failed Values":true},{"ID":132,"Name":"internal.metrics.resultSize","Update":14169,"Value":99183,"Internal":true,"Count Failed Values":true},{"ID":134,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":137,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":31195136,"Internal":true,"Count Failed Values":true},{"ID":139,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":8,"Value":50,"Internal":true,"Count Failed Values":true},{"ID":140,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":6,"Internal":true,"Count Failed Values":true},{"ID":141,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":84612,"Value":518943,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":62824,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2999,"Value":20604,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":218,"Executor Deserialize CPU Time":208729295,"Executor Run Time":270,"Executor CPU Time":253112853,"Peak Execution Memory":4456448,"Result Size":14169,"JVM GC Time":0,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":8,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":84612,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":2999},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":19,"Index":3,"Attempt":0,"Launch Time":1633789201271,"Executor ID":"3","Host":"10.1.0.15","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633789202924,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"8","Value":"58","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"84455","Value":"603398","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2994","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"194","Value":"571","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2994","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"35651584","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"41","Value":"176","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"88","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":128,"Name":"internal.metrics.executorDeserializeTime","Update":969,"Value":1468,"Internal":true,"Count Failed Values":true},{"ID":129,"Name":"internal.metrics.executorDeserializeCpuTime","Update":901040610,"Value":1356902302,"Internal":true,"Count Failed Values":true},{"ID":130,"Name":"internal.metrics.executorRunTime","Update":639,"Value":1929,"Internal":true,"Count Failed Values":true},{"ID":131,"Name":"internal.metrics.executorCpuTime","Update":615072661,"Value":1665790642,"Internal":true,"Count Failed Values":true},{"ID":132,"Name":"internal.metrics.resultSize","Update":14212,"Value":113395,"Internal":true,"Count Failed Values":true},{"ID":133,"Name":"internal.metrics.jvmGCTime","Update":33,"Value":33,"Internal":true,"Count Failed Values":true},{"ID":134,"Name":"internal.metrics.resultSerializationTime","Update":3,"Value":10,"Internal":true,"Count Failed Values":true},{"ID":137,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":35651584,"Internal":true,"Count Failed Values":true},{"ID":139,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":8,"Value":58,"Internal":true,"Count Failed Values":true},{"ID":140,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":6,"Internal":true,"Count Failed Values":true},{"ID":141,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":84455,"Value":603398,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":62824,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2994,"Value":23598,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":969,"Executor Deserialize CPU Time":901040610,"Executor Run Time":639,"Executor CPU Time":615072661,"Peak Execution Memory":4456448,"Result Size":14212,"JVM GC Time":33,"Result Serialization Time":3,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":8,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":84455,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":2994},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":2,"Stage Attempt ID":0,"Stage Name":"sql at ScalaSparkSQLBench.scala:43","Number of Tasks":8,"RDD Info":[{"RDD ID":16,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[15],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":12,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"12\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[11],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":15,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[14],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":14,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[1],"Details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1633789201258,"Completion Time":1633789202925,"Accumulables":[{"ID":26,"Name":"remote blocks read","Value":"58","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":27,"Name":"local blocks read","Value":"6","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Value":"603398","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"local bytes read","Value":"62824","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Value":"571","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Value":"35651584","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Value":"176","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Value":"88","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":128,"Name":"internal.metrics.executorDeserializeTime","Value":1468,"Internal":true,"Count Failed Values":true},{"ID":129,"Name":"internal.metrics.executorDeserializeCpuTime","Value":1356902302,"Internal":true,"Count Failed Values":true},{"ID":130,"Name":"internal.metrics.executorRunTime","Value":1929,"Internal":true,"Count Failed Values":true},{"ID":131,"Name":"internal.metrics.executorCpuTime","Value":1665790642,"Internal":true,"Count Failed Values":true},{"ID":132,"Name":"internal.metrics.resultSize","Value":113395,"Internal":true,"Count Failed Values":true},{"ID":133,"Name":"internal.metrics.jvmGCTime","Value":33,"Internal":true,"Count Failed Values":true},{"ID":134,"Name":"internal.metrics.resultSerializationTime","Value":10,"Internal":true,"Count Failed Values":true},{"ID":137,"Name":"internal.metrics.peakExecutionMemory","Value":35651584,"Internal":true,"Count Failed Values":true},{"ID":139,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":58,"Internal":true,"Count Failed Values":true},{"ID":140,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":6,"Internal":true,"Count Failed Values":true},{"ID":141,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":603398,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":62824,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.shuffle.read.recordsRead","Value":23598,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":1,"Completion Time":1633789202926,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"SparkListenerJobStart","Job ID":2,"Submission Time":1633789202997,"Stage Infos":[{"Stage ID":5,"Stage Attempt ID":0,"Stage Name":"sql at ScalaSparkSQLBench.scala:43","Number of Tasks":8,"RDD Info":[{"RDD ID":19,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[18],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":18,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[17],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[4],"Details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":3,"Stage Attempt ID":0,"Stage Name":"sql at ScalaSparkSQLBench.scala:43","Number of Tasks":8,"RDD Info":[{"RDD ID":11,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"12\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[10],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":10,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"13\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[9],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":6,"Name":"HadoopRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":4,"Stage Attempt ID":0,"Stage Name":"sql at ScalaSparkSQLBench.scala:43","Number of Tasks":8,"RDD Info":[{"RDD ID":17,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":12,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"12\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[11],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[3],"Details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[5,3,4],"Properties":{"spark.sql.warehouse.dir":"file:/home/usera112/projects/log-compressor/deployments/HiBench/spark-warehouse","spark.driver.host":"hostb7.dsrg.utoronto.ca","spark.eventLog.enabled":"true","spark.driver.port":"44011","__fetch_continuous_blocks_in_batch_enabled":"true","spark.jars":"file:/home/usera112/projects/log-compressor/deployments/HiBench/sparkbench/assembly/target/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.app.name":"ScalaJoin","spark.driver.memory":"4g","spark.submit.pyFiles":"","spark.default.parallelism":"8","spark.app.startTime":"1633789163045","spark.executor.id":"driver","spark.app.initial.jar.urls":"spark://hostb7.dsrg.utoronto.ca:44011/jars/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.submit.deployMode":"client","spark.master":"spark://hostb7:7077","spark.executor.memory":"4g","spark.eventLog.dir":"hdfs://hostb7:8020/directory","spark.sql.execution.id":"11","spark.sql.catalogImplementation":"hive","spark.app.id":"app-20211009101924-2782","spark.sql.shuffle.partitions":"8"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":4,"Stage Attempt ID":0,"Stage Name":"sql at ScalaSparkSQLBench.scala:43","Number of Tasks":8,"RDD Info":[{"RDD ID":17,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":12,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"12\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[11],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[3],"Details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1633789203002,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.sql.warehouse.dir":"file:/home/usera112/projects/log-compressor/deployments/HiBench/spark-warehouse","spark.driver.host":"hostb7.dsrg.utoronto.ca","spark.eventLog.enabled":"true","spark.driver.port":"44011","__fetch_continuous_blocks_in_batch_enabled":"true","spark.jars":"file:/home/usera112/projects/log-compressor/deployments/HiBench/sparkbench/assembly/target/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.app.name":"ScalaJoin","spark.driver.memory":"4g","spark.submit.pyFiles":"","spark.default.parallelism":"8","spark.app.startTime":"1633789163045","spark.executor.id":"driver","spark.app.initial.jar.urls":"spark://hostb7.dsrg.utoronto.ca:44011/jars/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.submit.deployMode":"client","spark.master":"spark://hostb7:7077","spark.executor.memory":"4g","spark.eventLog.dir":"hdfs://hostb7:8020/directory","spark.sql.execution.id":"11","spark.sql.catalogImplementation":"hive","spark.app.id":"app-20211009101924-2782","spark.sql.shuffle.partitions":"8"}} +{"Event":"SparkListenerTaskStart","Stage ID":4,"Stage Attempt ID":0,"Task Info":{"Task ID":24,"Index":0,"Attempt":0,"Launch Time":1633789203014,"Executor ID":"0","Host":"10.1.0.12","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":4,"Stage Attempt ID":0,"Task Info":{"Task ID":25,"Index":1,"Attempt":0,"Launch Time":1633789203014,"Executor ID":"7","Host":"10.1.0.9","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":4,"Stage Attempt ID":0,"Task Info":{"Task ID":26,"Index":2,"Attempt":0,"Launch Time":1633789203015,"Executor ID":"10","Host":"10.1.0.16","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":4,"Stage Attempt ID":0,"Task Info":{"Task ID":27,"Index":3,"Attempt":0,"Launch Time":1633789203015,"Executor ID":"11","Host":"10.1.0.13","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":4,"Stage Attempt ID":0,"Task Info":{"Task ID":28,"Index":4,"Attempt":0,"Launch Time":1633789203015,"Executor ID":"1","Host":"10.1.0.17","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":4,"Stage Attempt ID":0,"Task Info":{"Task ID":29,"Index":5,"Attempt":0,"Launch Time":1633789203015,"Executor ID":"6","Host":"10.1.0.18","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":4,"Stage Attempt ID":0,"Task Info":{"Task ID":30,"Index":6,"Attempt":0,"Launch Time":1633789203016,"Executor ID":"5","Host":"10.1.0.19","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":4,"Stage Attempt ID":0,"Task Info":{"Task ID":31,"Index":7,"Attempt":0,"Launch Time":1633789203016,"Executor ID":"2","Host":"10.1.0.10","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":4,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":28,"Index":4,"Attempt":0,"Launch Time":1633789203015,"Executor ID":"1","Host":"10.1.0.17","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633789203110,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"7","Value":"65","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":27,"Name":"local blocks read","Update":"1","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"72296","Value":"675694","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"local bytes read","Update":"10692","Value":"73516","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2937","Value":"26535","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":36,"Name":"data size","Update":"140976","Value":"140976","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":44,"Name":"shuffle bytes written","Update":"79984","Value":"79984","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":45,"Name":"shuffle records written","Update":"2937","Value":"2937","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":46,"Name":"shuffle write time","Update":"3729687","Value":"3729687","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"27","Value":"598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2937","Value":"26535","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"40108032","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"6","Value":"182","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"99","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":153,"Name":"internal.metrics.executorDeserializeTime","Update":46,"Value":46,"Internal":true,"Count Failed Values":true},{"ID":154,"Name":"internal.metrics.executorDeserializeCpuTime","Update":42482569,"Value":42482569,"Internal":true,"Count Failed Values":true},{"ID":155,"Name":"internal.metrics.executorRunTime","Update":38,"Value":38,"Internal":true,"Count Failed Values":true},{"ID":156,"Name":"internal.metrics.executorCpuTime","Update":37030227,"Value":37030227,"Internal":true,"Count Failed Values":true},{"ID":157,"Name":"internal.metrics.resultSize","Update":4236,"Value":4236,"Internal":true,"Count Failed Values":true},{"ID":162,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":4456448,"Internal":true,"Count Failed Values":true},{"ID":164,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":165,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":166,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":72296,"Value":72296,"Internal":true,"Count Failed Values":true},{"ID":167,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":168,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":10692,"Value":10692,"Internal":true,"Count Failed Values":true},{"ID":169,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":170,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2937,"Value":2937,"Internal":true,"Count Failed Values":true},{"ID":171,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":79984,"Value":79984,"Internal":true,"Count Failed Values":true},{"ID":172,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":2937,"Value":2937,"Internal":true,"Count Failed Values":true},{"ID":173,"Name":"internal.metrics.shuffle.write.writeTime","Update":3729687,"Value":3729687,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":46,"Executor Deserialize CPU Time":42482569,"Executor Run Time":38,"Executor CPU Time":37030227,"Peak Execution Memory":4456448,"Result Size":4236,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":72296,"Remote Bytes Read To Disk":0,"Local Bytes Read":10692,"Total Records Read":2937},"Shuffle Write Metrics":{"Shuffle Bytes Written":79984,"Shuffle Write Time":3729687,"Shuffle Records Written":2937},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":4,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":31,"Index":7,"Attempt":0,"Launch Time":1633789203016,"Executor ID":"2","Host":"10.1.0.10","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633789203111,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"7","Value":"72","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":27,"Name":"local blocks read","Update":"1","Value":"8","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"70979","Value":"746673","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"local bytes read","Update":"9717","Value":"83233","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2856","Value":"29391","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":36,"Name":"data size","Update":"137080","Value":"278056","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":44,"Name":"shuffle bytes written","Update":"77865","Value":"157849","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":45,"Name":"shuffle records written","Update":"2856","Value":"5793","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":46,"Name":"shuffle write time","Update":"3713439","Value":"7443126","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"27","Value":"625","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2856","Value":"29391","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"44564480","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"6","Value":"188","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"110","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":153,"Name":"internal.metrics.executorDeserializeTime","Update":48,"Value":94,"Internal":true,"Count Failed Values":true},{"ID":154,"Name":"internal.metrics.executorDeserializeCpuTime","Update":43150044,"Value":85632613,"Internal":true,"Count Failed Values":true},{"ID":155,"Name":"internal.metrics.executorRunTime","Update":39,"Value":77,"Internal":true,"Count Failed Values":true},{"ID":156,"Name":"internal.metrics.executorCpuTime","Update":37363811,"Value":74394038,"Internal":true,"Count Failed Values":true},{"ID":157,"Name":"internal.metrics.resultSize","Update":4236,"Value":8472,"Internal":true,"Count Failed Values":true},{"ID":162,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":8912896,"Internal":true,"Count Failed Values":true},{"ID":164,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":14,"Internal":true,"Count Failed Values":true},{"ID":165,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":166,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":70979,"Value":143275,"Internal":true,"Count Failed Values":true},{"ID":167,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":168,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":9717,"Value":20409,"Internal":true,"Count Failed Values":true},{"ID":169,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":170,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2856,"Value":5793,"Internal":true,"Count Failed Values":true},{"ID":171,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":77865,"Value":157849,"Internal":true,"Count Failed Values":true},{"ID":172,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":2856,"Value":5793,"Internal":true,"Count Failed Values":true},{"ID":173,"Name":"internal.metrics.shuffle.write.writeTime","Update":3713439,"Value":7443126,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":48,"Executor Deserialize CPU Time":43150044,"Executor Run Time":39,"Executor CPU Time":37363811,"Peak Execution Memory":4456448,"Result Size":4236,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":70979,"Remote Bytes Read To Disk":0,"Local Bytes Read":9717,"Total Records Read":2856},"Shuffle Write Metrics":{"Shuffle Bytes Written":77865,"Shuffle Write Time":3713439,"Shuffle Records Written":2856},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":4,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":27,"Index":3,"Attempt":0,"Launch Time":1633789203015,"Executor ID":"11","Host":"10.1.0.13","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633789203113,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"7","Value":"79","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":27,"Name":"local blocks read","Update":"1","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"74394","Value":"821067","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"local bytes read","Update":"10061","Value":"93294","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2994","Value":"32385","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":36,"Name":"data size","Update":"143712","Value":"421768","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":44,"Name":"shuffle bytes written","Update":"81445","Value":"239294","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":45,"Name":"shuffle records written","Update":"2994","Value":"8787","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":46,"Name":"shuffle write time","Update":"2747305","Value":"10190431","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"28","Value":"653","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2994","Value":"32385","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"49020928","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"6","Value":"194","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"121","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":153,"Name":"internal.metrics.executorDeserializeTime","Update":51,"Value":145,"Internal":true,"Count Failed Values":true},{"ID":154,"Name":"internal.metrics.executorDeserializeCpuTime","Update":46947629,"Value":132580242,"Internal":true,"Count Failed Values":true},{"ID":155,"Name":"internal.metrics.executorRunTime","Update":38,"Value":115,"Internal":true,"Count Failed Values":true},{"ID":156,"Name":"internal.metrics.executorCpuTime","Update":36474407,"Value":110868445,"Internal":true,"Count Failed Values":true},{"ID":157,"Name":"internal.metrics.resultSize","Update":4237,"Value":12709,"Internal":true,"Count Failed Values":true},{"ID":162,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":13369344,"Internal":true,"Count Failed Values":true},{"ID":164,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":21,"Internal":true,"Count Failed Values":true},{"ID":165,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":166,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":74394,"Value":217669,"Internal":true,"Count Failed Values":true},{"ID":167,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":168,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":10061,"Value":30470,"Internal":true,"Count Failed Values":true},{"ID":169,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":170,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2994,"Value":8787,"Internal":true,"Count Failed Values":true},{"ID":171,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":81445,"Value":239294,"Internal":true,"Count Failed Values":true},{"ID":172,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":2994,"Value":8787,"Internal":true,"Count Failed Values":true},{"ID":173,"Name":"internal.metrics.shuffle.write.writeTime","Update":2747305,"Value":10190431,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":51,"Executor Deserialize CPU Time":46947629,"Executor Run Time":38,"Executor CPU Time":36474407,"Peak Execution Memory":4456448,"Result Size":4237,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":74394,"Remote Bytes Read To Disk":0,"Local Bytes Read":10061,"Total Records Read":2994},"Shuffle Write Metrics":{"Shuffle Bytes Written":81445,"Shuffle Write Time":2747305,"Shuffle Records Written":2994},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":4,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":26,"Index":2,"Attempt":0,"Launch Time":1633789203015,"Executor ID":"10","Host":"10.1.0.16","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633789203124,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"7","Value":"86","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":27,"Name":"local blocks read","Update":"1","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"74808","Value":"895875","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"local bytes read","Update":"11096","Value":"104390","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"3049","Value":"35434","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":36,"Name":"data size","Update":"146336","Value":"568104","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":44,"Name":"shuffle bytes written","Update":"82790","Value":"322084","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":45,"Name":"shuffle records written","Update":"3049","Value":"11836","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":46,"Name":"shuffle write time","Update":"5131148","Value":"15321579","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"37","Value":"690","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"3049","Value":"35434","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"53477376","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"8","Value":"202","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"132","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":153,"Name":"internal.metrics.executorDeserializeTime","Update":47,"Value":192,"Internal":true,"Count Failed Values":true},{"ID":154,"Name":"internal.metrics.executorDeserializeCpuTime","Update":42448641,"Value":175028883,"Internal":true,"Count Failed Values":true},{"ID":155,"Name":"internal.metrics.executorRunTime","Update":52,"Value":167,"Internal":true,"Count Failed Values":true},{"ID":156,"Name":"internal.metrics.executorCpuTime","Update":49767354,"Value":160635799,"Internal":true,"Count Failed Values":true},{"ID":157,"Name":"internal.metrics.resultSize","Update":4237,"Value":16946,"Internal":true,"Count Failed Values":true},{"ID":162,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":17825792,"Internal":true,"Count Failed Values":true},{"ID":164,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":28,"Internal":true,"Count Failed Values":true},{"ID":165,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":166,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":74808,"Value":292477,"Internal":true,"Count Failed Values":true},{"ID":167,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":168,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":11096,"Value":41566,"Internal":true,"Count Failed Values":true},{"ID":169,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":170,"Name":"internal.metrics.shuffle.read.recordsRead","Update":3049,"Value":11836,"Internal":true,"Count Failed Values":true},{"ID":171,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":82790,"Value":322084,"Internal":true,"Count Failed Values":true},{"ID":172,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":3049,"Value":11836,"Internal":true,"Count Failed Values":true},{"ID":173,"Name":"internal.metrics.shuffle.write.writeTime","Update":5131148,"Value":15321579,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":47,"Executor Deserialize CPU Time":42448641,"Executor Run Time":52,"Executor CPU Time":49767354,"Peak Execution Memory":4456448,"Result Size":4237,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":74808,"Remote Bytes Read To Disk":0,"Local Bytes Read":11096,"Total Records Read":3049},"Shuffle Write Metrics":{"Shuffle Bytes Written":82790,"Shuffle Write Time":5131148,"Shuffle Records Written":3049},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":4,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":30,"Index":6,"Attempt":0,"Launch Time":1633789203016,"Executor ID":"5","Host":"10.1.0.19","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633789203158,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"8","Value":"94","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"82141","Value":"978016","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2910","Value":"38344","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":36,"Name":"data size","Update":"139680","Value":"707784","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":44,"Name":"shuffle bytes written","Update":"79272","Value":"401356","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":45,"Name":"shuffle records written","Update":"2910","Value":"14746","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":46,"Name":"shuffle write time","Update":"9436045","Value":"24757624","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"41","Value":"731","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2910","Value":"38344","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"57933824","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"7","Value":"209","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"143","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":153,"Name":"internal.metrics.executorDeserializeTime","Update":63,"Value":255,"Internal":true,"Count Failed Values":true},{"ID":154,"Name":"internal.metrics.executorDeserializeCpuTime","Update":57910631,"Value":232939514,"Internal":true,"Count Failed Values":true},{"ID":155,"Name":"internal.metrics.executorRunTime","Update":70,"Value":237,"Internal":true,"Count Failed Values":true},{"ID":156,"Name":"internal.metrics.executorCpuTime","Update":66488500,"Value":227124299,"Internal":true,"Count Failed Values":true},{"ID":157,"Name":"internal.metrics.resultSize","Update":4236,"Value":21182,"Internal":true,"Count Failed Values":true},{"ID":162,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":22282240,"Internal":true,"Count Failed Values":true},{"ID":164,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":8,"Value":36,"Internal":true,"Count Failed Values":true},{"ID":165,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":166,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":82141,"Value":374618,"Internal":true,"Count Failed Values":true},{"ID":167,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":168,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":41566,"Internal":true,"Count Failed Values":true},{"ID":169,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":170,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2910,"Value":14746,"Internal":true,"Count Failed Values":true},{"ID":171,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":79272,"Value":401356,"Internal":true,"Count Failed Values":true},{"ID":172,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":2910,"Value":14746,"Internal":true,"Count Failed Values":true},{"ID":173,"Name":"internal.metrics.shuffle.write.writeTime","Update":9436045,"Value":24757624,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":63,"Executor Deserialize CPU Time":57910631,"Executor Run Time":70,"Executor CPU Time":66488500,"Peak Execution Memory":4456448,"Result Size":4236,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":8,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":82141,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":2910},"Shuffle Write Metrics":{"Shuffle Bytes Written":79272,"Shuffle Write Time":9436045,"Shuffle Records Written":2910},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":4,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":29,"Index":5,"Attempt":0,"Launch Time":1633789203015,"Executor ID":"6","Host":"10.1.0.18","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633789203254,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"7","Value":"101","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":27,"Name":"local blocks read","Update":"1","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"72828","Value":"1050844","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"local bytes read","Update":"10381","Value":"114771","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2948","Value":"41292","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":36,"Name":"data size","Update":"141504","Value":"849288","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":44,"Name":"shuffle bytes written","Update":"80279","Value":"481635","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":45,"Name":"shuffle records written","Update":"2948","Value":"17694","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":46,"Name":"shuffle write time","Update":"2955698","Value":"27713322","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"45","Value":"776","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2948","Value":"41292","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"62390272","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"16","Value":"225","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"154","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":153,"Name":"internal.metrics.executorDeserializeTime","Update":76,"Value":331,"Internal":true,"Count Failed Values":true},{"ID":154,"Name":"internal.metrics.executorDeserializeCpuTime","Update":69573431,"Value":302512945,"Internal":true,"Count Failed Values":true},{"ID":155,"Name":"internal.metrics.executorRunTime","Update":152,"Value":389,"Internal":true,"Count Failed Values":true},{"ID":156,"Name":"internal.metrics.executorCpuTime","Update":142832467,"Value":369956766,"Internal":true,"Count Failed Values":true},{"ID":157,"Name":"internal.metrics.resultSize","Update":4236,"Value":25418,"Internal":true,"Count Failed Values":true},{"ID":162,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":26738688,"Internal":true,"Count Failed Values":true},{"ID":164,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":43,"Internal":true,"Count Failed Values":true},{"ID":165,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":166,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":72828,"Value":447446,"Internal":true,"Count Failed Values":true},{"ID":167,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":168,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":10381,"Value":51947,"Internal":true,"Count Failed Values":true},{"ID":169,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":170,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2948,"Value":17694,"Internal":true,"Count Failed Values":true},{"ID":171,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":80279,"Value":481635,"Internal":true,"Count Failed Values":true},{"ID":172,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":2948,"Value":17694,"Internal":true,"Count Failed Values":true},{"ID":173,"Name":"internal.metrics.shuffle.write.writeTime","Update":2955698,"Value":27713322,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":76,"Executor Deserialize CPU Time":69573431,"Executor Run Time":152,"Executor CPU Time":142832467,"Peak Execution Memory":4456448,"Result Size":4236,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":72828,"Remote Bytes Read To Disk":0,"Local Bytes Read":10381,"Total Records Read":2948},"Shuffle Write Metrics":{"Shuffle Bytes Written":80279,"Shuffle Write Time":2955698,"Shuffle Records Written":2948},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":4,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":25,"Index":1,"Attempt":0,"Launch Time":1633789203014,"Executor ID":"7","Host":"10.1.0.9","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633789203559,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"8","Value":"109","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"84612","Value":"1135456","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2999","Value":"44291","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":36,"Name":"data size","Update":"143952","Value":"993240","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":44,"Name":"shuffle bytes written","Update":"81642","Value":"563277","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":45,"Name":"shuffle records written","Update":"2999","Value":"20693","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":46,"Name":"shuffle write time","Update":"25512377","Value":"53225699","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"156","Value":"932","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2999","Value":"44291","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"66846720","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"32","Value":"257","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"165","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":153,"Name":"internal.metrics.executorDeserializeTime","Update":230,"Value":561,"Internal":true,"Count Failed Values":true},{"ID":154,"Name":"internal.metrics.executorDeserializeCpuTime","Update":219370826,"Value":521883771,"Internal":true,"Count Failed Values":true},{"ID":155,"Name":"internal.metrics.executorRunTime","Update":302,"Value":691,"Internal":true,"Count Failed Values":true},{"ID":156,"Name":"internal.metrics.executorCpuTime","Update":269611352,"Value":639568118,"Internal":true,"Count Failed Values":true},{"ID":157,"Name":"internal.metrics.resultSize","Update":4235,"Value":29653,"Internal":true,"Count Failed Values":true},{"ID":162,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":31195136,"Internal":true,"Count Failed Values":true},{"ID":164,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":8,"Value":51,"Internal":true,"Count Failed Values":true},{"ID":165,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":166,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":84612,"Value":532058,"Internal":true,"Count Failed Values":true},{"ID":167,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":168,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":51947,"Internal":true,"Count Failed Values":true},{"ID":169,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":170,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2999,"Value":20693,"Internal":true,"Count Failed Values":true},{"ID":171,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":81642,"Value":563277,"Internal":true,"Count Failed Values":true},{"ID":172,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":2999,"Value":20693,"Internal":true,"Count Failed Values":true},{"ID":173,"Name":"internal.metrics.shuffle.write.writeTime","Update":25512377,"Value":53225699,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":472141848,"JVMOffHeapMemory":84463448,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":536366,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":536366,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1193118,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":3,"MinorGCTime":31,"MajorGCCount":3,"MajorGCTime":144},"Task Metrics":{"Executor Deserialize Time":230,"Executor Deserialize CPU Time":219370826,"Executor Run Time":302,"Executor CPU Time":269611352,"Peak Execution Memory":4456448,"Result Size":4235,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":8,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":84612,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":2999},"Shuffle Write Metrics":{"Shuffle Bytes Written":81642,"Shuffle Write Time":25512377,"Shuffle Records Written":2999},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":4,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":24,"Index":0,"Attempt":0,"Launch Time":1633789203014,"Executor ID":"0","Host":"10.1.0.12","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633789204642,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"8","Value":"117","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"82217","Value":"1217673","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2905","Value":"47196","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":36,"Name":"data size","Update":"139440","Value":"1132680","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":44,"Name":"shuffle bytes written","Update":"79213","Value":"642490","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":45,"Name":"shuffle records written","Update":"2905","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":46,"Name":"shuffle write time","Update":"12231018","Value":"65456717","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"178","Value":"1110","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2905","Value":"47196","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"71303168","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"41","Value":"298","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"176","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":153,"Name":"internal.metrics.executorDeserializeTime","Update":1178,"Value":1739,"Internal":true,"Count Failed Values":true},{"ID":154,"Name":"internal.metrics.executorDeserializeCpuTime","Update":1106395401,"Value":1628279172,"Internal":true,"Count Failed Values":true},{"ID":155,"Name":"internal.metrics.executorRunTime","Update":412,"Value":1103,"Internal":true,"Count Failed Values":true},{"ID":156,"Name":"internal.metrics.executorCpuTime","Update":388572482,"Value":1028140600,"Internal":true,"Count Failed Values":true},{"ID":157,"Name":"internal.metrics.resultSize","Update":4279,"Value":33932,"Internal":true,"Count Failed Values":true},{"ID":158,"Name":"internal.metrics.jvmGCTime","Update":34,"Value":34,"Internal":true,"Count Failed Values":true},{"ID":162,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":35651584,"Internal":true,"Count Failed Values":true},{"ID":164,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":8,"Value":59,"Internal":true,"Count Failed Values":true},{"ID":165,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":166,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":82217,"Value":614275,"Internal":true,"Count Failed Values":true},{"ID":167,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":168,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":51947,"Internal":true,"Count Failed Values":true},{"ID":169,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":170,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2905,"Value":23598,"Internal":true,"Count Failed Values":true},{"ID":171,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":79213,"Value":642490,"Internal":true,"Count Failed Values":true},{"ID":172,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":2905,"Value":23598,"Internal":true,"Count Failed Values":true},{"ID":173,"Name":"internal.metrics.shuffle.write.writeTime","Update":12231018,"Value":65456717,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":1178,"Executor Deserialize CPU Time":1106395401,"Executor Run Time":412,"Executor CPU Time":388572482,"Peak Execution Memory":4456448,"Result Size":4279,"JVM GC Time":34,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":8,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":82217,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":2905},"Shuffle Write Metrics":{"Shuffle Bytes Written":79213,"Shuffle Write Time":12231018,"Shuffle Records Written":2905},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":4,"Stage Attempt ID":0,"Stage Name":"sql at ScalaSparkSQLBench.scala:43","Number of Tasks":8,"RDD Info":[{"RDD ID":17,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":12,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"12\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[11],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[3],"Details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1633789203002,"Completion Time":1633789204642,"Accumulables":[{"ID":26,"Name":"remote blocks read","Value":"117","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":27,"Name":"local blocks read","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Value":"1217673","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"local bytes read","Value":"114771","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Value":"47196","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":36,"Name":"data size","Value":"1132680","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":44,"Name":"shuffle bytes written","Value":"642490","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":45,"Name":"shuffle records written","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":46,"Name":"shuffle write time","Value":"65456717","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Value":"1110","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Value":"47196","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Value":"71303168","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Value":"298","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Value":"176","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":153,"Name":"internal.metrics.executorDeserializeTime","Value":1739,"Internal":true,"Count Failed Values":true},{"ID":154,"Name":"internal.metrics.executorDeserializeCpuTime","Value":1628279172,"Internal":true,"Count Failed Values":true},{"ID":155,"Name":"internal.metrics.executorRunTime","Value":1103,"Internal":true,"Count Failed Values":true},{"ID":156,"Name":"internal.metrics.executorCpuTime","Value":1028140600,"Internal":true,"Count Failed Values":true},{"ID":157,"Name":"internal.metrics.resultSize","Value":33932,"Internal":true,"Count Failed Values":true},{"ID":158,"Name":"internal.metrics.jvmGCTime","Value":34,"Internal":true,"Count Failed Values":true},{"ID":162,"Name":"internal.metrics.peakExecutionMemory","Value":35651584,"Internal":true,"Count Failed Values":true},{"ID":164,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":59,"Internal":true,"Count Failed Values":true},{"ID":165,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":5,"Internal":true,"Count Failed Values":true},{"ID":166,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":614275,"Internal":true,"Count Failed Values":true},{"ID":167,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":168,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":51947,"Internal":true,"Count Failed Values":true},{"ID":169,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":170,"Name":"internal.metrics.shuffle.read.recordsRead","Value":23598,"Internal":true,"Count Failed Values":true},{"ID":171,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":642490,"Internal":true,"Count Failed Values":true},{"ID":172,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":23598,"Internal":true,"Count Failed Values":true},{"ID":173,"Name":"internal.metrics.shuffle.write.writeTime","Value":65456717,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":5,"Stage Attempt ID":0,"Stage Name":"sql at ScalaSparkSQLBench.scala:43","Number of Tasks":8,"RDD Info":[{"RDD ID":19,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[18],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":18,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[17],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[4],"Details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1633789204645,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.sql.warehouse.dir":"file:/home/usera112/projects/log-compressor/deployments/HiBench/spark-warehouse","spark.driver.host":"hostb7.dsrg.utoronto.ca","spark.eventLog.enabled":"true","spark.driver.port":"44011","__fetch_continuous_blocks_in_batch_enabled":"true","spark.jars":"file:/home/usera112/projects/log-compressor/deployments/HiBench/sparkbench/assembly/target/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.app.name":"ScalaJoin","spark.driver.memory":"4g","spark.submit.pyFiles":"","spark.default.parallelism":"8","spark.app.startTime":"1633789163045","spark.executor.id":"driver","spark.app.initial.jar.urls":"spark://hostb7.dsrg.utoronto.ca:44011/jars/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.submit.deployMode":"client","spark.master":"spark://hostb7:7077","spark.executor.memory":"4g","spark.eventLog.dir":"hdfs://hostb7:8020/directory","spark.sql.execution.id":"11","spark.sql.catalogImplementation":"hive","spark.app.id":"app-20211009101924-2782","spark.sql.shuffle.partitions":"8"}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":32,"Index":0,"Attempt":0,"Launch Time":1633789204689,"Executor ID":"9","Host":"10.1.0.14","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":33,"Index":1,"Attempt":0,"Launch Time":1633789204689,"Executor ID":"8","Host":"10.1.0.11","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":34,"Index":2,"Attempt":0,"Launch Time":1633789204689,"Executor ID":"0","Host":"10.1.0.12","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":35,"Index":3,"Attempt":0,"Launch Time":1633789204689,"Executor ID":"4","Host":"10.1.0.8","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":36,"Index":4,"Attempt":0,"Launch Time":1633789204690,"Executor ID":"2","Host":"10.1.0.10","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":37,"Index":5,"Attempt":0,"Launch Time":1633789204690,"Executor ID":"3","Host":"10.1.0.15","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":38,"Index":6,"Attempt":0,"Launch Time":1633789204690,"Executor ID":"11","Host":"10.1.0.13","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":39,"Index":7,"Attempt":0,"Launch Time":1633789204690,"Executor ID":"10","Host":"10.1.0.16","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":38,"Index":6,"Attempt":0,"Launch Time":1633789204690,"Executor ID":"11","Host":"10.1.0.13","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633789205199,"Failed":false,"Killed":false,"Accumulables":[{"ID":37,"Name":"remote blocks read","Update":"7","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":38,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":39,"Name":"remote bytes read","Update":"68113","Value":"68113","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":41,"Name":"local bytes read","Update":"9554","Value":"9554","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":42,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":43,"Name":"records read","Update":"2846","Value":"2846","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":51,"Name":"duration","Update":"316","Value":"316","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":52,"Name":"sort time","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":53,"Name":"peak memory","Update":"4325376","Value":"4325376","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":54,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":178,"Name":"internal.metrics.executorDeserializeTime","Update":71,"Value":71,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.executorDeserializeCpuTime","Update":65347568,"Value":65347568,"Internal":true,"Count Failed Values":true},{"ID":180,"Name":"internal.metrics.executorRunTime","Update":421,"Value":421,"Internal":true,"Count Failed Values":true},{"ID":181,"Name":"internal.metrics.executorCpuTime","Update":334657806,"Value":334657806,"Internal":true,"Count Failed Values":true},{"ID":182,"Name":"internal.metrics.resultSize","Update":5952,"Value":5952,"Internal":true,"Count Failed Values":true},{"ID":184,"Name":"internal.metrics.resultSerializationTime","Update":3,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":187,"Name":"internal.metrics.peakExecutionMemory","Update":4325376,"Value":4325376,"Internal":true,"Count Failed Values":true},{"ID":189,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":190,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":191,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":68113,"Value":68113,"Internal":true,"Count Failed Values":true},{"ID":192,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":193,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":9554,"Value":9554,"Internal":true,"Count Failed Values":true},{"ID":194,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":195,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2846,"Value":2846,"Internal":true,"Count Failed Values":true},{"ID":201,"Name":"internal.metrics.output.bytesWritten","Update":120001,"Value":120001,"Internal":true,"Count Failed Values":true},{"ID":202,"Name":"internal.metrics.output.recordsWritten","Update":2846,"Value":2846,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":71,"Executor Deserialize CPU Time":65347568,"Executor Run Time":421,"Executor CPU Time":334657806,"Peak Execution Memory":4325376,"Result Size":5952,"JVM GC Time":0,"Result Serialization Time":3,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":68113,"Remote Bytes Read To Disk":0,"Local Bytes Read":9554,"Total Records Read":2846},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":120001,"Records Written":2846},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":35,"Index":3,"Attempt":0,"Launch Time":1633789204689,"Executor ID":"4","Host":"10.1.0.8","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633789205211,"Failed":false,"Killed":false,"Accumulables":[{"ID":37,"Name":"remote blocks read","Update":"8","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":39,"Name":"remote bytes read","Update":"83492","Value":"151605","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":42,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":43,"Name":"records read","Update":"3075","Value":"5921","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":51,"Name":"duration","Update":"327","Value":"643","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":52,"Name":"sort time","Update":"6","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":53,"Name":"peak memory","Update":"4325376","Value":"8650752","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":54,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":178,"Name":"internal.metrics.executorDeserializeTime","Update":77,"Value":148,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.executorDeserializeCpuTime","Update":71484546,"Value":136832114,"Internal":true,"Count Failed Values":true},{"ID":180,"Name":"internal.metrics.executorRunTime","Update":433,"Value":854,"Internal":true,"Count Failed Values":true},{"ID":181,"Name":"internal.metrics.executorCpuTime","Update":350450356,"Value":685108162,"Internal":true,"Count Failed Values":true},{"ID":182,"Name":"internal.metrics.resultSize","Update":5952,"Value":11904,"Internal":true,"Count Failed Values":true},{"ID":184,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":187,"Name":"internal.metrics.peakExecutionMemory","Update":4325376,"Value":8650752,"Internal":true,"Count Failed Values":true},{"ID":189,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":8,"Value":15,"Internal":true,"Count Failed Values":true},{"ID":190,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":191,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":83492,"Value":151605,"Internal":true,"Count Failed Values":true},{"ID":192,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":193,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":9554,"Internal":true,"Count Failed Values":true},{"ID":194,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":195,"Name":"internal.metrics.shuffle.read.recordsRead","Update":3075,"Value":5921,"Internal":true,"Count Failed Values":true},{"ID":201,"Name":"internal.metrics.output.bytesWritten","Update":128252,"Value":248253,"Internal":true,"Count Failed Values":true},{"ID":202,"Name":"internal.metrics.output.recordsWritten","Update":3075,"Value":5921,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":77,"Executor Deserialize CPU Time":71484546,"Executor Run Time":433,"Executor CPU Time":350450356,"Peak Execution Memory":4325376,"Result Size":5952,"JVM GC Time":0,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":8,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":83492,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":3075},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":128252,"Records Written":3075},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":39,"Index":7,"Attempt":0,"Launch Time":1633789204690,"Executor ID":"10","Host":"10.1.0.16","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633789205219,"Failed":false,"Killed":false,"Accumulables":[{"ID":37,"Name":"remote blocks read","Update":"7","Value":"22","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":38,"Name":"local blocks read","Update":"1","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":39,"Name":"remote bytes read","Update":"72297","Value":"223902","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":41,"Name":"local bytes read","Update":"10257","Value":"19811","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":42,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":43,"Name":"records read","Update":"3025","Value":"8946","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":51,"Name":"duration","Update":"296","Value":"939","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":52,"Name":"sort time","Update":"5","Value":"16","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":53,"Name":"peak memory","Update":"4325376","Value":"12976128","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":54,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":178,"Name":"internal.metrics.executorDeserializeTime","Update":92,"Value":240,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.executorDeserializeCpuTime","Update":85917179,"Value":222749293,"Internal":true,"Count Failed Values":true},{"ID":180,"Name":"internal.metrics.executorRunTime","Update":425,"Value":1279,"Internal":true,"Count Failed Values":true},{"ID":181,"Name":"internal.metrics.executorCpuTime","Update":318096729,"Value":1003204891,"Internal":true,"Count Failed Values":true},{"ID":182,"Name":"internal.metrics.resultSize","Update":5952,"Value":17856,"Internal":true,"Count Failed Values":true},{"ID":184,"Name":"internal.metrics.resultSerializationTime","Update":3,"Value":8,"Internal":true,"Count Failed Values":true},{"ID":187,"Name":"internal.metrics.peakExecutionMemory","Update":4325376,"Value":12976128,"Internal":true,"Count Failed Values":true},{"ID":189,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":22,"Internal":true,"Count Failed Values":true},{"ID":190,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":191,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":72297,"Value":223902,"Internal":true,"Count Failed Values":true},{"ID":192,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":193,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":10257,"Value":19811,"Internal":true,"Count Failed Values":true},{"ID":194,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":195,"Name":"internal.metrics.shuffle.read.recordsRead","Update":3025,"Value":8946,"Internal":true,"Count Failed Values":true},{"ID":201,"Name":"internal.metrics.output.bytesWritten","Update":129566,"Value":377819,"Internal":true,"Count Failed Values":true},{"ID":202,"Name":"internal.metrics.output.recordsWritten","Update":3025,"Value":8946,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":92,"Executor Deserialize CPU Time":85917179,"Executor Run Time":425,"Executor CPU Time":318096729,"Peak Execution Memory":4325376,"Result Size":5952,"JVM GC Time":0,"Result Serialization Time":3,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":72297,"Remote Bytes Read To Disk":0,"Local Bytes Read":10257,"Total Records Read":3025},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":129566,"Records Written":3025},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":36,"Index":4,"Attempt":0,"Launch Time":1633789204690,"Executor ID":"2","Host":"10.1.0.10","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633789205228,"Failed":false,"Killed":false,"Accumulables":[{"ID":37,"Name":"remote blocks read","Update":"7","Value":"29","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":38,"Name":"local blocks read","Update":"1","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":39,"Name":"remote bytes read","Update":"69831","Value":"293733","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":41,"Name":"local bytes read","Update":"9673","Value":"29484","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":42,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":43,"Name":"records read","Update":"2911","Value":"11857","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":51,"Name":"duration","Update":"312","Value":"1251","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":52,"Name":"sort time","Update":"6","Value":"22","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":53,"Name":"peak memory","Update":"4325376","Value":"17301504","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":54,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":178,"Name":"internal.metrics.executorDeserializeTime","Update":77,"Value":317,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.executorDeserializeCpuTime","Update":69361828,"Value":292111121,"Internal":true,"Count Failed Values":true},{"ID":180,"Name":"internal.metrics.executorRunTime","Update":449,"Value":1728,"Internal":true,"Count Failed Values":true},{"ID":181,"Name":"internal.metrics.executorCpuTime","Update":331558224,"Value":1334763115,"Internal":true,"Count Failed Values":true},{"ID":182,"Name":"internal.metrics.resultSize","Update":5952,"Value":23808,"Internal":true,"Count Failed Values":true},{"ID":184,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":10,"Internal":true,"Count Failed Values":true},{"ID":187,"Name":"internal.metrics.peakExecutionMemory","Update":4325376,"Value":17301504,"Internal":true,"Count Failed Values":true},{"ID":189,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":29,"Internal":true,"Count Failed Values":true},{"ID":190,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":191,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":69831,"Value":293733,"Internal":true,"Count Failed Values":true},{"ID":192,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":193,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":9673,"Value":29484,"Internal":true,"Count Failed Values":true},{"ID":194,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":195,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2911,"Value":11857,"Internal":true,"Count Failed Values":true},{"ID":201,"Name":"internal.metrics.output.bytesWritten","Update":122317,"Value":500136,"Internal":true,"Count Failed Values":true},{"ID":202,"Name":"internal.metrics.output.recordsWritten","Update":2911,"Value":11857,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":77,"Executor Deserialize CPU Time":69361828,"Executor Run Time":449,"Executor CPU Time":331558224,"Peak Execution Memory":4325376,"Result Size":5952,"JVM GC Time":0,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":69831,"Remote Bytes Read To Disk":0,"Local Bytes Read":9673,"Total Records Read":2911},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":122317,"Records Written":2911},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":32,"Index":0,"Attempt":0,"Launch Time":1633789204689,"Executor ID":"9","Host":"10.1.0.14","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633789205228,"Failed":false,"Killed":false,"Accumulables":[{"ID":37,"Name":"remote blocks read","Update":"8","Value":"37","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":39,"Name":"remote bytes read","Update":"78576","Value":"372309","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":42,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":43,"Name":"records read","Update":"2888","Value":"14745","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":51,"Name":"duration","Update":"314","Value":"1565","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":52,"Name":"sort time","Update":"6","Value":"28","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":53,"Name":"peak memory","Update":"4325376","Value":"21626880","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":54,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":178,"Name":"internal.metrics.executorDeserializeTime","Update":78,"Value":395,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.executorDeserializeCpuTime","Update":71457306,"Value":363568427,"Internal":true,"Count Failed Values":true},{"ID":180,"Name":"internal.metrics.executorRunTime","Update":448,"Value":2176,"Internal":true,"Count Failed Values":true},{"ID":181,"Name":"internal.metrics.executorCpuTime","Update":330505714,"Value":1665268829,"Internal":true,"Count Failed Values":true},{"ID":182,"Name":"internal.metrics.resultSize","Update":5952,"Value":29760,"Internal":true,"Count Failed Values":true},{"ID":184,"Name":"internal.metrics.resultSerializationTime","Update":3,"Value":13,"Internal":true,"Count Failed Values":true},{"ID":187,"Name":"internal.metrics.peakExecutionMemory","Update":4325376,"Value":21626880,"Internal":true,"Count Failed Values":true},{"ID":189,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":8,"Value":37,"Internal":true,"Count Failed Values":true},{"ID":190,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":191,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":78576,"Value":372309,"Internal":true,"Count Failed Values":true},{"ID":192,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":193,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":29484,"Internal":true,"Count Failed Values":true},{"ID":194,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":195,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2888,"Value":14745,"Internal":true,"Count Failed Values":true},{"ID":201,"Name":"internal.metrics.output.bytesWritten","Update":120473,"Value":620609,"Internal":true,"Count Failed Values":true},{"ID":202,"Name":"internal.metrics.output.recordsWritten","Update":2888,"Value":14745,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":78,"Executor Deserialize CPU Time":71457306,"Executor Run Time":448,"Executor CPU Time":330505714,"Peak Execution Memory":4325376,"Result Size":5952,"JVM GC Time":0,"Result Serialization Time":3,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":8,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":78576,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":2888},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":120473,"Records Written":2888},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":33,"Index":1,"Attempt":0,"Launch Time":1633789204689,"Executor ID":"8","Host":"10.1.0.11","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633789205294,"Failed":false,"Killed":false,"Accumulables":[{"ID":37,"Name":"remote blocks read","Update":"8","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":39,"Name":"remote bytes read","Update":"86048","Value":"458357","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":42,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":43,"Name":"records read","Update":"3175","Value":"17920","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":51,"Name":"duration","Update":"333","Value":"1898","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":52,"Name":"sort time","Update":"5","Value":"33","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":53,"Name":"peak memory","Update":"4325376","Value":"25952256","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":54,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":178,"Name":"internal.metrics.executorDeserializeTime","Update":89,"Value":484,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.executorDeserializeCpuTime","Update":82149783,"Value":445718210,"Internal":true,"Count Failed Values":true},{"ID":180,"Name":"internal.metrics.executorRunTime","Update":504,"Value":2680,"Internal":true,"Count Failed Values":true},{"ID":181,"Name":"internal.metrics.executorCpuTime","Update":417751654,"Value":2083020483,"Internal":true,"Count Failed Values":true},{"ID":182,"Name":"internal.metrics.resultSize","Update":5952,"Value":35712,"Internal":true,"Count Failed Values":true},{"ID":184,"Name":"internal.metrics.resultSerializationTime","Update":3,"Value":16,"Internal":true,"Count Failed Values":true},{"ID":187,"Name":"internal.metrics.peakExecutionMemory","Update":4325376,"Value":25952256,"Internal":true,"Count Failed Values":true},{"ID":189,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":8,"Value":45,"Internal":true,"Count Failed Values":true},{"ID":190,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":191,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":86048,"Value":458357,"Internal":true,"Count Failed Values":true},{"ID":192,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":193,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":29484,"Internal":true,"Count Failed Values":true},{"ID":194,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":195,"Name":"internal.metrics.shuffle.read.recordsRead","Update":3175,"Value":17920,"Internal":true,"Count Failed Values":true},{"ID":201,"Name":"internal.metrics.output.bytesWritten","Update":132377,"Value":752986,"Internal":true,"Count Failed Values":true},{"ID":202,"Name":"internal.metrics.output.recordsWritten","Update":3175,"Value":17920,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":89,"Executor Deserialize CPU Time":82149783,"Executor Run Time":504,"Executor CPU Time":417751654,"Peak Execution Memory":4325376,"Result Size":5952,"JVM GC Time":0,"Result Serialization Time":3,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":8,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":86048,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":3175},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":132377,"Records Written":3175},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":37,"Index":5,"Attempt":0,"Launch Time":1633789204690,"Executor ID":"3","Host":"10.1.0.15","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633789205962,"Failed":false,"Killed":false,"Accumulables":[{"ID":37,"Name":"remote blocks read","Update":"8","Value":"53","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":39,"Name":"remote bytes read","Update":"75179","Value":"533536","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":42,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":43,"Name":"records read","Update":"2747","Value":"20667","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":51,"Name":"duration","Update":"978","Value":"2876","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":52,"Name":"sort time","Update":"5","Value":"38","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":53,"Name":"peak memory","Update":"4325376","Value":"30277632","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":54,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":178,"Name":"internal.metrics.executorDeserializeTime","Update":130,"Value":614,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.executorDeserializeCpuTime","Update":121934949,"Value":567653159,"Internal":true,"Count Failed Values":true},{"ID":180,"Name":"internal.metrics.executorRunTime","Update":1128,"Value":3808,"Internal":true,"Count Failed Values":true},{"ID":181,"Name":"internal.metrics.executorCpuTime","Update":888495755,"Value":2971516238,"Internal":true,"Count Failed Values":true},{"ID":182,"Name":"internal.metrics.resultSize","Update":5995,"Value":41707,"Internal":true,"Count Failed Values":true},{"ID":183,"Name":"internal.metrics.jvmGCTime","Update":118,"Value":118,"Internal":true,"Count Failed Values":true},{"ID":184,"Name":"internal.metrics.resultSerializationTime","Update":3,"Value":19,"Internal":true,"Count Failed Values":true},{"ID":187,"Name":"internal.metrics.peakExecutionMemory","Update":4325376,"Value":30277632,"Internal":true,"Count Failed Values":true},{"ID":189,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":8,"Value":53,"Internal":true,"Count Failed Values":true},{"ID":190,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":191,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":75179,"Value":533536,"Internal":true,"Count Failed Values":true},{"ID":192,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":193,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":29484,"Internal":true,"Count Failed Values":true},{"ID":194,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":195,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2747,"Value":20667,"Internal":true,"Count Failed Values":true},{"ID":201,"Name":"internal.metrics.output.bytesWritten","Update":115563,"Value":868549,"Internal":true,"Count Failed Values":true},{"ID":202,"Name":"internal.metrics.output.recordsWritten","Update":2747,"Value":20667,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":130,"Executor Deserialize CPU Time":121934949,"Executor Run Time":1128,"Executor CPU Time":888495755,"Peak Execution Memory":4325376,"Result Size":5995,"JVM GC Time":118,"Result Serialization Time":3,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":8,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":75179,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":2747},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":115563,"Records Written":2747},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":34,"Index":2,"Attempt":0,"Launch Time":1633789204689,"Executor ID":"0","Host":"10.1.0.12","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633789206005,"Failed":false,"Killed":false,"Accumulables":[{"ID":37,"Name":"remote blocks read","Update":"7","Value":"60","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":38,"Name":"local blocks read","Update":"1","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":39,"Name":"remote bytes read","Update":"69341","Value":"602877","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":41,"Name":"local bytes read","Update":"10129","Value":"39613","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":42,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":43,"Name":"records read","Update":"2931","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":51,"Name":"duration","Update":"1025","Value":"3901","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":52,"Name":"sort time","Update":"7","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":53,"Name":"peak memory","Update":"4325376","Value":"34603008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":54,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":178,"Name":"internal.metrics.executorDeserializeTime","Update":119,"Value":733,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.executorDeserializeCpuTime","Update":107614273,"Value":675267432,"Internal":true,"Count Failed Values":true},{"ID":180,"Name":"internal.metrics.executorRunTime","Update":1181,"Value":4989,"Internal":true,"Count Failed Values":true},{"ID":181,"Name":"internal.metrics.executorCpuTime","Update":932909388,"Value":3904425626,"Internal":true,"Count Failed Values":true},{"ID":182,"Name":"internal.metrics.resultSize","Update":5995,"Value":47702,"Internal":true,"Count Failed Values":true},{"ID":183,"Name":"internal.metrics.jvmGCTime","Update":117,"Value":235,"Internal":true,"Count Failed Values":true},{"ID":184,"Name":"internal.metrics.resultSerializationTime","Update":3,"Value":22,"Internal":true,"Count Failed Values":true},{"ID":187,"Name":"internal.metrics.peakExecutionMemory","Update":4325376,"Value":34603008,"Internal":true,"Count Failed Values":true},{"ID":189,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":60,"Internal":true,"Count Failed Values":true},{"ID":190,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":191,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":69341,"Value":602877,"Internal":true,"Count Failed Values":true},{"ID":192,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":193,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":10129,"Value":39613,"Internal":true,"Count Failed Values":true},{"ID":194,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":195,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2931,"Value":23598,"Internal":true,"Count Failed Values":true},{"ID":201,"Name":"internal.metrics.output.bytesWritten","Update":122168,"Value":990717,"Internal":true,"Count Failed Values":true},{"ID":202,"Name":"internal.metrics.output.recordsWritten","Update":2931,"Value":23598,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":119,"Executor Deserialize CPU Time":107614273,"Executor Run Time":1181,"Executor CPU Time":932909388,"Peak Execution Memory":4325376,"Result Size":5995,"JVM GC Time":117,"Result Serialization Time":3,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":69341,"Remote Bytes Read To Disk":0,"Local Bytes Read":10129,"Total Records Read":2931},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":122168,"Records Written":2931},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":5,"Stage Attempt ID":0,"Stage Name":"sql at ScalaSparkSQLBench.scala:43","Number of Tasks":8,"RDD Info":[{"RDD ID":19,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[18],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":18,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[17],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[4],"Details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1633789204645,"Completion Time":1633789206005,"Accumulables":[{"ID":37,"Name":"remote blocks read","Value":"60","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":38,"Name":"local blocks read","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":39,"Name":"remote bytes read","Value":"602877","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":41,"Name":"local bytes read","Value":"39613","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":42,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":43,"Name":"records read","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":51,"Name":"duration","Value":"3901","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":52,"Name":"sort time","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":53,"Name":"peak memory","Value":"34603008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":54,"Name":"spill size","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":178,"Name":"internal.metrics.executorDeserializeTime","Value":733,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.executorDeserializeCpuTime","Value":675267432,"Internal":true,"Count Failed Values":true},{"ID":180,"Name":"internal.metrics.executorRunTime","Value":4989,"Internal":true,"Count Failed Values":true},{"ID":181,"Name":"internal.metrics.executorCpuTime","Value":3904425626,"Internal":true,"Count Failed Values":true},{"ID":182,"Name":"internal.metrics.resultSize","Value":47702,"Internal":true,"Count Failed Values":true},{"ID":183,"Name":"internal.metrics.jvmGCTime","Value":235,"Internal":true,"Count Failed Values":true},{"ID":184,"Name":"internal.metrics.resultSerializationTime","Value":22,"Internal":true,"Count Failed Values":true},{"ID":187,"Name":"internal.metrics.peakExecutionMemory","Value":34603008,"Internal":true,"Count Failed Values":true},{"ID":189,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":60,"Internal":true,"Count Failed Values":true},{"ID":190,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":4,"Internal":true,"Count Failed Values":true},{"ID":191,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":602877,"Internal":true,"Count Failed Values":true},{"ID":192,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":193,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":39613,"Internal":true,"Count Failed Values":true},{"ID":194,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":195,"Name":"internal.metrics.shuffle.read.recordsRead","Value":23598,"Internal":true,"Count Failed Values":true},{"ID":201,"Name":"internal.metrics.output.bytesWritten","Value":990717,"Internal":true,"Count Failed Values":true},{"ID":202,"Name":"internal.metrics.output.recordsWritten","Value":23598,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":2,"Completion Time":1633789206006,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":11,"accumUpdates":[[47,8],[48,990717],[49,23598],[50,0]]} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":11,"time":1633789206627} +{"Event":"SparkListenerApplicationEnd","Timestamp":1633789206627} +{"Event":"SparkListenerLogStart","Spark Version":"3.1.2"} +{"Event":"SparkListenerResourceProfileAdded","Resource Profile Id":0,"Executor Resource Requests":{"cores":{"Resource Name":"cores","Amount":1,"Discovery Script":"","Vendor":""},"memory":{"Resource Name":"memory","Amount":4096,"Discovery Script":"","Vendor":""},"offHeap":{"Resource Name":"offHeap","Amount":0,"Discovery Script":"","Vendor":""}},"Task Resource Requests":{"cpus":{"Resource Name":"cpus","Amount":1.0}}} +{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"driver","Host":"hostb7.dsrg.utoronto.ca","Port":34077},"Maximum Memory":2101975449,"Timestamp":1633886612494,"Maximum Onheap Memory":2101975449,"Maximum Offheap Memory":0} +{"Event":"SparkListenerEnvironmentUpdate","JVM Information":{"Java Home":"/usr/lib/jvm/java-8-openjdk-amd64/jre","Java Version":"1.8.0_162 (Oracle Corporation)","Scala Version":"version 2.12.10"},"Spark Properties":{"spark.driver.host":"hostb7.dsrg.utoronto.ca","spark.eventLog.enabled":"true","spark.driver.port":"42853","spark.jars":"file:/home/usera112/projects/log-compressor/deployments/HiBench/sparkbench/assembly/target/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.app.name":"ScalaJoin","spark.scheduler.mode":"FIFO","spark.driver.memory":"4g","spark.submit.pyFiles":"","spark.default.parallelism":"8","spark.app.startTime":"1633886611309","spark.executor.id":"driver","spark.app.initial.jar.urls":"spark://hostb7.dsrg.utoronto.ca:42853/jars/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.submit.deployMode":"client","spark.master":"spark://hostb7:7077","spark.executor.memory":"4g","spark.eventLog.dir":"hdfs://hostb7:8020/directory","spark.app.id":"app-20211010132332-4636","spark.sql.shuffle.partitions":"8"},"Hadoop Properties":{"hadoop.service.shutdown.timeout":"30s","yarn.resourcemanager.amlauncher.thread-count":"50","yarn.sharedcache.enabled":"false","fs.s3a.connection.maximum":"15","yarn.nodemanager.numa-awareness.numactl.cmd":"/usr/bin/numactl","fs.s3a.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","yarn.app.mapreduce.am.scheduler.heartbeat.interval-ms":"1000","yarn.timeline-service.timeline-client.number-of-async-entities-to-merge":"10","hadoop.security.kms.client.timeout":"60","hadoop.http.authentication.kerberos.principal":"HTTP/_HOST@LOCALHOST","mapreduce.jobhistory.loadedjob.tasks.max":"-1","mapreduce.framework.name":"yarn","yarn.sharedcache.uploader.server.thread-count":"50","yarn.nodemanager.linux-container-executor.nonsecure-mode.user-pattern":"^[_.A-Za-z0-9][-@_.A-Za-z0-9]{0,255}?[$]?$","tfile.fs.output.buffer.size":"262144","yarn.app.mapreduce.am.job.task.listener.thread-count":"30","hadoop.security.groups.cache.background.reload.threads":"3","yarn.resourcemanager.webapp.cross-origin.enabled":"false","fs.AbstractFileSystem.ftp.impl":"org.apache.hadoop.fs.ftp.FtpFs","hadoop.registry.secure":"false","hadoop.shell.safely.delete.limit.num.files":"100","dfs.bytes-per-checksum":"512","mapreduce.job.acl-view-job":" ","fs.s3a.s3guard.ddb.background.sleep":"25ms","fs.s3a.retry.limit":"${fs.s3a.attempts.maximum}","mapreduce.jobhistory.loadedjobs.cache.size":"5","fs.s3a.s3guard.ddb.table.create":"false","yarn.nodemanager.amrmproxy.enabled":"false","yarn.timeline-service.entity-group-fs-store.with-user-dir":"false","mapreduce.input.fileinputformat.split.minsize":"0","yarn.resourcemanager.container.liveness-monitor.interval-ms":"600000","yarn.resourcemanager.client.thread-count":"50","io.seqfile.compress.blocksize":"1000000","yarn.sharedcache.checksum.algo.impl":"org.apache.hadoop.yarn.sharedcache.ChecksumSHA256Impl","yarn.nodemanager.amrmproxy.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.nodemanager.amrmproxy.DefaultRequestInterceptor","dfs.replication":"3","yarn.timeline-service.entity-group-fs-store.leveldb-cache-read-cache-size":"10485760","mapreduce.reduce.shuffle.fetch.retry.interval-ms":"1000","mapreduce.task.profile.maps":"0-2","yarn.scheduler.include-port-in-node-name":"false","yarn.nodemanager.admin-env":"MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX","yarn.resourcemanager.node-removal-untracked.timeout-ms":"60000","mapreduce.am.max-attempts":"2","hadoop.security.kms.client.failover.sleep.base.millis":"100","mapreduce.jobhistory.webapp.https.address":"0.0.0.0:19890","yarn.node-labels.fs-store.impl.class":"org.apache.hadoop.yarn.nodelabels.FileSystemNodeLabelsStore","yarn.nodemanager.collector-service.address":"${yarn.nodemanager.hostname}:8048","fs.trash.checkpoint.interval":"0","mapreduce.job.map.output.collector.class":"org.apache.hadoop.mapred.MapTask$MapOutputBuffer","yarn.resourcemanager.node-ip-cache.expiry-interval-secs":"-1","hadoop.http.authentication.signature.secret.file":"*********(redacted)","hadoop.jetty.logs.serve.aliases":"true","yarn.resourcemanager.placement-constraints.handler":"disabled","yarn.timeline-service.handler-thread-count":"10","yarn.resourcemanager.max-completed-applications":"1000","yarn.resourcemanager.system-metrics-publisher.enabled":"false","yarn.resourcemanager.placement-constraints.algorithm.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm.DefaultPlacementAlgorithm","yarn.sharedcache.webapp.address":"0.0.0.0:8788","yarn.resourcemanager.delegation.token.renew-interval":"*********(redacted)","yarn.sharedcache.nm.uploader.replication.factor":"10","hadoop.security.groups.negative-cache.secs":"30","yarn.app.mapreduce.task.container.log.backups":"0","mapreduce.reduce.skip.proc-count.auto-incr":"true","hadoop.security.group.mapping.ldap.posix.attr.gid.name":"gidNumber","rpc.engine.org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolPB":"org.apache.hadoop.ipc.ProtobufRpcEngine","ipc.client.fallback-to-simple-auth-allowed":"false","yarn.nodemanager.resource.memory.enforced":"true","yarn.client.failover-proxy-provider":"org.apache.hadoop.yarn.client.ConfiguredRMFailoverProxyProvider","yarn.timeline-service.http-authentication.simple.anonymous.allowed":"true","ha.health-monitor.check-interval.ms":"1000","yarn.acl.reservation-enable":"false","yarn.resourcemanager.store.class":"org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore","yarn.app.mapreduce.am.hard-kill-timeout-ms":"10000","fs.s3a.etag.checksum.enabled":"false","yarn.nodemanager.container-metrics.enable":"true","yarn.timeline-service.client.fd-clean-interval-secs":"60","yarn.resourcemanager.nodemanagers.heartbeat-interval-ms":"1000","hadoop.common.configuration.version":"3.0.0","fs.s3a.s3guard.ddb.table.capacity.read":"500","yarn.nodemanager.remote-app-log-dir-suffix":"logs","yarn.nodemanager.windows-container.cpu-limit.enabled":"false","yarn.nodemanager.runtime.linux.docker.privileged-containers.allowed":"false","file.blocksize":"67108864","hadoop.registry.zk.retry.ceiling.ms":"60000","yarn.scheduler.configuration.leveldb-store.path":"${hadoop.tmp.dir}/yarn/system/confstore","yarn.sharedcache.store.in-memory.initial-delay-mins":"10","mapreduce.jobhistory.principal":"jhs/_HOST@REALM.TLD","mapreduce.map.skip.proc-count.auto-incr":"true","fs.s3a.committer.name":"file","mapreduce.task.profile.reduces":"0-2","hadoop.zk.num-retries":"1000","yarn.webapp.xfs-filter.enabled":"true","seq.io.sort.mb":"100","yarn.scheduler.configuration.max.version":"100","yarn.timeline-service.webapp.https.address":"${yarn.timeline-service.hostname}:8190","yarn.resourcemanager.scheduler.address":"${yarn.resourcemanager.hostname}:8030","yarn.node-labels.enabled":"false","yarn.resourcemanager.webapp.ui-actions.enabled":"true","mapreduce.task.timeout":"600000","yarn.sharedcache.client-server.thread-count":"50","hadoop.security.groups.shell.command.timeout":"0s","hadoop.security.crypto.cipher.suite":"AES/CTR/NoPadding","yarn.nodemanager.elastic-memory-control.oom-handler":"org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.DefaultOOMHandler","yarn.resourcemanager.connect.max-wait.ms":"900000","fs.defaultFS":"hdfs://hostb7:8020","yarn.minicluster.use-rpc":"false","fs.har.impl.disable.cache":"true","yarn.webapp.ui2.enable":"false","io.compression.codec.bzip2.library":"system-native","yarn.nodemanager.distributed-scheduling.enabled":"false","mapreduce.shuffle.connection-keep-alive.timeout":"5","yarn.resourcemanager.webapp.https.address":"${yarn.resourcemanager.hostname}:8090","mapreduce.jobhistory.address":"0.0.0.0:10020","yarn.resourcemanager.nm-tokens.master-key-rolling-interval-secs":"*********(redacted)","yarn.is.minicluster":"false","yarn.nodemanager.address":"${yarn.nodemanager.hostname}:0","fs.abfss.impl":"org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","fs.AbstractFileSystem.s3a.impl":"org.apache.hadoop.fs.s3a.S3A","mapreduce.task.combine.progress.records":"10000","yarn.resourcemanager.epoch.range":"0","yarn.resourcemanager.am.max-attempts":"2","yarn.nodemanager.linux-container-executor.cgroups.hierarchy":"/hadoop-yarn","fs.AbstractFileSystem.wasbs.impl":"org.apache.hadoop.fs.azure.Wasbs","yarn.timeline-service.entity-group-fs-store.cache-store-class":"org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore","fs.ftp.transfer.mode":"BLOCK_TRANSFER_MODE","ipc.server.log.slow.rpc":"false","yarn.resourcemanager.node-labels.provider.fetch-interval-ms":"1800000","yarn.router.webapp.https.address":"0.0.0.0:8091","yarn.nodemanager.webapp.cross-origin.enabled":"false","fs.wasb.impl":"org.apache.hadoop.fs.azure.NativeAzureFileSystem","yarn.resourcemanager.auto-update.containers":"false","yarn.app.mapreduce.am.job.committer.cancel-timeout":"60000","yarn.scheduler.configuration.zk-store.parent-path":"/confstore","yarn.nodemanager.default-container-executor.log-dirs.permissions":"710","yarn.app.attempt.diagnostics.limit.kc":"64","ftp.bytes-per-checksum":"512","yarn.nodemanager.resource.memory-mb":"-1","fs.AbstractFileSystem.abfs.impl":"org.apache.hadoop.fs.azurebfs.Abfs","yarn.timeline-service.writer.flush-interval-seconds":"60","fs.s3a.fast.upload.active.blocks":"4","hadoop.security.credential.clear-text-fallback":"true","yarn.nodemanager.collector-service.thread-count":"5","fs.azure.secure.mode":"false","mapreduce.jobhistory.joblist.cache.size":"20000","fs.ftp.host":"0.0.0.0","yarn.resourcemanager.fs.state-store.num-retries":"0","yarn.resourcemanager.nodemanager-connect-retries":"10","yarn.nodemanager.log-aggregation.num-log-files-per-app":"30","hadoop.security.kms.client.encrypted.key.cache.low-watermark":"0.3f","fs.s3a.committer.magic.enabled":"false","yarn.timeline-service.client.max-retries":"30","dfs.ha.fencing.ssh.connect-timeout":"30000","yarn.log-aggregation-enable":"false","yarn.system-metrics-publisher.enabled":"false","mapreduce.reduce.markreset.buffer.percent":"0.0","fs.AbstractFileSystem.viewfs.impl":"org.apache.hadoop.fs.viewfs.ViewFs","mapreduce.task.io.sort.factor":"10","yarn.nodemanager.amrmproxy.client.thread-count":"25","ha.failover-controller.new-active.rpc-timeout.ms":"60000","yarn.nodemanager.container-localizer.java.opts":"-Xmx256m","mapreduce.jobhistory.datestring.cache.size":"200000","mapreduce.job.acl-modify-job":" ","yarn.nodemanager.windows-container.memory-limit.enabled":"false","yarn.timeline-service.webapp.address":"${yarn.timeline-service.hostname}:8188","yarn.app.mapreduce.am.job.committer.commit-window":"10000","yarn.nodemanager.container-manager.thread-count":"20","yarn.minicluster.fixed.ports":"false","hadoop.tags.system":"YARN,HDFS,NAMENODE,DATANODE,REQUIRED,SECURITY,KERBEROS,PERFORMANCE,CLIENT\n ,SERVER,DEBUG,DEPRECATED,COMMON,OPTIONAL","yarn.cluster.max-application-priority":"0","yarn.timeline-service.ttl-enable":"true","mapreduce.jobhistory.recovery.store.fs.uri":"${hadoop.tmp.dir}/mapred/history/recoverystore","hadoop.caller.context.signature.max.size":"40","yarn.client.load.resource-types.from-server":"false","ha.zookeeper.session-timeout.ms":"10000","tfile.io.chunk.size":"1048576","fs.s3a.s3guard.ddb.table.capacity.write":"100","mapreduce.job.speculative.slowtaskthreshold":"1.0","io.serializations":"org.apache.hadoop.io.serializer.WritableSerialization, org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization, org.apache.hadoop.io.serializer.avro.AvroReflectSerialization","hadoop.security.kms.client.failover.sleep.max.millis":"2000","hadoop.security.group.mapping.ldap.directory.search.timeout":"10000","yarn.scheduler.configuration.store.max-logs":"1000","yarn.nodemanager.node-attributes.provider.fetch-interval-ms":"600000","fs.swift.impl":"org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem","yarn.nodemanager.local-cache.max-files-per-directory":"8192","hadoop.http.cross-origin.enabled":"false","hadoop.zk.acl":"world:anyone:rwcda","mapreduce.map.sort.spill.percent":"0.80","yarn.timeline-service.entity-group-fs-store.scan-interval-seconds":"60","yarn.node-attribute.fs-store.impl.class":"org.apache.hadoop.yarn.server.resourcemanager.nodelabels.FileSystemNodeAttributeStore","fs.s3a.retry.interval":"500ms","yarn.timeline-service.client.best-effort":"false","yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled":"*********(redacted)","hadoop.security.group.mapping.ldap.posix.attr.uid.name":"uidNumber","fs.AbstractFileSystem.swebhdfs.impl":"org.apache.hadoop.fs.SWebHdfs","yarn.nodemanager.elastic-memory-control.timeout-sec":"5","mapreduce.ifile.readahead":"true","yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms":"300000","yarn.timeline-service.reader.webapp.address":"${yarn.timeline-service.webapp.address}","yarn.resourcemanager.placement-constraints.algorithm.pool-size":"1","yarn.timeline-service.hbase.coprocessor.jar.hdfs.location":"/hbase/coprocessor/hadoop-yarn-server-timelineservice.jar","hadoop.security.kms.client.encrypted.key.cache.num.refill.threads":"2","yarn.resourcemanager.scheduler.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler","yarn.app.mapreduce.am.command-opts":"-Xmx1024m","mapreduce.cluster.local.dir":"${hadoop.tmp.dir}/mapred/local","io.mapfile.bloom.error.rate":"0.005","fs.client.resolve.topology.enabled":"false","yarn.nodemanager.runtime.linux.allowed-runtimes":"default","yarn.sharedcache.store.class":"org.apache.hadoop.yarn.server.sharedcachemanager.store.InMemorySCMStore","ha.failover-controller.graceful-fence.rpc-timeout.ms":"5000","ftp.replication":"3","hadoop.security.uid.cache.secs":"14400","mapreduce.job.maxtaskfailures.per.tracker":"3","fs.s3a.metadatastore.impl":"org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore","io.skip.checksum.errors":"false","yarn.app.mapreduce.client-am.ipc.max-retries-on-timeouts":"3","yarn.timeline-service.webapp.xfs-filter.xframe-options":"SAMEORIGIN","fs.s3a.connection.timeout":"200000","mapreduce.job.max.split.locations":"15","yarn.resourcemanager.nm-container-queuing.max-queue-length":"15","hadoop.registry.zk.session.timeout.ms":"60000","yarn.federation.cache-ttl.secs":"300","mapreduce.jvm.system-properties-to-log":"os.name,os.version,java.home,java.runtime.version,java.vendor,java.version,java.vm.name,java.class.path,java.io.tmpdir,user.dir,user.name","yarn.resourcemanager.opportunistic-container-allocation.nodes-used":"10","yarn.timeline-service.entity-group-fs-store.active-dir":"/tmp/entity-file-history/active","mapreduce.shuffle.transfer.buffer.size":"131072","yarn.timeline-service.client.retry-interval-ms":"1000","yarn.http.policy":"HTTP_ONLY","fs.s3a.socket.send.buffer":"8192","fs.AbstractFileSystem.abfss.impl":"org.apache.hadoop.fs.azurebfs.Abfss","yarn.sharedcache.uploader.server.address":"0.0.0.0:8046","yarn.resourcemanager.delegation-token.max-conf-size-bytes":"*********(redacted)","hadoop.http.authentication.token.validity":"*********(redacted)","mapreduce.shuffle.max.connections":"0","yarn.minicluster.yarn.nodemanager.resource.memory-mb":"4096","mapreduce.job.emit-timeline-data":"false","yarn.nodemanager.resource.system-reserved-memory-mb":"-1","hadoop.kerberos.min.seconds.before.relogin":"60","mapreduce.jobhistory.move.thread-count":"3","yarn.resourcemanager.admin.client.thread-count":"1","yarn.dispatcher.drain-events.timeout":"300000","fs.s3a.buffer.dir":"${hadoop.tmp.dir}/s3a","hadoop.ssl.enabled.protocols":"TLSv1,SSLv2Hello,TLSv1.1,TLSv1.2","mapreduce.jobhistory.admin.address":"0.0.0.0:10033","yarn.log-aggregation-status.time-out.ms":"600000","fs.s3a.assumed.role.sts.endpoint.region":"us-west-1","mapreduce.shuffle.port":"13562","yarn.resourcemanager.max-log-aggregation-diagnostics-in-memory":"10","yarn.nodemanager.health-checker.interval-ms":"600000","yarn.router.clientrm.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.clientrm.DefaultClientRequestInterceptor","yarn.resourcemanager.zk-appid-node.split-index":"0","ftp.blocksize":"67108864","yarn.nodemanager.runtime.linux.sandbox-mode.local-dirs.permissions":"read","yarn.router.rmadmin.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.rmadmin.DefaultRMAdminRequestInterceptor","yarn.nodemanager.log-container-debug-info.enabled":"true","yarn.client.max-cached-nodemanagers-proxies":"0","yarn.nodemanager.linux-container-executor.cgroups.delete-delay-ms":"20","yarn.nodemanager.delete.debug-delay-sec":"0","yarn.nodemanager.pmem-check-enabled":"true","yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage":"90.0","mapreduce.app-submission.cross-platform":"false","yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms":"10000","yarn.nodemanager.container-retry-minimum-interval-ms":"1000","hadoop.security.groups.cache.secs":"300","yarn.federation.enabled":"false","fs.azure.local.sas.key.mode":"false","ipc.maximum.data.length":"67108864","mapreduce.shuffle.max.threads":"0","yarn.router.pipeline.cache-max-size":"25","yarn.resourcemanager.nm-container-queuing.load-comparator":"QUEUE_LENGTH","hadoop.security.authorization":"false","mapreduce.job.complete.cancel.delegation.tokens":"*********(redacted)","fs.s3a.paging.maximum":"5000","nfs.exports.allowed.hosts":"* rw","yarn.nodemanager.amrmproxy.ha.enable":"false","mapreduce.jobhistory.http.policy":"HTTP_ONLY","yarn.sharedcache.store.in-memory.check-period-mins":"720","hadoop.security.group.mapping.ldap.ssl":"false","yarn.client.application-client-protocol.poll-interval-ms":"200","yarn.scheduler.configuration.leveldb-store.compaction-interval-secs":"86400","yarn.timeline-service.writer.class":"org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl","ha.zookeeper.parent-znode":"/hadoop-ha","yarn.nodemanager.log-aggregation.policy.class":"org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.AllContainerLogAggregationPolicy","mapreduce.reduce.shuffle.merge.percent":"0.66","hadoop.security.group.mapping.ldap.search.filter.group":"(objectClass=group)","yarn.resourcemanager.placement-constraints.scheduler.pool-size":"1","yarn.nodemanager.resourcemanager.minimum.version":"NONE","mapreduce.job.speculative.speculative-cap-running-tasks":"0.1","yarn.admin.acl":"*","yarn.nodemanager.recovery.supervised":"false","yarn.sharedcache.admin.thread-count":"1","yarn.resourcemanager.ha.automatic-failover.enabled":"true","mapreduce.reduce.skip.maxgroups":"0","mapreduce.reduce.shuffle.connect.timeout":"180000","yarn.resourcemanager.address":"${yarn.resourcemanager.hostname}:8032","ipc.client.ping":"true","mapreduce.task.local-fs.write-limit.bytes":"-1","fs.adl.oauth2.access.token.provider.type":"*********(redacted)","mapreduce.shuffle.ssl.file.buffer.size":"65536","yarn.resourcemanager.ha.automatic-failover.embedded":"true","yarn.nodemanager.resource-plugins.gpu.docker-plugin":"nvidia-docker-v1","hadoop.ssl.enabled":"false","fs.s3a.multipart.purge":"false","yarn.scheduler.configuration.store.class":"file","yarn.resourcemanager.nm-container-queuing.queue-limit-stdev":"1.0f","mapreduce.job.end-notification.max.attempts":"5","mapreduce.output.fileoutputformat.compress.codec":"org.apache.hadoop.io.compress.DefaultCodec","yarn.nodemanager.container-monitor.procfs-tree.smaps-based-rss.enabled":"false","ipc.client.bind.wildcard.addr":"false","yarn.resourcemanager.webapp.rest-csrf.enabled":"false","ha.health-monitor.connect-retry-interval.ms":"1000","yarn.nodemanager.keytab":"/etc/krb5.keytab","mapreduce.jobhistory.keytab":"/etc/security/keytab/jhs.service.keytab","fs.s3a.threads.max":"10","mapreduce.reduce.shuffle.input.buffer.percent":"0.70","yarn.nodemanager.runtime.linux.docker.allowed-container-networks":"host,none,bridge","yarn.nodemanager.node-labels.resync-interval-ms":"120000","hadoop.tmp.dir":"/tmp/hadoop-${user.name}","mapreduce.job.maps":"2","mapreduce.jobhistory.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.job.end-notification.max.retry.interval":"5000","yarn.log-aggregation.retain-check-interval-seconds":"-1","yarn.resourcemanager.resource-tracker.client.thread-count":"50","yarn.rm.system-metrics-publisher.emit-container-events":"false","yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size":"10000","yarn.resourcemanager.ha.automatic-failover.zk-base-path":"/yarn-leader-election","io.seqfile.local.dir":"${hadoop.tmp.dir}/io/local","fs.s3a.s3guard.ddb.throttle.retry.interval":"100ms","fs.AbstractFileSystem.wasb.impl":"org.apache.hadoop.fs.azure.Wasb","mapreduce.client.submit.file.replication":"10","mapreduce.jobhistory.minicluster.fixed.ports":"false","fs.s3a.multipart.threshold":"2147483647","yarn.resourcemanager.webapp.xfs-filter.xframe-options":"SAMEORIGIN","mapreduce.jobhistory.done-dir":"${yarn.app.mapreduce.am.staging-dir}/history/done","ipc.client.idlethreshold":"4000","yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage":"false","mapreduce.reduce.input.buffer.percent":"0.0","yarn.nodemanager.runtime.linux.docker.userremapping-gid-threshold":"1","yarn.nodemanager.webapp.rest-csrf.enabled":"false","fs.ftp.host.port":"21","ipc.ping.interval":"60000","yarn.resourcemanager.history-writer.multi-threaded-dispatcher.pool-size":"10","yarn.resourcemanager.admin.address":"${yarn.resourcemanager.hostname}:8033","file.client-write-packet-size":"65536","ipc.client.kill.max":"10","mapreduce.reduce.speculative":"true","hadoop.security.key.default.bitlength":"128","mapreduce.job.reducer.unconditional-preempt.delay.sec":"300","yarn.nodemanager.disk-health-checker.interval-ms":"120000","yarn.nodemanager.log.deletion-threads-count":"4","yarn.webapp.filter-entity-list-by-user":"false","ipc.client.connection.maxidletime":"10000","mapreduce.task.io.sort.mb":"100","yarn.nodemanager.localizer.client.thread-count":"5","io.erasurecode.codec.rs.rawcoders":"rs_native,rs_java","io.erasurecode.codec.rs-legacy.rawcoders":"rs-legacy_java","yarn.sharedcache.admin.address":"0.0.0.0:8047","yarn.resourcemanager.placement-constraints.algorithm.iterator":"SERIAL","yarn.nodemanager.localizer.cache.cleanup.interval-ms":"600000","hadoop.security.crypto.codec.classes.aes.ctr.nopadding":"org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec, org.apache.hadoop.crypto.JceAesCtrCryptoCodec","mapreduce.job.cache.limit.max-resources-mb":"0","fs.s3a.connection.ssl.enabled":"true","yarn.nodemanager.process-kill-wait.ms":"5000","mapreduce.job.hdfs-servers":"${fs.defaultFS}","hadoop.workaround.non.threadsafe.getpwuid":"true","fs.df.interval":"60000","fs.s3a.multiobjectdelete.enable":"true","yarn.sharedcache.cleaner.resource-sleep-ms":"0","yarn.nodemanager.disk-health-checker.min-healthy-disks":"0.25","hadoop.shell.missing.defaultFs.warning":"false","io.file.buffer.size":"65536","hadoop.security.group.mapping.ldap.search.attr.member":"member","hadoop.security.random.device.file.path":"/dev/urandom","hadoop.security.sensitive-config-keys":"*********(redacted)","fs.s3a.s3guard.ddb.max.retries":"9","hadoop.rpc.socket.factory.class.default":"org.apache.hadoop.net.StandardSocketFactory","yarn.intermediate-data-encryption.enable":"false","yarn.resourcemanager.connect.retry-interval.ms":"30000","yarn.nodemanager.container.stderr.pattern":"{*stderr*,*STDERR*}","yarn.scheduler.minimum-allocation-mb":"1024","yarn.app.mapreduce.am.staging-dir":"/tmp/hadoop-yarn/staging","mapreduce.reduce.shuffle.read.timeout":"180000","hadoop.http.cross-origin.max-age":"1800","io.erasurecode.codec.xor.rawcoders":"xor_native,xor_java","fs.s3a.connection.establish.timeout":"5000","mapreduce.job.running.map.limit":"0","yarn.minicluster.control-resource-monitoring":"false","hadoop.ssl.require.client.cert":"false","hadoop.kerberos.kinit.command":"kinit","yarn.federation.state-store.class":"org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore","mapreduce.reduce.log.level":"INFO","hadoop.security.dns.log-slow-lookups.threshold.ms":"1000","mapreduce.job.ubertask.enable":"false","adl.http.timeout":"-1","yarn.resourcemanager.placement-constraints.retry-attempts":"3","hadoop.caller.context.enabled":"false","yarn.nodemanager.vmem-pmem-ratio":"2.1","hadoop.rpc.protection":"authentication","ha.health-monitor.rpc-timeout.ms":"45000","yarn.nodemanager.remote-app-log-dir":"/tmp/logs","hadoop.zk.timeout-ms":"10000","fs.s3a.s3guard.cli.prune.age":"86400000","yarn.nodemanager.resource.pcores-vcores-multiplier":"1.0","yarn.nodemanager.runtime.linux.sandbox-mode":"disabled","yarn.app.mapreduce.am.containerlauncher.threadpool-initial-size":"10","fs.s3a.committer.threads":"8","hadoop.zk.retry-interval-ms":"1000","hadoop.security.crypto.buffer.size":"8192","yarn.nodemanager.node-labels.provider.fetch-interval-ms":"600000","mapreduce.jobhistory.recovery.store.leveldb.path":"${hadoop.tmp.dir}/mapred/history/recoverystore","yarn.client.failover-retries-on-socket-timeouts":"0","yarn.nodemanager.resource.memory.enabled":"false","fs.azure.authorization.caching.enable":"true","hadoop.security.instrumentation.requires.admin":"false","yarn.nodemanager.delete.thread-count":"4","mapreduce.job.finish-when-all-reducers-done":"true","hadoop.registry.jaas.context":"Client","yarn.timeline-service.leveldb-timeline-store.path":"${hadoop.tmp.dir}/yarn/timeline","io.map.index.interval":"128","yarn.resourcemanager.nm-container-queuing.max-queue-wait-time-ms":"100","fs.abfs.impl":"org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","mapreduce.job.counters.max":"120","mapreduce.jobhistory.webapp.rest-csrf.enabled":"false","yarn.timeline-service.store-class":"org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore","mapreduce.jobhistory.move.interval-ms":"180000","yarn.nodemanager.localizer.fetch.thread-count":"4","yarn.resourcemanager.scheduler.client.thread-count":"50","hadoop.ssl.hostname.verifier":"DEFAULT","yarn.timeline-service.leveldb-state-store.path":"${hadoop.tmp.dir}/yarn/timeline","mapreduce.job.classloader":"false","mapreduce.task.profile.map.params":"${mapreduce.task.profile.params}","ipc.client.connect.timeout":"20000","hadoop.security.auth_to_local.mechanism":"hadoop","yarn.timeline-service.app-collector.linger-period.ms":"60000","yarn.nm.liveness-monitor.expiry-interval-ms":"600000","yarn.resourcemanager.reservation-system.planfollower.time-step":"1000","yarn.nodemanager.runtime.linux.docker.enable-userremapping.allowed":"true","yarn.webapp.api-service.enable":"false","yarn.nodemanager.recovery.enabled":"false","mapreduce.job.end-notification.retry.interval":"1000","fs.du.interval":"600000","fs.ftp.impl":"org.apache.hadoop.fs.ftp.FTPFileSystem","yarn.nodemanager.container.stderr.tail.bytes":"4096","hadoop.security.group.mapping.ldap.read.timeout.ms":"60000","hadoop.security.groups.cache.warn.after.ms":"5000","file.bytes-per-checksum":"512","mapreduce.outputcommitter.factory.scheme.s3a":"org.apache.hadoop.fs.s3a.commit.S3ACommitterFactory","hadoop.security.groups.cache.background.reload":"false","yarn.nodemanager.container-monitor.enabled":"true","yarn.nodemanager.elastic-memory-control.enabled":"false","net.topology.script.number.args":"100","mapreduce.task.merge.progress.records":"10000","yarn.nodemanager.localizer.address":"${yarn.nodemanager.hostname}:8040","yarn.timeline-service.keytab":"/etc/krb5.keytab","mapreduce.reduce.shuffle.fetch.retry.timeout-ms":"30000","yarn.resourcemanager.rm.container-allocation.expiry-interval-ms":"600000","mapreduce.fileoutputcommitter.algorithm.version":"1","yarn.resourcemanager.work-preserving-recovery.enabled":"true","mapreduce.map.skip.maxrecords":"0","yarn.sharedcache.root-dir":"/sharedcache","fs.s3a.retry.throttle.limit":"${fs.s3a.attempts.maximum}","hadoop.http.authentication.type":"simple","mapreduce.job.cache.limit.max-resources":"0","mapreduce.task.userlog.limit.kb":"0","yarn.resourcemanager.scheduler.monitor.enable":"false","ipc.client.connect.max.retries":"10","hadoop.registry.zk.retry.times":"5","yarn.nodemanager.resource-monitor.interval-ms":"3000","yarn.nodemanager.resource-plugins.gpu.allowed-gpu-devices":"auto","mapreduce.job.sharedcache.mode":"disabled","yarn.nodemanager.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.shuffle.listen.queue.size":"128","yarn.scheduler.configuration.mutation.acl-policy.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy","mapreduce.map.cpu.vcores":"1","yarn.log-aggregation.file-formats":"TFile","yarn.timeline-service.client.fd-retain-secs":"300","hadoop.user.group.static.mapping.overrides":"dr.who=;","fs.azure.sas.expiry.period":"90d","mapreduce.jobhistory.recovery.store.class":"org.apache.hadoop.mapreduce.v2.hs.HistoryServerFileSystemStateStoreService","yarn.resourcemanager.fail-fast":"${yarn.fail-fast}","yarn.resourcemanager.proxy-user-privileges.enabled":"false","yarn.router.webapp.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.webapp.DefaultRequestInterceptorREST","yarn.nodemanager.resource.memory.cgroups.soft-limit-percentage":"90.0","mapreduce.job.reducer.preempt.delay.sec":"0","hadoop.util.hash.type":"murmur","yarn.nodemanager.disk-validator":"basic","yarn.app.mapreduce.client.job.max-retries":"3","mapreduce.reduce.shuffle.retry-delay.max.ms":"60000","hadoop.security.group.mapping.ldap.connection.timeout.ms":"60000","mapreduce.task.profile.params":"-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,verbose=n,file=%s","yarn.app.mapreduce.shuffle.log.backups":"0","yarn.nodemanager.container-diagnostics-maximum-size":"10000","hadoop.registry.zk.retry.interval.ms":"1000","yarn.nodemanager.linux-container-executor.cgroups.delete-timeout-ms":"1000","fs.AbstractFileSystem.file.impl":"org.apache.hadoop.fs.local.LocalFs","yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds":"-1","mapreduce.jobhistory.cleaner.interval-ms":"86400000","hadoop.registry.zk.quorum":"localhost:2181","mapreduce.output.fileoutputformat.compress":"false","yarn.resourcemanager.am-rm-tokens.master-key-rolling-interval-secs":"*********(redacted)","fs.s3a.assumed.role.session.duration":"30m","hadoop.security.group.mapping.ldap.conversion.rule":"none","hadoop.ssl.server.conf":"ssl-server.xml","fs.s3a.retry.throttle.interval":"1000ms","seq.io.sort.factor":"100","yarn.sharedcache.cleaner.initial-delay-mins":"10","mapreduce.client.completion.pollinterval":"5000","hadoop.ssl.keystores.factory.class":"org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory","yarn.app.mapreduce.am.resource.cpu-vcores":"1","yarn.timeline-service.enabled":"false","yarn.nodemanager.runtime.linux.docker.capabilities":"CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE","yarn.acl.enable":"false","yarn.timeline-service.entity-group-fs-store.done-dir":"/tmp/entity-file-history/done/","mapreduce.task.profile":"false","yarn.resourcemanager.fs.state-store.uri":"${hadoop.tmp.dir}/yarn/system/rmstore","mapreduce.jobhistory.always-scan-user-dir":"false","yarn.nodemanager.opportunistic-containers-use-pause-for-preemption":"false","yarn.nodemanager.linux-container-executor.nonsecure-mode.local-user":"nobody","yarn.timeline-service.reader.class":"org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineReaderImpl","yarn.resourcemanager.configuration.provider-class":"org.apache.hadoop.yarn.LocalConfigurationProvider","yarn.nodemanager.runtime.linux.docker.userremapping-uid-threshold":"1","yarn.resourcemanager.configuration.file-system-based-store":"/yarn/conf","mapreduce.job.cache.limit.max-single-resource-mb":"0","yarn.nodemanager.runtime.linux.docker.stop.grace-period":"10","yarn.resourcemanager.resource-profiles.source-file":"resource-profiles.json","yarn.nodemanager.resource.percentage-physical-cpu-limit":"100","mapreduce.jobhistory.client.thread-count":"10","tfile.fs.input.buffer.size":"262144","mapreduce.client.progressmonitor.pollinterval":"1000","yarn.nodemanager.log-dirs":"${yarn.log.dir}/userlogs","fs.automatic.close":"true","yarn.nodemanager.hostname":"0.0.0.0","yarn.nodemanager.resource.memory.cgroups.swappiness":"0","ftp.stream-buffer-size":"4096","yarn.fail-fast":"false","yarn.timeline-service.app-aggregation-interval-secs":"15","hadoop.security.group.mapping.ldap.search.filter.user":"(&(objectClass=user)(sAMAccountName={0}))","yarn.nodemanager.container-localizer.log.level":"INFO","yarn.timeline-service.address":"${yarn.timeline-service.hostname}:10200","mapreduce.job.ubertask.maxmaps":"9","fs.s3a.threads.keepalivetime":"60","mapreduce.jobhistory.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","mapreduce.task.files.preserve.failedtasks":"false","yarn.app.mapreduce.client.job.retry-interval":"2000","ha.failover-controller.graceful-fence.connection.retries":"1","yarn.resourcemanager.delegation.token.max-lifetime":"*********(redacted)","yarn.timeline-service.client.drain-entities.timeout.ms":"2000","yarn.nodemanager.resource-plugins.fpga.vendor-plugin.class":"org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.IntelFpgaOpenclPlugin","yarn.timeline-service.entity-group-fs-store.summary-store":"org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore","mapreduce.reduce.cpu.vcores":"1","mapreduce.job.encrypted-intermediate-data.buffer.kb":"128","fs.client.resolve.remote.symlinks":"true","yarn.nodemanager.webapp.https.address":"0.0.0.0:8044","hadoop.http.cross-origin.allowed-origins":"*","mapreduce.job.encrypted-intermediate-data":"false","yarn.timeline-service.entity-group-fs-store.retain-seconds":"604800","yarn.resourcemanager.metrics.runtime.buckets":"60,300,1440","yarn.timeline-service.generic-application-history.max-applications":"10000","yarn.nodemanager.local-dirs":"${hadoop.tmp.dir}/nm-local-dir","mapreduce.shuffle.connection-keep-alive.enable":"false","yarn.node-labels.configuration-type":"centralized","fs.s3a.path.style.access":"false","yarn.nodemanager.aux-services.mapreduce_shuffle.class":"org.apache.hadoop.mapred.ShuffleHandler","yarn.sharedcache.store.in-memory.staleness-period-mins":"10080","fs.adl.impl":"org.apache.hadoop.fs.adl.AdlFileSystem","yarn.resourcemanager.nodemanager.minimum.version":"NONE","mapreduce.jobhistory.webapp.xfs-filter.xframe-options":"SAMEORIGIN","yarn.app.mapreduce.am.staging-dir.erasurecoding.enabled":"false","net.topology.impl":"org.apache.hadoop.net.NetworkTopology","io.map.index.skip":"0","yarn.timeline-service.reader.webapp.https.address":"${yarn.timeline-service.webapp.https.address}","fs.ftp.data.connection.mode":"ACTIVE_LOCAL_DATA_CONNECTION_MODE","mapreduce.job.local-fs.single-disk-limit.check.kill-limit-exceed":"true","yarn.scheduler.maximum-allocation-vcores":"4","hadoop.http.cross-origin.allowed-headers":"X-Requested-With,Content-Type,Accept,Origin","yarn.nodemanager.log-aggregation.compression-type":"none","yarn.timeline-service.version":"1.0f","yarn.ipc.rpc.class":"org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC","mapreduce.reduce.maxattempts":"4","hadoop.security.dns.log-slow-lookups.enabled":"false","mapreduce.job.committer.setup.cleanup.needed":"true","mapreduce.job.running.reduce.limit":"0","ipc.maximum.response.length":"134217728","yarn.resourcemanager.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","mapreduce.job.token.tracking.ids.enabled":"*********(redacted)","hadoop.caller.context.max.size":"128","yarn.nodemanager.runtime.linux.docker.host-pid-namespace.allowed":"false","yarn.nodemanager.runtime.linux.docker.delayed-removal.allowed":"false","hadoop.registry.system.acls":"sasl:yarn@, sasl:mapred@, sasl:hdfs@","yarn.nodemanager.recovery.dir":"${hadoop.tmp.dir}/yarn-nm-recovery","fs.s3a.fast.upload.buffer":"disk","mapreduce.jobhistory.intermediate-done-dir":"${yarn.app.mapreduce.am.staging-dir}/history/done_intermediate","yarn.app.mapreduce.shuffle.log.separate":"true","fs.s3a.max.total.tasks":"5","fs.s3a.readahead.range":"64K","hadoop.http.authentication.simple.anonymous.allowed":"true","fs.s3a.attempts.maximum":"20","hadoop.registry.zk.connection.timeout.ms":"15000","yarn.resourcemanager.delegation-token-renewer.thread-count":"*********(redacted)","yarn.nodemanager.health-checker.script.timeout-ms":"1200000","yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size":"10000","yarn.resourcemanager.resource-profiles.enabled":"false","yarn.timeline-service.hbase-schema.prefix":"prod.","fs.azure.authorization":"false","mapreduce.map.log.level":"INFO","yarn.resourcemanager.decommissioning-nodes-watcher.poll-interval-secs":"20","mapreduce.output.fileoutputformat.compress.type":"RECORD","yarn.resourcemanager.leveldb-state-store.path":"${hadoop.tmp.dir}/yarn/system/rmstore","yarn.timeline-service.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.ifile.readahead.bytes":"4194304","yarn.sharedcache.app-checker.class":"org.apache.hadoop.yarn.server.sharedcachemanager.RemoteAppChecker","yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users":"true","yarn.nodemanager.resource.detect-hardware-capabilities":"false","mapreduce.cluster.acls.enabled":"false","mapreduce.job.speculative.retry-after-no-speculate":"1000","hadoop.security.group.mapping.ldap.search.group.hierarchy.levels":"0","yarn.resourcemanager.fs.state-store.retry-interval-ms":"1000","file.stream-buffer-size":"4096","yarn.resourcemanager.application-timeouts.monitor.interval-ms":"3000","mapreduce.map.output.compress.codec":"org.apache.hadoop.io.compress.DefaultCodec","mapreduce.map.speculative":"true","mapreduce.job.speculative.retry-after-speculate":"15000","yarn.nodemanager.linux-container-executor.cgroups.mount":"false","yarn.app.mapreduce.am.container.log.backups":"0","yarn.app.mapreduce.am.log.level":"INFO","mapreduce.job.reduce.slowstart.completedmaps":"0.05","yarn.timeline-service.http-authentication.type":"simple","hadoop.security.group.mapping.ldap.search.attr.group.name":"cn","yarn.nodemanager.resource-plugins.fpga.allowed-fpga-devices":"0,1","yarn.timeline-service.client.internal-timers-ttl-secs":"420","hadoop.http.logs.enabled":"true","fs.s3a.block.size":"32M","yarn.sharedcache.client-server.address":"0.0.0.0:8045","yarn.nodemanager.logaggregation.threadpool-size-max":"100","yarn.resourcemanager.hostname":"hostb7","yarn.resourcemanager.delegation.key.update-interval":"86400000","mapreduce.reduce.shuffle.fetch.retry.enabled":"${yarn.nodemanager.recovery.enabled}","mapreduce.map.memory.mb":"-1","mapreduce.task.skip.start.attempts":"2","fs.AbstractFileSystem.hdfs.impl":"org.apache.hadoop.fs.Hdfs","yarn.nodemanager.disk-health-checker.enable":"true","ipc.client.tcpnodelay":"true","ipc.client.rpc-timeout.ms":"0","yarn.nodemanager.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","ipc.client.low-latency":"false","mapreduce.input.lineinputformat.linespermap":"1","yarn.router.interceptor.user.threadpool-size":"5","ipc.client.connect.max.retries.on.timeouts":"45","yarn.timeline-service.leveldb-timeline-store.read-cache-size":"104857600","fs.AbstractFileSystem.har.impl":"org.apache.hadoop.fs.HarFs","mapreduce.job.split.metainfo.maxsize":"10000000","yarn.am.liveness-monitor.expiry-interval-ms":"600000","yarn.resourcemanager.container-tokens.master-key-rolling-interval-secs":"*********(redacted)","yarn.timeline-service.entity-group-fs-store.app-cache-size":"10","fs.s3a.socket.recv.buffer":"8192","yarn.resourcemanager.resource-tracker.address":"${yarn.resourcemanager.hostname}:8031","yarn.nodemanager.node-labels.provider.fetch-timeout-ms":"1200000","mapreduce.job.heap.memory-mb.ratio":"0.8","yarn.resourcemanager.leveldb-state-store.compaction-interval-secs":"3600","yarn.resourcemanager.webapp.rest-csrf.custom-header":"X-XSRF-Header","yarn.scheduler.configuration.fs.path":"file://${hadoop.tmp.dir}/yarn/system/schedconf","mapreduce.client.output.filter":"FAILED","hadoop.http.filter.initializers":"org.apache.hadoop.http.lib.StaticUserWebFilter","mapreduce.reduce.memory.mb":"-1","yarn.timeline-service.hostname":"0.0.0.0","file.replication":"1","yarn.nodemanager.container-metrics.unregister-delay-ms":"10000","yarn.nodemanager.container-metrics.period-ms":"-1","mapreduce.fileoutputcommitter.task.cleanup.enabled":"false","yarn.nodemanager.log.retain-seconds":"10800","yarn.timeline-service.entity-group-fs-store.cleaner-interval-seconds":"3600","yarn.resourcemanager.keytab":"/etc/krb5.keytab","hadoop.security.group.mapping.providers.combined":"true","mapreduce.reduce.merge.inmem.threshold":"1000","yarn.timeline-service.recovery.enabled":"false","fs.azure.saskey.usecontainersaskeyforallaccess":"true","yarn.sharedcache.nm.uploader.thread-count":"20","yarn.resourcemanager.nodemanager-graceful-decommission-timeout-secs":"3600","mapreduce.shuffle.ssl.enabled":"false","yarn.timeline-service.hbase.coprocessor.app-final-value-retention-milliseconds":"259200000","fs.s3a.committer.staging.abort.pending.uploads":"true","yarn.nodemanager.opportunistic-containers-max-queue-length":"0","yarn.resourcemanager.state-store.max-completed-applications":"${yarn.resourcemanager.max-completed-applications}","mapreduce.job.speculative.minimum-allowed-tasks":"10","yarn.log-aggregation.retain-seconds":"-1","yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb":"0","mapreduce.jobhistory.max-age-ms":"604800000","hadoop.http.cross-origin.allowed-methods":"GET,POST,HEAD","yarn.resourcemanager.opportunistic-container-allocation.enabled":"false","mapreduce.jobhistory.webapp.address":"0.0.0.0:19888","hadoop.system.tags":"YARN,HDFS,NAMENODE,DATANODE,REQUIRED,SECURITY,KERBEROS,PERFORMANCE,CLIENT\n ,SERVER,DEBUG,DEPRECATED,COMMON,OPTIONAL","yarn.log-aggregation.file-controller.TFile.class":"org.apache.hadoop.yarn.logaggregation.filecontroller.tfile.LogAggregationTFileController","yarn.client.nodemanager-connect.max-wait-ms":"180000","yarn.resourcemanager.webapp.address":"${yarn.resourcemanager.hostname}:8088","mapreduce.jobhistory.recovery.enable":"false","mapreduce.reduce.shuffle.parallelcopies":"5","fs.AbstractFileSystem.webhdfs.impl":"org.apache.hadoop.fs.WebHdfs","fs.trash.interval":"0","yarn.app.mapreduce.client.max-retries":"3","hadoop.security.authentication":"simple","mapreduce.task.profile.reduce.params":"${mapreduce.task.profile.params}","yarn.app.mapreduce.am.resource.mb":"1536","mapreduce.input.fileinputformat.list-status.num-threads":"1","yarn.nodemanager.container-executor.class":"org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor","io.mapfile.bloom.size":"1048576","yarn.timeline-service.ttl-ms":"604800000","yarn.resourcemanager.nm-container-queuing.min-queue-length":"5","yarn.nodemanager.resource.cpu-vcores":"-1","mapreduce.job.reduces":"1","fs.s3a.multipart.size":"100M","yarn.scheduler.minimum-allocation-vcores":"1","mapreduce.job.speculative.speculative-cap-total-tasks":"0.01","hadoop.ssl.client.conf":"ssl-client.xml","mapreduce.job.queuename":"default","mapreduce.job.encrypted-intermediate-data-key-size-bits":"128","fs.s3a.metadatastore.authoritative":"false","yarn.nodemanager.webapp.xfs-filter.xframe-options":"SAMEORIGIN","ha.health-monitor.sleep-after-disconnect.ms":"1000","yarn.app.mapreduce.shuffle.log.limit.kb":"0","hadoop.security.group.mapping":"org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback","yarn.client.application-client-protocol.poll-timeout-ms":"-1","mapreduce.jobhistory.jhist.format":"binary","yarn.resourcemanager.ha.enabled":"false","hadoop.http.staticuser.user":"dr.who","mapreduce.task.exit.timeout.check-interval-ms":"20000","mapreduce.jobhistory.intermediate-user-done-dir.permissions":"770","mapreduce.task.exit.timeout":"60000","yarn.nodemanager.linux-container-executor.resources-handler.class":"org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler","mapreduce.reduce.shuffle.memory.limit.percent":"0.25","yarn.resourcemanager.reservation-system.enable":"false","mapreduce.map.output.compress":"false","ha.zookeeper.acl":"world:anyone:rwcda","ipc.server.max.connections":"0","yarn.nodemanager.aux-services":"mapreduce_shuffle","yarn.nodemanager.runtime.linux.docker.default-container-network":"host","yarn.router.webapp.address":"0.0.0.0:8089","yarn.scheduler.maximum-allocation-mb":"8192","yarn.resourcemanager.scheduler.monitor.policies":"org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy","yarn.sharedcache.cleaner.period-mins":"1440","yarn.nodemanager.resource-plugins.gpu.docker-plugin.nvidia-docker-v1.endpoint":"http://localhost:3476/v1.0/docker/cli","yarn.app.mapreduce.am.container.log.limit.kb":"0","ipc.client.connect.retry.interval":"1000","yarn.timeline-service.http-cross-origin.enabled":"false","fs.wasbs.impl":"org.apache.hadoop.fs.azure.NativeAzureFileSystem$Secure","yarn.federation.subcluster-resolver.class":"org.apache.hadoop.yarn.server.federation.resolver.DefaultSubClusterResolverImpl","yarn.resourcemanager.zk-state-store.parent-path":"/rmstore","mapreduce.jobhistory.cleaner.enable":"true","yarn.timeline-service.client.fd-flush-interval-secs":"10","hadoop.security.kms.client.encrypted.key.cache.expiry":"43200000","yarn.client.nodemanager-client-async.thread-pool-max-size":"500","mapreduce.map.maxattempts":"4","yarn.resourcemanager.nm-container-queuing.sorting-nodes-interval-ms":"1000","fs.s3a.committer.staging.tmp.path":"tmp/staging","yarn.nodemanager.sleep-delay-before-sigkill.ms":"250","yarn.resourcemanager.nm-container-queuing.min-queue-wait-time-ms":"10","mapreduce.job.end-notification.retry.attempts":"0","yarn.nodemanager.resource.count-logical-processors-as-cores":"false","hadoop.registry.zk.root":"/registry","adl.feature.ownerandgroup.enableupn":"false","yarn.resourcemanager.zk-max-znode-size.bytes":"1048576","mapreduce.job.reduce.shuffle.consumer.plugin.class":"org.apache.hadoop.mapreduce.task.reduce.Shuffle","yarn.resourcemanager.delayed.delegation-token.removal-interval-ms":"*********(redacted)","yarn.nodemanager.localizer.cache.target-size-mb":"10240","fs.s3a.committer.staging.conflict-mode":"fail","mapreduce.client.libjars.wildcard":"true","fs.s3a.committer.staging.unique-filenames":"true","yarn.nodemanager.node-attributes.provider.fetch-timeout-ms":"1200000","fs.s3a.list.version":"2","ftp.client-write-packet-size":"65536","fs.AbstractFileSystem.adl.impl":"org.apache.hadoop.fs.adl.Adl","hadoop.security.key.default.cipher":"AES/CTR/NoPadding","yarn.client.failover-retries":"0","fs.s3a.multipart.purge.age":"86400","mapreduce.job.local-fs.single-disk-limit.check.interval-ms":"5000","net.topology.node.switch.mapping.impl":"org.apache.hadoop.net.ScriptBasedMapping","yarn.nodemanager.amrmproxy.address":"0.0.0.0:8049","ipc.server.listen.queue.size":"128","map.sort.class":"org.apache.hadoop.util.QuickSort","fs.viewfs.rename.strategy":"SAME_MOUNTPOINT","hadoop.security.kms.client.authentication.retry-count":"1","fs.permissions.umask-mode":"022","fs.s3a.assumed.role.credentials.provider":"org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider","yarn.nodemanager.vmem-check-enabled":"true","yarn.nodemanager.numa-awareness.enabled":"false","yarn.nodemanager.recovery.compaction-interval-secs":"3600","yarn.app.mapreduce.client-am.ipc.max-retries":"3","yarn.federation.registry.base-dir":"yarnfederation/","mapreduce.job.max.map":"-1","mapreduce.job.local-fs.single-disk-limit.bytes":"-1","mapreduce.job.ubertask.maxreduces":"1","hadoop.security.kms.client.encrypted.key.cache.size":"500","hadoop.security.java.secure.random.algorithm":"SHA1PRNG","ha.failover-controller.cli-check.rpc-timeout.ms":"20000","mapreduce.jobhistory.jobname.limit":"50","mapreduce.application.classpath":"$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*:$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*","yarn.client.nodemanager-connect.retry-interval-ms":"10000","yarn.timeline-service.state-store-class":"org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore","yarn.nodemanager.env-whitelist":"JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,PATH,LANG,TZ,HADOOP_MAPRED_HOME","yarn.sharedcache.nested-level":"3","yarn.timeline-service.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","fs.azure.user.agent.prefix":"unknown","yarn.resourcemanager.zk-delegation-token-node.split-index":"*********(redacted)","yarn.nodemanager.numa-awareness.read-topology":"false","yarn.nodemanager.webapp.address":"${yarn.nodemanager.hostname}:8042","rpc.metrics.quantile.enable":"false","yarn.registry.class":"org.apache.hadoop.registry.client.impl.FSRegistryOperationsService","mapreduce.jobhistory.admin.acl":"*","yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size":"10","yarn.scheduler.queue-placement-rules":"user-group","hadoop.http.authentication.kerberos.keytab":"${user.home}/hadoop.keytab","yarn.resourcemanager.recovery.enabled":"false","yarn.timeline-service.webapp.rest-csrf.enabled":"false"},"System Properties":{"java.io.tmpdir":"/tmp","line.separator":"\n","path.separator":":","sun.management.compiler":"HotSpot 64-Bit Tiered Compilers","SPARK_SUBMIT":"true","sun.cpu.endian":"little","java.specification.version":"1.8","java.vm.specification.name":"Java Virtual Machine Specification","java.vendor":"Oracle Corporation","java.vm.specification.version":"1.8","user.home":"/home/usera112","file.encoding.pkg":"sun.io","sun.nio.ch.bugLevel":"","sun.arch.data.model":"64","sun.boot.library.path":"/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64","user.dir":"/home/usera112/projects/log-compressor/deployments/HiBench","java.library.path":"/usr/java/packages/lib/amd64:/usr/lib/x86_64-linux-gnu/jni:/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:/usr/lib/jni:/lib:/usr/lib","sun.cpu.isalist":"","os.arch":"amd64","java.vm.version":"25.162-b12","jetty.git.hash":"b881a572662e1943a14ae12e7e1207989f218b74","java.endorsed.dirs":"/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/endorsed","java.runtime.version":"1.8.0_162-8u162-b12-1-b12","java.vm.info":"mixed mode","java.ext.dirs":"/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/ext:/usr/java/packages/lib/ext","java.runtime.name":"OpenJDK Runtime Environment","file.separator":"/","java.class.version":"52.0","java.specification.name":"Java Platform API Specification","sun.boot.class.path":"/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/resources.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/rt.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/sunrsasign.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/jsse.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/jce.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/charsets.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/jfr.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/classes","file.encoding":"UTF-8","user.timezone":"America/Toronto","java.specification.vendor":"Oracle Corporation","sun.java.launcher":"SUN_STANDARD","os.version":"4.15.0-29-generic","sun.os.patch.level":"unknown","java.vm.specification.vendor":"Oracle Corporation","user.country":"US","sun.jnu.encoding":"UTF-8","user.language":"en","java.vendor.url":"http://java.oracle.com/","java.awt.printerjob":"sun.print.PSPrinterJob","java.awt.graphicsenv":"sun.awt.X11GraphicsEnvironment","awt.toolkit":"sun.awt.X11.XToolkit","os.name":"Linux","java.vm.vendor":"Oracle Corporation","java.vendor.url.bug":"http://bugreport.sun.com/bugreport/","user.name":"usera112","java.vm.name":"OpenJDK 64-Bit Server VM","sun.java.command":"org.apache.spark.deploy.SparkSubmit --master spark://hostb7:7077 --properties-file /home/usera112/projects/log-compressor/deployments/HiBench/report/join/spark/conf/sparkbench/spark.conf --class com.intel.hibench.sparkbench.sql.ScalaSparkSQLBench /home/usera112/projects/log-compressor/deployments/HiBench/sparkbench/assembly/target/sparkbench-assembly-8.0-SNAPSHOT-dist.jar ScalaJoin /home/usera112/projects/log-compressor/deployments/HiBench/report/join/spark/conf/../rankings_uservisits_join.hive","java.home":"/usr/lib/jvm/java-8-openjdk-amd64/jre","java.version":"1.8.0_162","sun.io.unicode.encoding":"UnicodeLittle"},"Classpath Entries":{"/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jackson-mapper-asl-1.9.13.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-yarn-api-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hk2-locator-2.6.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/nimbus-jose-jwt-4.41.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/htrace-core4-4.1.0-incubating.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/cats-kernel_2.12-2.0.0-M4.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/httpclient-4.5.6.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-collections-3.2.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-beeline-2.3.7.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/json-smart-2.3.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/curator-recipes-2.13.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/stream-2.9.6.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-graphx_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jline-2.14.6.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-admissionregistration-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/guava-14.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-sql_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/parquet-common-1.10.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/chill-java-0.9.5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-beanutils-1.9.4.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kryo-shaded-4.0.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hk2-api-2.6.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-io-2.5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/datanucleus-rdbms-4.1.19.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/metrics-graphite-4.1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/breeze_2.12-1.0.jar":"System Classpath","spark://hostb7.dsrg.utoronto.ca:42853/jars/sparkbench-assembly-8.0-SNAPSHOT-dist.jar":"Added By User","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jakarta.xml.bind-api-2.3.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jakarta.activation-api-1.2.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-lang3-3.10.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-mesos_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/avro-mapred-1.8.2-hadoop2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jsr305-3.0.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/HikariCP-2.5.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/ehcache-3.3.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-cli-1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-core-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/joda-time-2.10.5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/javassist-3.25.0-GA.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerby-config-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/token-provider-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/leveldbjni-all-1.8.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-sketch_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-mllib-local_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-apps-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/json4s-ast_2.12-3.7.0-M5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/json-1.8.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/datanucleus-core-4.1.17.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spire-macros_2.12-0.17.0-M1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/parquet-jackson-1.10.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-yarn-server-web-proxy-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-math3-3.4.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-vector-code-gen-2.3.7.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-network-shuffle_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/RoaringBitmap-0.9.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-mapreduce-client-jobclient-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/super-csv-2.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/pyrolite-4.30.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-yarn-common-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/slf4j-log4j12-1.7.30.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/macro-compat_2.12-1.1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-discovery-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-hive_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-metastore-2.3.7.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-streaming_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jackson-databind-2.10.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/snakeyaml-1.24.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-lang-2.6.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/libfb303-0.9.3.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/curator-framework-2.13.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerby-xdr-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/xz-1.5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-shims-common-2.3.7.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/mesos-1.4.0-shaded-protobuf.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerb-crypto-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-llap-common-2.3.7.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jersey-server-2.30.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/minlog-1.3.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jackson-module-scala_2.12-2.10.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/okhttp-2.7.5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/transaction-api-1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-exec-2.3.7-core.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-tags_2.12-3.1.2-tests.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/shims-0.9.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-jdbc-2.3.7.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/paranamer-2.8.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/woodstox-core-5.0.3.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/generex-1.0.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jackson-annotations-2.10.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spire-platform_2.12-0.17.0-M1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/chill_2.12-0.9.5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/JLargeArrays-1.5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-net-3.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-service-rpc-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/py4j-0.10.9.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/okhttp-3.12.12.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-crypto-1.1.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-core_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/zjsonpatch-0.3.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/guice-servlet-4.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/derby-10.12.1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-pool-1.5.4.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/parquet-encoding-1.10.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/scala-xml_2.12-1.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jersey-client-2.30.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerby-pkix-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerb-common-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-annotations-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-shims-2.3.7.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-compiler-3.0.16.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-kvstore_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-repl_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jcip-annotations-1.0-1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/avro-1.8.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/osgi-resource-locator-1.0.3.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/javolution-5.5.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/re2j-1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/orc-core-1.5.12.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/javax.jdo-3.2.0-m3.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/bonecp-0.8.0.RELEASE.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jakarta.inject-2.6.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/objenesis-2.6.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jersey-hk2-2.30.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spire-util_2.12-0.17.0-M1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/datanucleus-api-jdo-4.2.4.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-settings-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/aopalliance-1.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-kubernetes_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/httpcore-4.4.12.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/janino-3.0.16.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/scala-reflect-2.12.10.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/aopalliance-repackaged-2.6.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/libthrift-0.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/json4s-jackson_2.12-3.7.0-M5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-policy-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jackson-module-paranamer-2.10.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jaxb-runtime-2.3.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/orc-mapreduce-1.5.12.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-storageclass-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jpam-1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/gson-2.2.4.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/automaton-1.11-8.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/curator-client-2.13.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-codec-1.10.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-httpclient-3.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/flatbuffers-java-1.9.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-apiextensions-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/zookeeper-3.4.14.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/netty-all-4.1.51.Final.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-yarn-client-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jersey-container-servlet-2.30.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jackson-module-jaxb-annotations-2.10.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jakarta.ws.rs-api-2.1.6.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/xbean-asm7-shaded-4.15.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-auth-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-batch-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/arrow-vector-2.0.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jakarta.servlet-api-4.0.3.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-unsafe_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-client-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-cli-2.3.7.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-mllib_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/json4s-core_2.12-3.7.0-M5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/scala-parser-combinators_2.12-1.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/arrow-memory-core-2.0.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jersey-common-2.30.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerby-util-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jdo-api-3.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-metrics-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-storage-api-2.7.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jackson-jaxrs-base-2.9.5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/ivy-2.4.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-extensions-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-networking-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-events-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hk2-utils-2.6.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-common-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-rbac-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/protobuf-java-2.5.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/scala-compiler-2.12.10.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/lz4-java-1.7.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/threeten-extra-1.5.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/stax-api-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-client-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/parquet-hadoop-1.10.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/breeze-macros_2.12-1.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/ST4-4.0.4.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-serde-2.3.7.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/hadoop-3.3.1/etc/hadoop/":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/snappy-java-1.1.8.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/core-1.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerb-identity-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/orc-shims-1.5.12.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-configuration2-2.1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jaxb-api-2.2.11.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/arrow-format-2.0.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jsp-api-2.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/activation-1.1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jakarta.annotation-api-1.3.5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/oro-2.0.8.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/shapeless_2.12-2.3.3.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jackson-core-2.10.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-daemon-1.0.13.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-launcher_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/opencsv-2.3.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-tags_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-text-1.6.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/arrow-memory-netty-2.0.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerb-util-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerby-asn1-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jackson-core-asl-1.9.13.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-coordination-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jackson-dataformat-yaml-2.10.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerb-admin-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/avro-ipc-1.8.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-mapreduce-client-common-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/antlr4-runtime-4.8-1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jackson-jaxrs-json-provider-2.9.5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-common-2.3.7.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/arpack_combined_all-0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/geronimo-jcache_1.0_spec-1.0-alpha-1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/machinist_2.12-0.6.8.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/aircompressor-0.10.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/univocity-parsers-2.9.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerb-server-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-dbcp-1.4.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/stax2-api-3.1.4.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/istack-commons-runtime-3.0.8.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-yarn-server-common-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerb-simplekdc-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/antlr-runtime-3.5.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/JTransforms-3.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/guice-4.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-yarn_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/slf4j-api-1.7.30.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jackson-datatype-jsr310-2.11.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/parquet-format-2.4.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-yarn-registry-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/velocity-1.5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/javax.inject-1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-scheduling-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-autoscaling-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerb-client-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spire_2.12-0.17.0-M1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/log4j-1.2.17.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-network-common_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/logging-interceptor-3.12.12.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/dnsjava-2.1.7.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/audience-annotations-0.5.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/metrics-jmx-4.1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jta-1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-shims-scheduler-2.3.7.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/json4s-scalap_2.12-3.7.0-M5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/conf/":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/metrics-json-4.1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jersey-container-servlet-core-2.30.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jodd-core-3.5.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/compress-lzf-1.0.3.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/metrics-jvm-4.1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/scala-collection-compat_2.12-2.1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/parquet-column-1.10.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/metrics-core-4.1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/scala-library-2.12.10.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-common-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-hive-thriftserver_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/zstd-jni-1.4.8-1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jakarta.validation-api-2.0.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerb-core-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/okio-1.14.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-logging-1.1.3.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jcl-over-slf4j-1.7.30.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jersey-media-jaxb-2.30.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/algebra_2.12-2.0.0-M2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-compress-1.20.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-hdfs-client-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-shims-0.23-2.3.7.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jul-to-slf4j-1.7.30.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-certificates-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-mapreduce-client-core-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-catalyst_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/accessors-smart-1.2.jar":"System Classpath"}} +{"Event":"SparkListenerApplicationStart","App Name":"ScalaJoin","App ID":"app-20211010132332-4636","Timestamp":1633886611309,"User":"usera112"} +{"Event":"SparkListenerExecutorAdded","Timestamp":1633886614425,"Executor ID":"5","Executor Info":{"Host":"10.1.0.19","Total Cores":16,"Log Urls":{"stdout":"http://10.1.0.19:8081/logPage/?appId=app-20211010132332-4636&executorId=5&logType=stdout","stderr":"http://10.1.0.19:8081/logPage/?appId=app-20211010132332-4636&executorId=5&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0}} +{"Event":"SparkListenerExecutorAdded","Timestamp":1633886614427,"Executor ID":"10","Executor Info":{"Host":"10.1.0.16","Total Cores":16,"Log Urls":{"stdout":"http://10.1.0.16:8081/logPage/?appId=app-20211010132332-4636&executorId=10&logType=stdout","stderr":"http://10.1.0.16:8081/logPage/?appId=app-20211010132332-4636&executorId=10&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0}} +{"Event":"SparkListenerExecutorAdded","Timestamp":1633886614453,"Executor ID":"4","Executor Info":{"Host":"10.1.0.8","Total Cores":16,"Log Urls":{"stdout":"http://10.1.0.8:8081/logPage/?appId=app-20211010132332-4636&executorId=4&logType=stdout","stderr":"http://10.1.0.8:8081/logPage/?appId=app-20211010132332-4636&executorId=4&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0}} +{"Event":"SparkListenerExecutorAdded","Timestamp":1633886614472,"Executor ID":"11","Executor Info":{"Host":"10.1.0.13","Total Cores":16,"Log Urls":{"stdout":"http://10.1.0.13:8081/logPage/?appId=app-20211010132332-4636&executorId=11&logType=stdout","stderr":"http://10.1.0.13:8081/logPage/?appId=app-20211010132332-4636&executorId=11&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0}} +{"Event":"SparkListenerExecutorAdded","Timestamp":1633886614477,"Executor ID":"1","Executor Info":{"Host":"10.1.0.17","Total Cores":16,"Log Urls":{"stdout":"http://10.1.0.17:8081/logPage/?appId=app-20211010132332-4636&executorId=1&logType=stdout","stderr":"http://10.1.0.17:8081/logPage/?appId=app-20211010132332-4636&executorId=1&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0}} +{"Event":"SparkListenerExecutorAdded","Timestamp":1633886614486,"Executor ID":"0","Executor Info":{"Host":"10.1.0.12","Total Cores":16,"Log Urls":{"stdout":"http://10.1.0.12:8081/logPage/?appId=app-20211010132332-4636&executorId=0&logType=stdout","stderr":"http://10.1.0.12:8081/logPage/?appId=app-20211010132332-4636&executorId=0&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0}} +{"Event":"SparkListenerExecutorAdded","Timestamp":1633886614498,"Executor ID":"8","Executor Info":{"Host":"10.1.0.11","Total Cores":16,"Log Urls":{"stdout":"http://10.1.0.11:8081/logPage/?appId=app-20211010132332-4636&executorId=8&logType=stdout","stderr":"http://10.1.0.11:8081/logPage/?appId=app-20211010132332-4636&executorId=8&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0}} +{"Event":"SparkListenerExecutorAdded","Timestamp":1633886614514,"Executor ID":"9","Executor Info":{"Host":"10.1.0.14","Total Cores":16,"Log Urls":{"stdout":"http://10.1.0.14:8081/logPage/?appId=app-20211010132332-4636&executorId=9&logType=stdout","stderr":"http://10.1.0.14:8081/logPage/?appId=app-20211010132332-4636&executorId=9&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0}} +{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"5","Host":"10.1.0.19","Port":40959},"Maximum Memory":2101975449,"Timestamp":1633886614518,"Maximum Onheap Memory":2101975449,"Maximum Offheap Memory":0} +{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"10","Host":"10.1.0.16","Port":44733},"Maximum Memory":2101975449,"Timestamp":1633886614520,"Maximum Onheap Memory":2101975449,"Maximum Offheap Memory":0} +{"Event":"SparkListenerExecutorAdded","Timestamp":1633886614529,"Executor ID":"2","Executor Info":{"Host":"10.1.0.10","Total Cores":16,"Log Urls":{"stdout":"http://10.1.0.10:8081/logPage/?appId=app-20211010132332-4636&executorId=2&logType=stdout","stderr":"http://10.1.0.10:8081/logPage/?appId=app-20211010132332-4636&executorId=2&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0}} +{"Event":"SparkListenerExecutorAdded","Timestamp":1633886614532,"Executor ID":"6","Executor Info":{"Host":"10.1.0.18","Total Cores":16,"Log Urls":{"stdout":"http://10.1.0.18:8081/logPage/?appId=app-20211010132332-4636&executorId=6&logType=stdout","stderr":"http://10.1.0.18:8081/logPage/?appId=app-20211010132332-4636&executorId=6&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0}} +{"Event":"SparkListenerExecutorAdded","Timestamp":1633886614544,"Executor ID":"3","Executor Info":{"Host":"10.1.0.15","Total Cores":16,"Log Urls":{"stdout":"http://10.1.0.15:8081/logPage/?appId=app-20211010132332-4636&executorId=3&logType=stdout","stderr":"http://10.1.0.15:8081/logPage/?appId=app-20211010132332-4636&executorId=3&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0}} +{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"4","Host":"10.1.0.8","Port":39057},"Maximum Memory":2101975449,"Timestamp":1633886614548,"Maximum Onheap Memory":2101975449,"Maximum Offheap Memory":0} +{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"11","Host":"10.1.0.13","Port":41379},"Maximum Memory":2101975449,"Timestamp":1633886614572,"Maximum Onheap Memory":2101975449,"Maximum Offheap Memory":0} +{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"1","Host":"10.1.0.17","Port":36477},"Maximum Memory":2101975449,"Timestamp":1633886614575,"Maximum Onheap Memory":2101975449,"Maximum Offheap Memory":0} +{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"0","Host":"10.1.0.12","Port":46419},"Maximum Memory":2101975449,"Timestamp":1633886614585,"Maximum Onheap Memory":2101975449,"Maximum Offheap Memory":0} +{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"8","Host":"10.1.0.11","Port":40375},"Maximum Memory":2101975449,"Timestamp":1633886614593,"Maximum Onheap Memory":2101975449,"Maximum Offheap Memory":0} +{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"9","Host":"10.1.0.14","Port":34719},"Maximum Memory":2101975449,"Timestamp":1633886614613,"Maximum Onheap Memory":2101975449,"Maximum Offheap Memory":0} +{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"2","Host":"10.1.0.10","Port":45933},"Maximum Memory":2101975449,"Timestamp":1633886614628,"Maximum Onheap Memory":2101975449,"Maximum Offheap Memory":0} +{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"6","Host":"10.1.0.18","Port":33835},"Maximum Memory":2101975449,"Timestamp":1633886614633,"Maximum Onheap Memory":2101975449,"Maximum Offheap Memory":0} +{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"3","Host":"10.1.0.15","Port":36115},"Maximum Memory":2101975449,"Timestamp":1633886614646,"Maximum Onheap Memory":2101975449,"Maximum Offheap Memory":0} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":0,"description":"sql at ScalaSparkSQLBench.scala:43","details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nSetCatalogAndNamespace (1)\n\n\n(1) SetCatalogAndNamespace\nArguments: org.apache.spark.sql.connector.catalog.CatalogManager@2b1a901d, spark_catalog, ArrayBuffer(DEFAULT)\n\n","sparkPlanInfo":{"nodeName":"SetCatalogAndNamespace","simpleString":"SetCatalogAndNamespace org.apache.spark.sql.connector.catalog.CatalogManager@2b1a901d, spark_catalog, ArrayBuffer(DEFAULT)","children":[],"metadata":{},"metrics":[]},"time":1633886616285} +{"Event":"SparkListenerExecutorAdded","Timestamp":1633886619475,"Executor ID":"7","Executor Info":{"Host":"10.1.0.9","Total Cores":16,"Log Urls":{"stdout":"http://10.1.0.9:8081/logPage/?appId=app-20211010132332-4636&executorId=7&logType=stdout","stderr":"http://10.1.0.9:8081/logPage/?appId=app-20211010132332-4636&executorId=7&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0}} +{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"7","Host":"10.1.0.9","Port":35243},"Maximum Memory":2101975449,"Timestamp":1633886619587,"Maximum Onheap Memory":2101975449,"Maximum Offheap Memory":0} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":0,"time":1633886639860} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":1,"description":"sql at ScalaSparkSQLBench.scala:43","details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nExecute SetCommand (1)\n +- SetCommand (2)\n\n\n(1) Execute SetCommand\nOutput [2]: [key#8, value#9]\n\n(2) SetCommand\nArguments: (hive.input.format,Some(org.apache.hadoop.hive.ql.io.HiveInputFormat))\n\n","sparkPlanInfo":{"nodeName":"Execute SetCommand","simpleString":"Execute SetCommand","children":[],"metadata":{},"metrics":[]},"time":1633886639916} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":1,"time":1633886639921} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":2,"description":"sql at ScalaSparkSQLBench.scala:43","details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nExecute SetCommand (1)\n +- SetCommand (2)\n\n\n(1) Execute SetCommand\nOutput [2]: [key#18, value#19]\n\n(2) SetCommand\nArguments: (mapreduce.job.maps,Some(8))\n\n","sparkPlanInfo":{"nodeName":"Execute SetCommand","simpleString":"Execute SetCommand","children":[],"metadata":{},"metrics":[]},"time":1633886639938} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":2,"time":1633886639938} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":3,"description":"sql at ScalaSparkSQLBench.scala:43","details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nExecute SetCommand (1)\n +- SetCommand (2)\n\n\n(1) Execute SetCommand\nOutput [2]: [key#28, value#29]\n\n(2) SetCommand\nArguments: (mapreduce.job.reduces,Some(8))\n\n","sparkPlanInfo":{"nodeName":"Execute SetCommand","simpleString":"Execute SetCommand","children":[],"metadata":{},"metrics":[]},"time":1633886639954} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":3,"time":1633886639954} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":4,"description":"sql at ScalaSparkSQLBench.scala:43","details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nExecute SetCommand (1)\n +- SetCommand (2)\n\n\n(1) Execute SetCommand\nOutput [2]: [key#38, value#39]\n\n(2) SetCommand\nArguments: (hive.stats.autogather,Some(false))\n\n","sparkPlanInfo":{"nodeName":"Execute SetCommand","simpleString":"Execute SetCommand","children":[],"metadata":{},"metrics":[]},"time":1633886639966} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":4,"time":1633886639966} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":5,"description":"sql at ScalaSparkSQLBench.scala:43","details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nLocalTableScan (1)\n\n\n(1) LocalTableScan\nOutput: []\nArguments: \n\n","sparkPlanInfo":{"nodeName":"LocalTableScan","simpleString":"LocalTableScan ","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":0,"metricType":"sum"}]},"time":1633886640023} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":5,"time":1633886640023} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":6,"description":"sql at ScalaSparkSQLBench.scala:43","details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nExecute CreateTableCommand (1)\n +- CreateTableCommand (2)\n\n\n(1) Execute CreateTableCommand\nOutput: []\n\n(2) CreateTableCommand\nArguments: `default`.`rankings`, org.apache.hadoop.hive.serde2.OpenCSVSerde, false\n\n","sparkPlanInfo":{"nodeName":"Execute CreateTableCommand","simpleString":"Execute CreateTableCommand","children":[],"metadata":{},"metrics":[]},"time":1633886640087} +{"Event":"org.apache.spark.sql.catalyst.catalog.CreateTablePreEvent","database":"default","name":"rankings"} +{"Event":"org.apache.spark.sql.catalyst.catalog.CreateTableEvent","database":"default","name":"rankings"} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":6,"time":1633886640911} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":7,"description":"sql at ScalaSparkSQLBench.scala:43","details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nLocalTableScan (1)\n\n\n(1) LocalTableScan\nOutput: []\nArguments: \n\n","sparkPlanInfo":{"nodeName":"LocalTableScan","simpleString":"LocalTableScan ","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1,"metricType":"sum"}]},"time":1633886640934} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":7,"time":1633886640934} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":8,"description":"sql at ScalaSparkSQLBench.scala:43","details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nExecute CreateTableCommand (1)\n +- CreateTableCommand (2)\n\n\n(1) Execute CreateTableCommand\nOutput: []\n\n(2) CreateTableCommand\nArguments: `default`.`uservisits_copy`, org.apache.hadoop.hive.serde2.OpenCSVSerde, false\n\n","sparkPlanInfo":{"nodeName":"Execute CreateTableCommand","simpleString":"Execute CreateTableCommand","children":[],"metadata":{},"metrics":[]},"time":1633886640947} +{"Event":"org.apache.spark.sql.catalyst.catalog.CreateTablePreEvent","database":"default","name":"uservisits_copy"} +{"Event":"org.apache.spark.sql.catalyst.catalog.CreateTableEvent","database":"default","name":"uservisits_copy"} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":8,"time":1633886641021} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":9,"description":"sql at ScalaSparkSQLBench.scala:43","details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nLocalTableScan (1)\n\n\n(1) LocalTableScan\nOutput: []\nArguments: \n\n","sparkPlanInfo":{"nodeName":"LocalTableScan","simpleString":"LocalTableScan ","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2,"metricType":"sum"}]},"time":1633886641033} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":9,"time":1633886641034} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":10,"description":"sql at ScalaSparkSQLBench.scala:43","details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nExecute CreateTableCommand (1)\n +- CreateTableCommand (2)\n\n\n(1) Execute CreateTableCommand\nOutput: []\n\n(2) CreateTableCommand\nArguments: `default`.`rankings_uservisits_join`, org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, false\n\n","sparkPlanInfo":{"nodeName":"Execute CreateTableCommand","simpleString":"Execute CreateTableCommand","children":[],"metadata":{},"metrics":[]},"time":1633886641046} +{"Event":"org.apache.spark.sql.catalyst.catalog.CreateTablePreEvent","database":"default","name":"rankings_uservisits_join"} +{"Event":"org.apache.spark.sql.catalyst.catalog.CreateTableEvent","database":"default","name":"rankings_uservisits_join"} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":10,"time":1633886641212} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":11,"description":"sql at ScalaSparkSQLBench.scala:43","details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nExecute InsertIntoHiveTable (15)\n+- * Project (14)\n +- * Sort (13)\n +- Exchange (12)\n +- * HashAggregate (11)\n +- Exchange (10)\n +- * HashAggregate (9)\n +- * Project (8)\n +- * BroadcastHashJoin Inner BuildLeft (7)\n :- BroadcastExchange (3)\n : +- * Filter (2)\n : +- Scan hive default.rankings (1)\n +- * Project (6)\n +- * Filter (5)\n +- Scan hive default.uservisits_copy (4)\n\n\n(1) Scan hive default.rankings\nOutput [2]: [pageurl#41, pagerank#42]\nArguments: [pageurl#41, pagerank#42], HiveTableRelation [`default`.`rankings`, org.apache.hadoop.hive.serde2.OpenCSVSerde, Data Cols: [pageurl#41, pagerank#42, avgduration#43], Partition Cols: []]\n\n(2) Filter [codegen id : 1]\nInput [2]: [pageurl#41, pagerank#42]\nCondition : isnotnull(pageURL#41)\n\n(3) BroadcastExchange\nInput [2]: [pageurl#41, pagerank#42]\nArguments: HashedRelationBroadcastMode(List(input[0, string, false]),false), [id=#88]\n\n(4) Scan hive default.uservisits_copy\nOutput [4]: [adrevenue#47, desturl#45, sourceip#44, visitdate#46]\nArguments: [adRevenue#47, destURL#45, sourceIP#44, visitDate#46], HiveTableRelation [`default`.`uservisits_copy`, org.apache.hadoop.hive.serde2.OpenCSVSerde, Data Cols: [sourceip#44, desturl#45, visitdate#46, adrevenue#47, useragent#48, countrycode#49, languagecode#..., Partition Cols: []]\n\n(5) Filter\nInput [4]: [adrevenue#47, desturl#45, sourceip#44, visitdate#46]\nCondition : (((isnotnull(visitDate#46) AND (datediff(cast(visitDate#46 as date), 10592) >= 0)) AND (datediff(cast(visitDate#46 as date), 10957) <= 0)) AND isnotnull(destURL#45))\n\n(6) Project\nOutput [3]: [sourceIP#44, destURL#45, adRevenue#47]\nInput [4]: [adrevenue#47, desturl#45, sourceip#44, visitdate#46]\n\n(7) BroadcastHashJoin [codegen id : 2]\nLeft keys [1]: [pageURL#41]\nRight keys [1]: [destURL#45]\nJoin condition: None\n\n(8) Project [codegen id : 2]\nOutput [3]: [pagerank#42, sourceIP#44, adRevenue#47]\nInput [5]: [pageurl#41, pagerank#42, sourceIP#44, destURL#45, adRevenue#47]\n\n(9) HashAggregate [codegen id : 2]\nInput [3]: [pagerank#42, sourceIP#44, adRevenue#47]\nKeys [1]: [sourceIP#44]\nFunctions [2]: [partial_avg(cast(pageRank#42 as double)), partial_sum(cast(adRevenue#47 as double))]\nAggregate Attributes [3]: [sum#62, count#63L, sum#64]\nResults [4]: [sourceIP#44, sum#65, count#66L, sum#67]\n\n(10) Exchange\nInput [4]: [sourceIP#44, sum#65, count#66L, sum#67]\nArguments: hashpartitioning(sourceIP#44, 8), ENSURE_REQUIREMENTS, [id=#97]\n\n(11) HashAggregate [codegen id : 3]\nInput [4]: [sourceIP#44, sum#65, count#66L, sum#67]\nKeys [1]: [sourceIP#44]\nFunctions [2]: [avg(cast(pageRank#42 as double)), sum(cast(adRevenue#47 as double))]\nAggregate Attributes [2]: [avg(cast(pageRank#42 as double))#53, sum(cast(adRevenue#47 as double))#54]\nResults [3]: [sourceIP#44, avg(cast(pageRank#42 as double))#53 AS avg(CAST(pageRank AS DOUBLE))#55, sum(cast(adRevenue#47 as double))#54 AS totalRevenue#40]\n\n(12) Exchange\nInput [3]: [sourceIP#44, avg(CAST(pageRank AS DOUBLE))#55, totalRevenue#40]\nArguments: rangepartitioning(totalRevenue#40 DESC NULLS LAST, 8), ENSURE_REQUIREMENTS, [id=#101]\n\n(13) Sort [codegen id : 4]\nInput [3]: [sourceIP#44, avg(CAST(pageRank AS DOUBLE))#55, totalRevenue#40]\nArguments: [totalRevenue#40 DESC NULLS LAST], true, 0\n\n(14) Project [codegen id : 4]\nOutput [3]: [ansi_cast(sourceIP#44 as string) AS sourceIP#60, ansi_cast(avg(CAST(pageRank AS DOUBLE))#55 as double) AS avgPageRank#61, totalRevenue#40]\nInput [3]: [sourceIP#44, avg(CAST(pageRank AS DOUBLE))#55, totalRevenue#40]\n\n(15) Execute InsertIntoHiveTable\nInput [3]: [sourceIP#60, avgPageRank#61, totalRevenue#40]\nArguments: `default`.`rankings_uservisits_join`, org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, true, false, [sourceIP, avgPageRank, totalRevenue]\n\n","sparkPlanInfo":{"nodeName":"Execute InsertIntoHiveTable","simpleString":"Execute InsertIntoHiveTable `default`.`rankings_uservisits_join`, org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, true, false, [sourceIP, avgPageRank, totalRevenue]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"Project","simpleString":"Project [ansi_cast(sourceIP#44 as string) AS sourceIP#60, ansi_cast(avg(CAST(pageRank AS DOUBLE))#55 as double) AS avgPageRank#61, totalRevenue#40]","children":[{"nodeName":"Sort","simpleString":"Sort [totalRevenue#40 DESC NULLS LAST], true, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Exchange","simpleString":"Exchange rangepartitioning(totalRevenue#40 DESC NULLS LAST, 8), ENSURE_REQUIREMENTS, [id=#101]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[sourceIP#44], functions=[avg(cast(pageRank#42 as double)), sum(cast(adRevenue#47 as double))])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(sourceIP#44, 8), ENSURE_REQUIREMENTS, [id=#97]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[sourceIP#44], functions=[partial_avg(cast(pageRank#42 as double)), partial_sum(cast(adRevenue#47 as double))])","children":[{"nodeName":"Project","simpleString":"Project [pagerank#42, sourceIP#44, adRevenue#47]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [pageURL#41], [destURL#45], Inner, BuildLeft, false","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, false]),false), [id=#88]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Filter","simpleString":"Filter isnotnull(pageURL#41)","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan hive default.rankings","simpleString":"Scan hive default.rankings [pageurl#41, pagerank#42], HiveTableRelation [`default`.`rankings`, org.apache.hadoop.hive.serde2.OpenCSVSerde, Data Cols: [pageurl#41, pagerank#42, avgduration#43], Partition Cols: []]","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":75,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":74,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":73,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":72,"metricType":"timing"},{"name":"time to build","accumulatorId":71,"metricType":"timing"},{"name":"time to collect","accumulatorId":70,"metricType":"timing"},{"name":"number of output rows","accumulatorId":69,"metricType":"sum"},{"name":"data size","accumulatorId":68,"metricType":"size"}]}],"metadata":{},"metrics":[]},{"nodeName":"Project","simpleString":"Project [sourceIP#44, destURL#45, adRevenue#47]","children":[{"nodeName":"Filter","simpleString":"Filter (((isnotnull(visitDate#46) AND (datediff(cast(visitDate#46 as date), 10592) >= 0)) AND (datediff(cast(visitDate#46 as date), 10957) <= 0)) AND isnotnull(destURL#45))","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan hive default.uservisits_copy","simpleString":"Scan hive default.uservisits_copy [adRevenue#47, destURL#45, sourceIP#44, visitDate#46], HiveTableRelation [`default`.`uservisits_copy`, org.apache.hadoop.hive.serde2.OpenCSVSerde, Data Cols: [sourceip#44, desturl#45, visitdate#46, adrevenue#47, useragent#48, countrycode#49, languagecode#..., Partition Cols: []]","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":77,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":76,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":67,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":64,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":65,"metricType":"timing"},{"name":"peak memory","accumulatorId":63,"metricType":"size"},{"name":"number of output rows","accumulatorId":62,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":66,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":61,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":34,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":35,"metricType":"nsTiming"},{"name":"records read","accumulatorId":32,"metricType":"sum"},{"name":"local bytes read","accumulatorId":30,"metricType":"size"},{"name":"fetch wait time","accumulatorId":31,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":28,"metricType":"size"},{"name":"local blocks read","accumulatorId":27,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":26,"metricType":"sum"},{"name":"data size","accumulatorId":25,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":29,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":33,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":58,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":59,"metricType":"timing"},{"name":"peak memory","accumulatorId":57,"metricType":"size"},{"name":"number of output rows","accumulatorId":56,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":60,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":55,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":45,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":46,"metricType":"nsTiming"},{"name":"records read","accumulatorId":43,"metricType":"sum"},{"name":"local bytes read","accumulatorId":41,"metricType":"size"},{"name":"fetch wait time","accumulatorId":42,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":39,"metricType":"size"},{"name":"local blocks read","accumulatorId":38,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":37,"metricType":"sum"},{"name":"data size","accumulatorId":36,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":40,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":44,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":52,"metricType":"timing"},{"name":"peak memory","accumulatorId":53,"metricType":"size"},{"name":"spill size","accumulatorId":54,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":51,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"number of written files","accumulatorId":47,"metricType":"sum"},{"name":"written output","accumulatorId":48,"metricType":"size"},{"name":"number of output rows","accumulatorId":49,"metricType":"sum"},{"name":"number of dynamic part","accumulatorId":50,"metricType":"sum"}]},"time":1633886642667} +{"Event":"SparkListenerJobStart","Job ID":0,"Submission Time":1633886643553,"Stage Infos":[{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Number of Tasks":8,"RDD Info":[{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"2\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"Scan hive default.rankings\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"HadoopRDD","Scope":"{\"id\":\"7\",\"name\":\"Scan hive default.rankings\"}","Callsite":"","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"Scan hive default.rankings\"}","Callsite":"","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"Scan hive default.rankings\"}","Callsite":"","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:185)\njava.util.concurrent.FutureTask.run(FutureTask.java:266)\njava.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\njava.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[0],"Properties":{"spark.sql.warehouse.dir":"file:/home/usera112/projects/log-compressor/deployments/HiBench/spark-warehouse","spark.driver.host":"hostb7.dsrg.utoronto.ca","spark.eventLog.enabled":"true","spark.job.interruptOnCancel":"true","spark.driver.port":"42853","spark.jars":"file:/home/usera112/projects/log-compressor/deployments/HiBench/sparkbench/assembly/target/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.app.name":"ScalaJoin","spark.rdd.scope":"{\"id\":\"0\",\"name\":\"WholeStageCodegen (4)\"}","spark.driver.memory":"4g","spark.rdd.scope.noOverride":"true","spark.submit.pyFiles":"","spark.default.parallelism":"8","spark.job.description":"broadcast exchange (runId 8616dbde-088b-499b-bd85-94ba59933204)","spark.app.startTime":"1633886611309","spark.executor.id":"driver","spark.jobGroup.id":"8616dbde-088b-499b-bd85-94ba59933204","spark.app.initial.jar.urls":"spark://hostb7.dsrg.utoronto.ca:42853/jars/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.submit.deployMode":"client","spark.master":"spark://hostb7:7077","spark.executor.memory":"4g","spark.eventLog.dir":"hdfs://hostb7:8020/directory","spark.sql.execution.id":"11","spark.sql.catalogImplementation":"hive","spark.app.id":"app-20211010132332-4636","spark.sql.shuffle.partitions":"8"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Number of Tasks":8,"RDD Info":[{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"2\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"Scan hive default.rankings\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"HadoopRDD","Scope":"{\"id\":\"7\",\"name\":\"Scan hive default.rankings\"}","Callsite":"","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"Scan hive default.rankings\"}","Callsite":"","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"Scan hive default.rankings\"}","Callsite":"","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:185)\njava.util.concurrent.FutureTask.run(FutureTask.java:266)\njava.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\njava.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1633886643571,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.sql.warehouse.dir":"file:/home/usera112/projects/log-compressor/deployments/HiBench/spark-warehouse","spark.driver.host":"hostb7.dsrg.utoronto.ca","spark.eventLog.enabled":"true","spark.job.interruptOnCancel":"true","spark.driver.port":"42853","spark.jars":"file:/home/usera112/projects/log-compressor/deployments/HiBench/sparkbench/assembly/target/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.app.name":"ScalaJoin","spark.rdd.scope":"{\"id\":\"0\",\"name\":\"WholeStageCodegen (4)\"}","spark.driver.memory":"4g","spark.rdd.scope.noOverride":"true","spark.submit.pyFiles":"","spark.default.parallelism":"8","spark.job.description":"broadcast exchange (runId 8616dbde-088b-499b-bd85-94ba59933204)","spark.app.startTime":"1633886611309","spark.executor.id":"driver","spark.jobGroup.id":"8616dbde-088b-499b-bd85-94ba59933204","spark.app.initial.jar.urls":"spark://hostb7.dsrg.utoronto.ca:42853/jars/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.submit.deployMode":"client","spark.master":"spark://hostb7:7077","spark.executor.memory":"4g","spark.eventLog.dir":"hdfs://hostb7:8020/directory","spark.sql.execution.id":"11","spark.sql.catalogImplementation":"hive","spark.app.id":"app-20211010132332-4636","spark.sql.shuffle.partitions":"8"}} +{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Launch Time":1633886643618,"Executor ID":"6","Host":"10.1.0.18","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":1,"Index":1,"Attempt":0,"Launch Time":1633886643635,"Executor ID":"10","Host":"10.1.0.16","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":2,"Index":2,"Attempt":0,"Launch Time":1633886643635,"Executor ID":"0","Host":"10.1.0.12","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":3,"Index":3,"Attempt":0,"Launch Time":1633886643636,"Executor ID":"9","Host":"10.1.0.14","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":4,"Index":4,"Attempt":0,"Launch Time":1633886643636,"Executor ID":"8","Host":"10.1.0.11","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":5,"Index":5,"Attempt":0,"Launch Time":1633886643637,"Executor ID":"11","Host":"10.1.0.13","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":6,"Index":6,"Attempt":0,"Launch Time":1633886643637,"Executor ID":"5","Host":"10.1.0.19","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":7,"Index":7,"Attempt":0,"Launch Time":1633886643638,"Executor ID":"2","Host":"10.1.0.10","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskGettingResult","Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Launch Time":1633886643618,"Executor ID":"6","Host":"10.1.0.18","Locality":"ANY","Speculative":false,"Getting Result Time":1633886645916,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskGettingResult","Task Info":{"Task ID":6,"Index":6,"Attempt":0,"Launch Time":1633886643637,"Executor ID":"5","Host":"10.1.0.19","Locality":"ANY","Speculative":false,"Getting Result Time":1633886645946,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskGettingResult","Task Info":{"Task ID":4,"Index":4,"Attempt":0,"Launch Time":1633886643636,"Executor ID":"8","Host":"10.1.0.11","Locality":"ANY","Speculative":false,"Getting Result Time":1633886645949,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskGettingResult","Task Info":{"Task ID":1,"Index":1,"Attempt":0,"Launch Time":1633886643635,"Executor ID":"10","Host":"10.1.0.16","Locality":"ANY","Speculative":false,"Getting Result Time":1633886645952,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":6,"Index":6,"Attempt":0,"Launch Time":1633886643637,"Executor ID":"5","Host":"10.1.0.19","Locality":"ANY","Speculative":false,"Getting Result Time":1633886645946,"Finish Time":1633886645975,"Failed":false,"Killed":false,"Accumulables":[{"ID":73,"Name":"duration","Update":"206","Value":"206","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"number of output rows","Update":"15000","Value":"15000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":75,"Name":"number of output rows","Update":"15000","Value":"15000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":78,"Name":"internal.metrics.executorDeserializeTime","Update":684,"Value":684,"Internal":true,"Count Failed Values":true},{"ID":79,"Name":"internal.metrics.executorDeserializeCpuTime","Update":608558395,"Value":608558395,"Internal":true,"Count Failed Values":true},{"ID":80,"Name":"internal.metrics.executorRunTime","Update":1573,"Value":1573,"Internal":true,"Count Failed Values":true},{"ID":81,"Name":"internal.metrics.executorCpuTime","Update":1435649112,"Value":1435649112,"Internal":true,"Count Failed Values":true},{"ID":82,"Name":"internal.metrics.resultSize","Update":1067500,"Value":1067500,"Internal":true,"Count Failed Values":true},{"ID":83,"Name":"internal.metrics.jvmGCTime","Update":131,"Value":131,"Internal":true,"Count Failed Values":true},{"ID":84,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":99,"Name":"internal.metrics.input.bytesRead","Update":655811,"Value":655811,"Internal":true,"Count Failed Values":true},{"ID":100,"Name":"internal.metrics.input.recordsRead","Update":15000,"Value":15000,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":684,"Executor Deserialize CPU Time":608558395,"Executor Run Time":1573,"Executor CPU Time":1435649112,"Peak Execution Memory":0,"Result Size":1067500,"JVM GC Time":131,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":655811,"Records Read":15000},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":1,"Index":1,"Attempt":0,"Launch Time":1633886643635,"Executor ID":"10","Host":"10.1.0.16","Locality":"ANY","Speculative":false,"Getting Result Time":1633886645952,"Finish Time":1633886645980,"Failed":false,"Killed":false,"Accumulables":[{"ID":73,"Name":"duration","Update":"190","Value":"396","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"number of output rows","Update":"15000","Value":"30000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":75,"Name":"number of output rows","Update":"15000","Value":"30000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":78,"Name":"internal.metrics.executorDeserializeTime","Update":699,"Value":1383,"Internal":true,"Count Failed Values":true},{"ID":79,"Name":"internal.metrics.executorDeserializeCpuTime","Update":631462249,"Value":1240020644,"Internal":true,"Count Failed Values":true},{"ID":80,"Name":"internal.metrics.executorRunTime","Update":1552,"Value":3125,"Internal":true,"Count Failed Values":true},{"ID":81,"Name":"internal.metrics.executorCpuTime","Update":1399479944,"Value":2835129056,"Internal":true,"Count Failed Values":true},{"ID":82,"Name":"internal.metrics.resultSize","Update":1060240,"Value":2127740,"Internal":true,"Count Failed Values":true},{"ID":83,"Name":"internal.metrics.jvmGCTime","Update":131,"Value":262,"Internal":true,"Count Failed Values":true},{"ID":84,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":99,"Name":"internal.metrics.input.bytesRead","Update":651866,"Value":1307677,"Internal":true,"Count Failed Values":true},{"ID":100,"Name":"internal.metrics.input.recordsRead","Update":15000,"Value":30000,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":699,"Executor Deserialize CPU Time":631462249,"Executor Run Time":1552,"Executor CPU Time":1399479944,"Peak Execution Memory":0,"Result Size":1060240,"JVM GC Time":131,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":651866,"Records Read":15000},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":4,"Index":4,"Attempt":0,"Launch Time":1633886643636,"Executor ID":"8","Host":"10.1.0.11","Locality":"ANY","Speculative":false,"Getting Result Time":1633886645949,"Finish Time":1633886645980,"Failed":false,"Killed":false,"Accumulables":[{"ID":73,"Name":"duration","Update":"188","Value":"584","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"number of output rows","Update":"15000","Value":"45000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":75,"Name":"number of output rows","Update":"15000","Value":"45000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":78,"Name":"internal.metrics.executorDeserializeTime","Update":692,"Value":2075,"Internal":true,"Count Failed Values":true},{"ID":79,"Name":"internal.metrics.executorDeserializeCpuTime","Update":609287687,"Value":1849308331,"Internal":true,"Count Failed Values":true},{"ID":80,"Name":"internal.metrics.executorRunTime","Update":1566,"Value":4691,"Internal":true,"Count Failed Values":true},{"ID":81,"Name":"internal.metrics.executorCpuTime","Update":1391180520,"Value":4226309576,"Internal":true,"Count Failed Values":true},{"ID":82,"Name":"internal.metrics.resultSize","Update":1062242,"Value":3189982,"Internal":true,"Count Failed Values":true},{"ID":83,"Name":"internal.metrics.jvmGCTime","Update":153,"Value":415,"Internal":true,"Count Failed Values":true},{"ID":84,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":99,"Name":"internal.metrics.input.bytesRead","Update":653437,"Value":1961114,"Internal":true,"Count Failed Values":true},{"ID":100,"Name":"internal.metrics.input.recordsRead","Update":15000,"Value":45000,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":692,"Executor Deserialize CPU Time":609287687,"Executor Run Time":1566,"Executor CPU Time":1391180520,"Peak Execution Memory":0,"Result Size":1062242,"JVM GC Time":153,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":653437,"Records Read":15000},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Launch Time":1633886643618,"Executor ID":"6","Host":"10.1.0.18","Locality":"ANY","Speculative":false,"Getting Result Time":1633886645916,"Finish Time":1633886645980,"Failed":false,"Killed":false,"Accumulables":[{"ID":73,"Name":"duration","Update":"185","Value":"769","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"number of output rows","Update":"15000","Value":"60000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":75,"Name":"number of output rows","Update":"15000","Value":"60000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":78,"Name":"internal.metrics.executorDeserializeTime","Update":692,"Value":2767,"Internal":true,"Count Failed Values":true},{"ID":79,"Name":"internal.metrics.executorDeserializeCpuTime","Update":624637032,"Value":2473945363,"Internal":true,"Count Failed Values":true},{"ID":80,"Name":"internal.metrics.executorRunTime","Update":1528,"Value":6219,"Internal":true,"Count Failed Values":true},{"ID":81,"Name":"internal.metrics.executorCpuTime","Update":1361454940,"Value":5587764516,"Internal":true,"Count Failed Values":true},{"ID":82,"Name":"internal.metrics.resultSize","Update":1063975,"Value":4253957,"Internal":true,"Count Failed Values":true},{"ID":83,"Name":"internal.metrics.jvmGCTime","Update":141,"Value":556,"Internal":true,"Count Failed Values":true},{"ID":84,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":6,"Internal":true,"Count Failed Values":true},{"ID":99,"Name":"internal.metrics.input.bytesRead","Update":652408,"Value":2613522,"Internal":true,"Count Failed Values":true},{"ID":100,"Name":"internal.metrics.input.recordsRead","Update":15000,"Value":60000,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":692,"Executor Deserialize CPU Time":624637032,"Executor Run Time":1528,"Executor CPU Time":1361454940,"Peak Execution Memory":0,"Result Size":1063975,"JVM GC Time":141,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":652408,"Records Read":15000},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskGettingResult","Task Info":{"Task ID":3,"Index":3,"Attempt":0,"Launch Time":1633886643636,"Executor ID":"9","Host":"10.1.0.14","Locality":"ANY","Speculative":false,"Getting Result Time":1633886645980,"Finish Time":1633886645998,"Failed":false,"Killed":false,"Accumulables":[{"ID":73,"Name":"duration","Update":"197","Value":"966","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"number of output rows","Update":"15000","Value":"75000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":75,"Name":"number of output rows","Update":"15000","Value":"75000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":78,"Name":"internal.metrics.executorDeserializeTime","Update":671,"Value":3438,"Internal":true,"Count Failed Values":true},{"ID":79,"Name":"internal.metrics.executorDeserializeCpuTime","Update":607404581,"Value":3081349944,"Internal":true,"Count Failed Values":true},{"ID":80,"Name":"internal.metrics.executorRunTime","Update":1598,"Value":7817,"Internal":true,"Count Failed Values":true},{"ID":81,"Name":"internal.metrics.executorCpuTime","Update":1448214525,"Value":7035979041,"Internal":true,"Count Failed Values":true},{"ID":82,"Name":"internal.metrics.resultSize","Update":1057646,"Value":5311603,"Internal":true,"Count Failed Values":true},{"ID":83,"Name":"internal.metrics.jvmGCTime","Update":146,"Value":702,"Internal":true,"Count Failed Values":true},{"ID":84,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":99,"Name":"internal.metrics.input.bytesRead","Update":650282,"Value":3263804,"Internal":true,"Count Failed Values":true},{"ID":100,"Name":"internal.metrics.input.recordsRead","Update":15000,"Value":75000,"Internal":true,"Count Failed Values":true}]}} +{"Event":"SparkListenerTaskGettingResult","Task Info":{"Task ID":2,"Index":2,"Attempt":0,"Launch Time":1633886643635,"Executor ID":"0","Host":"10.1.0.12","Locality":"ANY","Speculative":false,"Getting Result Time":1633886645981,"Finish Time":1633886646005,"Failed":false,"Killed":false,"Accumulables":[{"ID":73,"Name":"duration","Update":"203","Value":"1169","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"number of output rows","Update":"15000","Value":"90000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":75,"Name":"number of output rows","Update":"15000","Value":"90000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":78,"Name":"internal.metrics.executorDeserializeTime","Update":723,"Value":4161,"Internal":true,"Count Failed Values":true},{"ID":79,"Name":"internal.metrics.executorDeserializeCpuTime","Update":634415441,"Value":3715765385,"Internal":true,"Count Failed Values":true},{"ID":80,"Name":"internal.metrics.executorRunTime","Update":1559,"Value":9376,"Internal":true,"Count Failed Values":true},{"ID":81,"Name":"internal.metrics.executorCpuTime","Update":1423102479,"Value":8459081520,"Internal":true,"Count Failed Values":true},{"ID":82,"Name":"internal.metrics.resultSize","Update":1064422,"Value":6376025,"Internal":true,"Count Failed Values":true},{"ID":83,"Name":"internal.metrics.jvmGCTime","Update":131,"Value":833,"Internal":true,"Count Failed Values":true},{"ID":84,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":9,"Internal":true,"Count Failed Values":true},{"ID":99,"Name":"internal.metrics.input.bytesRead","Update":654511,"Value":3918315,"Internal":true,"Count Failed Values":true},{"ID":100,"Name":"internal.metrics.input.recordsRead","Update":15000,"Value":90000,"Internal":true,"Count Failed Values":true}]}} +{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":3,"Index":3,"Attempt":0,"Launch Time":1633886643636,"Executor ID":"9","Host":"10.1.0.14","Locality":"ANY","Speculative":false,"Getting Result Time":1633886645980,"Finish Time":1633886645998,"Failed":false,"Killed":false,"Accumulables":[{"ID":73,"Name":"duration","Update":"197","Value":"966","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"number of output rows","Update":"15000","Value":"75000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":75,"Name":"number of output rows","Update":"15000","Value":"75000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":78,"Name":"internal.metrics.executorDeserializeTime","Update":671,"Value":3438,"Internal":true,"Count Failed Values":true},{"ID":79,"Name":"internal.metrics.executorDeserializeCpuTime","Update":607404581,"Value":3081349944,"Internal":true,"Count Failed Values":true},{"ID":80,"Name":"internal.metrics.executorRunTime","Update":1598,"Value":7817,"Internal":true,"Count Failed Values":true},{"ID":81,"Name":"internal.metrics.executorCpuTime","Update":1448214525,"Value":7035979041,"Internal":true,"Count Failed Values":true},{"ID":82,"Name":"internal.metrics.resultSize","Update":1057646,"Value":5311603,"Internal":true,"Count Failed Values":true},{"ID":83,"Name":"internal.metrics.jvmGCTime","Update":146,"Value":702,"Internal":true,"Count Failed Values":true},{"ID":84,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":99,"Name":"internal.metrics.input.bytesRead","Update":650282,"Value":3263804,"Internal":true,"Count Failed Values":true},{"ID":100,"Name":"internal.metrics.input.recordsRead","Update":15000,"Value":75000,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":671,"Executor Deserialize CPU Time":607404581,"Executor Run Time":1598,"Executor CPU Time":1448214525,"Peak Execution Memory":0,"Result Size":1057646,"JVM GC Time":146,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":650282,"Records Read":15000},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":2,"Index":2,"Attempt":0,"Launch Time":1633886643635,"Executor ID":"0","Host":"10.1.0.12","Locality":"ANY","Speculative":false,"Getting Result Time":1633886645981,"Finish Time":1633886646005,"Failed":false,"Killed":false,"Accumulables":[{"ID":73,"Name":"duration","Update":"203","Value":"1169","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"number of output rows","Update":"15000","Value":"90000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":75,"Name":"number of output rows","Update":"15000","Value":"90000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":78,"Name":"internal.metrics.executorDeserializeTime","Update":723,"Value":4161,"Internal":true,"Count Failed Values":true},{"ID":79,"Name":"internal.metrics.executorDeserializeCpuTime","Update":634415441,"Value":3715765385,"Internal":true,"Count Failed Values":true},{"ID":80,"Name":"internal.metrics.executorRunTime","Update":1559,"Value":9376,"Internal":true,"Count Failed Values":true},{"ID":81,"Name":"internal.metrics.executorCpuTime","Update":1423102479,"Value":8459081520,"Internal":true,"Count Failed Values":true},{"ID":82,"Name":"internal.metrics.resultSize","Update":1064422,"Value":6376025,"Internal":true,"Count Failed Values":true},{"ID":83,"Name":"internal.metrics.jvmGCTime","Update":131,"Value":833,"Internal":true,"Count Failed Values":true},{"ID":84,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":9,"Internal":true,"Count Failed Values":true},{"ID":99,"Name":"internal.metrics.input.bytesRead","Update":654511,"Value":3918315,"Internal":true,"Count Failed Values":true},{"ID":100,"Name":"internal.metrics.input.recordsRead","Update":15000,"Value":90000,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":723,"Executor Deserialize CPU Time":634415441,"Executor Run Time":1559,"Executor CPU Time":1423102479,"Peak Execution Memory":0,"Result Size":1064422,"JVM GC Time":131,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":654511,"Records Read":15000},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskGettingResult","Task Info":{"Task ID":5,"Index":5,"Attempt":0,"Launch Time":1633886643637,"Executor ID":"11","Host":"10.1.0.13","Locality":"ANY","Speculative":false,"Getting Result Time":1633886646029,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskGettingResult","Task Info":{"Task ID":7,"Index":7,"Attempt":0,"Launch Time":1633886643638,"Executor ID":"2","Host":"10.1.0.10","Locality":"ANY","Speculative":false,"Getting Result Time":1633886646035,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":5,"Index":5,"Attempt":0,"Launch Time":1633886643637,"Executor ID":"11","Host":"10.1.0.13","Locality":"ANY","Speculative":false,"Getting Result Time":1633886646029,"Finish Time":1633886646053,"Failed":false,"Killed":false,"Accumulables":[{"ID":73,"Name":"duration","Update":"182","Value":"1351","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"number of output rows","Update":"15000","Value":"105000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":75,"Name":"number of output rows","Update":"15000","Value":"105000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":78,"Name":"internal.metrics.executorDeserializeTime","Update":702,"Value":4863,"Internal":true,"Count Failed Values":true},{"ID":79,"Name":"internal.metrics.executorDeserializeCpuTime","Update":632593447,"Value":4348358832,"Internal":true,"Count Failed Values":true},{"ID":80,"Name":"internal.metrics.executorRunTime","Update":1633,"Value":11009,"Internal":true,"Count Failed Values":true},{"ID":81,"Name":"internal.metrics.executorCpuTime","Update":1479670691,"Value":9938752211,"Internal":true,"Count Failed Values":true},{"ID":82,"Name":"internal.metrics.resultSize","Update":1059139,"Value":7435164,"Internal":true,"Count Failed Values":true},{"ID":83,"Name":"internal.metrics.jvmGCTime","Update":148,"Value":981,"Internal":true,"Count Failed Values":true},{"ID":84,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":10,"Internal":true,"Count Failed Values":true},{"ID":99,"Name":"internal.metrics.input.bytesRead","Update":651025,"Value":4569340,"Internal":true,"Count Failed Values":true},{"ID":100,"Name":"internal.metrics.input.recordsRead","Update":15000,"Value":105000,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":702,"Executor Deserialize CPU Time":632593447,"Executor Run Time":1633,"Executor CPU Time":1479670691,"Peak Execution Memory":0,"Result Size":1059139,"JVM GC Time":148,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":651025,"Records Read":15000},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":7,"Index":7,"Attempt":0,"Launch Time":1633886643638,"Executor ID":"2","Host":"10.1.0.10","Locality":"ANY","Speculative":false,"Getting Result Time":1633886646035,"Finish Time":1633886646056,"Failed":false,"Killed":false,"Accumulables":[{"ID":73,"Name":"duration","Update":"181","Value":"1532","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"number of output rows","Update":"15000","Value":"120000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":75,"Name":"number of output rows","Update":"15000","Value":"120000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":78,"Name":"internal.metrics.executorDeserializeTime","Update":692,"Value":5555,"Internal":true,"Count Failed Values":true},{"ID":79,"Name":"internal.metrics.executorDeserializeCpuTime","Update":607426390,"Value":4955785222,"Internal":true,"Count Failed Values":true},{"ID":80,"Name":"internal.metrics.executorRunTime","Update":1654,"Value":12663,"Internal":true,"Count Failed Values":true},{"ID":81,"Name":"internal.metrics.executorCpuTime","Update":1490277458,"Value":11429029669,"Internal":true,"Count Failed Values":true},{"ID":82,"Name":"internal.metrics.resultSize","Update":1058522,"Value":8493686,"Internal":true,"Count Failed Values":true},{"ID":83,"Name":"internal.metrics.jvmGCTime","Update":147,"Value":1128,"Internal":true,"Count Failed Values":true},{"ID":84,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":11,"Internal":true,"Count Failed Values":true},{"ID":99,"Name":"internal.metrics.input.bytesRead","Update":650945,"Value":5220285,"Internal":true,"Count Failed Values":true},{"ID":100,"Name":"internal.metrics.input.recordsRead","Update":15000,"Value":120000,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":692,"Executor Deserialize CPU Time":607426390,"Executor Run Time":1654,"Executor CPU Time":1490277458,"Peak Execution Memory":0,"Result Size":1058522,"JVM GC Time":147,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":650945,"Records Read":15000},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Number of Tasks":8,"RDD Info":[{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"2\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"Scan hive default.rankings\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"HadoopRDD","Scope":"{\"id\":\"7\",\"name\":\"Scan hive default.rankings\"}","Callsite":"","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"Scan hive default.rankings\"}","Callsite":"","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"Scan hive default.rankings\"}","Callsite":"","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:185)\njava.util.concurrent.FutureTask.run(FutureTask.java:266)\njava.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\njava.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1633886643571,"Completion Time":1633886646058,"Accumulables":[{"ID":73,"Name":"duration","Value":"1532","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"number of output rows","Value":"120000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":75,"Name":"number of output rows","Value":"120000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":78,"Name":"internal.metrics.executorDeserializeTime","Value":5555,"Internal":true,"Count Failed Values":true},{"ID":79,"Name":"internal.metrics.executorDeserializeCpuTime","Value":4955785222,"Internal":true,"Count Failed Values":true},{"ID":80,"Name":"internal.metrics.executorRunTime","Value":12663,"Internal":true,"Count Failed Values":true},{"ID":81,"Name":"internal.metrics.executorCpuTime","Value":11429029669,"Internal":true,"Count Failed Values":true},{"ID":82,"Name":"internal.metrics.resultSize","Value":8493686,"Internal":true,"Count Failed Values":true},{"ID":83,"Name":"internal.metrics.jvmGCTime","Value":1128,"Internal":true,"Count Failed Values":true},{"ID":84,"Name":"internal.metrics.resultSerializationTime","Value":11,"Internal":true,"Count Failed Values":true},{"ID":99,"Name":"internal.metrics.input.bytesRead","Value":5220285,"Internal":true,"Count Failed Values":true},{"ID":100,"Name":"internal.metrics.input.recordsRead","Value":120000,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":0,"Completion Time":1633886646065,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":11,"accumUpdates":[[72,111],[71,169],[70,3223],[69,120000],[68,29360128]]} +{"Event":"SparkListenerJobStart","Job ID":1,"Submission Time":1633886646652,"Stage Infos":[{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"sql at ScalaSparkSQLBench.scala:43","Number of Tasks":8,"RDD Info":[{"RDD ID":11,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"12\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[10],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":10,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"13\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[9],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":6,"Name":"HadoopRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":2,"Stage Attempt ID":0,"Stage Name":"sql at ScalaSparkSQLBench.scala:43","Number of Tasks":8,"RDD Info":[{"RDD ID":16,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[15],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":15,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[14],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":14,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":12,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"12\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[11],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[1],"Details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[1,2],"Properties":{"spark.sql.warehouse.dir":"file:/home/usera112/projects/log-compressor/deployments/HiBench/spark-warehouse","spark.driver.host":"hostb7.dsrg.utoronto.ca","spark.eventLog.enabled":"true","spark.driver.port":"42853","__fetch_continuous_blocks_in_batch_enabled":"true","spark.jars":"file:/home/usera112/projects/log-compressor/deployments/HiBench/sparkbench/assembly/target/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.app.name":"ScalaJoin","spark.rdd.scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","spark.driver.memory":"4g","spark.rdd.scope.noOverride":"true","spark.submit.pyFiles":"","spark.default.parallelism":"8","spark.app.startTime":"1633886611309","spark.executor.id":"driver","spark.app.initial.jar.urls":"spark://hostb7.dsrg.utoronto.ca:42853/jars/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.submit.deployMode":"client","spark.master":"spark://hostb7:7077","spark.executor.memory":"4g","spark.eventLog.dir":"hdfs://hostb7:8020/directory","spark.sql.execution.id":"11","spark.sql.catalogImplementation":"hive","spark.app.id":"app-20211010132332-4636","spark.sql.shuffle.partitions":"8"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"sql at ScalaSparkSQLBench.scala:43","Number of Tasks":8,"RDD Info":[{"RDD ID":11,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"12\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[10],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":10,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"13\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[9],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":6,"Name":"HadoopRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1633886646659,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.sql.warehouse.dir":"file:/home/usera112/projects/log-compressor/deployments/HiBench/spark-warehouse","spark.driver.host":"hostb7.dsrg.utoronto.ca","spark.eventLog.enabled":"true","spark.driver.port":"42853","__fetch_continuous_blocks_in_batch_enabled":"true","spark.jars":"file:/home/usera112/projects/log-compressor/deployments/HiBench/sparkbench/assembly/target/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.app.name":"ScalaJoin","spark.rdd.scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","spark.driver.memory":"4g","spark.rdd.scope.noOverride":"true","spark.submit.pyFiles":"","spark.default.parallelism":"8","spark.app.startTime":"1633886611309","spark.executor.id":"driver","spark.app.initial.jar.urls":"spark://hostb7.dsrg.utoronto.ca:42853/jars/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.submit.deployMode":"client","spark.master":"spark://hostb7:7077","spark.executor.memory":"4g","spark.eventLog.dir":"hdfs://hostb7:8020/directory","spark.sql.execution.id":"11","spark.sql.catalogImplementation":"hive","spark.app.id":"app-20211010132332-4636","spark.sql.shuffle.partitions":"8"}} +{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":8,"Index":0,"Attempt":0,"Launch Time":1633886646669,"Executor ID":"5","Host":"10.1.0.19","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":9,"Index":1,"Attempt":0,"Launch Time":1633886646670,"Executor ID":"2","Host":"10.1.0.10","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":10,"Index":2,"Attempt":0,"Launch Time":1633886646671,"Executor ID":"9","Host":"10.1.0.14","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":11,"Index":3,"Attempt":0,"Launch Time":1633886646671,"Executor ID":"3","Host":"10.1.0.15","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":12,"Index":4,"Attempt":0,"Launch Time":1633886646671,"Executor ID":"11","Host":"10.1.0.13","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":13,"Index":5,"Attempt":0,"Launch Time":1633886646672,"Executor ID":"6","Host":"10.1.0.18","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":14,"Index":6,"Attempt":0,"Launch Time":1633886646672,"Executor ID":"0","Host":"10.1.0.12","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":15,"Index":7,"Attempt":0,"Launch Time":1633886646672,"Executor ID":"10","Host":"10.1.0.16","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":12,"Index":4,"Attempt":0,"Launch Time":1633886646671,"Executor ID":"11","Host":"10.1.0.13","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":1633886648378,"Failed":false,"Killed":false,"Accumulables":[{"ID":25,"Name":"data size","Update":"161944","Value":"161944","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"shuffle bytes written","Update":"81819","Value":"81819","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"shuffle records written","Update":"2892","Value":"2892","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":35,"Name":"shuffle write time","Update":"7745450","Value":"7745450","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":61,"Name":"duration","Update":"1314","Value":"1314","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":62,"Name":"number of output rows","Update":"2892","Value":"2892","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":63,"Name":"peak memory","Update":"4456448","Value":"4456448","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":65,"Name":"time in aggregation build","Update":"999","Value":"999","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":66,"Name":"avg hash probe bucket list iters","Update":"10","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":67,"Name":"number of output rows","Update":"2892","Value":"2892","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":76,"Name":"number of output rows","Update":"2892","Value":"2892","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":77,"Name":"number of output rows","Update":"124603","Value":"124603","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":103,"Name":"internal.metrics.executorDeserializeTime","Update":194,"Value":194,"Internal":true,"Count Failed Values":true},{"ID":104,"Name":"internal.metrics.executorDeserializeCpuTime","Update":187312873,"Value":187312873,"Internal":true,"Count Failed Values":true},{"ID":105,"Name":"internal.metrics.executorRunTime","Update":1497,"Value":1497,"Internal":true,"Count Failed Values":true},{"ID":106,"Name":"internal.metrics.executorCpuTime","Update":1391505517,"Value":1391505517,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.resultSize","Update":2905,"Value":2905,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.jvmGCTime","Update":36,"Value":36,"Internal":true,"Count Failed Values":true},{"ID":112,"Name":"internal.metrics.peakExecutionMemory","Update":33816576,"Value":33816576,"Internal":true,"Count Failed Values":true},{"ID":121,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":81819,"Value":81819,"Internal":true,"Count Failed Values":true},{"ID":122,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":2892,"Value":2892,"Internal":true,"Count Failed Values":true},{"ID":123,"Name":"internal.metrics.shuffle.write.writeTime","Update":7745450,"Value":7745450,"Internal":true,"Count Failed Values":true},{"ID":124,"Name":"internal.metrics.input.bytesRead","Update":23064712,"Value":23064712,"Internal":true,"Count Failed Values":true},{"ID":125,"Name":"internal.metrics.input.recordsRead","Update":124603,"Value":124603,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":194,"Executor Deserialize CPU Time":187312873,"Executor Run Time":1497,"Executor CPU Time":1391505517,"Peak Execution Memory":33816576,"Result Size":2905,"JVM GC Time":36,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":81819,"Shuffle Write Time":7745450,"Shuffle Records Written":2892},"Input Metrics":{"Bytes Read":23064712,"Records Read":124603},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":15,"Index":7,"Attempt":0,"Launch Time":1633886646672,"Executor ID":"10","Host":"10.1.0.16","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":1633886648402,"Failed":false,"Killed":false,"Accumulables":[{"ID":25,"Name":"data size","Update":"167832","Value":"329776","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"shuffle bytes written","Update":"84637","Value":"166456","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"shuffle records written","Update":"2997","Value":"5889","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":35,"Name":"shuffle write time","Update":"7781822","Value":"15527272","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":61,"Name":"duration","Update":"1336","Value":"2650","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":62,"Name":"number of output rows","Update":"2997","Value":"5889","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":63,"Name":"peak memory","Update":"4456448","Value":"8912896","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":65,"Name":"time in aggregation build","Update":"1009","Value":"2008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":66,"Name":"avg hash probe bucket list iters","Update":"10","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":67,"Name":"number of output rows","Update":"2997","Value":"5889","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":76,"Name":"number of output rows","Update":"2997","Value":"5889","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":77,"Name":"number of output rows","Update":"125488","Value":"250091","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":103,"Name":"internal.metrics.executorDeserializeTime","Update":191,"Value":385,"Internal":true,"Count Failed Values":true},{"ID":104,"Name":"internal.metrics.executorDeserializeCpuTime","Update":184296885,"Value":371609758,"Internal":true,"Count Failed Values":true},{"ID":105,"Name":"internal.metrics.executorRunTime","Update":1526,"Value":3023,"Internal":true,"Count Failed Values":true},{"ID":106,"Name":"internal.metrics.executorCpuTime","Update":1410301401,"Value":2801806918,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.resultSize","Update":2905,"Value":5810,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.jvmGCTime","Update":38,"Value":74,"Internal":true,"Count Failed Values":true},{"ID":112,"Name":"internal.metrics.peakExecutionMemory","Update":33816576,"Value":67633152,"Internal":true,"Count Failed Values":true},{"ID":121,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":84637,"Value":166456,"Internal":true,"Count Failed Values":true},{"ID":122,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":2997,"Value":5889,"Internal":true,"Count Failed Values":true},{"ID":123,"Name":"internal.metrics.shuffle.write.writeTime","Update":7781822,"Value":15527272,"Internal":true,"Count Failed Values":true},{"ID":124,"Name":"internal.metrics.input.bytesRead","Update":23178313,"Value":46243025,"Internal":true,"Count Failed Values":true},{"ID":125,"Name":"internal.metrics.input.recordsRead","Update":125488,"Value":250091,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":191,"Executor Deserialize CPU Time":184296885,"Executor Run Time":1526,"Executor CPU Time":1410301401,"Peak Execution Memory":33816576,"Result Size":2905,"JVM GC Time":38,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":84637,"Shuffle Write Time":7781822,"Shuffle Records Written":2997},"Input Metrics":{"Bytes Read":23178313,"Records Read":125488},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":9,"Index":1,"Attempt":0,"Launch Time":1633886646670,"Executor ID":"2","Host":"10.1.0.10","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":1633886648410,"Failed":false,"Killed":false,"Accumulables":[{"ID":25,"Name":"data size","Update":"165920","Value":"495696","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"shuffle bytes written","Update":"83472","Value":"249928","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"shuffle records written","Update":"2963","Value":"8852","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":35,"Name":"shuffle write time","Update":"7910392","Value":"23437664","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":61,"Name":"duration","Update":"1353","Value":"4003","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":62,"Name":"number of output rows","Update":"2963","Value":"8852","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":63,"Name":"peak memory","Update":"4456448","Value":"13369344","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":65,"Name":"time in aggregation build","Update":"1030","Value":"3038","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":66,"Name":"avg hash probe bucket list iters","Update":"10","Value":"30","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":67,"Name":"number of output rows","Update":"2963","Value":"8852","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":76,"Name":"number of output rows","Update":"2963","Value":"8852","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":77,"Name":"number of output rows","Update":"125061","Value":"375152","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":103,"Name":"internal.metrics.executorDeserializeTime","Update":196,"Value":581,"Internal":true,"Count Failed Values":true},{"ID":104,"Name":"internal.metrics.executorDeserializeCpuTime","Update":189280000,"Value":560889758,"Internal":true,"Count Failed Values":true},{"ID":105,"Name":"internal.metrics.executorRunTime","Update":1531,"Value":4554,"Internal":true,"Count Failed Values":true},{"ID":106,"Name":"internal.metrics.executorCpuTime","Update":1417856615,"Value":4219663533,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.resultSize","Update":2904,"Value":8714,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.jvmGCTime","Update":36,"Value":110,"Internal":true,"Count Failed Values":true},{"ID":112,"Name":"internal.metrics.peakExecutionMemory","Update":33816576,"Value":101449728,"Internal":true,"Count Failed Values":true},{"ID":121,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":83472,"Value":249928,"Internal":true,"Count Failed Values":true},{"ID":122,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":2963,"Value":8852,"Internal":true,"Count Failed Values":true},{"ID":123,"Name":"internal.metrics.shuffle.write.writeTime","Update":7910392,"Value":23437664,"Internal":true,"Count Failed Values":true},{"ID":124,"Name":"internal.metrics.input.bytesRead","Update":23125676,"Value":69368701,"Internal":true,"Count Failed Values":true},{"ID":125,"Name":"internal.metrics.input.recordsRead","Update":125061,"Value":375152,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":196,"Executor Deserialize CPU Time":189280000,"Executor Run Time":1531,"Executor CPU Time":1417856615,"Peak Execution Memory":33816576,"Result Size":2904,"JVM GC Time":36,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":83472,"Shuffle Write Time":7910392,"Shuffle Records Written":2963},"Input Metrics":{"Bytes Read":23125676,"Records Read":125061},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":14,"Index":6,"Attempt":0,"Launch Time":1633886646672,"Executor ID":"0","Host":"10.1.0.12","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":1633886648439,"Failed":false,"Killed":false,"Accumulables":[{"ID":25,"Name":"data size","Update":"169456","Value":"665152","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"shuffle bytes written","Update":"85405","Value":"335333","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"shuffle records written","Update":"3026","Value":"11878","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":35,"Name":"shuffle write time","Update":"9110342","Value":"32548006","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":61,"Name":"duration","Update":"1344","Value":"5347","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":62,"Name":"number of output rows","Update":"3026","Value":"11878","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":63,"Name":"peak memory","Update":"4456448","Value":"17825792","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":65,"Name":"time in aggregation build","Update":"1018","Value":"4056","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":66,"Name":"avg hash probe bucket list iters","Update":"10","Value":"40","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":67,"Name":"number of output rows","Update":"3026","Value":"11878","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":76,"Name":"number of output rows","Update":"3026","Value":"11878","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":77,"Name":"number of output rows","Update":"125342","Value":"500494","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":103,"Name":"internal.metrics.executorDeserializeTime","Update":204,"Value":785,"Internal":true,"Count Failed Values":true},{"ID":104,"Name":"internal.metrics.executorDeserializeCpuTime","Update":197699444,"Value":758589202,"Internal":true,"Count Failed Values":true},{"ID":105,"Name":"internal.metrics.executorRunTime","Update":1549,"Value":6103,"Internal":true,"Count Failed Values":true},{"ID":106,"Name":"internal.metrics.executorCpuTime","Update":1437927392,"Value":5657590925,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.resultSize","Update":2904,"Value":11618,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.jvmGCTime","Update":37,"Value":147,"Internal":true,"Count Failed Values":true},{"ID":112,"Name":"internal.metrics.peakExecutionMemory","Update":33816576,"Value":135266304,"Internal":true,"Count Failed Values":true},{"ID":121,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":85405,"Value":335333,"Internal":true,"Count Failed Values":true},{"ID":122,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":3026,"Value":11878,"Internal":true,"Count Failed Values":true},{"ID":123,"Name":"internal.metrics.shuffle.write.writeTime","Update":9110342,"Value":32548006,"Internal":true,"Count Failed Values":true},{"ID":124,"Name":"internal.metrics.input.bytesRead","Update":23247031,"Value":92615732,"Internal":true,"Count Failed Values":true},{"ID":125,"Name":"internal.metrics.input.recordsRead","Update":125342,"Value":500494,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":204,"Executor Deserialize CPU Time":197699444,"Executor Run Time":1549,"Executor CPU Time":1437927392,"Peak Execution Memory":33816576,"Result Size":2904,"JVM GC Time":37,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":85405,"Shuffle Write Time":9110342,"Shuffle Records Written":3026},"Input Metrics":{"Bytes Read":23247031,"Records Read":125342},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":10,"Index":2,"Attempt":0,"Launch Time":1633886646671,"Executor ID":"9","Host":"10.1.0.14","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":1633886648463,"Failed":false,"Killed":false,"Accumulables":[{"ID":25,"Name":"data size","Update":"165872","Value":"831024","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"shuffle bytes written","Update":"83597","Value":"418930","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"shuffle records written","Update":"2962","Value":"14840","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":35,"Name":"shuffle write time","Update":"7636066","Value":"40184072","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":61,"Name":"duration","Update":"1391","Value":"6738","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":62,"Name":"number of output rows","Update":"2962","Value":"14840","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":63,"Name":"peak memory","Update":"4456448","Value":"22282240","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":65,"Name":"time in aggregation build","Update":"1061","Value":"5117","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":66,"Name":"avg hash probe bucket list iters","Update":"10","Value":"50","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":67,"Name":"number of output rows","Update":"2962","Value":"14840","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":76,"Name":"number of output rows","Update":"2962","Value":"14840","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":77,"Name":"number of output rows","Update":"125003","Value":"625497","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":103,"Name":"internal.metrics.executorDeserializeTime","Update":189,"Value":974,"Internal":true,"Count Failed Values":true},{"ID":104,"Name":"internal.metrics.executorDeserializeCpuTime","Update":184195217,"Value":942784419,"Internal":true,"Count Failed Values":true},{"ID":105,"Name":"internal.metrics.executorRunTime","Update":1590,"Value":7693,"Internal":true,"Count Failed Values":true},{"ID":106,"Name":"internal.metrics.executorCpuTime","Update":1481100516,"Value":7138691441,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.resultSize","Update":2904,"Value":14522,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.jvmGCTime","Update":37,"Value":184,"Internal":true,"Count Failed Values":true},{"ID":112,"Name":"internal.metrics.peakExecutionMemory","Update":33816576,"Value":169082880,"Internal":true,"Count Failed Values":true},{"ID":121,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":83597,"Value":418930,"Internal":true,"Count Failed Values":true},{"ID":122,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":2962,"Value":14840,"Internal":true,"Count Failed Values":true},{"ID":123,"Name":"internal.metrics.shuffle.write.writeTime","Update":7636066,"Value":40184072,"Internal":true,"Count Failed Values":true},{"ID":124,"Name":"internal.metrics.input.bytesRead","Update":23143498,"Value":115759230,"Internal":true,"Count Failed Values":true},{"ID":125,"Name":"internal.metrics.input.recordsRead","Update":125003,"Value":625497,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":189,"Executor Deserialize CPU Time":184195217,"Executor Run Time":1590,"Executor CPU Time":1481100516,"Peak Execution Memory":33816576,"Result Size":2904,"JVM GC Time":37,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":83597,"Shuffle Write Time":7636066,"Shuffle Records Written":2962},"Input Metrics":{"Bytes Read":23143498,"Records Read":125003},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":8,"Index":0,"Attempt":0,"Launch Time":1633886646669,"Executor ID":"5","Host":"10.1.0.19","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":1633886648477,"Failed":false,"Killed":false,"Accumulables":[{"ID":25,"Name":"data size","Update":"158088","Value":"989112","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"shuffle bytes written","Update":"79832","Value":"498762","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"shuffle records written","Update":"2823","Value":"17663","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":35,"Name":"shuffle write time","Update":"8489312","Value":"48673384","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":61,"Name":"duration","Update":"1391","Value":"8129","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":62,"Name":"number of output rows","Update":"2823","Value":"17663","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":63,"Name":"peak memory","Update":"262144","Value":"22544384","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":65,"Name":"time in aggregation build","Update":"1048","Value":"6165","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":67,"Name":"number of output rows","Update":"2823","Value":"17663","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":76,"Name":"number of output rows","Update":"2823","Value":"17663","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":77,"Name":"number of output rows","Update":"124858","Value":"750355","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":103,"Name":"internal.metrics.executorDeserializeTime","Update":198,"Value":1172,"Internal":true,"Count Failed Values":true},{"ID":104,"Name":"internal.metrics.executorDeserializeCpuTime","Update":190304790,"Value":1133089209,"Internal":true,"Count Failed Values":true},{"ID":105,"Name":"internal.metrics.executorRunTime","Update":1594,"Value":9287,"Internal":true,"Count Failed Values":true},{"ID":106,"Name":"internal.metrics.executorCpuTime","Update":1480262889,"Value":8618954330,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.resultSize","Update":2904,"Value":17426,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.jvmGCTime","Update":37,"Value":221,"Internal":true,"Count Failed Values":true},{"ID":112,"Name":"internal.metrics.peakExecutionMemory","Update":29622272,"Value":198705152,"Internal":true,"Count Failed Values":true},{"ID":121,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":79832,"Value":498762,"Internal":true,"Count Failed Values":true},{"ID":122,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":2823,"Value":17663,"Internal":true,"Count Failed Values":true},{"ID":123,"Name":"internal.metrics.shuffle.write.writeTime","Update":8489312,"Value":48673384,"Internal":true,"Count Failed Values":true},{"ID":124,"Name":"internal.metrics.input.bytesRead","Update":23122020,"Value":138881250,"Internal":true,"Count Failed Values":true},{"ID":125,"Name":"internal.metrics.input.recordsRead","Update":124858,"Value":750355,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":198,"Executor Deserialize CPU Time":190304790,"Executor Run Time":1594,"Executor CPU Time":1480262889,"Peak Execution Memory":29622272,"Result Size":2904,"JVM GC Time":37,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":79832,"Shuffle Write Time":8489312,"Shuffle Records Written":2823},"Input Metrics":{"Bytes Read":23122020,"Records Read":124858},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":13,"Index":5,"Attempt":0,"Launch Time":1633886646672,"Executor ID":"6","Host":"10.1.0.18","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":1633886648485,"Failed":false,"Killed":false,"Accumulables":[{"ID":25,"Name":"data size","Update":"170848","Value":"1159960","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"shuffle bytes written","Update":"85986","Value":"584748","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"shuffle records written","Update":"3051","Value":"20714","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":35,"Name":"shuffle write time","Update":"9141949","Value":"57815333","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":61,"Name":"duration","Update":"1393","Value":"9522","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":62,"Name":"number of output rows","Update":"3051","Value":"20714","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":63,"Name":"peak memory","Update":"4456448","Value":"27000832","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":65,"Name":"time in aggregation build","Update":"1049","Value":"7214","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":66,"Name":"avg hash probe bucket list iters","Update":"10","Value":"60","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":67,"Name":"number of output rows","Update":"3051","Value":"20714","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":76,"Name":"number of output rows","Update":"3051","Value":"20714","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":77,"Name":"number of output rows","Update":"124533","Value":"874888","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":103,"Name":"internal.metrics.executorDeserializeTime","Update":203,"Value":1375,"Internal":true,"Count Failed Values":true},{"ID":104,"Name":"internal.metrics.executorDeserializeCpuTime","Update":189570867,"Value":1322660076,"Internal":true,"Count Failed Values":true},{"ID":105,"Name":"internal.metrics.executorRunTime","Update":1597,"Value":10884,"Internal":true,"Count Failed Values":true},{"ID":106,"Name":"internal.metrics.executorCpuTime","Update":1475238135,"Value":10094192465,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.resultSize","Update":2904,"Value":20330,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.jvmGCTime","Update":38,"Value":259,"Internal":true,"Count Failed Values":true},{"ID":112,"Name":"internal.metrics.peakExecutionMemory","Update":33816576,"Value":232521728,"Internal":true,"Count Failed Values":true},{"ID":121,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":85986,"Value":584748,"Internal":true,"Count Failed Values":true},{"ID":122,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":3051,"Value":20714,"Internal":true,"Count Failed Values":true},{"ID":123,"Name":"internal.metrics.shuffle.write.writeTime","Update":9141949,"Value":57815333,"Internal":true,"Count Failed Values":true},{"ID":124,"Name":"internal.metrics.input.bytesRead","Update":23014586,"Value":161895836,"Internal":true,"Count Failed Values":true},{"ID":125,"Name":"internal.metrics.input.recordsRead","Update":124533,"Value":874888,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":119677952,"JVMOffHeapMemory":86570864,"OnHeapExecutionMemory":4456448,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":40864795,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":45321243,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1171015,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":4,"MinorGCTime":42,"MajorGCCount":3,"MajorGCTime":145},"Task Metrics":{"Executor Deserialize Time":203,"Executor Deserialize CPU Time":189570867,"Executor Run Time":1597,"Executor CPU Time":1475238135,"Peak Execution Memory":33816576,"Result Size":2904,"JVM GC Time":38,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":85986,"Shuffle Write Time":9141949,"Shuffle Records Written":3051},"Input Metrics":{"Bytes Read":23014586,"Records Read":124533},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":11,"Index":3,"Attempt":0,"Launch Time":1633886646671,"Executor ID":"3","Host":"10.1.0.15","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":1633886650484,"Failed":false,"Killed":false,"Accumulables":[{"ID":25,"Name":"data size","Update":"161504","Value":"1321464","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"shuffle bytes written","Update":"81474","Value":"666222","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"shuffle records written","Update":"2884","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":35,"Name":"shuffle write time","Update":"9766392","Value":"67581725","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":61,"Name":"duration","Update":"1536","Value":"11058","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":62,"Name":"number of output rows","Update":"2884","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":63,"Name":"peak memory","Update":"4456448","Value":"31457280","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":65,"Name":"time in aggregation build","Update":"1168","Value":"8382","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":66,"Name":"avg hash probe bucket list iters","Update":"10","Value":"70","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":67,"Name":"number of output rows","Update":"2884","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":76,"Name":"number of output rows","Update":"2884","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":77,"Name":"number of output rows","Update":"125112","Value":"1000000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":103,"Name":"internal.metrics.executorDeserializeTime","Update":901,"Value":2276,"Internal":true,"Count Failed Values":true},{"ID":104,"Name":"internal.metrics.executorDeserializeCpuTime","Update":831523308,"Value":2154183384,"Internal":true,"Count Failed Values":true},{"ID":105,"Name":"internal.metrics.executorRunTime","Update":2873,"Value":13757,"Internal":true,"Count Failed Values":true},{"ID":106,"Name":"internal.metrics.executorCpuTime","Update":2636702759,"Value":12730895224,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.resultSize","Update":2904,"Value":23234,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.jvmGCTime","Update":168,"Value":427,"Internal":true,"Count Failed Values":true},{"ID":112,"Name":"internal.metrics.peakExecutionMemory","Update":33816576,"Value":266338304,"Internal":true,"Count Failed Values":true},{"ID":121,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":81474,"Value":666222,"Internal":true,"Count Failed Values":true},{"ID":122,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":2884,"Value":23598,"Internal":true,"Count Failed Values":true},{"ID":123,"Name":"internal.metrics.shuffle.write.writeTime","Update":9766392,"Value":67581725,"Internal":true,"Count Failed Values":true},{"ID":124,"Name":"internal.metrics.input.bytesRead","Update":23103793,"Value":184999629,"Internal":true,"Count Failed Values":true},{"ID":125,"Name":"internal.metrics.input.recordsRead","Update":125112,"Value":1000000,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":901,"Executor Deserialize CPU Time":831523308,"Executor Run Time":2873,"Executor CPU Time":2636702759,"Peak Execution Memory":33816576,"Result Size":2904,"JVM GC Time":168,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":81474,"Shuffle Write Time":9766392,"Shuffle Records Written":2884},"Input Metrics":{"Bytes Read":23103793,"Records Read":125112},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"sql at ScalaSparkSQLBench.scala:43","Number of Tasks":8,"RDD Info":[{"RDD ID":11,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"12\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[10],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":10,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"13\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[9],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":6,"Name":"HadoopRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1633886646659,"Completion Time":1633886650485,"Accumulables":[{"ID":25,"Name":"data size","Value":"1321464","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"shuffle bytes written","Value":"666222","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"shuffle records written","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":35,"Name":"shuffle write time","Value":"67581725","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":61,"Name":"duration","Value":"11058","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":62,"Name":"number of output rows","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":63,"Name":"peak memory","Value":"31457280","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":65,"Name":"time in aggregation build","Value":"8382","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":66,"Name":"avg hash probe bucket list iters","Value":"70","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":67,"Name":"number of output rows","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":76,"Name":"number of output rows","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":77,"Name":"number of output rows","Value":"1000000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":103,"Name":"internal.metrics.executorDeserializeTime","Value":2276,"Internal":true,"Count Failed Values":true},{"ID":104,"Name":"internal.metrics.executorDeserializeCpuTime","Value":2154183384,"Internal":true,"Count Failed Values":true},{"ID":105,"Name":"internal.metrics.executorRunTime","Value":13757,"Internal":true,"Count Failed Values":true},{"ID":106,"Name":"internal.metrics.executorCpuTime","Value":12730895224,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.resultSize","Value":23234,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.jvmGCTime","Value":427,"Internal":true,"Count Failed Values":true},{"ID":112,"Name":"internal.metrics.peakExecutionMemory","Value":266338304,"Internal":true,"Count Failed Values":true},{"ID":121,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":666222,"Internal":true,"Count Failed Values":true},{"ID":122,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":23598,"Internal":true,"Count Failed Values":true},{"ID":123,"Name":"internal.metrics.shuffle.write.writeTime","Value":67581725,"Internal":true,"Count Failed Values":true},{"ID":124,"Name":"internal.metrics.input.bytesRead","Value":184999629,"Internal":true,"Count Failed Values":true},{"ID":125,"Name":"internal.metrics.input.recordsRead","Value":1000000,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":2,"Stage Attempt ID":0,"Stage Name":"sql at ScalaSparkSQLBench.scala:43","Number of Tasks":8,"RDD Info":[{"RDD ID":16,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[15],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":15,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[14],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":14,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":12,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"12\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[11],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[1],"Details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1633886650501,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.sql.warehouse.dir":"file:/home/usera112/projects/log-compressor/deployments/HiBench/spark-warehouse","spark.driver.host":"hostb7.dsrg.utoronto.ca","spark.eventLog.enabled":"true","spark.driver.port":"42853","__fetch_continuous_blocks_in_batch_enabled":"true","spark.jars":"file:/home/usera112/projects/log-compressor/deployments/HiBench/sparkbench/assembly/target/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.app.name":"ScalaJoin","spark.rdd.scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","spark.driver.memory":"4g","spark.rdd.scope.noOverride":"true","spark.submit.pyFiles":"","spark.default.parallelism":"8","spark.app.startTime":"1633886611309","spark.executor.id":"driver","spark.app.initial.jar.urls":"spark://hostb7.dsrg.utoronto.ca:42853/jars/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.submit.deployMode":"client","spark.master":"spark://hostb7:7077","spark.executor.memory":"4g","spark.eventLog.dir":"hdfs://hostb7:8020/directory","spark.sql.execution.id":"11","spark.sql.catalogImplementation":"hive","spark.app.id":"app-20211010132332-4636","spark.sql.shuffle.partitions":"8"}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":16,"Index":0,"Attempt":0,"Launch Time":1633886650509,"Executor ID":"6","Host":"10.1.0.18","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":17,"Index":1,"Attempt":0,"Launch Time":1633886650511,"Executor ID":"3","Host":"10.1.0.15","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":18,"Index":2,"Attempt":0,"Launch Time":1633886650511,"Executor ID":"0","Host":"10.1.0.12","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":19,"Index":3,"Attempt":0,"Launch Time":1633886650511,"Executor ID":"7","Host":"10.1.0.9","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":20,"Index":4,"Attempt":0,"Launch Time":1633886650512,"Executor ID":"10","Host":"10.1.0.16","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":21,"Index":5,"Attempt":0,"Launch Time":1633886650512,"Executor ID":"1","Host":"10.1.0.17","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":22,"Index":6,"Attempt":0,"Launch Time":1633886650513,"Executor ID":"4","Host":"10.1.0.8","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":23,"Index":7,"Attempt":0,"Launch Time":1633886650513,"Executor ID":"11","Host":"10.1.0.13","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":20,"Index":4,"Attempt":0,"Launch Time":1633886650512,"Executor ID":"10","Host":"10.1.0.16","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633886650735,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"7","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":27,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"72188","Value":"72188","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"local bytes read","Update":"10800","Value":"10800","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2937","Value":"2937","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"41","Value":"41","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2937","Value":"2937","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"4456448","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"16","Value":"16","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":128,"Name":"internal.metrics.executorDeserializeTime","Update":44,"Value":44,"Internal":true,"Count Failed Values":true},{"ID":129,"Name":"internal.metrics.executorDeserializeCpuTime","Update":39457154,"Value":39457154,"Internal":true,"Count Failed Values":true},{"ID":130,"Name":"internal.metrics.executorRunTime","Update":166,"Value":166,"Internal":true,"Count Failed Values":true},{"ID":131,"Name":"internal.metrics.executorCpuTime","Update":127369077,"Value":127369077,"Internal":true,"Count Failed Values":true},{"ID":132,"Name":"internal.metrics.resultSize","Update":14169,"Value":14169,"Internal":true,"Count Failed Values":true},{"ID":134,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":137,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":4456448,"Internal":true,"Count Failed Values":true},{"ID":139,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":140,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":141,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":72188,"Value":72188,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":10800,"Value":10800,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2937,"Value":2937,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":44,"Executor Deserialize CPU Time":39457154,"Executor Run Time":166,"Executor CPU Time":127369077,"Peak Execution Memory":4456448,"Result Size":14169,"JVM GC Time":0,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":72188,"Remote Bytes Read To Disk":0,"Local Bytes Read":10800,"Total Records Read":2937},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":16,"Index":0,"Attempt":0,"Launch Time":1633886650509,"Executor ID":"6","Host":"10.1.0.18","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633886650741,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"7","Value":"14","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":27,"Name":"local blocks read","Update":"1","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"71598","Value":"143786","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"local bytes read","Update":"10619","Value":"21419","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2905","Value":"5842","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"44","Value":"85","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2905","Value":"5842","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"8912896","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"18","Value":"34","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"22","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":128,"Name":"internal.metrics.executorDeserializeTime","Update":47,"Value":91,"Internal":true,"Count Failed Values":true},{"ID":129,"Name":"internal.metrics.executorDeserializeCpuTime","Update":40826189,"Value":80283343,"Internal":true,"Count Failed Values":true},{"ID":130,"Name":"internal.metrics.executorRunTime","Update":169,"Value":335,"Internal":true,"Count Failed Values":true},{"ID":131,"Name":"internal.metrics.executorCpuTime","Update":135097318,"Value":262466395,"Internal":true,"Count Failed Values":true},{"ID":132,"Name":"internal.metrics.resultSize","Update":14169,"Value":28338,"Internal":true,"Count Failed Values":true},{"ID":134,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":137,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":8912896,"Internal":true,"Count Failed Values":true},{"ID":139,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":14,"Internal":true,"Count Failed Values":true},{"ID":140,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":141,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":71598,"Value":143786,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":10619,"Value":21419,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2905,"Value":5842,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":47,"Executor Deserialize CPU Time":40826189,"Executor Run Time":169,"Executor CPU Time":135097318,"Peak Execution Memory":4456448,"Result Size":14169,"JVM GC Time":0,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":71598,"Remote Bytes Read To Disk":0,"Local Bytes Read":10619,"Total Records Read":2905},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":18,"Index":2,"Attempt":0,"Launch Time":1633886650511,"Executor ID":"0","Host":"10.1.0.12","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633886650741,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"7","Value":"21","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":27,"Name":"local blocks read","Update":"1","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"74808","Value":"218594","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"local bytes read","Update":"11096","Value":"32515","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"3049","Value":"8891","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"43","Value":"128","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"3049","Value":"8891","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"13369344","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"18","Value":"52","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"33","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":128,"Name":"internal.metrics.executorDeserializeTime","Update":46,"Value":137,"Internal":true,"Count Failed Values":true},{"ID":129,"Name":"internal.metrics.executorDeserializeCpuTime","Update":41224934,"Value":121508277,"Internal":true,"Count Failed Values":true},{"ID":130,"Name":"internal.metrics.executorRunTime","Update":167,"Value":502,"Internal":true,"Count Failed Values":true},{"ID":131,"Name":"internal.metrics.executorCpuTime","Update":132674598,"Value":395140993,"Internal":true,"Count Failed Values":true},{"ID":132,"Name":"internal.metrics.resultSize","Update":14169,"Value":42507,"Internal":true,"Count Failed Values":true},{"ID":134,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":137,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":13369344,"Internal":true,"Count Failed Values":true},{"ID":139,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":21,"Internal":true,"Count Failed Values":true},{"ID":140,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":141,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":74808,"Value":218594,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":11096,"Value":32515,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.shuffle.read.recordsRead","Update":3049,"Value":8891,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":46,"Executor Deserialize CPU Time":41224934,"Executor Run Time":167,"Executor CPU Time":132674598,"Peak Execution Memory":4456448,"Result Size":14169,"JVM GC Time":0,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":74808,"Remote Bytes Read To Disk":0,"Local Bytes Read":11096,"Total Records Read":3049},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":23,"Index":7,"Attempt":0,"Launch Time":1633886650513,"Executor ID":"11","Host":"10.1.0.13","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633886650747,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"7","Value":"28","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":27,"Name":"local blocks read","Update":"1","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"70635","Value":"289229","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"local bytes read","Update":"10061","Value":"42576","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2856","Value":"11747","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"43","Value":"171","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2856","Value":"11747","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"17825792","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"17","Value":"69","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"44","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":128,"Name":"internal.metrics.executorDeserializeTime","Update":47,"Value":184,"Internal":true,"Count Failed Values":true},{"ID":129,"Name":"internal.metrics.executorDeserializeCpuTime","Update":41600868,"Value":163109145,"Internal":true,"Count Failed Values":true},{"ID":130,"Name":"internal.metrics.executorRunTime","Update":174,"Value":676,"Internal":true,"Count Failed Values":true},{"ID":131,"Name":"internal.metrics.executorCpuTime","Update":136704415,"Value":531845408,"Internal":true,"Count Failed Values":true},{"ID":132,"Name":"internal.metrics.resultSize","Update":14169,"Value":56676,"Internal":true,"Count Failed Values":true},{"ID":134,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":137,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":17825792,"Internal":true,"Count Failed Values":true},{"ID":139,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":28,"Internal":true,"Count Failed Values":true},{"ID":140,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":141,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":70635,"Value":289229,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":10061,"Value":42576,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2856,"Value":11747,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":47,"Executor Deserialize CPU Time":41600868,"Executor Run Time":174,"Executor CPU Time":136704415,"Peak Execution Memory":4456448,"Result Size":14169,"JVM GC Time":0,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":70635,"Remote Bytes Read To Disk":0,"Local Bytes Read":10061,"Total Records Read":2856},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":17,"Index":1,"Attempt":0,"Launch Time":1633886650511,"Executor ID":"3","Host":"10.1.0.15","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633886650807,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"7","Value":"35","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":27,"Name":"local blocks read","Update":"1","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"74063","Value":"363292","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"local bytes read","Update":"10549","Value":"53125","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2999","Value":"14746","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"47","Value":"218","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2999","Value":"14746","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"22282240","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"19","Value":"88","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"55","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":128,"Name":"internal.metrics.executorDeserializeTime","Update":122,"Value":306,"Internal":true,"Count Failed Values":true},{"ID":129,"Name":"internal.metrics.executorDeserializeCpuTime","Update":116101291,"Value":279210436,"Internal":true,"Count Failed Values":true},{"ID":130,"Name":"internal.metrics.executorRunTime","Update":158,"Value":834,"Internal":true,"Count Failed Values":true},{"ID":131,"Name":"internal.metrics.executorCpuTime","Update":145272495,"Value":677117903,"Internal":true,"Count Failed Values":true},{"ID":132,"Name":"internal.metrics.resultSize","Update":14169,"Value":70845,"Internal":true,"Count Failed Values":true},{"ID":134,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":137,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":22282240,"Internal":true,"Count Failed Values":true},{"ID":139,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":35,"Internal":true,"Count Failed Values":true},{"ID":140,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":141,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":74063,"Value":363292,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":10549,"Value":53125,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2999,"Value":14746,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":122,"Executor Deserialize CPU Time":116101291,"Executor Run Time":158,"Executor CPU Time":145272495,"Peak Execution Memory":4456448,"Result Size":14169,"JVM GC Time":0,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":74063,"Remote Bytes Read To Disk":0,"Local Bytes Read":10549,"Total Records Read":2999},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":21,"Index":5,"Attempt":0,"Launch Time":1633886650512,"Executor ID":"1","Host":"10.1.0.17","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633886652103,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"8","Value":"43","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"83209","Value":"446501","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2948","Value":"17694","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"194","Value":"412","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2948","Value":"17694","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"26738688","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"44","Value":"132","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"66","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":128,"Name":"internal.metrics.executorDeserializeTime","Update":904,"Value":1210,"Internal":true,"Count Failed Values":true},{"ID":129,"Name":"internal.metrics.executorDeserializeCpuTime","Update":838498033,"Value":1117708469,"Internal":true,"Count Failed Values":true},{"ID":130,"Name":"internal.metrics.executorRunTime","Update":645,"Value":1479,"Internal":true,"Count Failed Values":true},{"ID":131,"Name":"internal.metrics.executorCpuTime","Update":616304248,"Value":1293422151,"Internal":true,"Count Failed Values":true},{"ID":132,"Name":"internal.metrics.resultSize","Update":14212,"Value":85057,"Internal":true,"Count Failed Values":true},{"ID":133,"Name":"internal.metrics.jvmGCTime","Update":33,"Value":33,"Internal":true,"Count Failed Values":true},{"ID":134,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":9,"Internal":true,"Count Failed Values":true},{"ID":137,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":26738688,"Internal":true,"Count Failed Values":true},{"ID":139,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":8,"Value":43,"Internal":true,"Count Failed Values":true},{"ID":140,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":141,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":83209,"Value":446501,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":53125,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2948,"Value":17694,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":904,"Executor Deserialize CPU Time":838498033,"Executor Run Time":645,"Executor CPU Time":616304248,"Peak Execution Memory":4456448,"Result Size":14212,"JVM GC Time":33,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":8,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":83209,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":2948},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":19,"Index":3,"Attempt":0,"Launch Time":1633886650511,"Executor ID":"7","Host":"10.1.0.9","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633886652176,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"8","Value":"51","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"84455","Value":"530956","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2994","Value":"20688","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"183","Value":"595","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2994","Value":"20688","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"31195136","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"39","Value":"171","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"77","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":128,"Name":"internal.metrics.executorDeserializeTime","Update":919,"Value":2129,"Internal":true,"Count Failed Values":true},{"ID":129,"Name":"internal.metrics.executorDeserializeCpuTime","Update":848576727,"Value":1966285196,"Internal":true,"Count Failed Values":true},{"ID":130,"Name":"internal.metrics.executorRunTime","Update":701,"Value":2180,"Internal":true,"Count Failed Values":true},{"ID":131,"Name":"internal.metrics.executorCpuTime","Update":649395251,"Value":1942817402,"Internal":true,"Count Failed Values":true},{"ID":132,"Name":"internal.metrics.resultSize","Update":14212,"Value":99269,"Internal":true,"Count Failed Values":true},{"ID":133,"Name":"internal.metrics.jvmGCTime","Update":33,"Value":66,"Internal":true,"Count Failed Values":true},{"ID":134,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":11,"Internal":true,"Count Failed Values":true},{"ID":137,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":31195136,"Internal":true,"Count Failed Values":true},{"ID":139,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":8,"Value":51,"Internal":true,"Count Failed Values":true},{"ID":140,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":141,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":84455,"Value":530956,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":53125,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2994,"Value":20688,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":919,"Executor Deserialize CPU Time":848576727,"Executor Run Time":701,"Executor CPU Time":649395251,"Peak Execution Memory":4456448,"Result Size":14212,"JVM GC Time":33,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":8,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":84455,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":2994},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":22,"Index":6,"Attempt":0,"Launch Time":1633886650513,"Executor ID":"4","Host":"10.1.0.8","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633886652199,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"8","Value":"59","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"82141","Value":"613097","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2910","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"196","Value":"791","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2910","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"35651584","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"40","Value":"211","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"88","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":128,"Name":"internal.metrics.executorDeserializeTime","Update":971,"Value":3100,"Internal":true,"Count Failed Values":true},{"ID":129,"Name":"internal.metrics.executorDeserializeCpuTime","Update":901930853,"Value":2868216049,"Internal":true,"Count Failed Values":true},{"ID":130,"Name":"internal.metrics.executorRunTime","Update":677,"Value":2857,"Internal":true,"Count Failed Values":true},{"ID":131,"Name":"internal.metrics.executorCpuTime","Update":642747927,"Value":2585565329,"Internal":true,"Count Failed Values":true},{"ID":132,"Name":"internal.metrics.resultSize","Update":14212,"Value":113481,"Internal":true,"Count Failed Values":true},{"ID":133,"Name":"internal.metrics.jvmGCTime","Update":32,"Value":98,"Internal":true,"Count Failed Values":true},{"ID":134,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":13,"Internal":true,"Count Failed Values":true},{"ID":137,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":35651584,"Internal":true,"Count Failed Values":true},{"ID":139,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":8,"Value":59,"Internal":true,"Count Failed Values":true},{"ID":140,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":141,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":82141,"Value":613097,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":53125,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2910,"Value":23598,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":137588888,"JVMOffHeapMemory":57420072,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":56143,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":56143,"OffHeapUnifiedMemory":0,"DirectPoolMemory":10053,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":2,"MinorGCTime":18,"MajorGCCount":2,"MajorGCTime":47},"Task Metrics":{"Executor Deserialize Time":971,"Executor Deserialize CPU Time":901930853,"Executor Run Time":677,"Executor CPU Time":642747927,"Peak Execution Memory":4456448,"Result Size":14212,"JVM GC Time":32,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":8,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":82141,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":2910},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":2,"Stage Attempt ID":0,"Stage Name":"sql at ScalaSparkSQLBench.scala:43","Number of Tasks":8,"RDD Info":[{"RDD ID":16,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[15],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":15,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[14],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":14,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":12,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"12\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[11],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[1],"Details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1633886650501,"Completion Time":1633886652200,"Accumulables":[{"ID":26,"Name":"remote blocks read","Value":"59","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":27,"Name":"local blocks read","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Value":"613097","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"local bytes read","Value":"53125","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Value":"791","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Value":"35651584","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Value":"211","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Value":"88","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":128,"Name":"internal.metrics.executorDeserializeTime","Value":3100,"Internal":true,"Count Failed Values":true},{"ID":129,"Name":"internal.metrics.executorDeserializeCpuTime","Value":2868216049,"Internal":true,"Count Failed Values":true},{"ID":130,"Name":"internal.metrics.executorRunTime","Value":2857,"Internal":true,"Count Failed Values":true},{"ID":131,"Name":"internal.metrics.executorCpuTime","Value":2585565329,"Internal":true,"Count Failed Values":true},{"ID":132,"Name":"internal.metrics.resultSize","Value":113481,"Internal":true,"Count Failed Values":true},{"ID":133,"Name":"internal.metrics.jvmGCTime","Value":98,"Internal":true,"Count Failed Values":true},{"ID":134,"Name":"internal.metrics.resultSerializationTime","Value":13,"Internal":true,"Count Failed Values":true},{"ID":137,"Name":"internal.metrics.peakExecutionMemory","Value":35651584,"Internal":true,"Count Failed Values":true},{"ID":139,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":59,"Internal":true,"Count Failed Values":true},{"ID":140,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":5,"Internal":true,"Count Failed Values":true},{"ID":141,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":613097,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":53125,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.shuffle.read.recordsRead","Value":23598,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":1,"Completion Time":1633886652201,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"SparkListenerJobStart","Job ID":2,"Submission Time":1633886652274,"Stage Infos":[{"Stage ID":5,"Stage Attempt ID":0,"Stage Name":"sql at ScalaSparkSQLBench.scala:43","Number of Tasks":8,"RDD Info":[{"RDD ID":19,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[18],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":18,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[17],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[4],"Details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":3,"Stage Attempt ID":0,"Stage Name":"sql at ScalaSparkSQLBench.scala:43","Number of Tasks":8,"RDD Info":[{"RDD ID":11,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"12\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[10],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":10,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"13\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[9],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":6,"Name":"HadoopRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":4,"Stage Attempt ID":0,"Stage Name":"sql at ScalaSparkSQLBench.scala:43","Number of Tasks":8,"RDD Info":[{"RDD ID":17,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":12,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"12\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[11],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[3],"Details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[5,3,4],"Properties":{"spark.sql.warehouse.dir":"file:/home/usera112/projects/log-compressor/deployments/HiBench/spark-warehouse","spark.driver.host":"hostb7.dsrg.utoronto.ca","spark.eventLog.enabled":"true","spark.driver.port":"42853","__fetch_continuous_blocks_in_batch_enabled":"true","spark.jars":"file:/home/usera112/projects/log-compressor/deployments/HiBench/sparkbench/assembly/target/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.app.name":"ScalaJoin","spark.driver.memory":"4g","spark.submit.pyFiles":"","spark.default.parallelism":"8","spark.app.startTime":"1633886611309","spark.executor.id":"driver","spark.app.initial.jar.urls":"spark://hostb7.dsrg.utoronto.ca:42853/jars/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.submit.deployMode":"client","spark.master":"spark://hostb7:7077","spark.executor.memory":"4g","spark.eventLog.dir":"hdfs://hostb7:8020/directory","spark.sql.execution.id":"11","spark.sql.catalogImplementation":"hive","spark.app.id":"app-20211010132332-4636","spark.sql.shuffle.partitions":"8"}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":4,"Stage Attempt ID":0,"Stage Name":"sql at ScalaSparkSQLBench.scala:43","Number of Tasks":8,"RDD Info":[{"RDD ID":17,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":12,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"12\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[11],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[3],"Details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1633886652279,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.sql.warehouse.dir":"file:/home/usera112/projects/log-compressor/deployments/HiBench/spark-warehouse","spark.driver.host":"hostb7.dsrg.utoronto.ca","spark.eventLog.enabled":"true","spark.driver.port":"42853","__fetch_continuous_blocks_in_batch_enabled":"true","spark.jars":"file:/home/usera112/projects/log-compressor/deployments/HiBench/sparkbench/assembly/target/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.app.name":"ScalaJoin","spark.driver.memory":"4g","spark.submit.pyFiles":"","spark.default.parallelism":"8","spark.app.startTime":"1633886611309","spark.executor.id":"driver","spark.app.initial.jar.urls":"spark://hostb7.dsrg.utoronto.ca:42853/jars/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.submit.deployMode":"client","spark.master":"spark://hostb7:7077","spark.executor.memory":"4g","spark.eventLog.dir":"hdfs://hostb7:8020/directory","spark.sql.execution.id":"11","spark.sql.catalogImplementation":"hive","spark.app.id":"app-20211010132332-4636","spark.sql.shuffle.partitions":"8"}} +{"Event":"SparkListenerTaskStart","Stage ID":4,"Stage Attempt ID":0,"Task Info":{"Task ID":24,"Index":0,"Attempt":0,"Launch Time":1633886652293,"Executor ID":"7","Host":"10.1.0.9","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":4,"Stage Attempt ID":0,"Task Info":{"Task ID":25,"Index":1,"Attempt":0,"Launch Time":1633886652293,"Executor ID":"10","Host":"10.1.0.16","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":4,"Stage Attempt ID":0,"Task Info":{"Task ID":26,"Index":2,"Attempt":0,"Launch Time":1633886652293,"Executor ID":"3","Host":"10.1.0.15","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":4,"Stage Attempt ID":0,"Task Info":{"Task ID":27,"Index":3,"Attempt":0,"Launch Time":1633886652293,"Executor ID":"11","Host":"10.1.0.13","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":4,"Stage Attempt ID":0,"Task Info":{"Task ID":28,"Index":4,"Attempt":0,"Launch Time":1633886652294,"Executor ID":"4","Host":"10.1.0.8","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":4,"Stage Attempt ID":0,"Task Info":{"Task ID":29,"Index":5,"Attempt":0,"Launch Time":1633886652294,"Executor ID":"9","Host":"10.1.0.14","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":4,"Stage Attempt ID":0,"Task Info":{"Task ID":30,"Index":6,"Attempt":0,"Launch Time":1633886652294,"Executor ID":"1","Host":"10.1.0.17","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":4,"Stage Attempt ID":0,"Task Info":{"Task ID":31,"Index":7,"Attempt":0,"Launch Time":1633886652294,"Executor ID":"8","Host":"10.1.0.11","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":4,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":25,"Index":1,"Attempt":0,"Launch Time":1633886652293,"Executor ID":"10","Host":"10.1.0.16","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633886652392,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"7","Value":"66","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":27,"Name":"local blocks read","Update":"1","Value":"6","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"73963","Value":"687060","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"local bytes read","Update":"10649","Value":"63774","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2999","Value":"26597","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":36,"Name":"data size","Update":"143952","Value":"143952","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":44,"Name":"shuffle bytes written","Update":"81470","Value":"81470","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":45,"Name":"shuffle records written","Update":"2999","Value":"2999","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":46,"Name":"shuffle write time","Update":"2979545","Value":"2979545","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"27","Value":"818","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2999","Value":"26597","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"40108032","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"6","Value":"217","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"99","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":153,"Name":"internal.metrics.executorDeserializeTime","Update":50,"Value":50,"Internal":true,"Count Failed Values":true},{"ID":154,"Name":"internal.metrics.executorDeserializeCpuTime","Update":42880086,"Value":42880086,"Internal":true,"Count Failed Values":true},{"ID":155,"Name":"internal.metrics.executorRunTime","Update":39,"Value":39,"Internal":true,"Count Failed Values":true},{"ID":156,"Name":"internal.metrics.executorCpuTime","Update":36997640,"Value":36997640,"Internal":true,"Count Failed Values":true},{"ID":157,"Name":"internal.metrics.resultSize","Update":4237,"Value":4237,"Internal":true,"Count Failed Values":true},{"ID":162,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":4456448,"Internal":true,"Count Failed Values":true},{"ID":164,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":165,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":166,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":73963,"Value":73963,"Internal":true,"Count Failed Values":true},{"ID":167,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":168,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":10649,"Value":10649,"Internal":true,"Count Failed Values":true},{"ID":169,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":170,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2999,"Value":2999,"Internal":true,"Count Failed Values":true},{"ID":171,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":81470,"Value":81470,"Internal":true,"Count Failed Values":true},{"ID":172,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":2999,"Value":2999,"Internal":true,"Count Failed Values":true},{"ID":173,"Name":"internal.metrics.shuffle.write.writeTime","Update":2979545,"Value":2979545,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":50,"Executor Deserialize CPU Time":42880086,"Executor Run Time":39,"Executor CPU Time":36997640,"Peak Execution Memory":4456448,"Result Size":4237,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":73963,"Remote Bytes Read To Disk":0,"Local Bytes Read":10649,"Total Records Read":2999},"Shuffle Write Metrics":{"Shuffle Bytes Written":81470,"Shuffle Write Time":2979545,"Shuffle Records Written":2999},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":4,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":26,"Index":2,"Attempt":0,"Launch Time":1633886652293,"Executor ID":"3","Host":"10.1.0.15","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633886652403,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"7","Value":"73","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":27,"Name":"local blocks read","Update":"1","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"75292","Value":"762352","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"local bytes read","Update":"10612","Value":"74386","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"3049","Value":"29646","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":36,"Name":"data size","Update":"146336","Value":"290288","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":44,"Name":"shuffle bytes written","Update":"82863","Value":"164333","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":45,"Name":"shuffle records written","Update":"3049","Value":"6048","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":46,"Name":"shuffle write time","Update":"4012945","Value":"6992490","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"34","Value":"852","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"3049","Value":"29646","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"44564480","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"7","Value":"224","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"110","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":153,"Name":"internal.metrics.executorDeserializeTime","Update":54,"Value":104,"Internal":true,"Count Failed Values":true},{"ID":154,"Name":"internal.metrics.executorDeserializeCpuTime","Update":45521713,"Value":88401799,"Internal":true,"Count Failed Values":true},{"ID":155,"Name":"internal.metrics.executorRunTime","Update":45,"Value":84,"Internal":true,"Count Failed Values":true},{"ID":156,"Name":"internal.metrics.executorCpuTime","Update":42313212,"Value":79310852,"Internal":true,"Count Failed Values":true},{"ID":157,"Name":"internal.metrics.resultSize","Update":4236,"Value":8473,"Internal":true,"Count Failed Values":true},{"ID":162,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":8912896,"Internal":true,"Count Failed Values":true},{"ID":164,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":14,"Internal":true,"Count Failed Values":true},{"ID":165,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":166,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":75292,"Value":149255,"Internal":true,"Count Failed Values":true},{"ID":167,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":168,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":10612,"Value":21261,"Internal":true,"Count Failed Values":true},{"ID":169,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":170,"Name":"internal.metrics.shuffle.read.recordsRead","Update":3049,"Value":6048,"Internal":true,"Count Failed Values":true},{"ID":171,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":82863,"Value":164333,"Internal":true,"Count Failed Values":true},{"ID":172,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":3049,"Value":6048,"Internal":true,"Count Failed Values":true},{"ID":173,"Name":"internal.metrics.shuffle.write.writeTime","Update":4012945,"Value":6992490,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":54,"Executor Deserialize CPU Time":45521713,"Executor Run Time":45,"Executor CPU Time":42313212,"Peak Execution Memory":4456448,"Result Size":4236,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":75292,"Remote Bytes Read To Disk":0,"Local Bytes Read":10612,"Total Records Read":3049},"Shuffle Write Metrics":{"Shuffle Bytes Written":82863,"Shuffle Write Time":4012945,"Shuffle Records Written":3049},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":4,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":27,"Index":3,"Attempt":0,"Launch Time":1633886652293,"Executor ID":"11","Host":"10.1.0.13","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633886652450,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"7","Value":"80","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":27,"Name":"local blocks read","Update":"1","Value":"8","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"73685","Value":"836037","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"local bytes read","Update":"10770","Value":"85156","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2994","Value":"32640","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":36,"Name":"data size","Update":"143712","Value":"434000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":44,"Name":"shuffle bytes written","Update":"81340","Value":"245673","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":45,"Name":"shuffle records written","Update":"2994","Value":"9042","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":46,"Name":"shuffle write time","Update":"3068160","Value":"10060650","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"30","Value":"882","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2994","Value":"32640","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"49020928","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"5","Value":"229","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"121","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":153,"Name":"internal.metrics.executorDeserializeTime","Update":104,"Value":208,"Internal":true,"Count Failed Values":true},{"ID":154,"Name":"internal.metrics.executorDeserializeCpuTime","Update":47825831,"Value":136227630,"Internal":true,"Count Failed Values":true},{"ID":155,"Name":"internal.metrics.executorRunTime","Update":42,"Value":126,"Internal":true,"Count Failed Values":true},{"ID":156,"Name":"internal.metrics.executorCpuTime","Update":38610610,"Value":117921462,"Internal":true,"Count Failed Values":true},{"ID":157,"Name":"internal.metrics.resultSize","Update":4237,"Value":12710,"Internal":true,"Count Failed Values":true},{"ID":162,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":13369344,"Internal":true,"Count Failed Values":true},{"ID":164,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":21,"Internal":true,"Count Failed Values":true},{"ID":165,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":166,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":73685,"Value":222940,"Internal":true,"Count Failed Values":true},{"ID":167,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":168,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":10770,"Value":32031,"Internal":true,"Count Failed Values":true},{"ID":169,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":170,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2994,"Value":9042,"Internal":true,"Count Failed Values":true},{"ID":171,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":81340,"Value":245673,"Internal":true,"Count Failed Values":true},{"ID":172,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":2994,"Value":9042,"Internal":true,"Count Failed Values":true},{"ID":173,"Name":"internal.metrics.shuffle.write.writeTime","Update":3068160,"Value":10060650,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":104,"Executor Deserialize CPU Time":47825831,"Executor Run Time":42,"Executor CPU Time":38610610,"Peak Execution Memory":4456448,"Result Size":4237,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":73685,"Remote Bytes Read To Disk":0,"Local Bytes Read":10770,"Total Records Read":2994},"Shuffle Write Metrics":{"Shuffle Bytes Written":81340,"Shuffle Write Time":3068160,"Shuffle Records Written":2994},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":4,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":30,"Index":6,"Attempt":0,"Launch Time":1633886652294,"Executor ID":"1","Host":"10.1.0.17","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633886652452,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"8","Value":"88","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"82141","Value":"918178","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2910","Value":"35550","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":36,"Name":"data size","Update":"139680","Value":"573680","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":44,"Name":"shuffle bytes written","Update":"79297","Value":"324970","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":45,"Name":"shuffle records written","Update":"2910","Value":"11952","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":46,"Name":"shuffle write time","Update":"12386955","Value":"22447605","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"42","Value":"924","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2910","Value":"35550","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"53477376","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"7","Value":"236","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"132","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":153,"Name":"internal.metrics.executorDeserializeTime","Update":61,"Value":269,"Internal":true,"Count Failed Values":true},{"ID":154,"Name":"internal.metrics.executorDeserializeCpuTime","Update":54460754,"Value":190688384,"Internal":true,"Count Failed Values":true},{"ID":155,"Name":"internal.metrics.executorRunTime","Update":86,"Value":212,"Internal":true,"Count Failed Values":true},{"ID":156,"Name":"internal.metrics.executorCpuTime","Update":82291839,"Value":200213301,"Internal":true,"Count Failed Values":true},{"ID":157,"Name":"internal.metrics.resultSize","Update":4236,"Value":16946,"Internal":true,"Count Failed Values":true},{"ID":162,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":17825792,"Internal":true,"Count Failed Values":true},{"ID":164,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":8,"Value":29,"Internal":true,"Count Failed Values":true},{"ID":165,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":166,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":82141,"Value":305081,"Internal":true,"Count Failed Values":true},{"ID":167,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":168,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":32031,"Internal":true,"Count Failed Values":true},{"ID":169,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":170,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2910,"Value":11952,"Internal":true,"Count Failed Values":true},{"ID":171,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":79297,"Value":324970,"Internal":true,"Count Failed Values":true},{"ID":172,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":2910,"Value":11952,"Internal":true,"Count Failed Values":true},{"ID":173,"Name":"internal.metrics.shuffle.write.writeTime","Update":12386955,"Value":22447605,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":61,"Executor Deserialize CPU Time":54460754,"Executor Run Time":86,"Executor CPU Time":82291839,"Peak Execution Memory":4456448,"Result Size":4236,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":8,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":82141,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":2910},"Shuffle Write Metrics":{"Shuffle Bytes Written":79297,"Shuffle Write Time":12386955,"Shuffle Records Written":2910},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":4,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":24,"Index":0,"Attempt":0,"Launch Time":1633886652293,"Executor ID":"7","Host":"10.1.0.9","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633886652456,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"8","Value":"96","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"82217","Value":"1000395","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2905","Value":"38455","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":36,"Name":"data size","Update":"139440","Value":"713120","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":44,"Name":"shuffle bytes written","Update":"79034","Value":"404004","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":45,"Name":"shuffle records written","Update":"2905","Value":"14857","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":46,"Name":"shuffle write time","Update":"11902912","Value":"34350517","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"42","Value":"966","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2905","Value":"38455","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"57933824","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"6","Value":"242","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"143","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":153,"Name":"internal.metrics.executorDeserializeTime","Update":69,"Value":338,"Internal":true,"Count Failed Values":true},{"ID":154,"Name":"internal.metrics.executorDeserializeCpuTime","Update":60065310,"Value":250753694,"Internal":true,"Count Failed Values":true},{"ID":155,"Name":"internal.metrics.executorRunTime","Update":83,"Value":295,"Internal":true,"Count Failed Values":true},{"ID":156,"Name":"internal.metrics.executorCpuTime","Update":80941330,"Value":281154631,"Internal":true,"Count Failed Values":true},{"ID":157,"Name":"internal.metrics.resultSize","Update":4235,"Value":21181,"Internal":true,"Count Failed Values":true},{"ID":162,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":22282240,"Internal":true,"Count Failed Values":true},{"ID":164,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":8,"Value":37,"Internal":true,"Count Failed Values":true},{"ID":165,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":166,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":82217,"Value":387298,"Internal":true,"Count Failed Values":true},{"ID":167,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":168,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":32031,"Internal":true,"Count Failed Values":true},{"ID":169,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":170,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2905,"Value":14857,"Internal":true,"Count Failed Values":true},{"ID":171,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":79034,"Value":404004,"Internal":true,"Count Failed Values":true},{"ID":172,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":2905,"Value":14857,"Internal":true,"Count Failed Values":true},{"ID":173,"Name":"internal.metrics.shuffle.write.writeTime","Update":11902912,"Value":34350517,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":69,"Executor Deserialize CPU Time":60065310,"Executor Run Time":83,"Executor CPU Time":80941330,"Peak Execution Memory":4456448,"Result Size":4235,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":8,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":82217,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":2905},"Shuffle Write Metrics":{"Shuffle Bytes Written":79034,"Shuffle Write Time":11902912,"Shuffle Records Written":2905},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":4,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":28,"Index":4,"Attempt":0,"Launch Time":1633886652294,"Executor ID":"4","Host":"10.1.0.8","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633886652456,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"8","Value":"104","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"82988","Value":"1083383","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2937","Value":"41392","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":36,"Name":"data size","Update":"140976","Value":"854096","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":44,"Name":"shuffle bytes written","Update":"79743","Value":"483747","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":45,"Name":"shuffle records written","Update":"2937","Value":"17794","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":46,"Name":"shuffle write time","Update":"11348691","Value":"45699208","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"43","Value":"1009","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2937","Value":"41392","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"62390272","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"7","Value":"249","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"154","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":153,"Name":"internal.metrics.executorDeserializeTime","Update":69,"Value":407,"Internal":true,"Count Failed Values":true},{"ID":154,"Name":"internal.metrics.executorDeserializeCpuTime","Update":60955899,"Value":311709593,"Internal":true,"Count Failed Values":true},{"ID":155,"Name":"internal.metrics.executorRunTime","Update":83,"Value":378,"Internal":true,"Count Failed Values":true},{"ID":156,"Name":"internal.metrics.executorCpuTime","Update":79088856,"Value":360243487,"Internal":true,"Count Failed Values":true},{"ID":157,"Name":"internal.metrics.resultSize","Update":4235,"Value":25416,"Internal":true,"Count Failed Values":true},{"ID":162,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":26738688,"Internal":true,"Count Failed Values":true},{"ID":164,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":8,"Value":45,"Internal":true,"Count Failed Values":true},{"ID":165,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":166,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":82988,"Value":470286,"Internal":true,"Count Failed Values":true},{"ID":167,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":168,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":32031,"Internal":true,"Count Failed Values":true},{"ID":169,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":170,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2937,"Value":17794,"Internal":true,"Count Failed Values":true},{"ID":171,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":79743,"Value":483747,"Internal":true,"Count Failed Values":true},{"ID":172,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":2937,"Value":17794,"Internal":true,"Count Failed Values":true},{"ID":173,"Name":"internal.metrics.shuffle.write.writeTime","Update":11348691,"Value":45699208,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":69,"Executor Deserialize CPU Time":60955899,"Executor Run Time":83,"Executor CPU Time":79088856,"Peak Execution Memory":4456448,"Result Size":4235,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":8,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":82988,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":2937},"Shuffle Write Metrics":{"Shuffle Bytes Written":79743,"Shuffle Write Time":11348691,"Shuffle Records Written":2937},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":4,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":29,"Index":5,"Attempt":0,"Launch Time":1633886652294,"Executor ID":"9","Host":"10.1.0.14","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633886652527,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"7","Value":"111","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":27,"Name":"local blocks read","Update":"1","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"72944","Value":"1156327","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"local bytes read","Update":"10265","Value":"95421","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2948","Value":"44340","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":36,"Name":"data size","Update":"141504","Value":"995600","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":44,"Name":"shuffle bytes written","Update":"80386","Value":"564133","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":45,"Name":"shuffle records written","Update":"2948","Value":"20742","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":46,"Name":"shuffle write time","Update":"3682339","Value":"49381547","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"47","Value":"1056","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2948","Value":"44340","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"66846720","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"17","Value":"266","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"165","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":153,"Name":"internal.metrics.executorDeserializeTime","Update":70,"Value":477,"Internal":true,"Count Failed Values":true},{"ID":154,"Name":"internal.metrics.executorDeserializeCpuTime","Update":63682790,"Value":375392383,"Internal":true,"Count Failed Values":true},{"ID":155,"Name":"internal.metrics.executorRunTime","Update":153,"Value":531,"Internal":true,"Count Failed Values":true},{"ID":156,"Name":"internal.metrics.executorCpuTime","Update":141854217,"Value":502097704,"Internal":true,"Count Failed Values":true},{"ID":157,"Name":"internal.metrics.resultSize","Update":4236,"Value":29652,"Internal":true,"Count Failed Values":true},{"ID":162,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":31195136,"Internal":true,"Count Failed Values":true},{"ID":164,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":52,"Internal":true,"Count Failed Values":true},{"ID":165,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":166,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":72944,"Value":543230,"Internal":true,"Count Failed Values":true},{"ID":167,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":168,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":10265,"Value":42296,"Internal":true,"Count Failed Values":true},{"ID":169,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":170,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2948,"Value":20742,"Internal":true,"Count Failed Values":true},{"ID":171,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":80386,"Value":564133,"Internal":true,"Count Failed Values":true},{"ID":172,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":2948,"Value":20742,"Internal":true,"Count Failed Values":true},{"ID":173,"Name":"internal.metrics.shuffle.write.writeTime","Update":3682339,"Value":49381547,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":70,"Executor Deserialize CPU Time":63682790,"Executor Run Time":153,"Executor CPU Time":141854217,"Peak Execution Memory":4456448,"Result Size":4236,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":72944,"Remote Bytes Read To Disk":0,"Local Bytes Read":10265,"Total Records Read":2948},"Shuffle Write Metrics":{"Shuffle Bytes Written":80386,"Shuffle Write Time":3682339,"Shuffle Records Written":2948},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":4,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":31,"Index":7,"Attempt":0,"Launch Time":1633886652294,"Executor ID":"8","Host":"10.1.0.11","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633886652866,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"8","Value":"119","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"80696","Value":"1237023","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2856","Value":"47196","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":36,"Name":"data size","Update":"137080","Value":"1132680","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":44,"Name":"shuffle bytes written","Update":"77822","Value":"641955","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":45,"Name":"shuffle records written","Update":"2856","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":46,"Name":"shuffle write time","Update":"10030040","Value":"59411587","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"143","Value":"1199","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2856","Value":"47196","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"71303168","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"37","Value":"303","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"176","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":153,"Name":"internal.metrics.executorDeserializeTime","Update":258,"Value":735,"Internal":true,"Count Failed Values":true},{"ID":154,"Name":"internal.metrics.executorDeserializeCpuTime","Update":235875863,"Value":611268246,"Internal":true,"Count Failed Values":true},{"ID":155,"Name":"internal.metrics.executorRunTime","Update":300,"Value":831,"Internal":true,"Count Failed Values":true},{"ID":156,"Name":"internal.metrics.executorCpuTime","Update":281219809,"Value":783317513,"Internal":true,"Count Failed Values":true},{"ID":157,"Name":"internal.metrics.resultSize","Update":4236,"Value":33888,"Internal":true,"Count Failed Values":true},{"ID":162,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":35651584,"Internal":true,"Count Failed Values":true},{"ID":164,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":8,"Value":60,"Internal":true,"Count Failed Values":true},{"ID":165,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":166,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":80696,"Value":623926,"Internal":true,"Count Failed Values":true},{"ID":167,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":168,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":42296,"Internal":true,"Count Failed Values":true},{"ID":169,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":170,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2856,"Value":23598,"Internal":true,"Count Failed Values":true},{"ID":171,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":77822,"Value":641955,"Internal":true,"Count Failed Values":true},{"ID":172,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":2856,"Value":23598,"Internal":true,"Count Failed Values":true},{"ID":173,"Name":"internal.metrics.shuffle.write.writeTime","Update":10030040,"Value":59411587,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":258,"Executor Deserialize CPU Time":235875863,"Executor Run Time":300,"Executor CPU Time":281219809,"Peak Execution Memory":4456448,"Result Size":4236,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":8,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":80696,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":2856},"Shuffle Write Metrics":{"Shuffle Bytes Written":77822,"Shuffle Write Time":10030040,"Shuffle Records Written":2856},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":4,"Stage Attempt ID":0,"Stage Name":"sql at ScalaSparkSQLBench.scala:43","Number of Tasks":8,"RDD Info":[{"RDD ID":17,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":12,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"12\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[11],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[3],"Details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1633886652279,"Completion Time":1633886652866,"Accumulables":[{"ID":26,"Name":"remote blocks read","Value":"119","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":27,"Name":"local blocks read","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Value":"1237023","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"local bytes read","Value":"95421","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Value":"47196","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":36,"Name":"data size","Value":"1132680","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":44,"Name":"shuffle bytes written","Value":"641955","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":45,"Name":"shuffle records written","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":46,"Name":"shuffle write time","Value":"59411587","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Value":"1199","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Value":"47196","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Value":"71303168","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Value":"303","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Value":"176","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":153,"Name":"internal.metrics.executorDeserializeTime","Value":735,"Internal":true,"Count Failed Values":true},{"ID":154,"Name":"internal.metrics.executorDeserializeCpuTime","Value":611268246,"Internal":true,"Count Failed Values":true},{"ID":155,"Name":"internal.metrics.executorRunTime","Value":831,"Internal":true,"Count Failed Values":true},{"ID":156,"Name":"internal.metrics.executorCpuTime","Value":783317513,"Internal":true,"Count Failed Values":true},{"ID":157,"Name":"internal.metrics.resultSize","Value":33888,"Internal":true,"Count Failed Values":true},{"ID":162,"Name":"internal.metrics.peakExecutionMemory","Value":35651584,"Internal":true,"Count Failed Values":true},{"ID":164,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":60,"Internal":true,"Count Failed Values":true},{"ID":165,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":4,"Internal":true,"Count Failed Values":true},{"ID":166,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":623926,"Internal":true,"Count Failed Values":true},{"ID":167,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":168,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":42296,"Internal":true,"Count Failed Values":true},{"ID":169,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":170,"Name":"internal.metrics.shuffle.read.recordsRead","Value":23598,"Internal":true,"Count Failed Values":true},{"ID":171,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":641955,"Internal":true,"Count Failed Values":true},{"ID":172,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":23598,"Internal":true,"Count Failed Values":true},{"ID":173,"Name":"internal.metrics.shuffle.write.writeTime","Value":59411587,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":5,"Stage Attempt ID":0,"Stage Name":"sql at ScalaSparkSQLBench.scala:43","Number of Tasks":8,"RDD Info":[{"RDD ID":19,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[18],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":18,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[17],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[4],"Details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1633886652870,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.sql.warehouse.dir":"file:/home/usera112/projects/log-compressor/deployments/HiBench/spark-warehouse","spark.driver.host":"hostb7.dsrg.utoronto.ca","spark.eventLog.enabled":"true","spark.driver.port":"42853","__fetch_continuous_blocks_in_batch_enabled":"true","spark.jars":"file:/home/usera112/projects/log-compressor/deployments/HiBench/sparkbench/assembly/target/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.app.name":"ScalaJoin","spark.driver.memory":"4g","spark.submit.pyFiles":"","spark.default.parallelism":"8","spark.app.startTime":"1633886611309","spark.executor.id":"driver","spark.app.initial.jar.urls":"spark://hostb7.dsrg.utoronto.ca:42853/jars/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.submit.deployMode":"client","spark.master":"spark://hostb7:7077","spark.executor.memory":"4g","spark.eventLog.dir":"hdfs://hostb7:8020/directory","spark.sql.execution.id":"11","spark.sql.catalogImplementation":"hive","spark.app.id":"app-20211010132332-4636","spark.sql.shuffle.partitions":"8"}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":32,"Index":0,"Attempt":0,"Launch Time":1633886652903,"Executor ID":"2","Host":"10.1.0.10","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":33,"Index":1,"Attempt":0,"Launch Time":1633886652904,"Executor ID":"11","Host":"10.1.0.13","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":34,"Index":2,"Attempt":0,"Launch Time":1633886652904,"Executor ID":"4","Host":"10.1.0.8","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":35,"Index":3,"Attempt":0,"Launch Time":1633886652904,"Executor ID":"7","Host":"10.1.0.9","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":36,"Index":4,"Attempt":0,"Launch Time":1633886652905,"Executor ID":"0","Host":"10.1.0.12","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":37,"Index":5,"Attempt":0,"Launch Time":1633886652905,"Executor ID":"5","Host":"10.1.0.19","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":38,"Index":6,"Attempt":0,"Launch Time":1633886652905,"Executor ID":"3","Host":"10.1.0.15","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":39,"Index":7,"Attempt":0,"Launch Time":1633886652905,"Executor ID":"9","Host":"10.1.0.14","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":33,"Index":1,"Attempt":0,"Launch Time":1633886652904,"Executor ID":"11","Host":"10.1.0.13","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633886653449,"Failed":false,"Killed":false,"Accumulables":[{"ID":37,"Name":"remote blocks read","Update":"7","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":38,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":39,"Name":"remote bytes read","Update":"72246","Value":"72246","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":41,"Name":"local bytes read","Update":"10594","Value":"10594","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":42,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":43,"Name":"records read","Update":"3059","Value":"3059","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":51,"Name":"duration","Update":"310","Value":"310","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":52,"Name":"sort time","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":53,"Name":"peak memory","Update":"4325376","Value":"4325376","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":54,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":178,"Name":"internal.metrics.executorDeserializeTime","Update":75,"Value":75,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.executorDeserializeCpuTime","Update":68491859,"Value":68491859,"Internal":true,"Count Failed Values":true},{"ID":180,"Name":"internal.metrics.executorRunTime","Update":453,"Value":453,"Internal":true,"Count Failed Values":true},{"ID":181,"Name":"internal.metrics.executorCpuTime","Update":330296732,"Value":330296732,"Internal":true,"Count Failed Values":true},{"ID":182,"Name":"internal.metrics.resultSize","Update":5952,"Value":5952,"Internal":true,"Count Failed Values":true},{"ID":184,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":187,"Name":"internal.metrics.peakExecutionMemory","Update":4325376,"Value":4325376,"Internal":true,"Count Failed Values":true},{"ID":189,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":190,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":191,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":72246,"Value":72246,"Internal":true,"Count Failed Values":true},{"ID":192,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":193,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":10594,"Value":10594,"Internal":true,"Count Failed Values":true},{"ID":194,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":195,"Name":"internal.metrics.shuffle.read.recordsRead","Update":3059,"Value":3059,"Internal":true,"Count Failed Values":true},{"ID":201,"Name":"internal.metrics.output.bytesWritten","Update":127517,"Value":127517,"Internal":true,"Count Failed Values":true},{"ID":202,"Name":"internal.metrics.output.recordsWritten","Update":3059,"Value":3059,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":75,"Executor Deserialize CPU Time":68491859,"Executor Run Time":453,"Executor CPU Time":330296732,"Peak Execution Memory":4325376,"Result Size":5952,"JVM GC Time":0,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":72246,"Remote Bytes Read To Disk":0,"Local Bytes Read":10594,"Total Records Read":3059},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":127517,"Records Written":3059},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":36,"Index":4,"Attempt":0,"Launch Time":1633886652905,"Executor ID":"0","Host":"10.1.0.12","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633886653449,"Failed":false,"Killed":false,"Accumulables":[{"ID":37,"Name":"remote blocks read","Update":"8","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":39,"Name":"remote bytes read","Update":"77456","Value":"149702","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":42,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":43,"Name":"records read","Update":"2839","Value":"5898","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":51,"Name":"duration","Update":"312","Value":"622","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":52,"Name":"sort time","Update":"6","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":53,"Name":"peak memory","Update":"4325376","Value":"8650752","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":54,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":178,"Name":"internal.metrics.executorDeserializeTime","Update":80,"Value":155,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.executorDeserializeCpuTime","Update":74216019,"Value":142707878,"Internal":true,"Count Failed Values":true},{"ID":180,"Name":"internal.metrics.executorRunTime","Update":449,"Value":902,"Internal":true,"Count Failed Values":true},{"ID":181,"Name":"internal.metrics.executorCpuTime","Update":338833998,"Value":669130730,"Internal":true,"Count Failed Values":true},{"ID":182,"Name":"internal.metrics.resultSize","Update":5952,"Value":11904,"Internal":true,"Count Failed Values":true},{"ID":184,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":187,"Name":"internal.metrics.peakExecutionMemory","Update":4325376,"Value":8650752,"Internal":true,"Count Failed Values":true},{"ID":189,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":8,"Value":15,"Internal":true,"Count Failed Values":true},{"ID":190,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":191,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":77456,"Value":149702,"Internal":true,"Count Failed Values":true},{"ID":192,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":193,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":10594,"Internal":true,"Count Failed Values":true},{"ID":194,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":195,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2839,"Value":5898,"Internal":true,"Count Failed Values":true},{"ID":201,"Name":"internal.metrics.output.bytesWritten","Update":119261,"Value":246778,"Internal":true,"Count Failed Values":true},{"ID":202,"Name":"internal.metrics.output.recordsWritten","Update":2839,"Value":5898,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":80,"Executor Deserialize CPU Time":74216019,"Executor Run Time":449,"Executor CPU Time":338833998,"Peak Execution Memory":4325376,"Result Size":5952,"JVM GC Time":0,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":8,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":77456,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":2839},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":119261,"Records Written":2839},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":39,"Index":7,"Attempt":0,"Launch Time":1633886652905,"Executor ID":"9","Host":"10.1.0.14","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633886653449,"Failed":false,"Killed":false,"Accumulables":[{"ID":37,"Name":"remote blocks read","Update":"7","Value":"22","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":38,"Name":"local blocks read","Update":"1","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":39,"Name":"remote bytes read","Update":"68649","Value":"218351","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":41,"Name":"local bytes read","Update":"10663","Value":"21257","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":42,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":43,"Name":"records read","Update":"2902","Value":"8800","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":51,"Name":"duration","Update":"308","Value":"930","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":52,"Name":"sort time","Update":"6","Value":"17","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":53,"Name":"peak memory","Update":"4325376","Value":"12976128","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":54,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":178,"Name":"internal.metrics.executorDeserializeTime","Update":80,"Value":235,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.executorDeserializeCpuTime","Update":74914209,"Value":217622087,"Internal":true,"Count Failed Values":true},{"ID":180,"Name":"internal.metrics.executorRunTime","Update":448,"Value":1350,"Internal":true,"Count Failed Values":true},{"ID":181,"Name":"internal.metrics.executorCpuTime","Update":323882943,"Value":993013673,"Internal":true,"Count Failed Values":true},{"ID":182,"Name":"internal.metrics.resultSize","Update":5952,"Value":17856,"Internal":true,"Count Failed Values":true},{"ID":184,"Name":"internal.metrics.resultSerializationTime","Update":3,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":187,"Name":"internal.metrics.peakExecutionMemory","Update":4325376,"Value":12976128,"Internal":true,"Count Failed Values":true},{"ID":189,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":22,"Internal":true,"Count Failed Values":true},{"ID":190,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":191,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":68649,"Value":218351,"Internal":true,"Count Failed Values":true},{"ID":192,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":193,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":10663,"Value":21257,"Internal":true,"Count Failed Values":true},{"ID":194,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":195,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2902,"Value":8800,"Internal":true,"Count Failed Values":true},{"ID":201,"Name":"internal.metrics.output.bytesWritten","Update":124359,"Value":371137,"Internal":true,"Count Failed Values":true},{"ID":202,"Name":"internal.metrics.output.recordsWritten","Update":2902,"Value":8800,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":80,"Executor Deserialize CPU Time":74914209,"Executor Run Time":448,"Executor CPU Time":323882943,"Peak Execution Memory":4325376,"Result Size":5952,"JVM GC Time":0,"Result Serialization Time":3,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":68649,"Remote Bytes Read To Disk":0,"Local Bytes Read":10663,"Total Records Read":2902},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":124359,"Records Written":2902},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":38,"Index":6,"Attempt":0,"Launch Time":1633886652905,"Executor ID":"3","Host":"10.1.0.15","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633886653462,"Failed":false,"Killed":false,"Accumulables":[{"ID":37,"Name":"remote blocks read","Update":"7","Value":"29","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":38,"Name":"local blocks read","Update":"1","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":39,"Name":"remote bytes read","Update":"69262","Value":"287613","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":41,"Name":"local bytes read","Update":"9703","Value":"30960","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":42,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":43,"Name":"records read","Update":"2891","Value":"11691","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":51,"Name":"duration","Update":"303","Value":"1233","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":52,"Name":"sort time","Update":"6","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":53,"Name":"peak memory","Update":"4325376","Value":"17301504","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":54,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":178,"Name":"internal.metrics.executorDeserializeTime","Update":100,"Value":335,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.executorDeserializeCpuTime","Update":93582698,"Value":311204785,"Internal":true,"Count Failed Values":true},{"ID":180,"Name":"internal.metrics.executorRunTime","Update":446,"Value":1796,"Internal":true,"Count Failed Values":true},{"ID":181,"Name":"internal.metrics.executorCpuTime","Update":335659027,"Value":1328672700,"Internal":true,"Count Failed Values":true},{"ID":182,"Name":"internal.metrics.resultSize","Update":5952,"Value":23808,"Internal":true,"Count Failed Values":true},{"ID":184,"Name":"internal.metrics.resultSerializationTime","Update":3,"Value":10,"Internal":true,"Count Failed Values":true},{"ID":187,"Name":"internal.metrics.peakExecutionMemory","Update":4325376,"Value":17301504,"Internal":true,"Count Failed Values":true},{"ID":189,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":29,"Internal":true,"Count Failed Values":true},{"ID":190,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":191,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":69262,"Value":287613,"Internal":true,"Count Failed Values":true},{"ID":192,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":193,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":9703,"Value":30960,"Internal":true,"Count Failed Values":true},{"ID":194,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":195,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2891,"Value":11691,"Internal":true,"Count Failed Values":true},{"ID":201,"Name":"internal.metrics.output.bytesWritten","Update":121918,"Value":493055,"Internal":true,"Count Failed Values":true},{"ID":202,"Name":"internal.metrics.output.recordsWritten","Update":2891,"Value":11691,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":100,"Executor Deserialize CPU Time":93582698,"Executor Run Time":446,"Executor CPU Time":335659027,"Peak Execution Memory":4325376,"Result Size":5952,"JVM GC Time":0,"Result Serialization Time":3,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":69262,"Remote Bytes Read To Disk":0,"Local Bytes Read":9703,"Total Records Read":2891},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":121918,"Records Written":2891},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":37,"Index":5,"Attempt":0,"Launch Time":1633886652905,"Executor ID":"5","Host":"10.1.0.19","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633886653487,"Failed":false,"Killed":false,"Accumulables":[{"ID":37,"Name":"remote blocks read","Update":"8","Value":"37","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":39,"Name":"remote bytes read","Update":"85304","Value":"372917","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":42,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":43,"Name":"records read","Update":"3132","Value":"14823","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":51,"Name":"duration","Update":"294","Value":"1527","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":52,"Name":"sort time","Update":"5","Value":"28","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":53,"Name":"peak memory","Update":"4325376","Value":"21626880","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":54,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":178,"Name":"internal.metrics.executorDeserializeTime","Update":87,"Value":422,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.executorDeserializeCpuTime","Update":81387229,"Value":392592014,"Internal":true,"Count Failed Values":true},{"ID":180,"Name":"internal.metrics.executorRunTime","Update":484,"Value":2280,"Internal":true,"Count Failed Values":true},{"ID":181,"Name":"internal.metrics.executorCpuTime","Update":381920658,"Value":1710593358,"Internal":true,"Count Failed Values":true},{"ID":182,"Name":"internal.metrics.resultSize","Update":5952,"Value":29760,"Internal":true,"Count Failed Values":true},{"ID":184,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":12,"Internal":true,"Count Failed Values":true},{"ID":187,"Name":"internal.metrics.peakExecutionMemory","Update":4325376,"Value":21626880,"Internal":true,"Count Failed Values":true},{"ID":189,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":8,"Value":37,"Internal":true,"Count Failed Values":true},{"ID":190,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":191,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":85304,"Value":372917,"Internal":true,"Count Failed Values":true},{"ID":192,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":193,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":30960,"Internal":true,"Count Failed Values":true},{"ID":194,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":195,"Name":"internal.metrics.shuffle.read.recordsRead","Update":3132,"Value":14823,"Internal":true,"Count Failed Values":true},{"ID":201,"Name":"internal.metrics.output.bytesWritten","Update":131756,"Value":624811,"Internal":true,"Count Failed Values":true},{"ID":202,"Name":"internal.metrics.output.recordsWritten","Update":3132,"Value":14823,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":87,"Executor Deserialize CPU Time":81387229,"Executor Run Time":484,"Executor CPU Time":381920658,"Peak Execution Memory":4325376,"Result Size":5952,"JVM GC Time":0,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":8,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":85304,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":3132},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":131756,"Records Written":3132},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":32,"Index":0,"Attempt":0,"Launch Time":1633886652903,"Executor ID":"2","Host":"10.1.0.10","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633886653497,"Failed":false,"Killed":false,"Accumulables":[{"ID":37,"Name":"remote blocks read","Update":"8","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":39,"Name":"remote bytes read","Update":"84078","Value":"456995","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":42,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":43,"Name":"records read","Update":"3101","Value":"17924","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":51,"Name":"duration","Update":"307","Value":"1834","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":52,"Name":"sort time","Update":"5","Value":"33","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":53,"Name":"peak memory","Update":"4325376","Value":"25952256","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":54,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":178,"Name":"internal.metrics.executorDeserializeTime","Update":90,"Value":512,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.executorDeserializeCpuTime","Update":85994483,"Value":478586497,"Internal":true,"Count Failed Values":true},{"ID":180,"Name":"internal.metrics.executorRunTime","Update":489,"Value":2769,"Internal":true,"Count Failed Values":true},{"ID":181,"Name":"internal.metrics.executorCpuTime","Update":393474482,"Value":2104067840,"Internal":true,"Count Failed Values":true},{"ID":182,"Name":"internal.metrics.resultSize","Update":5952,"Value":35712,"Internal":true,"Count Failed Values":true},{"ID":184,"Name":"internal.metrics.resultSerializationTime","Update":4,"Value":16,"Internal":true,"Count Failed Values":true},{"ID":187,"Name":"internal.metrics.peakExecutionMemory","Update":4325376,"Value":25952256,"Internal":true,"Count Failed Values":true},{"ID":189,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":8,"Value":45,"Internal":true,"Count Failed Values":true},{"ID":190,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":191,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":84078,"Value":456995,"Internal":true,"Count Failed Values":true},{"ID":192,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":193,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":30960,"Internal":true,"Count Failed Values":true},{"ID":194,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":195,"Name":"internal.metrics.shuffle.read.recordsRead","Update":3101,"Value":17924,"Internal":true,"Count Failed Values":true},{"ID":201,"Name":"internal.metrics.output.bytesWritten","Update":129343,"Value":754154,"Internal":true,"Count Failed Values":true},{"ID":202,"Name":"internal.metrics.output.recordsWritten","Update":3101,"Value":17924,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":90,"Executor Deserialize CPU Time":85994483,"Executor Run Time":489,"Executor CPU Time":393474482,"Peak Execution Memory":4325376,"Result Size":5952,"JVM GC Time":0,"Result Serialization Time":4,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":8,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":84078,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":3101},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":129343,"Records Written":3101},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":35,"Index":3,"Attempt":0,"Launch Time":1633886652904,"Executor ID":"7","Host":"10.1.0.9","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633886654337,"Failed":false,"Killed":false,"Accumulables":[{"ID":37,"Name":"remote blocks read","Update":"7","Value":"52","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":38,"Name":"local blocks read","Update":"1","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":39,"Name":"remote bytes read","Update":"76289","Value":"533284","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":41,"Name":"local bytes read","Update":"10900","Value":"41860","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":42,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":43,"Name":"records read","Update":"3223","Value":"21147","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":51,"Name":"duration","Update":"1157","Value":"2991","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":52,"Name":"sort time","Update":"6","Value":"39","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":53,"Name":"peak memory","Update":"4325376","Value":"30277632","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":54,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":178,"Name":"internal.metrics.executorDeserializeTime","Update":109,"Value":621,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.executorDeserializeCpuTime","Update":101726190,"Value":580312687,"Internal":true,"Count Failed Values":true},{"ID":180,"Name":"internal.metrics.executorRunTime","Update":1312,"Value":4081,"Internal":true,"Count Failed Values":true},{"ID":181,"Name":"internal.metrics.executorCpuTime","Update":1055555672,"Value":3159623512,"Internal":true,"Count Failed Values":true},{"ID":182,"Name":"internal.metrics.resultSize","Update":5995,"Value":41707,"Internal":true,"Count Failed Values":true},{"ID":183,"Name":"internal.metrics.jvmGCTime","Update":119,"Value":119,"Internal":true,"Count Failed Values":true},{"ID":184,"Name":"internal.metrics.resultSerializationTime","Update":3,"Value":19,"Internal":true,"Count Failed Values":true},{"ID":187,"Name":"internal.metrics.peakExecutionMemory","Update":4325376,"Value":30277632,"Internal":true,"Count Failed Values":true},{"ID":189,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":52,"Internal":true,"Count Failed Values":true},{"ID":190,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":191,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":76289,"Value":533284,"Internal":true,"Count Failed Values":true},{"ID":192,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":193,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":10900,"Value":41860,"Internal":true,"Count Failed Values":true},{"ID":194,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":195,"Name":"internal.metrics.shuffle.read.recordsRead","Update":3223,"Value":21147,"Internal":true,"Count Failed Values":true},{"ID":201,"Name":"internal.metrics.output.bytesWritten","Update":134365,"Value":888519,"Internal":true,"Count Failed Values":true},{"ID":202,"Name":"internal.metrics.output.recordsWritten","Update":3223,"Value":21147,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":109,"Executor Deserialize CPU Time":101726190,"Executor Run Time":1312,"Executor CPU Time":1055555672,"Peak Execution Memory":4325376,"Result Size":5995,"JVM GC Time":119,"Result Serialization Time":3,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":76289,"Remote Bytes Read To Disk":0,"Local Bytes Read":10900,"Total Records Read":3223},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":134365,"Records Written":3223},"Updated Blocks":[]}} +{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":34,"Index":2,"Attempt":0,"Launch Time":1633886652904,"Executor ID":"4","Host":"10.1.0.8","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633886654355,"Failed":false,"Killed":false,"Accumulables":[{"ID":37,"Name":"remote blocks read","Update":"7","Value":"59","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":38,"Name":"local blocks read","Update":"1","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":39,"Name":"remote bytes read","Update":"58219","Value":"591503","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":41,"Name":"local bytes read","Update":"8592","Value":"50452","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":42,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":43,"Name":"records read","Update":"2451","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":51,"Name":"duration","Update":"1139","Value":"4130","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":52,"Name":"sort time","Update":"5","Value":"44","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":53,"Name":"peak memory","Update":"4325376","Value":"34603008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":54,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":178,"Name":"internal.metrics.executorDeserializeTime","Update":118,"Value":739,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.executorDeserializeCpuTime","Update":108815452,"Value":689128139,"Internal":true,"Count Failed Values":true},{"ID":180,"Name":"internal.metrics.executorRunTime","Update":1319,"Value":5400,"Internal":true,"Count Failed Values":true},{"ID":181,"Name":"internal.metrics.executorCpuTime","Update":1051976887,"Value":4211600399,"Internal":true,"Count Failed Values":true},{"ID":182,"Name":"internal.metrics.resultSize","Update":5995,"Value":47702,"Internal":true,"Count Failed Values":true},{"ID":183,"Name":"internal.metrics.jvmGCTime","Update":109,"Value":228,"Internal":true,"Count Failed Values":true},{"ID":184,"Name":"internal.metrics.resultSerializationTime","Update":4,"Value":23,"Internal":true,"Count Failed Values":true},{"ID":187,"Name":"internal.metrics.peakExecutionMemory","Update":4325376,"Value":34603008,"Internal":true,"Count Failed Values":true},{"ID":189,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":59,"Internal":true,"Count Failed Values":true},{"ID":190,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":191,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":58219,"Value":591503,"Internal":true,"Count Failed Values":true},{"ID":192,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":193,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":8592,"Value":50452,"Internal":true,"Count Failed Values":true},{"ID":194,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":195,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2451,"Value":23598,"Internal":true,"Count Failed Values":true},{"ID":201,"Name":"internal.metrics.output.bytesWritten","Update":102178,"Value":990697,"Internal":true,"Count Failed Values":true},{"ID":202,"Name":"internal.metrics.output.recordsWritten","Update":2451,"Value":23598,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":118,"Executor Deserialize CPU Time":108815452,"Executor Run Time":1319,"Executor CPU Time":1051976887,"Peak Execution Memory":4325376,"Result Size":5995,"JVM GC Time":109,"Result Serialization Time":4,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":58219,"Remote Bytes Read To Disk":0,"Local Bytes Read":8592,"Total Records Read":2451},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":102178,"Records Written":2451},"Updated Blocks":[]}} +{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":5,"Stage Attempt ID":0,"Stage Name":"sql at ScalaSparkSQLBench.scala:43","Number of Tasks":8,"RDD Info":[{"RDD ID":19,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[18],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":18,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[17],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[4],"Details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1633886652870,"Completion Time":1633886654356,"Accumulables":[{"ID":37,"Name":"remote blocks read","Value":"59","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":38,"Name":"local blocks read","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":39,"Name":"remote bytes read","Value":"591503","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":41,"Name":"local bytes read","Value":"50452","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":42,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":43,"Name":"records read","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":51,"Name":"duration","Value":"4130","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":52,"Name":"sort time","Value":"44","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":53,"Name":"peak memory","Value":"34603008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":54,"Name":"spill size","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":178,"Name":"internal.metrics.executorDeserializeTime","Value":739,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.executorDeserializeCpuTime","Value":689128139,"Internal":true,"Count Failed Values":true},{"ID":180,"Name":"internal.metrics.executorRunTime","Value":5400,"Internal":true,"Count Failed Values":true},{"ID":181,"Name":"internal.metrics.executorCpuTime","Value":4211600399,"Internal":true,"Count Failed Values":true},{"ID":182,"Name":"internal.metrics.resultSize","Value":47702,"Internal":true,"Count Failed Values":true},{"ID":183,"Name":"internal.metrics.jvmGCTime","Value":228,"Internal":true,"Count Failed Values":true},{"ID":184,"Name":"internal.metrics.resultSerializationTime","Value":23,"Internal":true,"Count Failed Values":true},{"ID":187,"Name":"internal.metrics.peakExecutionMemory","Value":34603008,"Internal":true,"Count Failed Values":true},{"ID":189,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":59,"Internal":true,"Count Failed Values":true},{"ID":190,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":5,"Internal":true,"Count Failed Values":true},{"ID":191,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":591503,"Internal":true,"Count Failed Values":true},{"ID":192,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":193,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":50452,"Internal":true,"Count Failed Values":true},{"ID":194,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":195,"Name":"internal.metrics.shuffle.read.recordsRead","Value":23598,"Internal":true,"Count Failed Values":true},{"ID":201,"Name":"internal.metrics.output.bytesWritten","Value":990697,"Internal":true,"Count Failed Values":true},{"ID":202,"Name":"internal.metrics.output.recordsWritten","Value":23598,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} +{"Event":"SparkListenerJobEnd","Job ID":2,"Completion Time":1633886654356,"Job Result":{"Result":"JobSucceeded"}} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":11,"accumUpdates":[[47,8],[48,990697],[49,23598],[50,0]]} +{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":11,"time":1633886655028} +{"Event":"SparkListenerApplicationEnd","Timestamp":1633886655029} diff --git a/tests/test_ir/test_key_value_pair_log_event.py b/tests/test_ir/test_key_value_pair_log_event.py new file mode 100644 index 00000000..31d47482 --- /dev/null +++ b/tests/test_ir/test_key_value_pair_log_event.py @@ -0,0 +1,31 @@ +from pathlib import Path +from typing import Any, Dict + +from test_ir.test_utils import JsonFileReader, TestCLPBase + +from clp_ffi_py.ir import KeyValuePairLogEvent + + +class TestCaseKeyValuePairLogEvent(TestCLPBase): + """ + Class for testing `clp_ffi_py.ir.KeyValuePairLogEvent`. + """ + + input_src_dir: str = "test_json" + + def test_basic(self) -> None: + """ + Tests the conversion between a Python dictionary and a `KeyValuePairLogEvent` instance, + ensuring accurate serialization and deserialization in both directions. + """ + current_dir: Path = Path(__file__).resolve().parent + test_src_dir: Path = current_dir / Path(TestCaseKeyValuePairLogEvent.input_src_dir) + for file_path in test_src_dir.rglob("*"): + if not file_path.is_file(): + continue + json_file_reader: JsonFileReader = JsonFileReader(file_path) + for expected in json_file_reader.read_json_lines(): + self.assertEqual(isinstance(expected, dict), True) + actual: KeyValuePairLogEvent = KeyValuePairLogEvent(expected) + serialized_py_dict: Dict[Any, Any] = actual.to_dict() + self.assertEqual(expected, serialized_py_dict) diff --git a/tests/test_ir/test_utils.py b/tests/test_ir/test_utils.py index 19763361..282725ce 100644 --- a/tests/test_ir/test_utils.py +++ b/tests/test_ir/test_utils.py @@ -1,9 +1,11 @@ +import json import random import time import unittest from datetime import tzinfo from math import floor -from typing import IO, List, Optional, Set, Tuple, Union +from pathlib import Path +from typing import Any, Generator, IO, List, Optional, Set, Tuple, Union import dateutil.tz from smart_open import register_compressor # type: ignore @@ -22,6 +24,32 @@ from clp_ffi_py.wildcard_query import WildcardQuery +class JsonFileReader: + """ + Class for reading JSON files. + + It assumes each line in the file is a JSON string, and the parser parses each line into a JSON + object, and return then through a generator. + """ + + def __init__(self, file_path: Path): + """ + Initializes the `JSONFileReader` with the given file path. + + :param file_path: Path to the JSON file to read. + """ + self.file_path: Path = file_path + + def read_json_lines(self) -> Generator[Any, None, None]: + """ + Reads each line in the JSON file, parses it as a JSON object, and yields the JSON object. + + :yield: Parsed JSON object for each line in the file. + """ + with open(self.file_path, "r", encoding="utf-8") as file: + yield from (json.loads(line.strip()) for line in file) + + def _zstd_compressions_handler( file_obj: IO[bytes], mode: str ) -> Union[ZstdCompressionWriter, ZstdDecompressionReader]: diff --git a/tests/test_utils.py b/tests/test_utils.py index 2fdcb34d..57f69d81 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,9 +1,8 @@ import unittest from typing import Any, Dict -import msgpack - import clp_ffi_py.utils +import msgpack class TestUtils(unittest.TestCase): From 8d0f782d4fae2826526d53a93b6c477e8f8bd75c Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Sun, 3 Nov 2024 17:20:58 -0500 Subject: [PATCH 08/50] Fix msgpack configuration --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 676fd56b..75c60e10 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -61,8 +61,8 @@ set(CLP_FFI_PY_CLP_CORE_DIR "${CLP_FFI_PY_SRC_DIR}/clp/components/core") add_subdirectory(${CLP_FFI_PY_CLP_CORE_DIR}/src/clp/string_utils) -set(MSGPACK_CXX20 ON) -set(MSGPACK_USE_BOOST OFF) +set(MSGPACK_CXX20 ON CACHE BOOL "Enable C++20 in msgpack") +set(MSGPACK_USE_BOOST OFF CACHE BOOL "Disable Boost in msgpack") add_subdirectory(${CLP_FFI_PY_SRC_DIR}/msgpack EXCLUDE_FROM_ALL) add_subdirectory(${CLP_FFI_PY_SRC_DIR}/GSL) From 5662c86727722bbd5b68eff0a854ce8ad9e150ce Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Sun, 3 Nov 2024 19:52:03 -0500 Subject: [PATCH 09/50] Apply rabit's comments --- src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp | 6 +++--- src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.hpp | 9 ++++++--- tests/test_ir/test_key_value_pair_log_event.py | 2 +- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp b/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp index c287322c..1a40f54a 100644 --- a/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp +++ b/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp @@ -207,8 +207,8 @@ CLP_FFI_PY_METHOD auto PyKeyValuePairLogEvent_dealloc(PyKeyValuePairLogEvent* se CLP_FFI_PY_METHOD auto PyKeyValuePairLogEvent_to_dict(PyKeyValuePairLogEvent* self) -> PyObject* { // TODO: use an efficient algorithm to turn the underlying log event to a Python dictionary - auto const& kv_pair_log_event{self->get_kv_pair_log_event()}; - auto const serialized_json_result{kv_pair_log_event.serialize_to_json()}; + auto const* kv_pair_log_event{self->get_kv_pair_log_event()}; + auto const serialized_json_result{kv_pair_log_event->serialize_to_json()}; if (serialized_json_result.has_error()) { PyErr_Format( PyExc_RuntimeError, @@ -224,6 +224,7 @@ CLP_FFI_PY_METHOD auto PyKeyValuePairLogEvent_to_dict(PyKeyValuePairLogEvent* se } if (false == static_cast(PyDict_Check(parsed_json))) { PyErr_SetString(PyExc_TypeError, "Serialized JSON object is not a dictionary"); + return nullptr; } return parsed_json; } @@ -329,7 +330,6 @@ auto PyKeyValuePairLogEvent::get_py_type() -> PyTypeObject* { } auto PyKeyValuePairLogEvent::module_level_init(PyObject* py_module) -> bool { - // TODO: complete this function static_assert(std::is_trivially_destructible()); auto* type{py_reinterpret_cast(PyType_FromSpec(&PyKeyValuePairLogEvent_type_spec)) }; diff --git a/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.hpp b/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.hpp index b52cce66..c65fb1c2 100644 --- a/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.hpp +++ b/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.hpp @@ -37,10 +37,13 @@ class PyKeyValuePairLogEvent { /** * Releases the memory allocated for underlying data fields. */ - auto clean() -> void { delete m_kv_pair_log_event; } + auto clean() -> void { + delete m_kv_pair_log_event; + m_kv_pair_log_event = nullptr; + } - [[nodiscard]] auto get_kv_pair_log_event() const -> clp::ffi::KeyValuePairLogEvent const& { - return *m_kv_pair_log_event; + [[nodiscard]] auto get_kv_pair_log_event() const -> clp::ffi::KeyValuePairLogEvent const* { + return m_kv_pair_log_event; } /** diff --git a/tests/test_ir/test_key_value_pair_log_event.py b/tests/test_ir/test_key_value_pair_log_event.py index 31d47482..fac8f6fb 100644 --- a/tests/test_ir/test_key_value_pair_log_event.py +++ b/tests/test_ir/test_key_value_pair_log_event.py @@ -25,7 +25,7 @@ def test_basic(self) -> None: continue json_file_reader: JsonFileReader = JsonFileReader(file_path) for expected in json_file_reader.read_json_lines(): - self.assertEqual(isinstance(expected, dict), True) + self.assertIsInstance(expected, dict, "Input must be a dictionary") actual: KeyValuePairLogEvent = KeyValuePairLogEvent(expected) serialized_py_dict: Dict[Any, Any] = actual.to_dict() self.assertEqual(expected, serialized_py_dict) From 800e7332952720bdb12999917bf86754c8979f7f Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Sun, 3 Nov 2024 20:58:53 -0500 Subject: [PATCH 10/50] Fix comment --- src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.hpp b/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.hpp index c65fb1c2..8f89349c 100644 --- a/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.hpp +++ b/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.hpp @@ -12,7 +12,7 @@ namespace clp_ffi_py::ir::native { /** * A PyObject structure functioning as a Python-compatible interface to retrieve a key-value pair - * log event. The underlying data is pointed to by `m_kv_pair_log_event`. + * log event. The underlying data is pointed by `m_kv_pair_log_event`. */ class PyKeyValuePairLogEvent { public: From a21bc2e23b05e62a1a3bf6aeb9e78fa3a73f2446 Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Sun, 3 Nov 2024 21:59:13 -0500 Subject: [PATCH 11/50] Add comment --- src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp b/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp index 1a40f54a..0c8da638 100644 --- a/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp +++ b/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp @@ -107,6 +107,12 @@ PyDoc_STRVAR( ":return: Serialized log event in a Python dictionary.\n" ); +/** + * Callback of `PyKeyValuePairLogEvent`'s `to_dict` method: + * @param self + * @return Serialized log event in a Python dictionary on success. + * @return nullptr on failure with the relevant Python exception and error set. + */ CLP_FFI_PY_METHOD auto PyKeyValuePairLogEvent_to_dict(PyKeyValuePairLogEvent* self) -> PyObject*; // NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays) From 04112b553da53ef94d1acb88fb8bdd7a7d095b43 Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Sun, 3 Nov 2024 23:02:47 -0500 Subject: [PATCH 12/50] Reordering... --- .../ir/native/PyKeyValuePairLogEvent.cpp | 37 +++++++------------ 1 file changed, 13 insertions(+), 24 deletions(-) diff --git a/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp b/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp index 0c8da638..eb3a3285 100644 --- a/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp +++ b/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp @@ -62,6 +62,9 @@ class IrUnitHandler { std::optional m_log_event; }; +/** + * Callback of `PyKeyValuePairLogEvent`'s `__init__` method. + */ // NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays) PyDoc_STRVAR( cPyKeyValuePairLogEventDoc, @@ -77,15 +80,6 @@ PyDoc_STRVAR( ":param dictionary: A dictionary representing the key-value log event, where all keys are" " expected to be of string type, including keys inside any sub-dictionaries.\n" ); - -/** - * Callback of `PyKeyValuePairLogEvent`'s `__init__` method: - * @param self - * @param args - * @param keywords - * @return 0 on success. - * @return -1 on failure with the relevant Python exception and error set. - */ CLP_FFI_PY_METHOD auto PyKeyValuePairLogEvent_init( PyKeyValuePairLogEvent* self, PyObject* args, @@ -93,11 +87,8 @@ CLP_FFI_PY_METHOD auto PyKeyValuePairLogEvent_init( ) -> int; /** - * Callback of `PyKeyValuePairLogEvent`'s deallocator. - * @param self + * Callback of `PyKeyValuePairLogEvent`'s `to_dict` method. */ -CLP_FFI_PY_METHOD auto PyKeyValuePairLogEvent_dealloc(PyKeyValuePairLogEvent* self) -> void; - // NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays) PyDoc_STRVAR( cPyKeyValuePairLogEventToDictDoc, @@ -106,14 +97,12 @@ PyDoc_STRVAR( "Converts the underlying key-value pair log event into a Python dictionary.\n\n" ":return: Serialized log event in a Python dictionary.\n" ); +CLP_FFI_PY_METHOD auto PyKeyValuePairLogEvent_to_dict(PyKeyValuePairLogEvent* self) -> PyObject*; /** - * Callback of `PyKeyValuePairLogEvent`'s `to_dict` method: - * @param self - * @return Serialized log event in a Python dictionary on success. - * @return nullptr on failure with the relevant Python exception and error set. + * Callback of `PyKeyValuePairLogEvent`'s deallocator. */ -CLP_FFI_PY_METHOD auto PyKeyValuePairLogEvent_to_dict(PyKeyValuePairLogEvent* self) -> PyObject*; +CLP_FFI_PY_METHOD auto PyKeyValuePairLogEvent_dealloc(PyKeyValuePairLogEvent* self) -> void; // NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays) PyMethodDef PyKeyValuePairLogEvent_method_table[]{ @@ -132,7 +121,7 @@ PyType_Slot PyKeyValuePairLogEvent_slots[]{ {Py_tp_new, reinterpret_cast(PyType_GenericNew)}, {Py_tp_init, reinterpret_cast(PyKeyValuePairLogEvent_init)}, {Py_tp_methods, static_cast(PyKeyValuePairLogEvent_method_table)}, - {Py_tp_doc, const_cast(static_cast(cPyKeyValuePairLogEventToDictDoc))}, + {Py_tp_doc, const_cast(static_cast(cPyKeyValuePairLogEventDoc))}, {0, nullptr} }; // NOLINTEND(cppcoreguidelines-avoid-c-arrays, cppcoreguidelines-pro-type-*-cast) @@ -206,11 +195,6 @@ CLP_FFI_PY_METHOD auto PyKeyValuePairLogEvent_init( return 0; } -CLP_FFI_PY_METHOD auto PyKeyValuePairLogEvent_dealloc(PyKeyValuePairLogEvent* self) -> void { - self->clean(); - PyObject_Del(self); -} - CLP_FFI_PY_METHOD auto PyKeyValuePairLogEvent_to_dict(PyKeyValuePairLogEvent* self) -> PyObject* { // TODO: use an efficient algorithm to turn the underlying log event to a Python dictionary auto const* kv_pair_log_event{self->get_kv_pair_log_event()}; @@ -324,6 +308,11 @@ auto convert_py_dict_to_key_value_pair_log_event(PyDictObject* py_dict return std::move(ir_unit_handler.m_log_event); } + +CLP_FFI_PY_METHOD auto PyKeyValuePairLogEvent_dealloc(PyKeyValuePairLogEvent* self) -> void { + self->clean(); + PyObject_Del(self); +} } // namespace auto PyKeyValuePairLogEvent::init(clp::ffi::KeyValuePairLogEvent kv_pair_log_event) -> bool { From 389b79f426b9136f0a1745baf558569323e6a30f Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Sun, 3 Nov 2024 23:30:18 -0500 Subject: [PATCH 13/50] Reordering... --- src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp b/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp index eb3a3285..c603cc9d 100644 --- a/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp +++ b/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp @@ -160,6 +160,10 @@ CLP_FFI_PY_METHOD auto PyKeyValuePairLogEvent_init( static char keyword_dictionary[]{"dictionary"}; static char* keyword_table[]{static_cast(keyword_dictionary), nullptr}; + // If the argument parsing fails, `self` will be deallocated. We must reset all pointers to + // nullptr in advance, otherwise the deallocator might trigger segmentation fault. + self->default_init(); + PyObject* dictionary{Py_None}; if (false == static_cast(PyArg_ParseTupleAndKeywords( @@ -173,10 +177,6 @@ CLP_FFI_PY_METHOD auto PyKeyValuePairLogEvent_init( return -1; } - // If the argument parsing fails, `self` will be deallocated. We must reset all pointers to - // nullptr in advance, otherwise the deallocator might trigger segmentation fault. - self->default_init(); - if (false == static_cast(PyDict_Check(dictionary))) { PyErr_SetString(PyExc_TypeError, "`dictionary` must be a Python dictionary object"); return -1; From 8b41d8022b489406cefed4f19e3304b9687dba12 Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Mon, 4 Nov 2024 00:05:04 -0500 Subject: [PATCH 14/50] Fix init issue... --- src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp b/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp index c603cc9d..d4aa72c5 100644 --- a/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp +++ b/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -188,11 +189,7 @@ CLP_FFI_PY_METHOD auto PyKeyValuePairLogEvent_init( return -1; } - if (false == self->init(std::move(optional_kv_pair_log_event.value()))) { - return -1; - } - - return 0; + return self->init(std::move(optional_kv_pair_log_event.value())) ? 0 : -1; } CLP_FFI_PY_METHOD auto PyKeyValuePairLogEvent_to_dict(PyKeyValuePairLogEvent* self) -> PyObject* { @@ -317,7 +314,11 @@ CLP_FFI_PY_METHOD auto PyKeyValuePairLogEvent_dealloc(PyKeyValuePairLogEvent* se auto PyKeyValuePairLogEvent::init(clp::ffi::KeyValuePairLogEvent kv_pair_log_event) -> bool { m_kv_pair_log_event = new clp::ffi::KeyValuePairLogEvent{std::move(kv_pair_log_event)}; - return nullptr != m_kv_pair_log_event; + if (nullptr == m_kv_pair_log_event) { + PyErr_SetString(PyExc_RuntimeError, clp_ffi_py::cOutofMemoryError); + return false; + } + return true; } auto PyKeyValuePairLogEvent::get_py_type() -> PyTypeObject* { From 15fcb413c5e6fa9fa46d656ff7e41aad9c31b077 Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Mon, 4 Nov 2024 03:17:08 -0500 Subject: [PATCH 15/50] Implement PySerializer --- CMakeLists.txt | 2 + clp_ffi_py/ir/__init__.py | 1 + clp_ffi_py/ir/native.pyi | 8 + src/clp_ffi_py/PyObjectCast.hpp | 2 + src/clp_ffi_py/Python.hpp | 4 + src/clp_ffi_py/ir/native/PySerializer.cpp | 458 ++++++++++++++++++++++ src/clp_ffi_py/ir/native/PySerializer.hpp | 149 +++++++ src/clp_ffi_py/modules/ir_native.cpp | 6 + 8 files changed, 630 insertions(+) create mode 100644 src/clp_ffi_py/ir/native/PySerializer.cpp create mode 100644 src/clp_ffi_py/ir/native/PySerializer.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 75c60e10..130aa66c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -108,6 +108,8 @@ set(CLP_FFI_PY_LIB_IR_SOURCES ${CLP_FFI_PY_LIB_SRC_DIR}/ir/native/PyMetadata.hpp ${CLP_FFI_PY_LIB_SRC_DIR}/ir/native/PyQuery.cpp ${CLP_FFI_PY_LIB_SRC_DIR}/ir/native/PyQuery.hpp + ${CLP_FFI_PY_LIB_SRC_DIR}/ir/native/PySerializer.cpp + ${CLP_FFI_PY_LIB_SRC_DIR}/ir/native/PySerializer.hpp ${CLP_FFI_PY_LIB_SRC_DIR}/ir/native/Query.cpp ${CLP_FFI_PY_LIB_SRC_DIR}/ir/native/Query.hpp ${CLP_FFI_PY_LIB_SRC_DIR}/ir/native/serialization_methods.cpp diff --git a/clp_ffi_py/ir/__init__.py b/clp_ffi_py/ir/__init__.py index defc702e..b89ffda4 100644 --- a/clp_ffi_py/ir/__init__.py +++ b/clp_ffi_py/ir/__init__.py @@ -18,6 +18,7 @@ "Metadata", # native "Query", # native "QueryBuilder", # query_builder + "Serializer", # native "ClpIrFileReader", # readers "ClpIrStreamReader", # readers ] diff --git a/clp_ffi_py/ir/native.pyi b/clp_ffi_py/ir/native.pyi index 16e2e9c2..3f59883a 100644 --- a/clp_ffi_py/ir/native.pyi +++ b/clp_ffi_py/ir/native.pyi @@ -86,4 +86,12 @@ class KeyValuePairLogEvent: def __init__(self, dictionary: Dict[Any, Any]): ... def to_dict(self) -> Dict[Any, Any]: ... +class Serializer: + def __init__(self, output_stream: IO[bytes]): ... + def serialize_msgpack_byte_sequence(self, msgpack_byte_sequence: bytes) -> None: ... + def write_to_stream(self) -> int: ... + def get_buffer_size(self) -> int: ... + def flush_stream(self) -> None: ... + def close(self, flush_stream: bool = True) -> int: ... + class IncompleteStreamError(Exception): ... diff --git a/src/clp_ffi_py/PyObjectCast.hpp b/src/clp_ffi_py/PyObjectCast.hpp index 909793cf..196c47f8 100644 --- a/src/clp_ffi_py/PyObjectCast.hpp +++ b/src/clp_ffi_py/PyObjectCast.hpp @@ -118,6 +118,7 @@ class PyKeyValuePairLogEvent; class PyLogEvent; class PyMetadata; class PyQuery; +class PySerializer; } // namespace ir::native CLP_FFI_PY_MARK_AS_PYOBJECT(ir::native::PyDeserializerBuffer); @@ -126,6 +127,7 @@ CLP_FFI_PY_MARK_AS_PYOBJECT(ir::native::PyKeyValuePairLogEvent); CLP_FFI_PY_MARK_AS_PYOBJECT(ir::native::PyLogEvent); CLP_FFI_PY_MARK_AS_PYOBJECT(ir::native::PyMetadata); CLP_FFI_PY_MARK_AS_PYOBJECT(ir::native::PyQuery); +CLP_FFI_PY_MARK_AS_PYOBJECT(ir::native::PySerializer); CLP_FFI_PY_MARK_AS_PYOBJECT(PyBytesObject); CLP_FFI_PY_MARK_AS_PYOBJECT(PyDictObject); CLP_FFI_PY_MARK_AS_PYOBJECT(PyTypeObject); diff --git a/src/clp_ffi_py/Python.hpp b/src/clp_ffi_py/Python.hpp index 9c626e19..1283043e 100644 --- a/src/clp_ffi_py/Python.hpp +++ b/src/clp_ffi_py/Python.hpp @@ -13,14 +13,18 @@ #ifdef CLP_FFI_PY_ENABLE_LINTING // The following headers are added to export Python headers to get rid of clang-tidy warnings. // IWYU pragma: begin_exports +#include #include #include +#include +#include #include #include #include #include #include #include +#include #include // IWYU pragma: end_exports #endif diff --git a/src/clp_ffi_py/ir/native/PySerializer.cpp b/src/clp_ffi_py/ir/native/PySerializer.cpp new file mode 100644 index 00000000..25a180e1 --- /dev/null +++ b/src/clp_ffi_py/ir/native/PySerializer.cpp @@ -0,0 +1,458 @@ +#include // Must always be included before any other header files + +#include "PySerializer.hpp" + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include + +namespace clp_ffi_py::ir::native { +namespace { +/** + * Callback of `PySerializer`'s `__init__` method: + */ +// NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays) +PyDoc_STRVAR( + cPySerializerDoc, + "Serializer for CLP key-value pair IR stream.\n" + "This class serializes log events into CLP IR format (using four-byte-encoding) and writes" + " the serialized data to a specified byte stream.\n\n" + "__init__(self, output_stream)\n\n" + "Initializes a `Serializer` instance with the given output stream. Notice that each object" + " should be strictly initialized only once. Double initialization will result in memory" + " leak.\n\n" + ":param output_stream: A writable byte stream to which the serializer will write serialized" + " IR byte sequence.\n" +); +CLP_FFI_PY_METHOD auto +PySerializer_init(PySerializer* self, PyObject* args, PyObject* keywords) -> int; + +/** + * Callback of `PySerializer`'s `serialize_msgpack` method. + */ +// NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays) +PyDoc_STRVAR( + cPySerializerSerializeMsgpackDoc, + "serialize_msgpack(self, msgpack_byte_sequence)\n" + "--\n\n" + "Serializes the given msgpack byte sequence into as a log event.\n" + "NOTE: the serialization results will be buffered inside the serializer. `write_to_stream`" + " must be called to write buffered bytes into the output stream.\n\n" + ":param msgpack_byte_sequence: A byte sequence encoded in msgpack as the input log event." + " The unpacked result must be a msgpack map with all keys as strings.\n" + ":raise IOError: If the serializer has already been closed.\n" + ":raise TypeError: If the unpacked result is not a msgpack map.\n" + ":raise RuntimeError: If it fails to unpack the given msgpack byte sequence, or the" + " serialization method returns failure.\n" +); +CLP_FFI_PY_METHOD auto +PySerializer_serialize_msgpack(PySerializer* self, PyObject* msgpack_byte_sequence) -> PyObject*; + +/** + * Callback of `PySerializer`'s `write_to_stream` method. + */ +// NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays) +PyDoc_STRVAR( + cPySerializerWriteToStreamDoc, + "write_to_stream(self)\n" + "--\n\n" + "Writes the buffered results to the output stream and clears the buffer.\n\n" + ":return: Number of bytes written.\n" + ":raise IOError: If the serializer has already been closed.\n" +); +CLP_FFI_PY_METHOD auto PySerializer_write_to_stream(PySerializer* self) -> PyObject*; + +/** + * Callback of `PySerializer`'s `get_buffer_size` method. + */ +// NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays) +PyDoc_STRVAR( + cPySerializerGetBufferSizeDoc, + "get_buffer_size(self)\n" + "--\n\n" + "Gets the size of the result buffer.\n\n" + ":return: The buffer size in bytes.\n" + ":raise IOError: If the serializer has already been closed.\n" +); +CLP_FFI_PY_METHOD auto PySerializer_get_buffer_size(PySerializer* self) -> PyObject*; + +/** + * Callback of `PySerializer`'s `flush_stream` method. + */ +// NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays) +PyDoc_STRVAR( + cPySerializerFlushStreamDoc, + "flush_stream(self)\n" + "--\n\n" + "Flushes the output stream.\n\n" + ":raise IOError: If the serializer has already been closed.\n" +); +CLP_FFI_PY_METHOD auto PySerializer_flush_stream(PySerializer* self) -> PyObject*; + +/** + * Callback of `PySerializer`'s `close` method. + */ +// NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays) +PyDoc_STRVAR( + cPySerializerCloseDoc, + "close(self, flush_stream=True)\n" + "--\n\n" + "Closes the serializer by writing the results into the output stream with the byte sequence" + " in the end that indicates the end of a CLP IR stream. This method must be called to" + " terminate an IR stream. Otherwise, the stream will be considered incomplete.\n\n" + ":param flush_stream: Whether to flush the output stream.\n" + ":return: Forwards :meth:`write_to_stream`'s return value.\n" + ":raise IOError: If the serializer has already been closed.\n" +); +CLP_FFI_PY_METHOD auto +PySerializer_close(PySerializer* self, PyObject* args, PyObject* keywords) -> PyObject*; + +/** + * Callback of `PySerializer`'s deallocator. + */ +CLP_FFI_PY_METHOD auto PySerializer_dealloc(PySerializer* self) -> void; + +// NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays) +PyMethodDef PySerializer_method_table[]{ + {"serialize_msgpack", + py_c_function_cast(PySerializer_serialize_msgpack), + METH_O, + static_cast(cPySerializerSerializeMsgpackDoc)}, + + {"write_to_stream", + py_c_function_cast(PySerializer_write_to_stream), + METH_NOARGS, + static_cast(cPySerializerWriteToStreamDoc)}, + + {"get_buffer_size", + py_c_function_cast(PySerializer_get_buffer_size), + METH_NOARGS, + static_cast(cPySerializerGetBufferSizeDoc)}, + + {"flush_stream", + py_c_function_cast(PySerializer_flush_stream), + METH_NOARGS, + static_cast(cPySerializerFlushStreamDoc)}, + + {"close", + py_c_function_cast(PySerializer_close), + METH_VARARGS | METH_KEYWORDS, + static_cast(cPySerializerCloseDoc)}, + + {nullptr} +}; + +// NOLINTBEGIN(cppcoreguidelines-avoid-c-arrays, cppcoreguidelines-pro-type-*-cast) +PyType_Slot PySerializer_slots[]{ + {Py_tp_alloc, reinterpret_cast(PyType_GenericAlloc)}, + {Py_tp_dealloc, reinterpret_cast(PySerializer_dealloc)}, + {Py_tp_new, reinterpret_cast(PyType_GenericNew)}, + {Py_tp_init, reinterpret_cast(PySerializer_init)}, + {Py_tp_methods, static_cast(PySerializer_method_table)}, + {Py_tp_doc, const_cast(static_cast(cPySerializerDoc))}, + {0, nullptr} +}; +// NOLINTEND(cppcoreguidelines-avoid-c-arrays, cppcoreguidelines-pro-type-*-cast) + +/** + * `PySerializer`'s Python type specifications. + */ +PyType_Spec PySerializer_type_spec{ + "clp_ffi_py.ir.native.Serializer", + sizeof(PySerializer), + 0, + Py_TPFLAGS_DEFAULT, + static_cast(PySerializer_slots) +}; + +CLP_FFI_PY_METHOD auto +PySerializer_init(PySerializer* self, PyObject* args, PyObject* keywords) -> int { + static char keyword_output_stream[]{"output_stream"}; + static char* keyword_table[]{static_cast(keyword_output_stream), nullptr}; + + // If the argument parsing fails, `self` will be deallocated. We must reset all pointers to + // nullptr in advance, otherwise the deallocator might trigger segmentation fault. + self->default_init(); + + PyObject* output_stream{Py_None}; + if (false + == static_cast(PyArg_ParseTupleAndKeywords( + args, + keywords, + "O", + static_cast(keyword_table), + &output_stream + ))) + { + return -1; + } + + // Ensure the `output_stream` has `write` and `flush` methods + PyObjectPtr const write_method{PyObject_GetAttrString(output_stream, "write")}; + if (nullptr == write_method) { + return -1; + } + if (false == static_cast(PyCallable_Check(write_method.get()))) { + PyErr_SetString( + PyExc_TypeError, + "The attribute `write` of the given output stream object is not callable." + ); + return -1; + } + + PyObjectPtr const flush_method{PyObject_GetAttrString(output_stream, "flush")}; + if (nullptr == flush_method) { + return -1; + } + if (false == static_cast(PyCallable_Check(flush_method.get()))) { + PyErr_SetString( + PyExc_TypeError, + "The attribute `flush` of the given output stream object is not callable." + ); + return -1; + } + + auto serializer_result{PySerializer::ClpIrSerializer::create()}; + if (serializer_result.has_error()) { + PyErr_Format( + PyExc_RuntimeError, + cSerializerCreateErrorFormatStr.data(), + serializer_result.error().message().c_str() + ); + return -1; + } + + return self->init(output_stream, std::move(serializer_result.value())) ? 0 : -1; +} + +CLP_FFI_PY_METHOD auto +PySerializer_serialize_msgpack(PySerializer* self, PyObject* msgpack_byte_sequence) -> PyObject* { + if (false == static_cast(PyBytes_Check(msgpack_byte_sequence))) { + PyErr_SetString( + PyExc_TypeError, + "`msgpack_byte_sequence` is supposed to return a `bytes` object" + ); + return nullptr; + } + + auto* py_bytes_msgpack_byte_sequence{py_reinterpret_cast(msgpack_byte_sequence)}; + // Since the type is already checked, we can use the macro to avoid duplicated type checking. + if (false + == self->serialize_msgpack_map( + {PyBytes_AS_STRING(py_bytes_msgpack_byte_sequence), + static_cast(PyBytes_GET_SIZE(py_bytes_msgpack_byte_sequence))} + )) + { + return nullptr; + } + + Py_RETURN_NONE; +} + +CLP_FFI_PY_METHOD auto PySerializer_write_to_stream(PySerializer* self) -> PyObject* { + auto const optional_num_bytes_write{self->write_ir_buf_to_output_stream()}; + if (false == optional_num_bytes_write.has_value()) { + return nullptr; + } + return PyLong_FromSsize_t(optional_num_bytes_write.value()); +} + +CLP_FFI_PY_METHOD auto PySerializer_get_buffer_size(PySerializer* self) -> PyObject* { + if (false == self->assert_is_not_closed()) { + return nullptr; + } + return PyLong_FromSsize_t(self->get_ir_buf_size()); +} + +CLP_FFI_PY_METHOD auto PySerializer_flush_stream(PySerializer* self) -> PyObject* { + if (false == self->flush_output_stream()) { + return nullptr; + } + Py_RETURN_NONE; +} + +CLP_FFI_PY_METHOD auto +PySerializer_close(PySerializer* self, PyObject* args, PyObject* keywords) -> PyObject* { + static char keyword_flush_stream[]{"flush_stream"}; + static char* keyword_table[]{static_cast(keyword_flush_stream), nullptr}; + + int flush_stream{1}; + + if (false + == static_cast(PyArg_ParseTupleAndKeywords( + args, + keywords, + "|p", + static_cast(keyword_table), + &flush_stream + ))) + { + return nullptr; + } + + auto const optional_num_bytes_written{self->close(static_cast(flush_stream))}; + if (false == optional_num_bytes_written.has_value()) { + return nullptr; + } + + return PyLong_FromSsize_t(optional_num_bytes_written.value()); +} + +CLP_FFI_PY_METHOD auto PySerializer_dealloc(PySerializer* self) -> void { + self->clean(); +} +} // namespace + +auto PySerializer::init(PyObject* output_stream, PySerializer::ClpIrSerializer serializer) -> bool { + m_output_stream = output_stream; + Py_INCREF(output_stream); + m_serializer = new PySerializer::ClpIrSerializer{std::move(serializer)}; + if (nullptr == m_serializer) { + PyErr_SetString(PyExc_RuntimeError, clp_ffi_py::cOutofMemoryError); + return false; + } + return true; +} + +auto PySerializer::assert_is_not_closed() const -> bool { + if (is_closed()) { + PyErr_SetString(PyExc_IOError, "Serializer has already been closed."); + return false; + } + return true; +} + +auto PySerializer::serialize_msgpack_map(std::span msgpack_byte_sequence) -> bool { + if (false == assert_is_not_closed()) { + return false; + } + + auto const unpack_result{unpack_msgpack(msgpack_byte_sequence)}; + if (unpack_result.has_error()) { + PyErr_SetString(PyExc_RuntimeError, unpack_result.error().c_str()); + return false; + } + + auto const& msgpack_obj{unpack_result.value().get()}; + if (msgpack::type::MAP != msgpack_obj.type) { + PyErr_SetString(PyExc_TypeError, "Unpacked msgpack is not a map"); + return false; + } + + if (false == m_serializer->serialize_msgpack_map(msgpack_obj.via.map)) { + PyErr_SetString(PyExc_RuntimeError, cSerializerSerializeMsgpackMapError.data()); + return false; + } + + return true; +} + +auto PySerializer::write_ir_buf_to_output_stream() -> std::optional { + if (false == assert_is_not_closed()) { + return std::nullopt; + } + + auto const optional_num_bytes_written{write_to_output_stream(m_serializer->get_ir_buf_view())}; + if (false == optional_num_bytes_written.has_value()) { + return std::nullopt; + } + + m_serializer->clear_ir_buf(); + return optional_num_bytes_written; +} + +auto PySerializer::flush_output_stream() -> bool { + if (false == assert_is_not_closed()) { + return false; + } + if (nullptr == PyObject_CallMethod(m_output_stream, "flush", "")) { + return false; + } + return true; +} + +auto PySerializer::close(bool flush_stream) -> std::optional { + if (false == assert_is_not_closed()) { + return std::nullopt; + } + + // Write the IR buffer into the stream. + auto const optional_num_bytes_written_from_ir_buf{write_ir_buf_to_output_stream()}; + if (false == optional_num_bytes_written_from_ir_buf.has_value()) { + return std::nullopt; + } + + // Write end-of-stream + constexpr std::array cEndOfStreamBuf{clp::ffi::ir_stream::cProtocol::Eof}; + auto const optional_num_bytes_written_from_end_of_stream_buf{ + write_to_output_stream({cEndOfStreamBuf.cbegin(), cEndOfStreamBuf.cend()}) + }; + if (false == optional_num_bytes_written_from_end_of_stream_buf.has_value()) { + return std::nullopt; + } + + // Flush the output stream if needed + if (flush_stream && false == flush_output_stream()) { + return std::nullopt; + } + + close_serializer(); + return optional_num_bytes_written_from_ir_buf.value() + + optional_num_bytes_written_from_end_of_stream_buf.value(); +} + +auto PySerializer::module_level_init(PyObject* py_module) -> bool { + static_assert(std::is_trivially_destructible()); + auto* type{py_reinterpret_cast(PyType_FromSpec(&PySerializer_type_spec))}; + m_py_type.reset(type); + if (nullptr == type) { + return false; + } + return add_python_type(get_py_type(), "Serializer", py_module); +} + +auto PySerializer::write_to_output_stream(PySerializer::BufferView buf +) -> std::optional { + if (buf.empty()) { + return 0; + } + + // `PyBUF_READ` ensures the buffer is read-only, so it should be safe to cast `char const*` to + // `char*` + PyObjectPtr const ir_buf_mem_view{PyMemoryView_FromMemory( + // NOLINTNEXTLINE(bugprone-casting-through-void, cppcoreguidelines-pro-type-*-cast) + static_cast(const_cast(static_cast(buf.data()))), + static_cast(buf.size()), + PyBUF_READ + )}; + if (nullptr == ir_buf_mem_view) { + return std::nullopt; + } + + PyObject* py_num_bytes_written{ + PyObject_CallMethod(m_output_stream, "write", "O", ir_buf_mem_view.get()) + }; + if (nullptr == py_num_bytes_written) { + return std::nullopt; + } + + Py_ssize_t num_bytes_written{}; + if (false == parse_py_int(py_num_bytes_written, num_bytes_written)) { + return std::nullopt; + } + return num_bytes_written; +} +} // namespace clp_ffi_py::ir::native diff --git a/src/clp_ffi_py/ir/native/PySerializer.hpp b/src/clp_ffi_py/ir/native/PySerializer.hpp new file mode 100644 index 00000000..d6684017 --- /dev/null +++ b/src/clp_ffi_py/ir/native/PySerializer.hpp @@ -0,0 +1,149 @@ +#ifndef CLP_FFI_PY_IR_NATIVE_PYSERIALIZER_HPP +#define CLP_FFI_PY_IR_NATIVE_PYSERIALIZER_HPP + +#include // Must always be included before any other header files + +#include +#include +#include + +#include +#include + +#include + +namespace clp_ffi_py::ir::native { +/** + * A PyObject structure for CLP key-value pair IR format serialization (using four-byte encoding). + * The underlying serializer is pointed by `m_serializer`, and the serialized IR stream is written + * into an `IO[byte]` stream pointed by `m_output_stream`. + */ +class PySerializer { +public: + using ClpIrSerializer = clp::ffi::ir_stream::Serializer; + using BufferView = ClpIrSerializer::BufferView; + + /** + * Initializes the underlying data with the given inputs. Since the memory allocation of + * `PySerializer` is handled by CPython's allocator, cpp constructors will not be explicitly + * called. This function serves as the default constructor initialize the underlying serializer. + * It has to be manually called whenever creating a new `PySerializer` object through + * CPython APIs. + * @param output_stream + * @param serializer + * @return true on success. + * @return false on failure with the relevant Python exception and error set. + */ + [[nodiscard]] auto init(PyObject* output_stream, ClpIrSerializer serializer) -> bool; + + /** + * Initializes the pointers to nullptr by default. Should be called once the object is + * allocated. + */ + auto default_init() -> void { + m_output_stream = nullptr; + m_serializer = nullptr; + } + + /** + * Releases the memory allocated for underlying data fields. + */ + auto clean() -> void { + Py_XDECREF(m_output_stream); + close_serializer(); + } + + [[nodiscard]] auto is_closed() const -> bool { return nullptr == m_serializer; } + + /** + * Asserts the serializer has not been closed. + * @return true on success, false if it's already been closed with `IOError` set. + */ + [[nodiscard]] auto assert_is_not_closed() const -> bool; + + /** + * Serializes the given msgpack byte sequence as a msgpack map into IR format. + * NOTE: the serializer must not be closed to call this method. + * @param msgpack_byte_sequence + * @return true on success. + * @return false on error with the relevant Python exception and error set. + */ + [[nodiscard]] auto serialize_msgpack_map(std::span msgpack_byte_sequence) -> bool; + + /** + * Writes the underlying IR buffer into `m_output_stream`. + * NOTE: the serializer must not be closed to call this method. + * @return The number of bytes written on success. + * @return std::nullopt on failure with the relevant Python exception and error set. + */ + [[nodiscard]] auto write_ir_buf_to_output_stream() -> std::optional; + + [[nodiscard]] auto get_ir_buf_size() const -> Py_ssize_t { + return static_cast(m_serializer->get_ir_buf_view().size()); + } + + /** + * Flushes `m_output_stream`. + * NOTE: the serializer must not be closed to call this method. + * @return true on success. + * @return false on failure with the relevant Python exception and error set. + */ + [[nodiscard]] auto flush_output_stream() -> bool; + + /** + * Closes the serializer by writing the buffered results into the output stream with + * end-of-stream IR Unit appended in the end. + * NOTE: the serializer must not be closed to call this method. + * @param flush_stream Whether to flush `output_stream`. + * @return number of bytes written on success. + * @return std::nullopt on failure with the relevant Python exception and error set. + */ + [[nodiscard]] auto close(bool flush_stream) -> std::optional; + + /** + * Gets the `PyTypeObject` that represents `PySerializer`'s Python type. This type is + * dynamically created and initialized during the execution of `module_level_init`. + * @return Python type object associated with `PySerializer`. + */ + [[nodiscard]] static auto get_py_type() -> PyTypeObject* { return m_py_type.get(); } + + /** + * Creates and initializes `PySerializer` as a Python type, and then incorporates this + * type as a Python object into the py_module module. + * @param py_module This is the Python module where the initialized `PySerializer` will be + * incorporated. + * @return true on success. + * @return false on failure with the relevant Python exception and error set. + */ + [[nodiscard]] static auto module_level_init(PyObject* py_module) -> bool; + +private: + /** + * Writes the data from given buffer to `m_output_stream` by calling `m_output_stream`'s `write` + * Python method. + * @param buf + * @return The number of bytes written on success. + * @return std::nullopt on failure with the relevant Python exception and error set. + */ + [[nodiscard]] auto write_to_output_stream(BufferView buf) -> std::optional; + + /** + * Closes `m_serializer` by releasing the allocated memory. + * NOTE: it is safe to call this method more than once as it resets `m_serializer` to nullptr. + */ + auto close_serializer() -> void { + delete m_serializer; + m_serializer = nullptr; + } + + // Variables + PyObject_HEAD; + PyObject* m_output_stream; + // NOLINTNEXTLINE(cppcoreguidelines-owning-memory) + gsl::owner m_serializer; + + static inline PyObjectStaticPtr m_py_type{nullptr}; +}; +} // namespace clp_ffi_py::ir::native + +#endif // CLP_FFI_PY_IR_NATIVE_PYSERIALIZER_HPP diff --git a/src/clp_ffi_py/modules/ir_native.cpp b/src/clp_ffi_py/modules/ir_native.cpp index c362d7be..c2b802b4 100644 --- a/src/clp_ffi_py/modules/ir_native.cpp +++ b/src/clp_ffi_py/modules/ir_native.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include namespace { @@ -75,5 +76,10 @@ PyMODINIT_FUNC PyInit_native() { return nullptr; } + if (false == clp_ffi_py::ir::native::PySerializer::module_level_init(new_module)) { + Py_DECREF(new_module); + return nullptr; + } + return new_module; } From 941a26a8e95f88462a48b1508736d998067a7d72 Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Mon, 4 Nov 2024 18:05:04 -0500 Subject: [PATCH 16/50] Add desctructor --- src/clp_ffi_py/PyObjectUtils.hpp | 28 +++++++++++++++++++++++ src/clp_ffi_py/ir/native/PySerializer.cpp | 23 +++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/src/clp_ffi_py/PyObjectUtils.hpp b/src/clp_ffi_py/PyObjectUtils.hpp index 914e24ad..0b671af0 100644 --- a/src/clp_ffi_py/PyObjectUtils.hpp +++ b/src/clp_ffi_py/PyObjectUtils.hpp @@ -47,6 +47,34 @@ using PyObjectPtr = std::unique_ptr> */ template using PyObjectStaticPtr = std::unique_ptr>; + +/** + * A guard class for Python exceptions. In certain CPython methods, such as `tp_finalize`, + * the exception state must remain unchanged throughout execution. This class saves the current + * exception state upon initialization and restores it upon destruction, ensuring the exception + * status is preserved. + * Docs: https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_finalize + */ +class PyErrGuard { +public: + // Constructor + PyErrGuard() { PyErr_Fetch(&m_error_type, &m_error_value, &m_error_traceback); } + + // Destructor + ~PyErrGuard() { PyErr_Restore(m_error_type, m_error_value, m_error_traceback); } + + // Delete copy/move constructor and assignment + PyErrGuard(PyErrGuard const&) = delete; + PyErrGuard(PyErrGuard&&) = delete; + auto operator=(PyErrGuard const&) -> PyErrGuard& = delete; + auto operator=(PyErrGuard&&) -> PyErrGuard& = delete; + +private: + // Variables + PyObject* m_error_type{nullptr}; + PyObject* m_error_value{nullptr}; + PyObject* m_error_traceback{nullptr}; +}; } // namespace clp_ffi_py #endif // CLP_FFI_PY_PY_OBJECT_UTILS_HPP diff --git a/src/clp_ffi_py/ir/native/PySerializer.cpp b/src/clp_ffi_py/ir/native/PySerializer.cpp index 25a180e1..c0db9745 100644 --- a/src/clp_ffi_py/ir/native/PySerializer.cpp +++ b/src/clp_ffi_py/ir/native/PySerializer.cpp @@ -125,6 +125,11 @@ PySerializer_close(PySerializer* self, PyObject* args, PyObject* keywords) -> Py */ CLP_FFI_PY_METHOD auto PySerializer_dealloc(PySerializer* self) -> void; +/** + * Callback of `PySerializer`'s finalization (destructor). + */ +CLP_FFI_PY_METHOD auto PySerializer_finalize(PySerializer* self) -> void; + // NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays) PyMethodDef PySerializer_method_table[]{ {"serialize_msgpack", @@ -161,6 +166,7 @@ PyType_Slot PySerializer_slots[]{ {Py_tp_dealloc, reinterpret_cast(PySerializer_dealloc)}, {Py_tp_new, reinterpret_cast(PyType_GenericNew)}, {Py_tp_init, reinterpret_cast(PySerializer_init)}, + {Py_tp_finalize, reinterpret_cast(PySerializer_finalize)}, {Py_tp_methods, static_cast(PySerializer_method_table)}, {Py_tp_doc, const_cast(static_cast(cPySerializerDoc))}, {0, nullptr} @@ -314,6 +320,23 @@ PySerializer_close(PySerializer* self, PyObject* args, PyObject* keywords) -> Py CLP_FFI_PY_METHOD auto PySerializer_dealloc(PySerializer* self) -> void { self->clean(); } + +CLP_FFI_PY_METHOD auto PySerializer_finalize(PySerializer* self) -> void { + PyErrGuard const err_guard; + if (self->is_closed()) { + return; + } + + if (0 + != PyErr_WarnEx( + PyExc_RuntimeWarning, + "`Serializer.close()` is not called before object destruction", + 1 + )) + { + PyErr_Clear(); + } +} } // namespace auto PySerializer::init(PyObject* output_stream, PySerializer::ClpIrSerializer serializer) -> bool { From b56216016409177e269cbd8042be3f0c3fe53d15 Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Mon, 4 Nov 2024 20:35:08 -0500 Subject: [PATCH 17/50] Update stub file --- clp_ffi_py/ir/native.pyi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clp_ffi_py/ir/native.pyi b/clp_ffi_py/ir/native.pyi index 3f59883a..4ae8d1e7 100644 --- a/clp_ffi_py/ir/native.pyi +++ b/clp_ffi_py/ir/native.pyi @@ -88,7 +88,7 @@ class KeyValuePairLogEvent: class Serializer: def __init__(self, output_stream: IO[bytes]): ... - def serialize_msgpack_byte_sequence(self, msgpack_byte_sequence: bytes) -> None: ... + def serialize_msgpack(self, msgpack_byte_sequence: bytes) -> None: ... def write_to_stream(self) -> int: ... def get_buffer_size(self) -> int: ... def flush_stream(self) -> None: ... From b42a6a32ea53c42f364e832b6149ba95a546862d Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Mon, 4 Nov 2024 20:35:35 -0500 Subject: [PATCH 18/50] Add finalize for testing... --- src/clp_ffi_py/ir/native/PySerializer.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/clp_ffi_py/ir/native/PySerializer.cpp b/src/clp_ffi_py/ir/native/PySerializer.cpp index c0db9745..50b42040 100644 --- a/src/clp_ffi_py/ir/native/PySerializer.cpp +++ b/src/clp_ffi_py/ir/native/PySerializer.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -180,7 +181,7 @@ PyType_Spec PySerializer_type_spec{ "clp_ffi_py.ir.native.Serializer", sizeof(PySerializer), 0, - Py_TPFLAGS_DEFAULT, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_FINALIZE, static_cast(PySerializer_slots) }; @@ -318,15 +319,18 @@ PySerializer_close(PySerializer* self, PyObject* args, PyObject* keywords) -> Py } CLP_FFI_PY_METHOD auto PySerializer_dealloc(PySerializer* self) -> void { + std::cerr << "Dealloc...\n"; self->clean(); } CLP_FFI_PY_METHOD auto PySerializer_finalize(PySerializer* self) -> void { + std::cerr << "Finalizing...\n"; PyErrGuard const err_guard; if (self->is_closed()) { return; } + std::cerr << "Not closed!\n"; if (0 != PyErr_WarnEx( PyExc_RuntimeWarning, @@ -334,6 +338,7 @@ CLP_FFI_PY_METHOD auto PySerializer_finalize(PySerializer* self) -> void { 1 )) { + std::cerr << "Exception captured!\n"; PyErr_Clear(); } } From 32db0ff533edeedf329fed9a670767bd3929dc7f Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Mon, 4 Nov 2024 21:47:11 -0500 Subject: [PATCH 19/50] Remove cerr --- src/clp_ffi_py/ir/native/PySerializer.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/clp_ffi_py/ir/native/PySerializer.cpp b/src/clp_ffi_py/ir/native/PySerializer.cpp index 50b42040..30a22175 100644 --- a/src/clp_ffi_py/ir/native/PySerializer.cpp +++ b/src/clp_ffi_py/ir/native/PySerializer.cpp @@ -5,7 +5,6 @@ #include #include #include -#include #include #include #include @@ -319,18 +318,15 @@ PySerializer_close(PySerializer* self, PyObject* args, PyObject* keywords) -> Py } CLP_FFI_PY_METHOD auto PySerializer_dealloc(PySerializer* self) -> void { - std::cerr << "Dealloc...\n"; self->clean(); } CLP_FFI_PY_METHOD auto PySerializer_finalize(PySerializer* self) -> void { - std::cerr << "Finalizing...\n"; PyErrGuard const err_guard; if (self->is_closed()) { return; } - std::cerr << "Not closed!\n"; if (0 != PyErr_WarnEx( PyExc_RuntimeWarning, @@ -338,7 +334,6 @@ CLP_FFI_PY_METHOD auto PySerializer_finalize(PySerializer* self) -> void { 1 )) { - std::cerr << "Exception captured!\n"; PyErr_Clear(); } } From ebcc46193fc08c62c6553fb4dab7c0de89480a21 Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Mon, 4 Nov 2024 22:23:48 -0500 Subject: [PATCH 20/50] Add support for context manager --- clp_ffi_py/ir/native.pyi | 12 +++- src/clp_ffi_py/ir/native/PySerializer.cpp | 81 +++++++++++++++++++++++ 2 files changed, 92 insertions(+), 1 deletion(-) diff --git a/clp_ffi_py/ir/native.pyi b/clp_ffi_py/ir/native.pyi index 4ae8d1e7..2460e06d 100644 --- a/clp_ffi_py/ir/native.pyi +++ b/clp_ffi_py/ir/native.pyi @@ -1,5 +1,8 @@ +from __future__ import annotations + from datetime import tzinfo -from typing import Any, Dict, IO, List, Optional +from types import TracebackType +from typing import Any, Dict, IO, List, Optional, Type from clp_ffi_py.wildcard_query import WildcardQuery @@ -88,6 +91,13 @@ class KeyValuePairLogEvent: class Serializer: def __init__(self, output_stream: IO[bytes]): ... + def __enter__(self) -> Serializer: ... + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + exc_value: Optional[BaseException], + traceback: Optional[TracebackType], + ) -> None: ... def serialize_msgpack(self, msgpack_byte_sequence: bytes) -> None: ... def write_to_stream(self) -> int: ... def get_buffer_size(self) -> int: ... diff --git a/src/clp_ffi_py/ir/native/PySerializer.cpp b/src/clp_ffi_py/ir/native/PySerializer.cpp index 30a22175..64ab2078 100644 --- a/src/clp_ffi_py/ir/native/PySerializer.cpp +++ b/src/clp_ffi_py/ir/native/PySerializer.cpp @@ -120,6 +120,35 @@ PyDoc_STRVAR( CLP_FFI_PY_METHOD auto PySerializer_close(PySerializer* self, PyObject* args, PyObject* keywords) -> PyObject*; +/** + * Callback of `PySerializer`'s `__enter__` method. + */ +// NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays) +PyDoc_STRVAR( + cPySerializerEnterDoc, + "__enter__(self)\n" + "--\n\n" + "Enters the runtime context.\n\n" + ":return: self.\n" +); +CLP_FFI_PY_METHOD auto PySerializer_enter(PySerializer* self) -> PyObject*; + +/** + * Callback of `PySerializer`'s `__exit__` method. + */ +PyDoc_STRVAR( + cPySerializerExitDoc, + "__exit__(self, exc_type, exc_value, traceback)\n" + "--\n\n" + "Exits the runtime context, automatically calling :meth:`close` to flush all buffered data" + " into the output stream." + ":param exc_type: The type of the exception caused the context to be exited. Unused.\n" + ":param exc_value: The value of the exception caused the context to be exited. Unused.\n" + ":param exc_traceable: The traceback. Unused.\n" +); +CLP_FFI_PY_METHOD auto +PySerializer_exit(PySerializer* self, PyObject* args, PyObject* keywords) -> PyObject*; + /** * Callback of `PySerializer`'s deallocator. */ @@ -157,6 +186,16 @@ PyMethodDef PySerializer_method_table[]{ METH_VARARGS | METH_KEYWORDS, static_cast(cPySerializerCloseDoc)}, + {"__enter__", + py_c_function_cast(PySerializer_enter), + METH_NOARGS, + static_cast(cPySerializerEnterDoc)}, + + {"__exit__", + py_c_function_cast(PySerializer_exit), + METH_VARARGS | METH_KEYWORDS, + static_cast(cPySerializerExitDoc)}, + {nullptr} }; @@ -317,6 +356,48 @@ PySerializer_close(PySerializer* self, PyObject* args, PyObject* keywords) -> Py return PyLong_FromSsize_t(optional_num_bytes_written.value()); } +CLP_FFI_PY_METHOD auto PySerializer_enter(PySerializer* self) -> PyObject* { + return py_reinterpret_cast(self); +} + +CLP_FFI_PY_METHOD auto +PySerializer_exit(PySerializer* self, PyObject* args, PyObject* keywords) -> PyObject* { + static char keyword_exc_type[]{"exc_type"}; + static char keyword_exc_value[]{"exc_value"}; + static char keyword_traceback[]{"traceback"}; + static char* keyword_table[]{ + static_cast(keyword_exc_type), + static_cast(keyword_exc_value), + static_cast(keyword_traceback), + nullptr + }; + + PyObject* py_exc_type{}; + PyObject* py_exc_value{}; + PyObject* py_traceback{}; + if (false + == static_cast(PyArg_ParseTupleAndKeywords( + args, + keywords, + "|OOO", + static_cast(keyword_table), + &py_exc_type, + &py_exc_value, + &py_traceback + ))) + { + return nullptr; + } + + // We don't do anything with the given exception. It is the caller's responsibility to raise + // the exceptions + if (false == self->close(true).has_value()) { + return nullptr; + } + + Py_RETURN_NONE; +} + CLP_FFI_PY_METHOD auto PySerializer_dealloc(PySerializer* self) -> void { self->clean(); } From 76f98e364a7e4054d3bda226c7458696238e9625 Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Mon, 4 Nov 2024 22:27:03 -0500 Subject: [PATCH 21/50] Add comment --- src/clp_ffi_py/ir/native/PySerializer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/clp_ffi_py/ir/native/PySerializer.cpp b/src/clp_ffi_py/ir/native/PySerializer.cpp index 64ab2078..bb80ff16 100644 --- a/src/clp_ffi_py/ir/native/PySerializer.cpp +++ b/src/clp_ffi_py/ir/native/PySerializer.cpp @@ -390,7 +390,7 @@ PySerializer_exit(PySerializer* self, PyObject* args, PyObject* keywords) -> PyO } // We don't do anything with the given exception. It is the caller's responsibility to raise - // the exceptions + // the exceptions: https://docs.python.org/3/reference/datamodel.html#object.__exit__ if (false == self->close(true).has_value()) { return nullptr; } From 5507570c389904ac6d3b1a6807ac1cd4131c3b84 Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Mon, 4 Nov 2024 23:35:52 -0500 Subject: [PATCH 22/50] Add simple unit tests --- tests/test_ir/test_serializer.py | 81 +++++++++++++++++++++++++++++++- 1 file changed, 79 insertions(+), 2 deletions(-) diff --git a/tests/test_ir/test_serializer.py b/tests/test_ir/test_serializer.py index beb4b92f..2dc33f9a 100644 --- a/tests/test_ir/test_serializer.py +++ b/tests/test_ir/test_serializer.py @@ -1,6 +1,11 @@ -from test_ir.test_utils import TestCLPBase -from clp_ffi_py.ir import FourByteSerializer +from io import BytesIO +from pathlib import Path + +from test_ir.test_utils import JsonFileReader, TestCLPBase + +from clp_ffi_py.ir import FourByteSerializer, Serializer +from clp_ffi_py.utils import serialize_dict_to_msgpack class TestCaseFourByteSerializer(TestCLPBase): @@ -38,3 +43,75 @@ def test_serialization_methods_consistency(self) -> None: timestamp_delta ) self.assertEqual(serialized_message_and_ts_delta, serialized_message + serialized_ts_delta) + + +class TestCaseSerializer(TestCLPBase): + """ + Class for testing `clp_ffi_py.ir.Serializer`. + """ + + input_src_dir: str = "test_json" + + def test_serialize_json(self) -> None: + """ + Tests serializing JSON files. + + The JSON parser will parse the file into Python dictionaries, + and then convert them into msgpack and feed into `clp_ffi_py.ir.Serializer`. + """ + current_dir: Path = Path(__file__).resolve().parent + test_src_dir: Path = current_dir / Path(TestCaseSerializer.input_src_dir) + + byte_buffer: BytesIO + num_bytes_serialized: int + serializer: Serializer + + # Test with context manager + for file_path in test_src_dir.rglob("*"): + if not file_path.is_file(): + continue + byte_buffer = BytesIO() + num_bytes_serialized = 0 + with Serializer(byte_buffer) as serializer: + for json_obj in JsonFileReader(file_path).read_json_lines(): + serializer.serialize_msgpack(serialize_dict_to_msgpack(json_obj)) + num_bytes_serialized += serializer.write_to_stream() + self.assertEqual(0, serializer.write_to_stream()) + serializer.flush_stream() + self.assertEqual(num_bytes_serialized + 1, len(byte_buffer.getvalue())) + byte_buffer.close() + + # Test without context manager + for file_path in test_src_dir.rglob("*"): + if not file_path.is_file(): + continue + byte_buffer = BytesIO() + num_bytes_serialized = 0 + serializer = Serializer(byte_buffer) + for json_obj in JsonFileReader(file_path).read_json_lines(): + serializer.serialize_msgpack(serialize_dict_to_msgpack(json_obj)) + num_bytes_serialized += serializer.write_to_stream() + self.assertEqual(0, serializer.write_to_stream()) + serializer.flush_stream() + self.assertEqual(num_bytes_serialized, len(byte_buffer.getvalue())) + serializer.close() + byte_buffer.close() + + def test_closing_empty(self) -> None: + """ + Tests closing an empty serializer. + """ + byte_buffer: BytesIO + serializer: Serializer + + byte_buffer = BytesIO() + with Serializer(byte_buffer) as serializer: + serializer.flush_stream() + self.assertTrue(1 < len(byte_buffer.getvalue())) + byte_buffer.close() + + byte_buffer = BytesIO() + serializer = Serializer(byte_buffer) + serializer.close() + self.assertTrue(1 < len(byte_buffer.getvalue())) + byte_buffer.close() From cc5a42139755f70d161a66198c613601db7a3c64 Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Tue, 5 Nov 2024 01:56:36 -0500 Subject: [PATCH 23/50] refactor --- clp_ffi_py/ir/native.pyi | 11 +- src/clp_ffi_py/ir/native/PySerializer.cpp | 371 +++++++++--------- src/clp_ffi_py/ir/native/PySerializer.hpp | 82 ++-- tests/test_ir/test_json/spark-event-logs.json | 2 + tests/test_ir/test_serializer.py | 92 +++-- 5 files changed, 313 insertions(+), 245 deletions(-) diff --git a/clp_ffi_py/ir/native.pyi b/clp_ffi_py/ir/native.pyi index 2460e06d..a3fc6331 100644 --- a/clp_ffi_py/ir/native.pyi +++ b/clp_ffi_py/ir/native.pyi @@ -90,7 +90,7 @@ class KeyValuePairLogEvent: def to_dict(self) -> Dict[Any, Any]: ... class Serializer: - def __init__(self, output_stream: IO[bytes]): ... + def __init__(self, output_stream: IO[bytes], buffer_size_limit: int = 65536): ... def __enter__(self) -> Serializer: ... def __exit__( self, @@ -98,10 +98,9 @@ class Serializer: exc_value: Optional[BaseException], traceback: Optional[TracebackType], ) -> None: ... - def serialize_msgpack(self, msgpack_byte_sequence: bytes) -> None: ... - def write_to_stream(self) -> int: ... - def get_buffer_size(self) -> int: ... - def flush_stream(self) -> None: ... - def close(self, flush_stream: bool = True) -> int: ... + def serialize_msgpack_map(self, msgpack_map: bytes) -> int: ... + def get_num_bytes_serialized(self) -> int: ... + def flush(self) -> None: ... + def close(self) -> None: ... class IncompleteStreamError(Exception): ... diff --git a/src/clp_ffi_py/ir/native/PySerializer.cpp b/src/clp_ffi_py/ir/native/PySerializer.cpp index bb80ff16..e94d7d4d 100644 --- a/src/clp_ffi_py/ir/native/PySerializer.cpp +++ b/src/clp_ffi_py/ir/native/PySerializer.cpp @@ -29,13 +29,17 @@ PyDoc_STRVAR( cPySerializerDoc, "Serializer for CLP key-value pair IR stream.\n" "This class serializes log events into CLP IR format (using four-byte-encoding) and writes" - " the serialized data to a specified byte stream.\n\n" - "__init__(self, output_stream)\n\n" - "Initializes a `Serializer` instance with the given output stream. Notice that each object" - " should be strictly initialized only once. Double initialization will result in memory" - " leak.\n\n" + " the serialized data to a specified byte stream object.\n\n" + "__init__(self, output_stream, buffer_size_limit)\n\n" + "Initializes a :class:`Serializer` instance with the given output stream. Notice that each" + " object should be strictly initialized only once. Double initialization will result in" + " memory leak.\n\n" ":param output_stream: A writable byte stream to which the serializer will write serialized" " IR byte sequence.\n" + ":param buffer_size_limit: The buffer size in bytes that will trigger the internal buffer" + " flush, defaults to 65536. The serialized log events will be first buffered inside the" + " internal buffer. When the size of the buffer exceeds this limit, the internal buffer will" + " be written to the output stream.\n" ); CLP_FFI_PY_METHOD auto PySerializer_init(PySerializer* self, PyObject* args, PyObject* keywords) -> int; @@ -45,62 +49,46 @@ PySerializer_init(PySerializer* self, PyObject* args, PyObject* keywords) -> int */ // NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays) PyDoc_STRVAR( - cPySerializerSerializeMsgpackDoc, - "serialize_msgpack(self, msgpack_byte_sequence)\n" + cPySerializerSerializeMsgpackMapDoc, + "serialize_msgpack_map(self, msgpack_map)\n" "--\n\n" - "Serializes the given msgpack byte sequence into as a log event.\n" - "NOTE: the serialization results will be buffered inside the serializer. `write_to_stream`" - " must be called to write buffered bytes into the output stream.\n\n" - ":param msgpack_byte_sequence: A byte sequence encoded in msgpack as the input log event." + "Serializes the given msgpack byte sequence into as a log event into CLP IR format.\n\n" + ":param msgpack_map: A byte sequence encoded in msgpack as the input log event." " The unpacked result must be a msgpack map with all keys as strings.\n" + ":return: The number of bytes serialized.\n" ":raise IOError: If the serializer has already been closed.\n" ":raise TypeError: If the unpacked result is not a msgpack map.\n" ":raise RuntimeError: If it fails to unpack the given msgpack byte sequence, or the" " serialization method returns failure.\n" ); CLP_FFI_PY_METHOD auto -PySerializer_serialize_msgpack(PySerializer* self, PyObject* msgpack_byte_sequence) -> PyObject*; +PySerializer_serialize_msgpack_map(PySerializer* self, PyObject* msgpack_map) -> PyObject*; /** - * Callback of `PySerializer`'s `write_to_stream` method. + * Callback of `PySerializer`'s `get_num_bytes_serialized` method. */ // NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays) PyDoc_STRVAR( - cPySerializerWriteToStreamDoc, - "write_to_stream(self)\n" + cPySerializerGetNumBytesSerializedDoc, + "get_num_bytes_serialized(self)\n" "--\n\n" - "Writes the buffered results to the output stream and clears the buffer.\n\n" - ":return: Number of bytes written.\n" + ":return: The total number of bytes serialized.\n" ":raise IOError: If the serializer has already been closed.\n" ); -CLP_FFI_PY_METHOD auto PySerializer_write_to_stream(PySerializer* self) -> PyObject*; +CLP_FFI_PY_METHOD auto PySerializer_get_num_bytes_serialized(PySerializer* self) -> PyObject*; /** - * Callback of `PySerializer`'s `get_buffer_size` method. + * Callback of `PySerializer`'s `flush` method. */ // NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays) PyDoc_STRVAR( - cPySerializerGetBufferSizeDoc, - "get_buffer_size(self)\n" + cPySerializerFlushDoc, + "flush(self)\n" "--\n\n" - "Gets the size of the result buffer.\n\n" - ":return: The buffer size in bytes.\n" + "Flushes the internal buffer and the output stream.\n\n" ":raise IOError: If the serializer has already been closed.\n" ); -CLP_FFI_PY_METHOD auto PySerializer_get_buffer_size(PySerializer* self) -> PyObject*; - -/** - * Callback of `PySerializer`'s `flush_stream` method. - */ -// NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays) -PyDoc_STRVAR( - cPySerializerFlushStreamDoc, - "flush_stream(self)\n" - "--\n\n" - "Flushes the output stream.\n\n" - ":raise IOError: If the serializer has already been closed.\n" -); -CLP_FFI_PY_METHOD auto PySerializer_flush_stream(PySerializer* self) -> PyObject*; +CLP_FFI_PY_METHOD auto PySerializer_flush(PySerializer* self) -> PyObject*; /** * Callback of `PySerializer`'s `close` method. @@ -108,17 +96,17 @@ CLP_FFI_PY_METHOD auto PySerializer_flush_stream(PySerializer* self) -> PyObject // NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays) PyDoc_STRVAR( cPySerializerCloseDoc, - "close(self, flush_stream=True)\n" + "close(self)\n" "--\n\n" - "Closes the serializer by writing the results into the output stream with the byte sequence" - " in the end that indicates the end of a CLP IR stream. This method must be called to" - " terminate an IR stream. Otherwise, the stream will be considered incomplete.\n\n" - ":param flush_stream: Whether to flush the output stream.\n" - ":return: Forwards :meth:`write_to_stream`'s return value.\n" + "Closes the serializer, writing any remaining data to the output stream and appending a" + " byte sequence to mark the end of a CLP IR stream. The output stream is then flushed and" + " closed.\n" + "NOTE: This method must be called to properly terminate an IR stream. Failing to call it" + " may leave the stream incomplete, potentially resulting in data loss due to data" + " buffering\n\n" ":raise IOError: If the serializer has already been closed.\n" ); -CLP_FFI_PY_METHOD auto -PySerializer_close(PySerializer* self, PyObject* args, PyObject* keywords) -> PyObject*; +CLP_FFI_PY_METHOD auto PySerializer_close(PySerializer* self) -> PyObject*; /** * Callback of `PySerializer`'s `__enter__` method. @@ -154,36 +142,26 @@ PySerializer_exit(PySerializer* self, PyObject* args, PyObject* keywords) -> PyO */ CLP_FFI_PY_METHOD auto PySerializer_dealloc(PySerializer* self) -> void; -/** - * Callback of `PySerializer`'s finalization (destructor). - */ -CLP_FFI_PY_METHOD auto PySerializer_finalize(PySerializer* self) -> void; - // NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays) PyMethodDef PySerializer_method_table[]{ - {"serialize_msgpack", - py_c_function_cast(PySerializer_serialize_msgpack), + {"serialize_msgpack_map", + py_c_function_cast(PySerializer_serialize_msgpack_map), METH_O, - static_cast(cPySerializerSerializeMsgpackDoc)}, + static_cast(cPySerializerSerializeMsgpackMapDoc)}, - {"write_to_stream", - py_c_function_cast(PySerializer_write_to_stream), + {"get_num_bytes_serialized", + py_c_function_cast(PySerializer_get_num_bytes_serialized), METH_NOARGS, - static_cast(cPySerializerWriteToStreamDoc)}, + static_cast(cPySerializerGetNumBytesSerializedDoc)}, - {"get_buffer_size", - py_c_function_cast(PySerializer_get_buffer_size), + {"flush", + py_c_function_cast(PySerializer_flush), METH_NOARGS, - static_cast(cPySerializerGetBufferSizeDoc)}, - - {"flush_stream", - py_c_function_cast(PySerializer_flush_stream), - METH_NOARGS, - static_cast(cPySerializerFlushStreamDoc)}, + static_cast(cPySerializerFlushDoc)}, {"close", py_c_function_cast(PySerializer_close), - METH_VARARGS | METH_KEYWORDS, + METH_NOARGS, static_cast(cPySerializerCloseDoc)}, {"__enter__", @@ -205,7 +183,6 @@ PyType_Slot PySerializer_slots[]{ {Py_tp_dealloc, reinterpret_cast(PySerializer_dealloc)}, {Py_tp_new, reinterpret_cast(PyType_GenericNew)}, {Py_tp_init, reinterpret_cast(PySerializer_init)}, - {Py_tp_finalize, reinterpret_cast(PySerializer_finalize)}, {Py_tp_methods, static_cast(PySerializer_method_table)}, {Py_tp_doc, const_cast(static_cast(cPySerializerDoc))}, {0, nullptr} @@ -219,54 +196,70 @@ PyType_Spec PySerializer_type_spec{ "clp_ffi_py.ir.native.Serializer", sizeof(PySerializer), 0, - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_FINALIZE, + Py_TPFLAGS_DEFAULT, static_cast(PySerializer_slots) }; CLP_FFI_PY_METHOD auto PySerializer_init(PySerializer* self, PyObject* args, PyObject* keywords) -> int { static char keyword_output_stream[]{"output_stream"}; - static char* keyword_table[]{static_cast(keyword_output_stream), nullptr}; + static char keyword_buffer_size_limit[]{"buffer_size_limit"}; + static char* keyword_table[]{ + static_cast(keyword_output_stream), + static_cast(keyword_buffer_size_limit), + nullptr + }; // If the argument parsing fails, `self` will be deallocated. We must reset all pointers to // nullptr in advance, otherwise the deallocator might trigger segmentation fault. self->default_init(); PyObject* output_stream{Py_None}; + Py_ssize_t buffer_size_limit{PySerializer::cDefaultBufferSizeLimit}; if (false == static_cast(PyArg_ParseTupleAndKeywords( args, keywords, - "O", + "O|L", static_cast(keyword_table), - &output_stream + &output_stream, + &buffer_size_limit ))) { return -1; } - // Ensure the `output_stream` has `write` and `flush` methods - PyObjectPtr const write_method{PyObject_GetAttrString(output_stream, "write")}; - if (nullptr == write_method) { + // Ensure the `output_stream` has `write`, `flush`, and `close` methods + auto output_stream_has_method = [&](char const* method_name) -> bool { + PyObjectPtr const write_method{PyObject_GetAttrString(output_stream, method_name) + }; + if (nullptr == write_method) { + return false; + } + if (false == static_cast(PyCallable_Check(write_method.get()))) { + PyErr_SetString(PyExc_TypeError, ""); + PyErr_Format( + PyExc_TypeError, + "The attribute `%s` of the given output stream object is not callable.", + method_name + ); + return false; + } + return true; + }; + + if (false == output_stream_has_method("write")) { return -1; } - if (false == static_cast(PyCallable_Check(write_method.get()))) { - PyErr_SetString( - PyExc_TypeError, - "The attribute `write` of the given output stream object is not callable." - ); + if (false == output_stream_has_method("flush")) { return -1; } - - PyObjectPtr const flush_method{PyObject_GetAttrString(output_stream, "flush")}; - if (nullptr == flush_method) { + if (false == output_stream_has_method("close")) { return -1; } - if (false == static_cast(PyCallable_Check(flush_method.get()))) { - PyErr_SetString( - PyExc_TypeError, - "The attribute `flush` of the given output stream object is not callable." - ); + + if (0 > buffer_size_limit) { + PyErr_SetString(PyExc_ValueError, "The buffer size limit cannot be negative"); return -1; } @@ -280,12 +273,17 @@ PySerializer_init(PySerializer* self, PyObject* args, PyObject* keywords) -> int return -1; } - return self->init(output_stream, std::move(serializer_result.value())) ? 0 : -1; + if (false == self->init(output_stream, std::move(serializer_result.value()), buffer_size_limit)) + { + return -1; + } + + return 0; } CLP_FFI_PY_METHOD auto -PySerializer_serialize_msgpack(PySerializer* self, PyObject* msgpack_byte_sequence) -> PyObject* { - if (false == static_cast(PyBytes_Check(msgpack_byte_sequence))) { +PySerializer_serialize_msgpack_map(PySerializer* self, PyObject* msgpack_map) -> PyObject* { + if (false == static_cast(PyBytes_Check(msgpack_map))) { PyErr_SetString( PyExc_TypeError, "`msgpack_byte_sequence` is supposed to return a `bytes` object" @@ -293,67 +291,35 @@ PySerializer_serialize_msgpack(PySerializer* self, PyObject* msgpack_byte_sequen return nullptr; } - auto* py_bytes_msgpack_byte_sequence{py_reinterpret_cast(msgpack_byte_sequence)}; + auto* py_bytes_msgpack_map{py_reinterpret_cast(msgpack_map)}; // Since the type is already checked, we can use the macro to avoid duplicated type checking. - if (false - == self->serialize_msgpack_map( - {PyBytes_AS_STRING(py_bytes_msgpack_byte_sequence), - static_cast(PyBytes_GET_SIZE(py_bytes_msgpack_byte_sequence))} - )) - { + auto const num_byte_serialized{self->serialize_msgpack_map( + {PyBytes_AS_STRING(py_bytes_msgpack_map), + static_cast(PyBytes_GET_SIZE(py_bytes_msgpack_map))} + )}; + if (false == num_byte_serialized.has_value()) { return nullptr; } - Py_RETURN_NONE; -} - -CLP_FFI_PY_METHOD auto PySerializer_write_to_stream(PySerializer* self) -> PyObject* { - auto const optional_num_bytes_write{self->write_ir_buf_to_output_stream()}; - if (false == optional_num_bytes_write.has_value()) { - return nullptr; - } - return PyLong_FromSsize_t(optional_num_bytes_write.value()); + return PyLong_FromSsize_t(num_byte_serialized.value()); } -CLP_FFI_PY_METHOD auto PySerializer_get_buffer_size(PySerializer* self) -> PyObject* { - if (false == self->assert_is_not_closed()) { - return nullptr; - } - return PyLong_FromSsize_t(self->get_ir_buf_size()); +CLP_FFI_PY_METHOD auto PySerializer_get_num_bytes_serialized(PySerializer* self) -> PyObject* { + return PyLong_FromSsize_t(self->get_num_bytes_serialized()); } -CLP_FFI_PY_METHOD auto PySerializer_flush_stream(PySerializer* self) -> PyObject* { - if (false == self->flush_output_stream()) { +CLP_FFI_PY_METHOD auto PySerializer_flush(PySerializer* self) -> PyObject* { + if (false == self->flush()) { return nullptr; } Py_RETURN_NONE; } -CLP_FFI_PY_METHOD auto -PySerializer_close(PySerializer* self, PyObject* args, PyObject* keywords) -> PyObject* { - static char keyword_flush_stream[]{"flush_stream"}; - static char* keyword_table[]{static_cast(keyword_flush_stream), nullptr}; - - int flush_stream{1}; - - if (false - == static_cast(PyArg_ParseTupleAndKeywords( - args, - keywords, - "|p", - static_cast(keyword_table), - &flush_stream - ))) - { +CLP_FFI_PY_METHOD auto PySerializer_close(PySerializer* self) -> PyObject* { + if (false == self->close()) { return nullptr; } - - auto const optional_num_bytes_written{self->close(static_cast(flush_stream))}; - if (false == optional_num_bytes_written.has_value()) { - return nullptr; - } - - return PyLong_FromSsize_t(optional_num_bytes_written.value()); + Py_RETURN_NONE; } CLP_FFI_PY_METHOD auto PySerializer_enter(PySerializer* self) -> PyObject* { @@ -391,7 +357,7 @@ PySerializer_exit(PySerializer* self, PyObject* args, PyObject* keywords) -> PyO // We don't do anything with the given exception. It is the caller's responsibility to raise // the exceptions: https://docs.python.org/3/reference/datamodel.html#object.__exit__ - if (false == self->close(true).has_value()) { + if (false == self->close()) { return nullptr; } @@ -399,35 +365,42 @@ PySerializer_exit(PySerializer* self, PyObject* args, PyObject* keywords) -> PyO } CLP_FFI_PY_METHOD auto PySerializer_dealloc(PySerializer* self) -> void { - self->clean(); -} - -CLP_FFI_PY_METHOD auto PySerializer_finalize(PySerializer* self) -> void { PyErrGuard const err_guard; - if (self->is_closed()) { - return; - } - if (0 - != PyErr_WarnEx( - PyExc_RuntimeWarning, - "`Serializer.close()` is not called before object destruction", - 1 - )) - { - PyErr_Clear(); + if (false == self->is_closed()) { + if (0 + != PyErr_WarnEx( + PyExc_RuntimeWarning, + "`Serializer.close()` is not called before object destruction", + 1 + )) + { + PyErr_Clear(); + } } + + self->clean(); } } // namespace -auto PySerializer::init(PyObject* output_stream, PySerializer::ClpIrSerializer serializer) -> bool { +auto PySerializer::init( + PyObject* output_stream, + PySerializer::ClpIrSerializer serializer, + Py_ssize_t buffer_size_limit +) -> bool { m_output_stream = output_stream; Py_INCREF(output_stream); m_serializer = new PySerializer::ClpIrSerializer{std::move(serializer)}; + m_buffer_size_limit = buffer_size_limit; if (nullptr == m_serializer) { PyErr_SetString(PyExc_RuntimeError, clp_ffi_py::cOutofMemoryError); return false; } + auto const preamble_size{get_ir_buf_size()}; + if (preamble_size > m_buffer_size_limit && false == write_ir_buf_to_output_stream()) { + return false; + } + m_num_total_bytes_serialized += preamble_size; return true; } @@ -439,83 +412,75 @@ auto PySerializer::assert_is_not_closed() const -> bool { return true; } -auto PySerializer::serialize_msgpack_map(std::span msgpack_byte_sequence) -> bool { +auto PySerializer::serialize_msgpack_map(std::span msgpack_byte_sequence +) -> std::optional { if (false == assert_is_not_closed()) { - return false; + return std::nullopt; } auto const unpack_result{unpack_msgpack(msgpack_byte_sequence)}; if (unpack_result.has_error()) { PyErr_SetString(PyExc_RuntimeError, unpack_result.error().c_str()); - return false; + return std::nullopt; } auto const& msgpack_obj{unpack_result.value().get()}; if (msgpack::type::MAP != msgpack_obj.type) { PyErr_SetString(PyExc_TypeError, "Unpacked msgpack is not a map"); - return false; + return std::nullopt; } + auto const buffer_size_before_serialization{get_ir_buf_size()}; if (false == m_serializer->serialize_msgpack_map(msgpack_obj.via.map)) { PyErr_SetString(PyExc_RuntimeError, cSerializerSerializeMsgpackMapError.data()); - return false; - } - - return true; -} - -auto PySerializer::write_ir_buf_to_output_stream() -> std::optional { - if (false == assert_is_not_closed()) { return std::nullopt; } + auto const buffer_size_after_serialization{get_ir_buf_size()}; + auto const num_bytes_serialized{ + buffer_size_after_serialization - buffer_size_before_serialization + }; + m_num_total_bytes_serialized += num_bytes_serialized; - auto const optional_num_bytes_written{write_to_output_stream(m_serializer->get_ir_buf_view())}; - if (false == optional_num_bytes_written.has_value()) { + if (buffer_size_after_serialization > m_buffer_size_limit + && false == write_ir_buf_to_output_stream()) + { return std::nullopt; } - - m_serializer->clear_ir_buf(); - return optional_num_bytes_written; + return num_bytes_serialized; } -auto PySerializer::flush_output_stream() -> bool { +auto PySerializer::flush() -> bool { if (false == assert_is_not_closed()) { return false; } - if (nullptr == PyObject_CallMethod(m_output_stream, "flush", "")) { + if (false == write_ir_buf_to_output_stream()) { return false; } - return true; + return flush_output_stream(); } -auto PySerializer::close(bool flush_stream) -> std::optional { +auto PySerializer::close() -> bool { if (false == assert_is_not_closed()) { - return std::nullopt; + return false; } - // Write the IR buffer into the stream. - auto const optional_num_bytes_written_from_ir_buf{write_ir_buf_to_output_stream()}; - if (false == optional_num_bytes_written_from_ir_buf.has_value()) { - return std::nullopt; + if (false == write_ir_buf_to_output_stream()) { + return false; } // Write end-of-stream constexpr std::array cEndOfStreamBuf{clp::ffi::ir_stream::cProtocol::Eof}; - auto const optional_num_bytes_written_from_end_of_stream_buf{ - write_to_output_stream({cEndOfStreamBuf.cbegin(), cEndOfStreamBuf.cend()}) - }; - if (false == optional_num_bytes_written_from_end_of_stream_buf.has_value()) { - return std::nullopt; + if (false == write_to_output_stream({cEndOfStreamBuf.cbegin(), cEndOfStreamBuf.cend()})) { + return false; } + m_num_total_bytes_serialized += cEndOfStreamBuf.size(); - // Flush the output stream if needed - if (flush_stream && false == flush_output_stream()) { - return std::nullopt; + if (false == (flush_output_stream() && close_output_stream())) { + return false; } close_serializer(); - return optional_num_bytes_written_from_ir_buf.value() - + optional_num_bytes_written_from_end_of_stream_buf.value(); + return true; } auto PySerializer::module_level_init(PyObject* py_module) -> bool { @@ -528,6 +493,28 @@ auto PySerializer::module_level_init(PyObject* py_module) -> bool { return add_python_type(get_py_type(), "Serializer", py_module); } +auto PySerializer::write_ir_buf_to_output_stream() -> bool { + if (false == assert_is_not_closed()) { + return false; + } + + auto const optional_num_bytes_written{write_to_output_stream(m_serializer->get_ir_buf_view())}; + if (false == optional_num_bytes_written.has_value()) { + return false; + } + if (optional_num_bytes_written.value() != get_ir_buf_size()) { + PyErr_SetString( + PyExc_RuntimeError, + "The number of bytes written to the output stream doesn't match the size of the " + "internal buffer" + ); + return false; + } + + m_serializer->clear_ir_buf(); + return true; +} + auto PySerializer::write_to_output_stream(PySerializer::BufferView buf ) -> std::optional { if (buf.empty()) { @@ -559,4 +546,12 @@ auto PySerializer::write_to_output_stream(PySerializer::BufferView buf } return num_bytes_written; } + +auto PySerializer::flush_output_stream() -> bool { + return nullptr != PyObject_CallMethod(m_output_stream, "flush", ""); +} + +auto PySerializer::close_output_stream() -> bool { + return nullptr != PyObject_CallMethod(m_output_stream, "close", ""); +} } // namespace clp_ffi_py::ir::native diff --git a/src/clp_ffi_py/ir/native/PySerializer.hpp b/src/clp_ffi_py/ir/native/PySerializer.hpp index d6684017..99cd0ae7 100644 --- a/src/clp_ffi_py/ir/native/PySerializer.hpp +++ b/src/clp_ffi_py/ir/native/PySerializer.hpp @@ -3,6 +3,7 @@ #include // Must always be included before any other header files +#include #include #include #include @@ -23,6 +24,12 @@ class PySerializer { using ClpIrSerializer = clp::ffi::ir_stream::Serializer; using BufferView = ClpIrSerializer::BufferView; + /** + * The default buffer size limit. Any change to the value should also be applied to `__init__`'s + * doc string and Python stub file. + */ + static constexpr size_t cDefaultBufferSizeLimit{65'536}; + /** * Initializes the underlying data with the given inputs. Since the memory allocation of * `PySerializer` is handled by CPython's allocator, cpp constructors will not be explicitly @@ -31,10 +38,12 @@ class PySerializer { * CPython APIs. * @param output_stream * @param serializer + * @param buffer_size_limit * @return true on success. * @return false on failure with the relevant Python exception and error set. */ - [[nodiscard]] auto init(PyObject* output_stream, ClpIrSerializer serializer) -> bool; + [[nodiscard]] auto + init(PyObject* output_stream, ClpIrSerializer serializer, Py_ssize_t buffer_size_limit) -> bool; /** * Initializes the pointers to nullptr by default. Should be called once the object is @@ -43,6 +52,8 @@ class PySerializer { auto default_init() -> void { m_output_stream = nullptr; m_serializer = nullptr; + m_num_total_bytes_serialized = 0; + m_buffer_size_limit = 0; } /** @@ -65,40 +76,32 @@ class PySerializer { * Serializes the given msgpack byte sequence as a msgpack map into IR format. * NOTE: the serializer must not be closed to call this method. * @param msgpack_byte_sequence - * @return true on success. - * @return false on error with the relevant Python exception and error set. - */ - [[nodiscard]] auto serialize_msgpack_map(std::span msgpack_byte_sequence) -> bool; - - /** - * Writes the underlying IR buffer into `m_output_stream`. - * NOTE: the serializer must not be closed to call this method. - * @return The number of bytes written on success. - * @return std::nullopt on failure with the relevant Python exception and error set. + * @return the number of bytes serialized on success. + * @return std::nullptr on failure with the relevant Python exception and error set. */ - [[nodiscard]] auto write_ir_buf_to_output_stream() -> std::optional; + [[nodiscard]] auto serialize_msgpack_map(std::span msgpack_byte_sequence + ) -> std::optional; - [[nodiscard]] auto get_ir_buf_size() const -> Py_ssize_t { - return static_cast(m_serializer->get_ir_buf_view().size()); + [[nodiscard]] auto get_num_bytes_serialized() const -> Py_ssize_t { + return m_num_total_bytes_serialized; } /** - * Flushes `m_output_stream`. + * Flushes the underlying IR buffer and `m_output_stream`. * NOTE: the serializer must not be closed to call this method. * @return true on success. * @return false on failure with the relevant Python exception and error set. */ - [[nodiscard]] auto flush_output_stream() -> bool; + [[nodiscard]] auto flush() -> bool; /** * Closes the serializer by writing the buffered results into the output stream with * end-of-stream IR Unit appended in the end. * NOTE: the serializer must not be closed to call this method. - * @param flush_stream Whether to flush `output_stream`. - * @return number of bytes written on success. - * @return std::nullopt on failure with the relevant Python exception and error set. + * @return true on success. + * @return false on failure with the relevant Python exception and error set. */ - [[nodiscard]] auto close(bool flush_stream) -> std::optional; + [[nodiscard]] auto close() -> bool; /** * Gets the `PyTypeObject` that represents `PySerializer`'s Python type. This type is @@ -118,14 +121,17 @@ class PySerializer { [[nodiscard]] static auto module_level_init(PyObject* py_module) -> bool; private: + [[nodiscard]] auto get_ir_buf_size() const -> Py_ssize_t { + return static_cast(m_serializer->get_ir_buf_view().size()); + } + /** - * Writes the data from given buffer to `m_output_stream` by calling `m_output_stream`'s `write` - * Python method. - * @param buf - * @return The number of bytes written on success. - * @return std::nullopt on failure with the relevant Python exception and error set. + * Writes the underlying IR buffer into `m_output_stream`. + * NOTE: the serializer must not be closed to call this method. + * @return true on success. + * @return false on failure with the relevant Python exception and error set. */ - [[nodiscard]] auto write_to_output_stream(BufferView buf) -> std::optional; + [[nodiscard]] auto write_ir_buf_to_output_stream() -> bool; /** * Closes `m_serializer` by releasing the allocated memory. @@ -136,11 +142,35 @@ class PySerializer { m_serializer = nullptr; } + /** + * Wrapper of `output_stream`'s `write` method. + * @param buf + * @return The number of bytes written on success. + * @return std::nullopt on failure with the relevant Python exception and error set. + */ + [[nodiscard]] auto write_to_output_stream(BufferView buf) -> std::optional; + + /** + * Wrapper of `output_stream`'s `flush` method. + * @return true on success. + * @return false on failure with the relevant Python exception and error set. + */ + [[nodiscard]] auto flush_output_stream() -> bool; + + /** + * Wrapper of `output_stream`'s `close` method. + * @return true on success. + * @return false on failure with the relevant Python exception and error set. + */ + [[nodiscard]] auto close_output_stream() -> bool; + // Variables PyObject_HEAD; PyObject* m_output_stream; // NOLINTNEXTLINE(cppcoreguidelines-owning-memory) gsl::owner m_serializer; + Py_ssize_t m_num_total_bytes_serialized; + Py_ssize_t m_buffer_size_limit; static inline PyObjectStaticPtr m_py_type{nullptr}; }; diff --git a/tests/test_ir/test_json/spark-event-logs.json b/tests/test_ir/test_json/spark-event-logs.json index a9d33a68..06f851f0 100644 --- a/tests/test_ir/test_json/spark-event-logs.json +++ b/tests/test_ir/test_json/spark-event-logs.json @@ -330,3 +330,5 @@ {"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":11,"accumUpdates":[[47,8],[48,990697],[49,23598],[50,0]]} {"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":11,"time":1633886655028} {"Event":"SparkListenerApplicationEnd","Timestamp":1633886655029} +{} +{"empty":{},"null":null} diff --git a/tests/test_ir/test_serializer.py b/tests/test_ir/test_serializer.py index 2dc33f9a..6f3a6ad3 100644 --- a/tests/test_ir/test_serializer.py +++ b/tests/test_ir/test_serializer.py @@ -1,4 +1,3 @@ - from io import BytesIO from pathlib import Path @@ -51,6 +50,8 @@ class TestCaseSerializer(TestCLPBase): """ input_src_dir: str = "test_json" + current_dir: Path = Path(__file__).resolve().parent + test_src_dir: Path = current_dir / Path(input_src_dir) def test_serialize_json(self) -> None: """ @@ -59,43 +60,80 @@ def test_serialize_json(self) -> None: The JSON parser will parse the file into Python dictionaries, and then convert them into msgpack and feed into `clp_ffi_py.ir.Serializer`. """ - current_dir: Path = Path(__file__).resolve().parent - test_src_dir: Path = current_dir / Path(TestCaseSerializer.input_src_dir) byte_buffer: BytesIO num_bytes_serialized: int serializer: Serializer # Test with context manager - for file_path in test_src_dir.rglob("*"): + for file_path in TestCaseSerializer.test_src_dir.rglob("*"): if not file_path.is_file(): continue byte_buffer = BytesIO() - num_bytes_serialized = 0 with Serializer(byte_buffer) as serializer: + num_bytes_serialized = serializer.get_num_bytes_serialized() for json_obj in JsonFileReader(file_path).read_json_lines(): - serializer.serialize_msgpack(serialize_dict_to_msgpack(json_obj)) - num_bytes_serialized += serializer.write_to_stream() - self.assertEqual(0, serializer.write_to_stream()) - serializer.flush_stream() - self.assertEqual(num_bytes_serialized + 1, len(byte_buffer.getvalue())) - byte_buffer.close() + num_bytes_serialized += serializer.serialize_msgpack_map( + serialize_dict_to_msgpack(json_obj) + ) + serializer.flush() + self.assertEqual( + len(byte_buffer.getvalue()), serializer.get_num_bytes_serialized() + ) + self.assertEqual(num_bytes_serialized, serializer.get_num_bytes_serialized()) # Test without context manager - for file_path in test_src_dir.rglob("*"): + for file_path in TestCaseSerializer.test_src_dir.rglob("*"): if not file_path.is_file(): continue byte_buffer = BytesIO() - num_bytes_serialized = 0 serializer = Serializer(byte_buffer) + num_bytes_serialized = serializer.get_num_bytes_serialized() for json_obj in JsonFileReader(file_path).read_json_lines(): - serializer.serialize_msgpack(serialize_dict_to_msgpack(json_obj)) - num_bytes_serialized += serializer.write_to_stream() - self.assertEqual(0, serializer.write_to_stream()) - serializer.flush_stream() - self.assertEqual(num_bytes_serialized, len(byte_buffer.getvalue())) + num_bytes_serialized += serializer.serialize_msgpack_map( + serialize_dict_to_msgpack(json_obj) + ) + serializer.flush() + self.assertEqual(len(byte_buffer.getvalue()), serializer.get_num_bytes_serialized()) + self.assertEqual(num_bytes_serialized, serializer.get_num_bytes_serialized()) serializer.close() - byte_buffer.close() + self.assertEqual( + num_bytes_serialized + 1, + serializer.get_num_bytes_serialized(), + "End-of-stream byte is missing", + ) + + def test_serialize_with_customized_buffer_size_limit(self) -> None: + """ + Tests serializing with customized buffer size limit. + """ + buffer_size_limit: int = 3000 + for file_path in TestCaseSerializer.test_src_dir.rglob("*"): + if not file_path.is_file(): + continue + byte_buffer: BytesIO = BytesIO() + with Serializer( + buffer_size_limit=buffer_size_limit, output_stream=byte_buffer + ) as serializer: + serializer.flush() + num_bytes_in_ir_buffer: int = 0 + for json_obj in JsonFileReader(file_path).read_json_lines(): + num_bytes_serialized: int = serializer.serialize_msgpack_map( + serialize_dict_to_msgpack(json_obj) + ) + self.assertNotEqual(0, num_bytes_serialized) + num_bytes_in_ir_buffer += num_bytes_serialized + if num_bytes_in_ir_buffer > buffer_size_limit: + # The IR buffer should already be written to the stream + self.assertEqual( + serializer.get_num_bytes_serialized(), len(byte_buffer.getvalue()) + ) + num_bytes_in_ir_buffer = 0 + else: + self.assertEqual( + serializer.get_num_bytes_serialized(), + num_bytes_in_ir_buffer + len(byte_buffer.getvalue()), + ) def test_closing_empty(self) -> None: """ @@ -105,13 +143,17 @@ def test_closing_empty(self) -> None: serializer: Serializer byte_buffer = BytesIO() - with Serializer(byte_buffer) as serializer: - serializer.flush_stream() - self.assertTrue(1 < len(byte_buffer.getvalue())) - byte_buffer.close() + preamble_size: int + with Serializer(byte_buffer, 0) as serializer: + self.assertTrue(len(byte_buffer.getvalue()) > 0) + self.assertEqual(len(byte_buffer.getvalue()), serializer.get_num_bytes_serialized()) + preamble_size = serializer.get_num_bytes_serialized() byte_buffer = BytesIO() serializer = Serializer(byte_buffer) serializer.close() - self.assertTrue(1 < len(byte_buffer.getvalue())) - byte_buffer.close() + self.assertEqual( + serializer.get_num_bytes_serialized(), + preamble_size + 1, + "End-of-stream byte is missing", + ) From 270626066c153e3bd6d94411a50dd24e1ebe81fc Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Tue, 5 Nov 2024 02:09:49 -0500 Subject: [PATCH 24/50] Add unit tests for unclosed serializer --- src/clp_ffi_py/ir/native/PySerializer.cpp | 12 +++++++----- tests/test_ir/test_serializer.py | 6 ++++++ 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/clp_ffi_py/ir/native/PySerializer.cpp b/src/clp_ffi_py/ir/native/PySerializer.cpp index e94d7d4d..18e82c4e 100644 --- a/src/clp_ffi_py/ir/native/PySerializer.cpp +++ b/src/clp_ffi_py/ir/native/PySerializer.cpp @@ -101,9 +101,9 @@ PyDoc_STRVAR( "Closes the serializer, writing any remaining data to the output stream and appending a" " byte sequence to mark the end of a CLP IR stream. The output stream is then flushed and" " closed.\n" - "NOTE: This method must be called to properly terminate an IR stream. Failing to call it" - " may leave the stream incomplete, potentially resulting in data loss due to data" - " buffering\n\n" + "NOTE: This method must be called to properly terminate an IR stream. Forgetting/failing" + " to call it will leave the stream incomplete, and potentially resulting in data loss due" + " to data buffering.\n\n" ":raise IOError: If the serializer has already been closed.\n" ); CLP_FFI_PY_METHOD auto PySerializer_close(PySerializer* self) -> PyObject*; @@ -370,8 +370,10 @@ CLP_FFI_PY_METHOD auto PySerializer_dealloc(PySerializer* self) -> void { if (false == self->is_closed()) { if (0 != PyErr_WarnEx( - PyExc_RuntimeWarning, - "`Serializer.close()` is not called before object destruction", + PyExc_ResourceWarning, + "`Serializer.close()` is not called before object destruction. Forget to call " + "`close` will leave the stream incomplete, and potentially resulting in data " + "loss due to data buffering", 1 )) { diff --git a/tests/test_ir/test_serializer.py b/tests/test_ir/test_serializer.py index 6f3a6ad3..947db073 100644 --- a/tests/test_ir/test_serializer.py +++ b/tests/test_ir/test_serializer.py @@ -1,5 +1,6 @@ from io import BytesIO from pathlib import Path +from typing import Optional from test_ir.test_utils import JsonFileReader, TestCLPBase @@ -157,3 +158,8 @@ def test_closing_empty(self) -> None: preamble_size + 1, "End-of-stream byte is missing", ) + + def test_not_closed(self) -> None: + serializer: Optional[Serializer] = Serializer(BytesIO()) + with self.assertWarns(ResourceWarning) as not_closed_warning: + serializer = None From 003f19d5c127a115f9c58c1e705b834b8f47f7eb Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Tue, 5 Nov 2024 18:31:26 -0500 Subject: [PATCH 25/50] Fix dangling reference --- src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp b/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp index d4aa72c5..85aefea8 100644 --- a/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp +++ b/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp @@ -218,15 +218,17 @@ CLP_FFI_PY_METHOD auto PyKeyValuePairLogEvent_to_dict(PyKeyValuePairLogEvent* se auto convert_py_dict_to_key_value_pair_log_event(PyDictObject* py_dict ) -> std::optional { - auto* serialized_msgpack_byte_sequence{py_utils_serialize_dict_to_msgpack(py_dict)}; + PyObjectPtr const serialized_msgpack_byte_sequence{ + py_utils_serialize_dict_to_msgpack(py_dict) + }; if (nullptr == serialized_msgpack_byte_sequence) { return std::nullopt; } // Since the type is already checked, we can use the macro to avoid duplicated type checking. std::span const data_view{ - PyBytes_AS_STRING(serialized_msgpack_byte_sequence), - static_cast(PyBytes_GET_SIZE(serialized_msgpack_byte_sequence)) + PyBytes_AS_STRING(serialized_msgpack_byte_sequence.get()), + static_cast(PyBytes_GET_SIZE(serialized_msgpack_byte_sequence.get())) }; auto const unpack_result{unpack_msgpack(data_view)}; if (unpack_result.has_error()) { From 98c95e21d9a9192cc43d1688223d429c1c7868e9 Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Tue, 5 Nov 2024 18:42:27 -0500 Subject: [PATCH 26/50] Apply rabit's comment --- src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp b/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp index 85aefea8..648e20a8 100644 --- a/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp +++ b/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp @@ -310,7 +310,7 @@ auto convert_py_dict_to_key_value_pair_log_event(PyDictObject* py_dict CLP_FFI_PY_METHOD auto PyKeyValuePairLogEvent_dealloc(PyKeyValuePairLogEvent* self) -> void { self->clean(); - PyObject_Del(self); + Py_TYPE(self)->tp_free(py_reinterpret_cast(self)); } } // namespace From 942c0ce3378ff65c2a3eb8e75d3a39af06469184 Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Tue, 5 Nov 2024 18:48:03 -0500 Subject: [PATCH 27/50] Forgot to free self in dealloc... --- src/clp_ffi_py/ir/native/PySerializer.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/clp_ffi_py/ir/native/PySerializer.cpp b/src/clp_ffi_py/ir/native/PySerializer.cpp index 18e82c4e..86287d4e 100644 --- a/src/clp_ffi_py/ir/native/PySerializer.cpp +++ b/src/clp_ffi_py/ir/native/PySerializer.cpp @@ -382,6 +382,7 @@ CLP_FFI_PY_METHOD auto PySerializer_dealloc(PySerializer* self) -> void { } self->clean(); + Py_TYPE(self)->tp_free(py_reinterpret_cast(self)); } } // namespace From 9a1a26aa18eb3fb7f71a8265e01aac4c46088ea9 Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Tue, 5 Nov 2024 20:49:39 -0500 Subject: [PATCH 28/50] Bug fixing... --- pyproject.toml | 2 +- src/clp_ffi_py/ir/native/PySerializer.cpp | 2 +- src/clp_ffi_py/ir/native/PySerializer.hpp | 11 ++++++++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b83b25e7..21fb40e3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,7 +97,7 @@ select = ["E", "I", "F"] isort.order-by-type = false [tool.scikit-build] -cmake.build-type = "Release" +cmake.build-type = "Debug" logging.level = "INFO" # By default, sdist-build-core uses `.gitignore` to select what to exclude from the source # distribution. This list is to explicitly exclude files not specified in `.gitignore`. diff --git a/src/clp_ffi_py/ir/native/PySerializer.cpp b/src/clp_ffi_py/ir/native/PySerializer.cpp index 86287d4e..216566a9 100644 --- a/src/clp_ffi_py/ir/native/PySerializer.cpp +++ b/src/clp_ffi_py/ir/native/PySerializer.cpp @@ -382,7 +382,7 @@ CLP_FFI_PY_METHOD auto PySerializer_dealloc(PySerializer* self) -> void { } self->clean(); - Py_TYPE(self)->tp_free(py_reinterpret_cast(self)); + PySerializer::get_py_type()->tp_free(py_reinterpret_cast(self)); } } // namespace diff --git a/src/clp_ffi_py/ir/native/PySerializer.hpp b/src/clp_ffi_py/ir/native/PySerializer.hpp index 99cd0ae7..5adb78d4 100644 --- a/src/clp_ffi_py/ir/native/PySerializer.hpp +++ b/src/clp_ffi_py/ir/native/PySerializer.hpp @@ -24,6 +24,15 @@ class PySerializer { using ClpIrSerializer = clp::ffi::ir_stream::Serializer; using BufferView = ClpIrSerializer::BufferView; + // Delete default constructor to disable direct instantiation. + PySerializer() = delete; + + // Delete copy/move constructors and assignments + PySerializer(PySerializer const&) = delete; + PySerializer(PySerializer&&) = delete; + auto operator=(PySerializer const&) -> PySerializer& = delete; + auto operator=(PySerializer&&) -> PySerializer& = delete; + /** * The default buffer size limit. Any change to the value should also be applied to `__init__`'s * doc string and Python stub file. @@ -60,8 +69,8 @@ class PySerializer { * Releases the memory allocated for underlying data fields. */ auto clean() -> void { - Py_XDECREF(m_output_stream); close_serializer(); + Py_XDECREF(m_output_stream); } [[nodiscard]] auto is_closed() const -> bool { return nullptr == m_serializer; } From 94f76ec5c1ee7d2c854c0da14399b3cf4e2fd44d Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Tue, 5 Nov 2024 22:37:19 -0500 Subject: [PATCH 29/50] Fix the reference bug --- src/clp_ffi_py/ir/native/PySerializer.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/clp_ffi_py/ir/native/PySerializer.cpp b/src/clp_ffi_py/ir/native/PySerializer.cpp index 216566a9..8333eac5 100644 --- a/src/clp_ffi_py/ir/native/PySerializer.cpp +++ b/src/clp_ffi_py/ir/native/PySerializer.cpp @@ -323,6 +323,7 @@ CLP_FFI_PY_METHOD auto PySerializer_close(PySerializer* self) -> PyObject* { } CLP_FFI_PY_METHOD auto PySerializer_enter(PySerializer* self) -> PyObject* { + Py_INCREF(self); return py_reinterpret_cast(self); } @@ -382,7 +383,7 @@ CLP_FFI_PY_METHOD auto PySerializer_dealloc(PySerializer* self) -> void { } self->clean(); - PySerializer::get_py_type()->tp_free(py_reinterpret_cast(self)); + Py_TYPE(self)->tp_free(py_reinterpret_cast(self)); } } // namespace From b1d0b5332ee9dcf1f574797780aa706bffe0480f Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Tue, 5 Nov 2024 22:38:56 -0500 Subject: [PATCH 30/50] Update to release --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 21fb40e3..b83b25e7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,7 +97,7 @@ select = ["E", "I", "F"] isort.order-by-type = false [tool.scikit-build] -cmake.build-type = "Debug" +cmake.build-type = "Release" logging.level = "INFO" # By default, sdist-build-core uses `.gitignore` to select what to exclude from the source # distribution. This list is to explicitly exclude files not specified in `.gitignore`. From c72acb132de2e5d3640fcb74472d357d6772fa12 Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Tue, 5 Nov 2024 22:43:15 -0500 Subject: [PATCH 31/50] Fix ruff --- tests/test_ir/test_serializer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_ir/test_serializer.py b/tests/test_ir/test_serializer.py index 947db073..58908a5a 100644 --- a/tests/test_ir/test_serializer.py +++ b/tests/test_ir/test_serializer.py @@ -161,5 +161,5 @@ def test_closing_empty(self) -> None: def test_not_closed(self) -> None: serializer: Optional[Serializer] = Serializer(BytesIO()) - with self.assertWarns(ResourceWarning) as not_closed_warning: - serializer = None + with self.assertWarns(ResourceWarning) as _: + serializer = None # noqa From 43180e9c752e53b992ddf52eb533bff7e89b2564 Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Tue, 5 Nov 2024 22:45:44 -0500 Subject: [PATCH 32/50] Add destructor --- src/clp_ffi_py/ir/native/PySerializer.hpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/clp_ffi_py/ir/native/PySerializer.hpp b/src/clp_ffi_py/ir/native/PySerializer.hpp index 5adb78d4..01d7a203 100644 --- a/src/clp_ffi_py/ir/native/PySerializer.hpp +++ b/src/clp_ffi_py/ir/native/PySerializer.hpp @@ -33,6 +33,9 @@ class PySerializer { auto operator=(PySerializer const&) -> PySerializer& = delete; auto operator=(PySerializer&&) -> PySerializer& = delete; + // Destructor + ~PySerializer() = default; + /** * The default buffer size limit. Any change to the value should also be applied to `__init__`'s * doc string and Python stub file. From 163925557eebbb3cd1e8f7ef4361693337e970e8 Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Tue, 5 Nov 2024 22:48:04 -0500 Subject: [PATCH 33/50] Add big five --- src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.hpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.hpp b/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.hpp index 8f89349c..308ca74e 100644 --- a/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.hpp +++ b/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.hpp @@ -16,6 +16,18 @@ namespace clp_ffi_py::ir::native { */ class PyKeyValuePairLogEvent { public: + // Delete default constructor to disable direct instantiation. + PyKeyValuePairLogEvent() = delete; + + // Delete copy/move constructors and assignments + PyKeyValuePairLogEvent(PyKeyValuePairLogEvent const&) = delete; + PyKeyValuePairLogEvent(PyKeyValuePairLogEvent&&) = delete; + auto operator=(PyKeyValuePairLogEvent const&) -> PyKeyValuePairLogEvent& = delete; + auto operator=(PyKeyValuePairLogEvent&&) -> PyKeyValuePairLogEvent& = delete; + + // Destructor + ~PyKeyValuePairLogEvent() = default; + /** * Initializes the underlying data with the given inputs. Since the memory allocation of * `PyKeyValuePairLogEvent` is handled by CPython's allocator, cpp constructors will not be From 802608f14093e914fd61087c432baa136b286b82 Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Tue, 5 Nov 2024 22:48:50 -0500 Subject: [PATCH 34/50] Run black --- tests/test_ir/test_serializer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_ir/test_serializer.py b/tests/test_ir/test_serializer.py index 58908a5a..7915613b 100644 --- a/tests/test_ir/test_serializer.py +++ b/tests/test_ir/test_serializer.py @@ -162,4 +162,4 @@ def test_closing_empty(self) -> None: def test_not_closed(self) -> None: serializer: Optional[Serializer] = Serializer(BytesIO()) with self.assertWarns(ResourceWarning) as _: - serializer = None # noqa + serializer = None # noqa From cad3888006c56ad7430389db48ede6d7112154aa Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Wed, 6 Nov 2024 16:45:41 -0500 Subject: [PATCH 35/50] Apply code review comments --- src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp b/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp index 648e20a8..58f0ac73 100644 --- a/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp +++ b/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp @@ -290,14 +290,14 @@ auto convert_py_dict_to_key_value_pair_log_event(PyDictObject* py_dict return std::nullopt; } auto const ir_unit_type{result.value()}; - if (ir_unit_type == clp::ffi::ir_stream::IrUnitType::SchemaTreeNodeInsertion) { - continue; + if (clp::ffi::ir_stream::IrUnitType::LogEvent == ir_unit_type) { + break; } - if (ir_unit_type != clp::ffi::ir_stream::IrUnitType::LogEvent) { + if (clp::ffi::ir_stream::IrUnitType::SchemaTreeNodeInsertion != ir_unit_type) { + continue; PyErr_SetString(PyExc_RuntimeError, "Unexpected Ir Unit Type"); return std::nullopt; } - break; } if (false == ir_unit_handler.m_log_event.has_value()) { From 7d83cba63935763136f14b9fb2108d08cd15d99f Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Wed, 6 Nov 2024 19:09:29 -0500 Subject: [PATCH 36/50] Fix... --- src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp b/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp index 58f0ac73..5b7d862d 100644 --- a/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp +++ b/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp @@ -294,7 +294,6 @@ auto convert_py_dict_to_key_value_pair_log_event(PyDictObject* py_dict break; } if (clp::ffi::ir_stream::IrUnitType::SchemaTreeNodeInsertion != ir_unit_type) { - continue; PyErr_SetString(PyExc_RuntimeError, "Unexpected Ir Unit Type"); return std::nullopt; } From 118cce4a59e090cfab7ca938a2966cea073c219a Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Wed, 6 Nov 2024 20:40:17 -0500 Subject: [PATCH 37/50] Fix issue --- src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp b/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp index 5b7d862d..9e4a2a79 100644 --- a/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp +++ b/src/clp_ffi_py/ir/native/PyKeyValuePairLogEvent.cpp @@ -205,15 +205,15 @@ CLP_FFI_PY_METHOD auto PyKeyValuePairLogEvent_to_dict(PyKeyValuePairLogEvent* se return nullptr; } auto const json_str{serialized_json_result.value().dump()}; - auto* parsed_json{py_utils_parse_json_str(json_str)}; + PyObjectPtr parsed_json{py_utils_parse_json_str(json_str)}; if (nullptr == parsed_json) { return nullptr; } - if (false == static_cast(PyDict_Check(parsed_json))) { + if (false == static_cast(PyDict_Check(parsed_json.get()))) { PyErr_SetString(PyExc_TypeError, "Serialized JSON object is not a dictionary"); return nullptr; } - return parsed_json; + return parsed_json.release(); } auto convert_py_dict_to_key_value_pair_log_event(PyDictObject* py_dict From 369395e39f6c6d0c58c14be1c39b429f518a6673 Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Wed, 6 Nov 2024 22:18:43 -0500 Subject: [PATCH 38/50] Apply code review comments --- .../test_json/{elasticsearch.json => elasticsearch.jsonl} | 0 .../{spark-event-logs.json => spark-event-logs.jsonl} | 2 ++ tests/test_ir/test_key_value_pair_log_event.py | 6 +++--- tests/test_ir/test_utils.py | 4 ++-- 4 files changed, 7 insertions(+), 5 deletions(-) rename tests/test_ir/test_json/{elasticsearch.json => elasticsearch.jsonl} (100%) rename tests/test_ir/test_json/{spark-event-logs.json => spark-event-logs.jsonl} (99%) diff --git a/tests/test_ir/test_json/elasticsearch.json b/tests/test_ir/test_json/elasticsearch.jsonl similarity index 100% rename from tests/test_ir/test_json/elasticsearch.json rename to tests/test_ir/test_json/elasticsearch.jsonl diff --git a/tests/test_ir/test_json/spark-event-logs.json b/tests/test_ir/test_json/spark-event-logs.jsonl similarity index 99% rename from tests/test_ir/test_json/spark-event-logs.json rename to tests/test_ir/test_json/spark-event-logs.jsonl index a9d33a68..06f851f0 100644 --- a/tests/test_ir/test_json/spark-event-logs.json +++ b/tests/test_ir/test_json/spark-event-logs.jsonl @@ -330,3 +330,5 @@ {"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":11,"accumUpdates":[[47,8],[48,990697],[49,23598],[50,0]]} {"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":11,"time":1633886655028} {"Event":"SparkListenerApplicationEnd","Timestamp":1633886655029} +{} +{"empty":{},"null":null} diff --git a/tests/test_ir/test_key_value_pair_log_event.py b/tests/test_ir/test_key_value_pair_log_event.py index fac8f6fb..ca69b785 100644 --- a/tests/test_ir/test_key_value_pair_log_event.py +++ b/tests/test_ir/test_key_value_pair_log_event.py @@ -1,7 +1,7 @@ from pathlib import Path from typing import Any, Dict -from test_ir.test_utils import JsonFileReader, TestCLPBase +from test_ir.test_utils import JsonLinesFileReader, TestCLPBase from clp_ffi_py.ir import KeyValuePairLogEvent @@ -23,8 +23,8 @@ def test_basic(self) -> None: for file_path in test_src_dir.rglob("*"): if not file_path.is_file(): continue - json_file_reader: JsonFileReader = JsonFileReader(file_path) - for expected in json_file_reader.read_json_lines(): + json_file_reader: JsonLinesFileReader = JsonLinesFileReader(file_path) + for expected in json_file_reader.read_line(): self.assertIsInstance(expected, dict, "Input must be a dictionary") actual: KeyValuePairLogEvent = KeyValuePairLogEvent(expected) serialized_py_dict: Dict[Any, Any] = actual.to_dict() diff --git a/tests/test_ir/test_utils.py b/tests/test_ir/test_utils.py index 282725ce..2a3daa1b 100644 --- a/tests/test_ir/test_utils.py +++ b/tests/test_ir/test_utils.py @@ -24,7 +24,7 @@ from clp_ffi_py.wildcard_query import WildcardQuery -class JsonFileReader: +class JsonLinesFileReader: """ Class for reading JSON files. @@ -40,7 +40,7 @@ def __init__(self, file_path: Path): """ self.file_path: Path = file_path - def read_json_lines(self) -> Generator[Any, None, None]: + def read_line(self) -> Generator[Any, None, None]: """ Reads each line in the JSON file, parses it as a JSON object, and yields the JSON object. From 1474c96ff7dcc7c9bd8f7d647f2d05af96d2f258 Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Thu, 7 Nov 2024 00:04:00 -0500 Subject: [PATCH 39/50] fix --- tests/test_ir/test_key_value_pair_log_event.py | 2 +- tests/test_ir/test_utils.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_ir/test_key_value_pair_log_event.py b/tests/test_ir/test_key_value_pair_log_event.py index ca69b785..4c07a3cf 100644 --- a/tests/test_ir/test_key_value_pair_log_event.py +++ b/tests/test_ir/test_key_value_pair_log_event.py @@ -24,7 +24,7 @@ def test_basic(self) -> None: if not file_path.is_file(): continue json_file_reader: JsonLinesFileReader = JsonLinesFileReader(file_path) - for expected in json_file_reader.read_line(): + for expected in json_file_reader.read_lines(): self.assertIsInstance(expected, dict, "Input must be a dictionary") actual: KeyValuePairLogEvent = KeyValuePairLogEvent(expected) serialized_py_dict: Dict[Any, Any] = actual.to_dict() diff --git a/tests/test_ir/test_utils.py b/tests/test_ir/test_utils.py index 2a3daa1b..ced05e0c 100644 --- a/tests/test_ir/test_utils.py +++ b/tests/test_ir/test_utils.py @@ -40,14 +40,15 @@ def __init__(self, file_path: Path): """ self.file_path: Path = file_path - def read_line(self) -> Generator[Any, None, None]: + def read_lines(self) -> Generator[Any, None, None]: """ Reads each line in the JSON file, parses it as a JSON object, and yields the JSON object. :yield: Parsed JSON object for each line in the file. """ with open(self.file_path, "r", encoding="utf-8") as file: - yield from (json.loads(line.strip()) for line in file) + for line in file: + yield json.loads(line.strip()) def _zstd_compressions_handler( From 770827d287245508a0bfdbba1c911a7a8b5afe77 Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Thu, 7 Nov 2024 00:07:06 -0500 Subject: [PATCH 40/50] Merge with kv_log_event's changes --- tests/test_ir/test_serializer.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_ir/test_serializer.py b/tests/test_ir/test_serializer.py index 7915613b..bd13b8b2 100644 --- a/tests/test_ir/test_serializer.py +++ b/tests/test_ir/test_serializer.py @@ -2,7 +2,7 @@ from pathlib import Path from typing import Optional -from test_ir.test_utils import JsonFileReader, TestCLPBase +from test_ir.test_utils import JsonLinesFileReader, TestCLPBase from clp_ffi_py.ir import FourByteSerializer, Serializer from clp_ffi_py.utils import serialize_dict_to_msgpack @@ -73,7 +73,7 @@ def test_serialize_json(self) -> None: byte_buffer = BytesIO() with Serializer(byte_buffer) as serializer: num_bytes_serialized = serializer.get_num_bytes_serialized() - for json_obj in JsonFileReader(file_path).read_json_lines(): + for json_obj in JsonLinesFileReader(file_path).read_lines(): num_bytes_serialized += serializer.serialize_msgpack_map( serialize_dict_to_msgpack(json_obj) ) @@ -90,7 +90,7 @@ def test_serialize_json(self) -> None: byte_buffer = BytesIO() serializer = Serializer(byte_buffer) num_bytes_serialized = serializer.get_num_bytes_serialized() - for json_obj in JsonFileReader(file_path).read_json_lines(): + for json_obj in JsonLinesFileReader(file_path).read_lines(): num_bytes_serialized += serializer.serialize_msgpack_map( serialize_dict_to_msgpack(json_obj) ) @@ -118,7 +118,7 @@ def test_serialize_with_customized_buffer_size_limit(self) -> None: ) as serializer: serializer.flush() num_bytes_in_ir_buffer: int = 0 - for json_obj in JsonFileReader(file_path).read_json_lines(): + for json_obj in JsonLinesFileReader(file_path).read_lines(): num_bytes_serialized: int = serializer.serialize_msgpack_map( serialize_dict_to_msgpack(json_obj) ) From a1086c4494b431b0c8c742d9aac699cce772fd76 Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Fri, 8 Nov 2024 23:58:54 -0500 Subject: [PATCH 41/50] Fix leak --- src/clp_ffi_py/ir/native/PySerializer.cpp | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/clp_ffi_py/ir/native/PySerializer.cpp b/src/clp_ffi_py/ir/native/PySerializer.cpp index 8333eac5..f4a9c8e9 100644 --- a/src/clp_ffi_py/ir/native/PySerializer.cpp +++ b/src/clp_ffi_py/ir/native/PySerializer.cpp @@ -537,7 +537,7 @@ auto PySerializer::write_to_output_stream(PySerializer::BufferView buf return std::nullopt; } - PyObject* py_num_bytes_written{ + PyObjectPtr const py_num_bytes_written{ PyObject_CallMethod(m_output_stream, "write", "O", ir_buf_mem_view.get()) }; if (nullptr == py_num_bytes_written) { @@ -545,17 +545,25 @@ auto PySerializer::write_to_output_stream(PySerializer::BufferView buf } Py_ssize_t num_bytes_written{}; - if (false == parse_py_int(py_num_bytes_written, num_bytes_written)) { + if (false == parse_py_int(py_num_bytes_written.get(), num_bytes_written)) { return std::nullopt; } return num_bytes_written; } auto PySerializer::flush_output_stream() -> bool { - return nullptr != PyObject_CallMethod(m_output_stream, "flush", ""); + PyObjectPtr const ret_val{PyObject_CallMethod(m_output_stream, "flush", "")}; + if (nullptr == ret_val) { + return false; + } + return true; } auto PySerializer::close_output_stream() -> bool { - return nullptr != PyObject_CallMethod(m_output_stream, "close", ""); + PyObjectPtr const ret_val{PyObject_CallMethod(m_output_stream, "close", "")}; + if (nullptr == ret_val) { + return false; + } + return true; } } // namespace clp_ffi_py::ir::native From 2eb113be9de517726f9f9ef2f6838cd68c9ecda5 Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Mon, 18 Nov 2024 19:46:00 -0500 Subject: [PATCH 42/50] Merge beta branch --- clp_ffi_py/ir/native.pyi | 2 +- clp_ffi_py/utils.py | 1 - src/clp_ffi_py/ir/native/PySerializer.cpp | 83 ++++++++++++----------- src/clp_ffi_py/ir/native/PySerializer.hpp | 7 +- tests/test_ir/test_serializer.py | 16 ++--- tests/test_utils.py | 3 +- 6 files changed, 60 insertions(+), 52 deletions(-) diff --git a/clp_ffi_py/ir/native.pyi b/clp_ffi_py/ir/native.pyi index a3fc6331..7ba19f14 100644 --- a/clp_ffi_py/ir/native.pyi +++ b/clp_ffi_py/ir/native.pyi @@ -98,7 +98,7 @@ class Serializer: exc_value: Optional[BaseException], traceback: Optional[TracebackType], ) -> None: ... - def serialize_msgpack_map(self, msgpack_map: bytes) -> int: ... + def serialize_log_event_from_msgpack_map(self, msgpack_map: bytes) -> int: ... def get_num_bytes_serialized(self) -> int: ... def flush(self) -> None: ... def close(self) -> None: ... diff --git a/clp_ffi_py/utils.py b/clp_ffi_py/utils.py index 2a8264eb..a36ea9eb 100644 --- a/clp_ffi_py/utils.py +++ b/clp_ffi_py/utils.py @@ -3,7 +3,6 @@ from typing import Any, Dict, Optional import dateutil.tz - import msgpack diff --git a/src/clp_ffi_py/ir/native/PySerializer.cpp b/src/clp_ffi_py/ir/native/PySerializer.cpp index f4a9c8e9..4699ae20 100644 --- a/src/clp_ffi_py/ir/native/PySerializer.cpp +++ b/src/clp_ffi_py/ir/native/PySerializer.cpp @@ -27,19 +27,19 @@ namespace { // NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays) PyDoc_STRVAR( cPySerializerDoc, - "Serializer for CLP key-value pair IR stream.\n" - "This class serializes log events into CLP IR format (using four-byte-encoding) and writes" - " the serialized data to a specified byte stream object.\n\n" - "__init__(self, output_stream, buffer_size_limit)\n\n" - "Initializes a :class:`Serializer` instance with the given output stream. Notice that each" - " object should be strictly initialized only once. Double initialization will result in" - " memory leak.\n\n" - ":param output_stream: A writable byte stream to which the serializer will write serialized" - " IR byte sequence.\n" - ":param buffer_size_limit: The buffer size in bytes that will trigger the internal buffer" - " flush, defaults to 65536. The serialized log events will be first buffered inside the" - " internal buffer. When the size of the buffer exceeds this limit, the internal buffer will" - " be written to the output stream.\n" + "Serializer for serializing CLP key-value pair IR streams.\n" + "This class serializes log events using the CLP key-value pair IR format and writes the" + " serialized data to a specified byte stream object.\n\n" + "__init__(self, output_stream, buffer_size_limit=65536)\n\n" + "Initializes a :class:`Serializer` instance with the given output stream. Note that each" + " object should only be initialized once. Double initialization will result in a memory" + " leak.\n\n" + ":param output_stream: A writable byte output stream to which the serializer will write the" + " serialized IR byte sequences.\n" + ":type output_stream: IO[bytes]\n" + ":param buffer_size_limit: Threshold of how much serialized data to buffer before flushing" + " it to `output_stream`. Defaults to 64 KiB.\n" + ":type buffer_size_limit: int\n" ); CLP_FFI_PY_METHOD auto PySerializer_init(PySerializer* self, PyObject* args, PyObject* keywords) -> int; @@ -49,20 +49,24 @@ PySerializer_init(PySerializer* self, PyObject* args, PyObject* keywords) -> int */ // NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays) PyDoc_STRVAR( - cPySerializerSerializeMsgpackMapDoc, - "serialize_msgpack_map(self, msgpack_map)\n" + cPySerializerSerializeLogEventFromMsgpackMapDoc, + "serialize_log_event_from_msgpack_map(self, msgpack_map)\n" "--\n\n" - "Serializes the given msgpack byte sequence into as a log event into CLP IR format.\n\n" - ":param msgpack_map: A byte sequence encoded in msgpack as the input log event." - " The unpacked result must be a msgpack map with all keys as strings.\n" + "Serializes the given log event.\n\n" + ":param msgpack_map: The log event as a packed msgpack map where all keys are" + " strings.\n" + ":type msgpack_map: bytes\n" ":return: The number of bytes serialized.\n" + ":rtype: int\n" ":raise IOError: If the serializer has already been closed.\n" - ":raise TypeError: If the unpacked result is not a msgpack map.\n" - ":raise RuntimeError: If it fails to unpack the given msgpack byte sequence, or the" - " serialization method returns failure.\n" + ":raise TypeError: If `msgpack_map` is not a packed msgpack map.\n" + ":raise RuntimeError: If `msgpack_map` couldn't be unpacked or serialization into the IR" + " stream failed.\n" ); -CLP_FFI_PY_METHOD auto -PySerializer_serialize_msgpack_map(PySerializer* self, PyObject* msgpack_map) -> PyObject*; +CLP_FFI_PY_METHOD auto PySerializer_serialize_log_event_from_msgpack_map( + PySerializer* self, + PyObject* msgpack_map +) -> PyObject*; /** * Callback of `PySerializer`'s `get_num_bytes_serialized` method. @@ -73,6 +77,7 @@ PyDoc_STRVAR( "get_num_bytes_serialized(self)\n" "--\n\n" ":return: The total number of bytes serialized.\n" + ":rtype: int\n" ":raise IOError: If the serializer has already been closed.\n" ); CLP_FFI_PY_METHOD auto PySerializer_get_num_bytes_serialized(PySerializer* self) -> PyObject*; @@ -85,7 +90,7 @@ PyDoc_STRVAR( cPySerializerFlushDoc, "flush(self)\n" "--\n\n" - "Flushes the internal buffer and the output stream.\n\n" + "Flushes any buffered data and the output stream.\n\n" ":raise IOError: If the serializer has already been closed.\n" ); CLP_FFI_PY_METHOD auto PySerializer_flush(PySerializer* self) -> PyObject*; @@ -98,12 +103,11 @@ PyDoc_STRVAR( cPySerializerCloseDoc, "close(self)\n" "--\n\n" - "Closes the serializer, writing any remaining data to the output stream and appending a" - " byte sequence to mark the end of a CLP IR stream. The output stream is then flushed and" + "Closes the serializer, writing any buffered data to the output stream and appending a byte" + " sequence to mark the end of the CLP IR stream. The output stream is then flushed and" " closed.\n" - "NOTE: This method must be called to properly terminate an IR stream. Forgetting/failing" - " to call it will leave the stream incomplete, and potentially resulting in data loss due" - " to data buffering.\n\n" + "NOTE: This method must be called to properly terminate an IR stream. If it isn't called," + " the stream will be incomplete, and any buffered data may be lost.\n\n" ":raise IOError: If the serializer has already been closed.\n" ); CLP_FFI_PY_METHOD auto PySerializer_close(PySerializer* self) -> PyObject*; @@ -118,6 +122,7 @@ PyDoc_STRVAR( "--\n\n" "Enters the runtime context.\n\n" ":return: self.\n" + ":rtype: :class:`Serializer`\n" ); CLP_FFI_PY_METHOD auto PySerializer_enter(PySerializer* self) -> PyObject*; @@ -130,8 +135,8 @@ PyDoc_STRVAR( "--\n\n" "Exits the runtime context, automatically calling :meth:`close` to flush all buffered data" " into the output stream." - ":param exc_type: The type of the exception caused the context to be exited. Unused.\n" - ":param exc_value: The value of the exception caused the context to be exited. Unused.\n" + ":param exc_type: The type of the exception that caused the exit. Unused.\n" + ":param exc_value: The value of the exception that caused the exit. Unused.\n" ":param exc_traceable: The traceback. Unused.\n" ); CLP_FFI_PY_METHOD auto @@ -144,10 +149,10 @@ CLP_FFI_PY_METHOD auto PySerializer_dealloc(PySerializer* self) -> void; // NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays) PyMethodDef PySerializer_method_table[]{ - {"serialize_msgpack_map", - py_c_function_cast(PySerializer_serialize_msgpack_map), + {"serialize_log_event_from_msgpack_map", + py_c_function_cast(PySerializer_serialize_log_event_from_msgpack_map), METH_O, - static_cast(cPySerializerSerializeMsgpackMapDoc)}, + static_cast(cPySerializerSerializeLogEventFromMsgpackMapDoc)}, {"get_num_bytes_serialized", py_c_function_cast(PySerializer_get_num_bytes_serialized), @@ -281,8 +286,10 @@ PySerializer_init(PySerializer* self, PyObject* args, PyObject* keywords) -> int return 0; } -CLP_FFI_PY_METHOD auto -PySerializer_serialize_msgpack_map(PySerializer* self, PyObject* msgpack_map) -> PyObject* { +CLP_FFI_PY_METHOD auto PySerializer_serialize_log_event_from_msgpack_map( + PySerializer* self, + PyObject* msgpack_map +) -> PyObject* { if (false == static_cast(PyBytes_Check(msgpack_map))) { PyErr_SetString( PyExc_TypeError, @@ -293,7 +300,7 @@ PySerializer_serialize_msgpack_map(PySerializer* self, PyObject* msgpack_map) -> auto* py_bytes_msgpack_map{py_reinterpret_cast(msgpack_map)}; // Since the type is already checked, we can use the macro to avoid duplicated type checking. - auto const num_byte_serialized{self->serialize_msgpack_map( + auto const num_byte_serialized{self->serialize_log_event_from_msgpack_map( {PyBytes_AS_STRING(py_bytes_msgpack_map), static_cast(PyBytes_GET_SIZE(py_bytes_msgpack_map))} )}; @@ -416,7 +423,7 @@ auto PySerializer::assert_is_not_closed() const -> bool { return true; } -auto PySerializer::serialize_msgpack_map(std::span msgpack_byte_sequence +auto PySerializer::serialize_log_event_from_msgpack_map(std::span msgpack_byte_sequence ) -> std::optional { if (false == assert_is_not_closed()) { return std::nullopt; diff --git a/src/clp_ffi_py/ir/native/PySerializer.hpp b/src/clp_ffi_py/ir/native/PySerializer.hpp index 01d7a203..e25e44a8 100644 --- a/src/clp_ffi_py/ir/native/PySerializer.hpp +++ b/src/clp_ffi_py/ir/native/PySerializer.hpp @@ -4,12 +4,12 @@ #include // Must always be included before any other header files #include -#include #include #include #include #include +#include #include @@ -85,13 +85,14 @@ class PySerializer { [[nodiscard]] auto assert_is_not_closed() const -> bool; /** - * Serializes the given msgpack byte sequence as a msgpack map into IR format. + * Serializes the log event from the given msgpack map into IR format. * NOTE: the serializer must not be closed to call this method. * @param msgpack_byte_sequence * @return the number of bytes serialized on success. * @return std::nullptr on failure with the relevant Python exception and error set. */ - [[nodiscard]] auto serialize_msgpack_map(std::span msgpack_byte_sequence + [[nodiscard]] auto serialize_log_event_from_msgpack_map( + std::span msgpack_byte_sequence ) -> std::optional; [[nodiscard]] auto get_num_bytes_serialized() const -> Py_ssize_t { diff --git a/tests/test_ir/test_serializer.py b/tests/test_ir/test_serializer.py index bd13b8b2..48c377af 100644 --- a/tests/test_ir/test_serializer.py +++ b/tests/test_ir/test_serializer.py @@ -50,9 +50,9 @@ class TestCaseSerializer(TestCLPBase): Class for testing `clp_ffi_py.ir.Serializer`. """ - input_src_dir: str = "test_json" + jsonl_test_data_dir: Path = Path("test_data") / "jsonl" current_dir: Path = Path(__file__).resolve().parent - test_src_dir: Path = current_dir / Path(input_src_dir) + test_data_dir: Path = current_dir / jsonl_test_data_dir def test_serialize_json(self) -> None: """ @@ -67,14 +67,14 @@ def test_serialize_json(self) -> None: serializer: Serializer # Test with context manager - for file_path in TestCaseSerializer.test_src_dir.rglob("*"): + for file_path in TestCaseSerializer.test_data_dir.rglob("*"): if not file_path.is_file(): continue byte_buffer = BytesIO() with Serializer(byte_buffer) as serializer: num_bytes_serialized = serializer.get_num_bytes_serialized() for json_obj in JsonLinesFileReader(file_path).read_lines(): - num_bytes_serialized += serializer.serialize_msgpack_map( + num_bytes_serialized += serializer.serialize_log_event_from_msgpack_map( serialize_dict_to_msgpack(json_obj) ) serializer.flush() @@ -84,14 +84,14 @@ def test_serialize_json(self) -> None: self.assertEqual(num_bytes_serialized, serializer.get_num_bytes_serialized()) # Test without context manager - for file_path in TestCaseSerializer.test_src_dir.rglob("*"): + for file_path in TestCaseSerializer.test_data_dir.rglob("*"): if not file_path.is_file(): continue byte_buffer = BytesIO() serializer = Serializer(byte_buffer) num_bytes_serialized = serializer.get_num_bytes_serialized() for json_obj in JsonLinesFileReader(file_path).read_lines(): - num_bytes_serialized += serializer.serialize_msgpack_map( + num_bytes_serialized += serializer.serialize_log_event_from_msgpack_map( serialize_dict_to_msgpack(json_obj) ) serializer.flush() @@ -109,7 +109,7 @@ def test_serialize_with_customized_buffer_size_limit(self) -> None: Tests serializing with customized buffer size limit. """ buffer_size_limit: int = 3000 - for file_path in TestCaseSerializer.test_src_dir.rglob("*"): + for file_path in TestCaseSerializer.test_data_dir.rglob("*"): if not file_path.is_file(): continue byte_buffer: BytesIO = BytesIO() @@ -119,7 +119,7 @@ def test_serialize_with_customized_buffer_size_limit(self) -> None: serializer.flush() num_bytes_in_ir_buffer: int = 0 for json_obj in JsonLinesFileReader(file_path).read_lines(): - num_bytes_serialized: int = serializer.serialize_msgpack_map( + num_bytes_serialized: int = serializer.serialize_log_event_from_msgpack_map( serialize_dict_to_msgpack(json_obj) ) self.assertNotEqual(0, num_bytes_serialized) diff --git a/tests/test_utils.py b/tests/test_utils.py index 57f69d81..2fdcb34d 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,9 +1,10 @@ import unittest from typing import Any, Dict -import clp_ffi_py.utils import msgpack +import clp_ffi_py.utils + class TestUtils(unittest.TestCase): """ From 68e088d7d475ff77f97f19c654a31c59f7ab6135 Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Mon, 18 Nov 2024 20:00:21 -0500 Subject: [PATCH 43/50] Apply new review suggestions from last PR --- src/clp_ffi_py/ir/native/PySerializer.hpp | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/clp_ffi_py/ir/native/PySerializer.hpp b/src/clp_ffi_py/ir/native/PySerializer.hpp index e25e44a8..37bfcb94 100644 --- a/src/clp_ffi_py/ir/native/PySerializer.hpp +++ b/src/clp_ffi_py/ir/native/PySerializer.hpp @@ -27,7 +27,7 @@ class PySerializer { // Delete default constructor to disable direct instantiation. PySerializer() = delete; - // Delete copy/move constructors and assignments + // Delete copy & move constructors and assignment operators PySerializer(PySerializer const&) = delete; PySerializer(PySerializer&&) = delete; auto operator=(PySerializer const&) -> PySerializer& = delete; @@ -46,8 +46,7 @@ class PySerializer { * Initializes the underlying data with the given inputs. Since the memory allocation of * `PySerializer` is handled by CPython's allocator, cpp constructors will not be explicitly * called. This function serves as the default constructor initialize the underlying serializer. - * It has to be manually called whenever creating a new `PySerializer` object through - * CPython APIs. + * It has to be called manually to create a `PySerializer` object through CPython APIs. * @param output_stream * @param serializer * @param buffer_size_limit @@ -126,8 +125,7 @@ class PySerializer { /** * Creates and initializes `PySerializer` as a Python type, and then incorporates this * type as a Python object into the py_module module. - * @param py_module This is the Python module where the initialized `PySerializer` will be - * incorporated. + * @param py_module The Python module where the initialized `PySerializer` will be incorporated. * @return true on success. * @return false on failure with the relevant Python exception and error set. */ From ea64e8ad0f6f0e84f863501d6a6f53b064c47a64 Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Mon, 18 Nov 2024 20:10:47 -0500 Subject: [PATCH 44/50] Apply code review comments from last PR --- tests/test_ir/test_serializer.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/tests/test_ir/test_serializer.py b/tests/test_ir/test_serializer.py index 48c377af..9a2fb3f3 100644 --- a/tests/test_ir/test_serializer.py +++ b/tests/test_ir/test_serializer.py @@ -1,6 +1,6 @@ from io import BytesIO from pathlib import Path -from typing import Optional +from typing import List, Optional from test_ir.test_utils import JsonLinesFileReader, TestCLPBase @@ -62,14 +62,13 @@ def test_serialize_json(self) -> None: and then convert them into msgpack and feed into `clp_ffi_py.ir.Serializer`. """ + test_files: List[Path] = self.__get_test_files() byte_buffer: BytesIO num_bytes_serialized: int serializer: Serializer # Test with context manager - for file_path in TestCaseSerializer.test_data_dir.rglob("*"): - if not file_path.is_file(): - continue + for file_path in test_files: byte_buffer = BytesIO() with Serializer(byte_buffer) as serializer: num_bytes_serialized = serializer.get_num_bytes_serialized() @@ -84,9 +83,7 @@ def test_serialize_json(self) -> None: self.assertEqual(num_bytes_serialized, serializer.get_num_bytes_serialized()) # Test without context manager - for file_path in TestCaseSerializer.test_data_dir.rglob("*"): - if not file_path.is_file(): - continue + for file_path in test_files: byte_buffer = BytesIO() serializer = Serializer(byte_buffer) num_bytes_serialized = serializer.get_num_bytes_serialized() @@ -109,9 +106,7 @@ def test_serialize_with_customized_buffer_size_limit(self) -> None: Tests serializing with customized buffer size limit. """ buffer_size_limit: int = 3000 - for file_path in TestCaseSerializer.test_data_dir.rglob("*"): - if not file_path.is_file(): - continue + for file_path in self.__get_test_files(): byte_buffer: BytesIO = BytesIO() with Serializer( buffer_size_limit=buffer_size_limit, output_stream=byte_buffer @@ -163,3 +158,12 @@ def test_not_closed(self) -> None: serializer: Optional[Serializer] = Serializer(BytesIO()) with self.assertWarns(ResourceWarning) as _: serializer = None # noqa + + def __get_test_files(self) -> List[Path]: + test_files: List[Path] = [] + for file_path in TestCaseSerializer.test_data_dir.rglob("*"): + if not file_path.is_file(): + continue + test_files.append(file_path) + self.assertFalse(0 == len(test_files), "No test files found") + return test_files From c4e36fbce842eb5c18e66d6d512c64f2864bc1b8 Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Mon, 18 Nov 2024 20:14:22 -0500 Subject: [PATCH 45/50] Remove unused unit test inputs --- tests/test_ir/test_json/elasticsearch.jsonl | 100 ------ .../test_ir/test_json/spark-event-logs.jsonl | 334 ------------------ 2 files changed, 434 deletions(-) delete mode 100644 tests/test_ir/test_json/elasticsearch.jsonl delete mode 100644 tests/test_ir/test_json/spark-event-logs.jsonl diff --git a/tests/test_ir/test_json/elasticsearch.jsonl b/tests/test_ir/test_json/elasticsearch.jsonl deleted file mode 100644 index c6368011..00000000 --- a/tests/test_ir/test_json/elasticsearch.jsonl +++ /dev/null @@ -1,100 +0,0 @@ -{"@timestamp":"2023-03-28T04:00:00.040Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#7]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [abacus_go]"]} -{"@timestamp":"2023-03-28T04:00:00.202Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#1]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [rider_product_cored]"]} -{"@timestamp":"2023-03-28T04:00:00.202Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#4]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [fares_management]"]} -{"@timestamp":"2023-03-28T04:00:00.201Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#8]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [fulfillment_compatibled]"]} -{"@timestamp":"2023-03-28T04:00:00.201Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#3]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [k8s_apiserver]"]} -{"@timestamp":"2023-03-28T04:00:00.202Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#6]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [event_logs]"]} -{"@timestamp":"2023-03-28T04:00:00.202Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#2]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [k8s_kubelet]"]} -{"@timestamp":"2023-03-28T04:00:00.203Z", "log.level":"TRACE", "message":"Processing scheduled tasks started", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][generic][T#9]","log.logger":"org.elasticsearch.xpack.transform.transforms.scheduling.TransformScheduler","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:00.240Z", "log.level":"TRACE", "message":"checking jobs [2023-03-28T04:00:00.240275019Z]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"ticker-schedule-trigger-engine","log.logger":"org.elasticsearch.xpack.watcher.trigger.schedule.engine.TickerScheduleTriggerEngine","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:00.069Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#5]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [driverapp]"]} -{"@timestamp":"2023-03-28T04:00:01.119Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#7]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [michelangelo_gateway]"]} -{"@timestamp":"2023-03-28T04:00:01.119Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#6]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [finprod_rewards_eligibility]"]} -{"@timestamp":"2023-03-28T04:00:01.119Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#8]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [.geoip_databases]"]} -{"@timestamp":"2023-03-28T04:00:01.119Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#2]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [fares_intelligence]"]} -{"@timestamp":"2023-03-28T04:00:01.892Z", "log.level":"TRACE", "message":"Processing scheduled tasks finished, took 0ms", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][generic][T#9]","log.logger":"org.elasticsearch.xpack.transform.transforms.scheduling.TransformScheduler","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:01.892Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#6]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [access_platform_core]"]} -{"@timestamp":"2023-03-28T04:00:01.320Z", "log.level":"TRACE", "message":"scheduling global_checkpoint_sync every 30s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][generic][T#7]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [k8s_kubelet]"]} -{"@timestamp":"2023-03-28T04:00:01.329Z", "log.level":"DEBUG", "message":"no retention leases are expired from current retention leases [RetentionLeases{primaryTerm=1, version=1, leases={peer_recovery/iKPGCkp9RVOKXOj20uOt4g=RetentionLease{id='peer_recovery/iKPGCkp9RVOKXOj20uOt4g', retainingSequenceNumber=0, timestamp=1679973869034, source='peer recovery'}}}]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][management][T#2]","log.logger":"org.elasticsearch.index.seqno.ReplicationTracker","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [k8s_kubelet][0]"]} -{"@timestamp":"2023-03-28T04:00:01.619Z", "log.level":"TRACE", "message":"checking jobs [2023-03-28T04:00:01.619921956Z]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"ticker-schedule-trigger-engine","log.logger":"org.elasticsearch.xpack.watcher.trigger.schedule.engine.TickerScheduleTriggerEngine","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:01.892Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#8]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [web_wok]"]} -{"@timestamp":"2023-03-28T04:00:01.893Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#1]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [fare_route]"]} -{"@timestamp":"2023-03-28T04:00:01.119Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#5]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [invoice_gen]"]} -{"@timestamp":"2023-03-28T04:00:01.893Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#7]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [ad_bidder]"]} -{"@timestamp":"2023-03-28T04:00:01.893Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#3]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [apphealth]"]} -{"@timestamp":"2023-03-28T04:00:01.893Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#4]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [eats_store]"]} -{"@timestamp":"2023-03-28T04:00:01.894Z", "log.level":"TRACE", "message":"background syncing retention leases [RetentionLeases{primaryTerm=1, version=1, leases={peer_recovery/iKPGCkp9RVOKXOj20uOt4g=RetentionLease{id='peer_recovery/iKPGCkp9RVOKXOj20uOt4g', retainingSequenceNumber=0, timestamp=1679973869034, source='peer recovery'}}}] after expiration check", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][management][T#2]","log.logger":"org.elasticsearch.index.shard.IndexShard","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [k8s_kubelet][0]"]} -{"@timestamp":"2023-03-28T04:00:01.895Z", "log.level":"TRACE", "message":"register 617 [transport] [retention_lease_background_sync] [retention_lease_background_sync shardId=[k8s_kubelet][0]]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][management][T#2]","log.logger":"org.elasticsearch.tasks.TaskManager","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:01.895Z", "log.level":"TRACE", "message":"[324][indices:admin/seq_no/retention_lease_background_sync[p]] sent to [{hostb9}{iKPGCkp9RVOKXOj20uOt4g}{cPRh9FqiSlGP9jClhm8Ugg}{hostb9}{127.0.0.1}{127.0.0.1:9300}{cdfhilmrstw}{ml.max_jvm_size=33285996544, ml.allocated_processors_double=16.0, xpack.installed=true, ml.machine_memory=135062048768, ml.allocated_processors=16}] (timeout: [null])", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][management][T#2]","log.logger":"org.elasticsearch.transport.TransportService.tracer","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:01.896Z", "log.level":"TRACE", "message":"[324][indices:admin/seq_no/retention_lease_background_sync[p]] received request", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][management][T#2]","log.logger":"org.elasticsearch.transport.TransportService.tracer","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:01.896Z", "log.level":"TRACE", "message":"scheduling retention_lease_sync every 30s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][management][T#2]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [k8s_kubelet]"]} -{"@timestamp":"2023-03-28T04:00:01.896Z", "log.level":"TRACE", "message":"register 618 [direct] [indices:admin/seq_no/retention_lease_background_sync[p]] [retention_lease_background_sync shardId=[k8s_kubelet][0]]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][management][T#1]","log.logger":"org.elasticsearch.tasks.TaskManager","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:01.896Z", "log.level":"TRACE", "message":"skipping persisting retention leases [RetentionLeases{primaryTerm=1, version=1, leases={peer_recovery/iKPGCkp9RVOKXOj20uOt4g=RetentionLease{id='peer_recovery/iKPGCkp9RVOKXOj20uOt4g', retainingSequenceNumber=0, timestamp=1679973869034, source='peer recovery'}}}], already persisted", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][management][T#1]","log.logger":"org.elasticsearch.index.seqno.ReplicationTracker","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [k8s_kubelet][0]"]} -{"@timestamp":"2023-03-28T04:00:01.897Z", "log.level":"TRACE", "message":"[[k8s_kubelet][0]] op [indices:admin/seq_no/retention_lease_background_sync] completed on primary for request [RetentionLeaseBackgroundSyncAction.Request{retentionLeases=RetentionLeases{primaryTerm=1, version=1, leases={peer_recovery/iKPGCkp9RVOKXOj20uOt4g=RetentionLease{id='peer_recovery/iKPGCkp9RVOKXOj20uOt4g', retainingSequenceNumber=0, timestamp=1679973869034, source='peer recovery'}}}, shardId=[k8s_kubelet][0], timeout=1m, index='k8s_kubelet', waitForActiveShards=0}]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][management][T#1]","log.logger":"org.elasticsearch.index.seqno.RetentionLeaseBackgroundSyncAction","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:01.897Z", "log.level":"TRACE", "message":"skipped updating local checkpoint of [Q4Xb11JATPaFZqmgTQHcVQ] from [-1] to [-1], current checkpoint is higher", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][management][T#1]","log.logger":"org.elasticsearch.index.seqno.ReplicationTracker","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [k8s_kubelet][0]"]} -{"@timestamp":"2023-03-28T04:00:01.897Z", "log.level":"TRACE", "message":"unregister task for id: 618", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][management][T#1]","log.logger":"org.elasticsearch.tasks.TaskManager","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:01.898Z", "log.level":"TRACE", "message":"[324][indices:admin/seq_no/retention_lease_background_sync[p]] sent response", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][management][T#1]","log.logger":"org.elasticsearch.transport.TransportService.tracer","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:01.898Z", "log.level":"TRACE", "message":"[324][indices:admin/seq_no/retention_lease_background_sync[p]] received response from [{hostb9}{iKPGCkp9RVOKXOj20uOt4g}{cPRh9FqiSlGP9jClhm8Ugg}{hostb9}{127.0.0.1}{127.0.0.1:9300}{cdfhilmrstw}{ml.max_jvm_size=33285996544, ml.allocated_processors_double=16.0, xpack.installed=true, ml.machine_memory=135062048768, ml.allocated_processors=16}]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][management][T#1]","log.logger":"org.elasticsearch.transport.TransportService.tracer","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:01.898Z", "log.level":"TRACE", "message":"unregister task for id: 617", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][management][T#1]","log.logger":"org.elasticsearch.tasks.TaskManager","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:02.039Z", "log.level":"TRACE", "message":"[apphealth][0] top warming [ElasticsearchDirectoryReader()]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][generic][T#3]","log.logger":"org.elasticsearch.index.IndexWarmer","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:02.040Z", "log.level":"TRACE", "message":"top warming took [43.9micros]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][generic][T#3]","log.logger":"org.elasticsearch.index.warmer.ShardIndexWarmerService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [apphealth][0]"]} -{"@timestamp":"2023-03-28T04:00:02.040Z", "log.level":"DEBUG", "message":"state: [RECOVERING]->[POST_RECOVERY], reason [post recovery from shard_store]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][generic][T#3]","log.logger":"org.elasticsearch.index.shard.IndexShard","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [apphealth][0]"]} -{"@timestamp":"2023-03-28T04:00:02.040Z", "log.level":"TRACE", "message":"recovery completed from [shard_store], took [13.4s]\n index : files [0] with total_size [0b], took[9.5s]\n : recovered_files [0] with total_size [0b]\n : reusing_files [0] with total_size [0b]\n verify_index : took [0s], check_index [0s]\n translog : number_of_operations [0], took [3.7s]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][generic][T#3]","log.logger":"org.elasticsearch.index.shard.IndexShard","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [apphealth][0]"]} -{"@timestamp":"2023-03-28T04:00:02.041Z", "log.level":"DEBUG", "message":"sending [internal:cluster/shard/started] to [iKPGCkp9RVOKXOj20uOt4g] for shard entry [StartedShardEntry{shardId [[apphealth][0]], allocationId [Abip50RxQTmavqi4NTwSqQ], primary term [1], message [after new shard recovery]}]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][generic][T#3]","log.logger":"org.elasticsearch.cluster.action.shard.ShardStateAction","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:02.041Z", "log.level":"TRACE", "message":"[325][internal:cluster/shard/started] sent to [{hostb9}{iKPGCkp9RVOKXOj20uOt4g}{cPRh9FqiSlGP9jClhm8Ugg}{hostb9}{127.0.0.1}{127.0.0.1:9300}{cdfhilmrstw}{ml.max_jvm_size=33285996544, ml.allocated_processors_double=16.0, xpack.installed=true, ml.machine_memory=135062048768, ml.allocated_processors=16}] (timeout: [null])", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][generic][T#3]","log.logger":"org.elasticsearch.transport.TransportService.tracer","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:02.041Z", "log.level":"TRACE", "message":"[325][internal:cluster/shard/started] received request", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][generic][T#3]","log.logger":"org.elasticsearch.transport.TransportService.tracer","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:02.041Z", "log.level":"TRACE", "message":"register 619 [direct] [internal:cluster/shard/started] []", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][generic][T#3]","log.logger":"org.elasticsearch.tasks.TaskManager","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:02.042Z", "log.level":"DEBUG", "message":"[apphealth][0] received shard started for [StartedShardEntry{shardId [[apphealth][0]], allocationId [Abip50RxQTmavqi4NTwSqQ], primary term [1], message [after new shard recovery]}]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][generic][T#3]","log.logger":"org.elasticsearch.cluster.action.shard.ShardStateAction","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:02.042Z", "log.level":"TRACE", "message":"will process [shard-started StartedShardEntry{shardId [[apphealth][0]], allocationId [Abip50RxQTmavqi4NTwSqQ], primary term [1], message [after new shard recovery]}[StartedShardUpdateTask{entry=StartedShardEntry{shardId [[apphealth][0]], allocationId [Abip50RxQTmavqi4NTwSqQ], primary term [1], message [after new shard recovery]}, listener=ChannelActionListener{org.elasticsearch.transport.TaskTransportChannel@1aeede97}{StartedShardEntry{shardId [[apphealth][0]], allocationId [Abip50RxQTmavqi4NTwSqQ], primary term [1], message [after new shard recovery]}}{internal:cluster/shard/started}}]]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.service.MasterService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:02.042Z", "log.level":"DEBUG", "message":"executing cluster state update for [shard-started StartedShardEntry{shardId [[apphealth][0]], allocationId [Abip50RxQTmavqi4NTwSqQ], primary term [1], message [after new shard recovery]}[StartedShardUpdateTask{entry=StartedShardEntry{shardId [[apphealth][0]], allocationId [Abip50RxQTmavqi4NTwSqQ], primary term [1], message [after new shard recovery]}, listener=ChannelActionListener{org.elasticsearch.transport.TaskTransportChannel@1aeede97}{StartedShardEntry{shardId [[apphealth][0]], allocationId [Abip50RxQTmavqi4NTwSqQ], primary term [1], message [after new shard recovery]}}{internal:cluster/shard/started}}]]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.service.MasterService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:02.043Z", "log.level":"DEBUG", "message":"[apphealth][0] starting shard [apphealth][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], recovery_source[new shard recovery], s[INITIALIZING], a[id=Abip50RxQTmavqi4NTwSqQ], unassigned_info[[reason=INDEX_CREATED], at[2023-03-28T03:59:38.780Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0], expected_shard_size[225] (shard started task: [StartedShardEntry{shardId [[apphealth][0]], allocationId [Abip50RxQTmavqi4NTwSqQ], primary term [1], message [after new shard recovery]}])", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.action.shard.ShardStateAction","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:02.043Z", "log.level":"TRACE", "message":"[apphealth][0] marked shard as started (routing: [apphealth][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], recovery_source[new shard recovery], s[INITIALIZING], a[id=Abip50RxQTmavqi4NTwSqQ], unassigned_info[[reason=INDEX_CREATED], at[2023-03-28T03:59:38.780Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0], expected_shard_size[225])", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.routing.allocation.AllocationService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:02.045Z", "log.level":"DEBUG", "message":"took [2ms] to compute cluster state update for [shard-started StartedShardEntry{shardId [[apphealth][0]], allocationId [Abip50RxQTmavqi4NTwSqQ], primary term [1], message [after new shard recovery]}[StartedShardUpdateTask{entry=StartedShardEntry{shardId [[apphealth][0]], allocationId [Abip50RxQTmavqi4NTwSqQ], primary term [1], message [after new shard recovery]}, listener=ChannelActionListener{org.elasticsearch.transport.TaskTransportChannel@1aeede97}{StartedShardEntry{shardId [[apphealth][0]], allocationId [Abip50RxQTmavqi4NTwSqQ], primary term [1], message [after new shard recovery]}}{internal:cluster/shard/started}}]]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.service.MasterService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:02.045Z", "log.level":"TRACE", "message":"register 620 [master] [publish_cluster_state_update] [publication of cluster state [154]]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.tasks.TaskManager","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:02.054Z", "log.level":"TRACE", "message":"cluster state updated, source [shard-started StartedShardEntry{shardId [[apphealth][0]], allocationId [Abip50RxQTmavqi4NTwSqQ], primary term [1], message [after new shard recovery]}[StartedShardUpdateTask{entry=StartedShardEntry{shardId [[apphealth][0]], allocationId [Abip50RxQTmavqi4NTwSqQ], primary term [1], message [after new shard recovery]}, listener=ChannelActionListener{org.elasticsearch.transport.TaskTransportChannel@1aeede97}{StartedShardEntry{shardId [[apphealth][0]], allocationId [Abip50RxQTmavqi4NTwSqQ], primary term [1], message [after new shard recovery]}}{internal:cluster/shard/started}}]]\ncluster uuid: bp-xkJD1S1iyBlj6I_JRHA [committed: true]\nversion: 154\nstate uuid: Xx549IAcSMCY30xGiugtFg\nfrom_diff: false\nmeta data version: 56\n coordination_metadata:\n term: 6\n last_committed_config: VotingConfiguration{iKPGCkp9RVOKXOj20uOt4g}\n last_accepted_config: VotingConfiguration{iKPGCkp9RVOKXOj20uOt4g}\n voting tombstones: []\n [k8s_kubelet/fQ9Iy7wlTKahsWLbs-oE_Q]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [Q4Xb11JATPaFZqmgTQHcVQ]\n [fares_intelligence/pSRJWxyXSqefhwLVhmT9nQ]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [-dD8ROsgS6qaYMoalciocA]\n [event_logs/DcomuHoGQzOjZq2bX67TFg]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [1uCl_LIDSBCWh6koVA0fjQ]\n [rider_product_cored/AdGiEBLBQbicsXDomsa3mw]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [GwRH_ZlFRtCV7m_n626z7g]\n [access_platform_core/l4kdkOEQRaKfWGLpTZURCQ]: v[6], mv[1], sv[2], av[1]\n 0: p_term [1], isa_ids [6ajNSTQZSr6tP3YtmRaxgw]\n [.geoip_databases/5k1GHwKvQBKersi12DFh4A]: v[6], mv[1], sv[1], av[1]\n 0: p_term [3], isa_ids [UdvCKUGqTEmYFzZlnSu_tA]\n [web_wok/R1IULAXSQ4SC0pLFCU5oJA]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [JXruVMhdRVGvP1kQCbUNSQ]\n [abacus_go/cfvmY-Y_Su63-qSkPODDzg]: v[6], mv[1], sv[2], av[1]\n 0: p_term [1], isa_ids [AMhgJMlSTPOoKSsL7s3OKA]\n [k8s_apiserver/qQasynZgQ6e6AmD5ZWqh3w]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [Aaa0F0jiStCkteGDYR8yRQ]\n [michelangelo_gateway/ZV2vZ8PmS0uysg-xzKO7QA]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [k5vDE54CSTSakhfgtGaTcA]\n [driverapp/cqbade6KQFqLEK-wPKlkBg]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [b5_Bhd3ITjOkl-xLF8H1jg]\n [apphealth/rcPrX1aWRmeED9W5PU6A4g]: v[4], mv[1], sv[1], av[1]\n 0: p_term [1], isa_ids [Abip50RxQTmavqi4NTwSqQ]\n [fares_management/F1lH-QvyTkS5hnbDFwE2Xg]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [sOxfKX3nReynYN5HN88cuQ]\n [finprod_rewards_eligibility/dJNRGPwrQS2bIPlpsXgUoQ]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [Hg4Dm9O8QEqXWKY7rH1X0A]\n [fare_route/a4fjivWhQRee8JW7ZbXiXw]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [ltSLFgu4Rry0_0vXq28DbA]\n [ad_bidder/tLHfw5r2QH-o7lG5tapKxw]: v[6], mv[1], sv[2], av[1]\n 0: p_term [1], isa_ids [YtEUwPgfQDGM0Cm_C4hXlw]\n [fulfillment_compatibled/tMgrGBWeR3irgR5PyN8Oig]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [D_3C_3kMSNKa-XJk7pyoGg]\n [eats_store/hF-kN4XFRy--z_gUhYLwdA]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [WRNwRQ6gTNGfSNQok4iSyQ]\n [invoice_gen/RPu5EzJVRfuqyI8MQ7iTjQ]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [onWbjhHuTD2C3AEayRJq1w]\nmetadata customs:\n licenses: LicensesMetadata{license={\"uid\":\"8e65ab16-3f3f-48cc-ab5f-e20097ceb81e\",\"type\":\"basic\",\"issue_date_in_millis\":1679968689560,\"max_nodes\":1000,\"max_resource_units\":null,\"issued_to\":\"elasticsearch\",\"issuer\":\"elasticsearch\",\"signature\":\"////+wAAAOBYLHp18OvjZrRTi0AmCZ6DOpkrd2cXJB/ffBB+AxaDsF+z7lkfdPbgYofMvUB7XbiD5j95mIfUcdlTJHVdk9NpPTFMKL5wC7BXn4CTS6kNox+Xi0+qSLpcgHzelOT63alJr1L5j/2pqffHnuMLG88Ye15eFhAb9H2A5MS3hX0HG9uTxyEidFCyu4gWV4qrZJDQKLpzC49MMnpP+ob2bNH/pA7dGVdWqEJpVNPe5/hAtGtCdVdUX6DvV0Q3dgwuejc0hRg/vl0YUtp45bQPBHDHECNKgxWmJCZrfstl/prRHA==\",\"start_date_in_millis\":-1}, trialVersion=null} component_template: {\"component_template\":{\"metrics-mappings\":{\"template\":{\"mappings\":{\"dynamic_templates\":[{\"long_metrics\":{\"mapping\":{\"index\":false,\"type\":\"long\"},\"match_mapping_type\":\"long\"}},{\"double_metrics\":{\"mapping\":{\"index\":false,\"type\":\"float\"},\"match_mapping_type\":\"double\"}}],\"properties\":{\"data_stream\":{\"properties\":{\"type\":{\"type\":\"constant_keyword\",\"value\":\"metrics\"}}}}}},\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default mappings for the metrics index template installed by x-pack\"}},\".deprecation-indexing-mappings\":{\"template\":{\"mappings\":{\"dynamic_templates\":[{\"strings_as_keyword\":{\"mapping\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"match_mapping_type\":\"string\"}}],\"date_detection\":false,\"properties\":{\"@timestamp\":{\"type\":\"date\"},\"ecs\":{\"properties\":{\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"data_stream\":{\"properties\":{\"namespace\":{\"type\":\"constant_keyword\"},\"type\":{\"type\":\"constant_keyword\",\"value\":\"logs\"},\"dataset\":{\"type\":\"constant_keyword\"}}},\"host\":{\"properties\":{\"ip\":{\"type\":\"ip\"}}},\"message\":{\"type\":\"text\"}}}},\"version\":1,\"_meta\":{\"managed\":true,\"description\":\"default mappings for Stack deprecation logs index template installed by x-pack\"}},\"synthetics-mappings\":{\"template\":{\"mappings\":{\"properties\":{\"data_stream\":{\"properties\":{\"type\":{\"type\":\"constant_keyword\",\"value\":\"synthetics\"}}}}}},\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default mappings for the synthetics index template installed by x-pack\"}},\"metrics-settings\":{\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\"metrics\"},\"codec\":\"best_compression\",\"query\":{\"default_field\":[\"message\"]}}}},\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default settings for the metrics index template installed by x-pack\"}},\"logs-settings\":{\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\"logs\"},\"codec\":\"best_compression\",\"query\":{\"default_field\":[\"message\"]}}}},\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default settings for the logs index template installed by x-pack\"}},\".deprecation-indexing-settings\":{\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\".deprecation-indexing-ilm-policy\"},\"codec\":\"best_compression\",\"hidden\":\"true\",\"auto_expand_replicas\":\"0-1\",\"query\":{\"default_field\":[\"message\"]}}}},\"version\":1,\"_meta\":{\"managed\":true,\"description\":\"default settings for Stack deprecation logs index template installed by x-pack\"}},\"synthetics-settings\":{\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\"synthetics\"},\"codec\":\"best_compression\"}}},\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default settings for the synthetics index template installed by x-pack\"}},\"logs-mappings\":{\"template\":{\"mappings\":{\"properties\":{\"data_stream\":{\"properties\":{\"type\":{\"type\":\"constant_keyword\",\"value\":\"logs\"}}}}}},\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default mappings for the logs index template installed by x-pack\"}},\"data-streams-mappings\":{\"template\":{\"mappings\":{\"dynamic_templates\":[{\"match_ip\":{\"mapping\":{\"type\":\"ip\"},\"match_mapping_type\":\"string\",\"match\":\"ip\"}},{\"match_message\":{\"mapping\":{\"type\":\"match_only_text\"},\"match_mapping_type\":\"string\",\"match\":\"message\"}},{\"strings_as_keyword\":{\"mapping\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"match_mapping_type\":\"string\"}}],\"date_detection\":false,\"properties\":{\"@timestamp\":{\"type\":\"date\"},\"ecs\":{\"properties\":{\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"data_stream\":{\"properties\":{\"namespace\":{\"type\":\"constant_keyword\"},\"dataset\":{\"type\":\"constant_keyword\"}}},\"host\":{\"type\":\"object\"}}}},\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"general mapping conventions for data streams\"}}}} persistent_tasks: {\"last_allocation_id\":6,\"tasks\":[{\"id\":\"health-node\",\"task\":{\"health-node\":{\"params\":{}}},\"allocation_id\":5,\"assignment\":{\"executor_node\":\"iKPGCkp9RVOKXOj20uOt4g\",\"explanation\":\"\"},\"allocation_id_on_last_status_update\":0},{\"id\":\"geoip-downloader\",\"task\":{\"geoip-downloader\":{\"params\":{},\"state\":{\"databases\":{\"GeoLite2-ASN.mmdb\":{\"last_update\":1679968651799,\"last_check\":1679968651799,\"first_chunk\":0,\"last_chunk\":4,\"md5\":\"33268a0d7cb3ba8d9d581bf1e5dc0980\"},\"GeoLite2-Country.mmdb\":{\"last_update\":1679968809323,\"last_check\":1679968809323,\"first_chunk\":0,\"last_chunk\":2,\"md5\":\"da6d05a7b044ea35910f20f756c057ad\"},\"GeoLite2-City.mmdb\":{\"last_update\":1679968693385,\"last_check\":1679968693385,\"first_chunk\":0,\"last_chunk\":33,\"md5\":\"2d91f3e42689a12c2f496199befdfccf\"}}}}},\"allocation_id\":6,\"assignment\":{\"executor_node\":\"iKPGCkp9RVOKXOj20uOt4g\",\"explanation\":\"\"},\"allocation_id_on_last_status_update\":0}]} index_lifecycle: {\n \"policies\" : {\n \"90-days-default\" : {\n \"policy\" : {\n \"phases\" : {\n \"warm\" : {\n \"min_age\" : \"2d\",\n \"actions\" : {\n \"shrink\" : {\n \"number_of_shards\" : 1\n },\n \"forcemerge\" : {\n \"max_num_segments\" : 1\n }\n }\n },\n \"cold\" : {\n \"min_age\" : \"30d\",\n \"actions\" : { }\n },\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"90d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"built-in ILM policy using the hot, warm, and cold phases with a retention of 90 days\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968678193,\n \"modified_date_string\" : \"2023-03-28T01:57:58.193Z\"\n },\n \"ilm-history-ilm-policy\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"90d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"default policy for the ILM history indices\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968685095,\n \"modified_date_string\" : \"2023-03-28T01:58:05.095Z\"\n },\n \"watch-history-ilm-policy-16\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"3d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"4d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"default policy for the watcher history indices\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968647327,\n \"modified_date_string\" : \"2023-03-28T01:57:27.327Z\"\n },\n \"30-days-default\" : {\n \"policy\" : {\n \"phases\" : {\n \"warm\" : {\n \"min_age\" : \"2d\",\n \"actions\" : {\n \"shrink\" : {\n \"number_of_shards\" : 1\n },\n \"forcemerge\" : {\n \"max_num_segments\" : 1\n }\n }\n },\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"30d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"built-in ILM policy using the hot and warm phases with a retention of 30 days\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968662496,\n \"modified_date_string\" : \"2023-03-28T01:57:42.496Z\"\n },\n \"synthetics\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"default policy for the synthetics index template installed by x-pack\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968682454,\n \"modified_date_string\" : \"2023-03-28T01:58:02.454Z\"\n },\n \".monitoring-8-ilm-policy\" : {\n \"policy\" : {\n \"phases\" : {\n \"warm\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"forcemerge\" : {\n \"max_num_segments\" : 1\n }\n }\n },\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"3d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"3d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"defaults\" : {\n \"delete_min_age\" : \"Using value of [3d] based on the monitoring plugin default\"\n },\n \"description\" : \"Index lifecycle policy generated for [monitoring-*-8] data streams\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968676492,\n \"modified_date_string\" : \"2023-03-28T01:57:56.492Z\"\n },\n \"slm-history-ilm-policy\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"90d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"default policy for the SLM history indices\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968644040,\n \"modified_date_string\" : \"2023-03-28T01:57:24.040Z\"\n },\n \".fleet-actions-results-ilm-policy\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_size\" : \"300gb\",\n \"max_age\" : \"30d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"90d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"default policy for fleet action results indices\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968686060,\n \"modified_date_string\" : \"2023-03-28T01:58:06.060Z\"\n },\n \"365-days-default\" : {\n \"policy\" : {\n \"phases\" : {\n \"warm\" : {\n \"min_age\" : \"2d\",\n \"actions\" : {\n \"shrink\" : {\n \"number_of_shards\" : 1\n },\n \"forcemerge\" : {\n \"max_num_segments\" : 1\n }\n }\n },\n \"cold\" : {\n \"min_age\" : \"30d\",\n \"actions\" : { }\n },\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"365d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"built-in ILM policy using the hot, warm, and cold phases with a retention of 365 days\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968675542,\n \"modified_date_string\" : \"2023-03-28T01:57:55.542Z\"\n },\n \"7-days-default\" : {\n \"policy\" : {\n \"phases\" : {\n \"warm\" : {\n \"min_age\" : \"2d\",\n \"actions\" : {\n \"shrink\" : {\n \"number_of_shards\" : 1\n },\n \"forcemerge\" : {\n \"max_num_segments\" : 1\n }\n }\n },\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"7d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"7d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"built-in ILM policy using the hot and warm phases with a retention of 7 days\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968672404,\n \"modified_date_string\" : \"2023-03-28T01:57:52.404Z\"\n },\n \".deprecation-indexing-ilm-policy\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"10gb\",\n \"max_age\" : \"30d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"30d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"ILM policy used for deprecation logs\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968670134,\n \"modified_date_string\" : \"2023-03-28T01:57:50.134Z\"\n },\n \"metrics\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"default policy for the metrics index template installed by x-pack\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968660707,\n \"modified_date_string\" : \"2023-03-28T01:57:40.707Z\"\n },\n \"ml-size-based-ilm-policy\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\"\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"default policy for machine learning state and stats indices\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968650499,\n \"modified_date_string\" : \"2023-03-28T01:57:30.499Z\"\n },\n \"180-days-default\" : {\n \"policy\" : {\n \"phases\" : {\n \"warm\" : {\n \"min_age\" : \"2d\",\n \"actions\" : {\n \"shrink\" : {\n \"number_of_shards\" : 1\n },\n \"forcemerge\" : {\n \"max_num_segments\" : 1\n }\n }\n },\n \"cold\" : {\n \"min_age\" : \"30d\",\n \"actions\" : { }\n },\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"180d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"built-in ILM policy using the hot, warm, and cold phases with a retention of 180 days\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968681386,\n \"modified_date_string\" : \"2023-03-28T01:58:01.386Z\"\n },\n \"logs\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"default policy for the logs index template installed by x-pack\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968679651,\n \"modified_date_string\" : \"2023-03-28T01:57:59.651Z\"\n }\n },\n \"operation_mode\" : \"RUNNING\"\n} index_template: {\"index_template\":{\".watch-history-16\":{\"index_patterns\":[\".watcher-history-16*\"],\"template\":{\"settings\":{\"index\":{\"format\":\"6\",\"lifecycle\":{\"name\":\"watch-history-ilm-policy-16\"},\"hidden\":\"true\",\"number_of_shards\":\"1\",\"auto_expand_replicas\":\"0-1\",\"number_of_replicas\":\"0\"}},\"mappings\":{\"_meta\":{\"watcher-history-version\":\"16\"},\"dynamic\":false,\"dynamic_templates\":[{\"disabled_payload_fields\":{\"match_pattern\":\"regex\",\"path_match\":\"result\\\\.(input(\\\\..+)*|(transform(\\\\..+)*)|(actions\\\\.transform(\\\\..+)*))\\\\.payload\",\"mapping\":{\"type\":\"object\",\"enabled\":false}}},{\"disabled_search_request_body_fields\":{\"match_pattern\":\"regex\",\"path_match\":\"result\\\\.(input(\\\\..+)*|(transform(\\\\..+)*)|(actions\\\\.transform(\\\\..+)*))\\\\.search\\\\.request\\\\.(body|template)\",\"mapping\":{\"type\":\"object\",\"enabled\":false}}},{\"disabled_exception_fields\":{\"match_pattern\":\"regex\",\"path_match\":\"result\\\\.(input(\\\\..+)*|(transform(\\\\..+)*)|(actions\\\\.transform(\\\\..+)*)|actions)\\\\.error\",\"mapping\":{\"type\":\"object\",\"enabled\":false}}},{\"disabled_jira_custom_fields\":{\"path_match\":\"result.actions.jira.fields.customfield_*\",\"mapping\":{\"type\":\"object\",\"enabled\":false}}}],\"properties\":{\"exception\":{\"type\":\"object\",\"enabled\":false},\"metadata\":{\"dynamic\":true,\"type\":\"object\"},\"trigger_event\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"schedule\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"scheduled_time\":{\"type\":\"date\"}}},\"triggered_time\":{\"type\":\"date\"},\"type\":{\"type\":\"keyword\"},\"manual\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"schedule\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"scheduled_time\":{\"type\":\"date\"}}}}}}},\"result\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"input\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"search\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"request\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"indices\":{\"type\":\"keyword\"},\"types\":{\"type\":\"keyword\"},\"search_type\":{\"type\":\"keyword\"}}}}},\"payload\":{\"type\":\"object\",\"enabled\":false},\"http\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"request\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"path\":{\"type\":\"keyword\"},\"host\":{\"type\":\"keyword\"}}}}},\"type\":{\"type\":\"keyword\"},\"status\":{\"type\":\"keyword\"}}},\"condition\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"compare\":{\"type\":\"object\",\"enabled\":false},\"array_compare\":{\"type\":\"object\",\"enabled\":false},\"type\":{\"type\":\"keyword\"},\"met\":{\"type\":\"boolean\"},\"script\":{\"type\":\"object\",\"enabled\":false},\"status\":{\"type\":\"keyword\"}}},\"transform\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"search\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"request\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"indices\":{\"type\":\"keyword\"},\"types\":{\"type\":\"keyword\"}}}}},\"type\":{\"type\":\"keyword\"}}},\"execution_duration\":{\"type\":\"long\"},\"actions\":{\"include_in_parent\":true,\"dynamic\":true,\"type\":\"nested\",\"properties\":{\"reason\":{\"type\":\"keyword\"},\"foreach\":{\"type\":\"object\",\"enabled\":false},\"webhook\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"request\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"path\":{\"type\":\"keyword\"},\"host\":{\"type\":\"keyword\"}}}}},\"number_of_actions_executed\":{\"type\":\"integer\"},\"slack\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"sent_messages\":{\"include_in_parent\":true,\"dynamic\":true,\"type\":\"nested\",\"properties\":{\"reason\":{\"type\":\"text\"},\"request\":{\"type\":\"object\",\"enabled\":false},\"response\":{\"type\":\"object\",\"enabled\":false},\"to\":{\"type\":\"keyword\"},\"message\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"attachments\":{\"include_in_parent\":true,\"dynamic\":true,\"type\":\"nested\",\"properties\":{\"color\":{\"type\":\"keyword\"},\"fields\":{\"properties\":{\"value\":{\"type\":\"text\"}}}}},\"icon\":{\"type\":\"keyword\"},\"from\":{\"type\":\"text\"},\"text\":{\"type\":\"text\"}}},\"status\":{\"type\":\"keyword\"}}},\"account\":{\"type\":\"keyword\"}}},\"index\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"response\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"index\":{\"type\":\"keyword\"},\"id\":{\"type\":\"keyword\"},\"type\":{\"type\":\"keyword\"}}}}},\"pagerduty\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"sent_event\":{\"include_in_parent\":true,\"dynamic\":true,\"type\":\"nested\",\"properties\":{\"reason\":{\"type\":\"text\"},\"request\":{\"type\":\"object\",\"enabled\":false},\"response\":{\"type\":\"object\",\"enabled\":false},\"event\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"client_url\":{\"type\":\"keyword\"},\"context\":{\"include_in_parent\":true,\"dynamic\":true,\"type\":\"nested\",\"properties\":{\"src\":{\"type\":\"keyword\"},\"alt\":{\"type\":\"text\"},\"href\":{\"type\":\"keyword\"},\"type\":{\"type\":\"keyword\"}}},\"client\":{\"type\":\"text\"},\"description\":{\"type\":\"text\"},\"attach_payload\":{\"type\":\"boolean\"},\"incident_key\":{\"type\":\"keyword\"},\"type\":{\"type\":\"keyword\"},\"account\":{\"type\":\"keyword\"}}}}},\"account\":{\"type\":\"keyword\"}}},\"id\":{\"type\":\"keyword\"},\"type\":{\"type\":\"keyword\"},\"email\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"message\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"cc\":{\"type\":\"keyword\"},\"bcc\":{\"type\":\"keyword\"},\"reply_to\":{\"type\":\"keyword\"},\"from\":{\"type\":\"keyword\"},\"id\":{\"type\":\"keyword\"},\"to\":{\"type\":\"keyword\"}}}}},\"status\":{\"type\":\"keyword\"},\"jira\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"result\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"self\":{\"type\":\"keyword\"},\"id\":{\"type\":\"keyword\"},\"key\":{\"type\":\"keyword\"}}},\"reason\":{\"type\":\"text\"},\"request\":{\"type\":\"object\",\"enabled\":false},\"response\":{\"type\":\"object\",\"enabled\":false},\"fields\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"summary\":{\"type\":\"text\"},\"issuetype\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"name\":{\"type\":\"keyword\"},\"id\":{\"type\":\"keyword\"}}},\"description\":{\"type\":\"text\"},\"project\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"id\":{\"type\":\"keyword\"},\"key\":{\"type\":\"keyword\"}}},\"labels\":{\"type\":\"text\"}}},\"account\":{\"type\":\"keyword\"}}}}},\"execution_time\":{\"type\":\"date\"}}},\"node\":{\"type\":\"keyword\"},\"input\":{\"type\":\"object\",\"enabled\":false},\"condition\":{\"type\":\"object\",\"enabled\":false},\"watch_id\":{\"type\":\"keyword\"},\"messages\":{\"type\":\"text\"},\"vars\":{\"type\":\"object\",\"enabled\":false},\"state\":{\"type\":\"keyword\"},\"user\":{\"type\":\"text\"},\"status\":{\"dynamic\":true,\"type\":\"object\",\"enabled\":false}}}},\"composed_of\":[],\"priority\":2147483647,\"version\":16,\"_meta\":{\"managed\":true,\"description\":\"index template for watcher history indices\"},\"data_stream\":{\"hidden\":true,\"allow_custom_routing\":false},\"allow_auto_create\":true},\".monitoring-beats-mb\":{\"index_patterns\":[\".monitoring-beats-8-*\"],\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\".monitoring-8-ilm-policy\"},\"mapping\":{\"total_fields\":{\"limit\":\"2000\"}}}},\"mappings\":{\"dynamic\":false,\"properties\":{\"@timestamp\":{\"type\":\"date\"},\"cluster_uuid\":{\"path\":\"beat.elasticsearch.cluster.id\",\"type\":\"alias\"},\"service\":{\"properties\":{\"node\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"environment\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"hostname\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"origin\":{\"properties\":{\"node\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"environment\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"target\":{\"properties\":{\"node\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"environment\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"beat\":{\"properties\":{\"elasticsearch\":{\"properties\":{\"cluster\":{\"properties\":{\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"stats\":{\"properties\":{\"handles\":{\"properties\":{\"limit\":{\"properties\":{\"hard\":{\"type\":\"long\"},\"soft\":{\"type\":\"long\"}}},\"open\":{\"type\":\"long\"}}},\"apm_server\":{\"properties\":{\"server\":{\"properties\":{\"request\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"response\":{\"properties\":{\"valid\":{\"properties\":{\"count\":{\"type\":\"long\"},\"accepted\":{\"type\":\"long\"},\"ok\":{\"type\":\"long\"}}},\"count\":{\"type\":\"long\"},\"errors\":{\"properties\":{\"ratelimit\":{\"type\":\"long\"},\"internal\":{\"type\":\"long\"},\"toolarge\":{\"type\":\"long\"},\"method\":{\"type\":\"long\"},\"unauthorized\":{\"type\":\"long\"},\"forbidden\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"},\"closed\":{\"type\":\"long\"},\"decode\":{\"type\":\"long\"},\"queue\":{\"type\":\"long\"},\"concurrency\":{\"type\":\"long\"},\"validate\":{\"type\":\"long\"}}}}},\"concurrent\":{\"properties\":{\"wait\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}}}},\"acm\":{\"properties\":{\"request\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"response\":{\"properties\":{\"valid\":{\"properties\":{\"notmodified\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"},\"accepted\":{\"type\":\"long\"},\"ok\":{\"type\":\"long\"}}},\"request\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"count\":{\"type\":\"long\"},\"unset\":{\"type\":\"long\"},\"errors\":{\"properties\":{\"ratelimit\":{\"type\":\"long\"},\"internal\":{\"type\":\"long\"},\"toolarge\":{\"type\":\"long\"},\"method\":{\"type\":\"long\"},\"unavailable\":{\"type\":\"long\"},\"forbidden\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"},\"decode\":{\"type\":\"long\"},\"unauthorized\":{\"type\":\"long\"},\"notfound\":{\"type\":\"long\"},\"closed\":{\"type\":\"long\"},\"invalidquery\":{\"type\":\"long\"},\"queue\":{\"type\":\"long\"},\"validate\":{\"type\":\"long\"}}}}}}},\"decoder\":{\"properties\":{\"deflate\":{\"properties\":{\"content-length\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"reader\":{\"properties\":{\"size\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"gzip\":{\"properties\":{\"content-length\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"uncompressed\":{\"properties\":{\"content-length\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"missing-content-length\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"processor\":{\"properties\":{\"metric\":{\"properties\":{\"transformations\":{\"type\":\"long\"},\"decoding\":{\"properties\":{\"count\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}},\"validation\":{\"properties\":{\"count\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}}}},\"sourcemap\":{\"properties\":{\"counter\":{\"type\":\"long\"},\"decoding\":{\"properties\":{\"count\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}},\"validation\":{\"properties\":{\"count\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}}}},\"error\":{\"properties\":{\"stacktraces\":{\"type\":\"long\"},\"spans\":{\"type\":\"long\"},\"frames\":{\"type\":\"long\"},\"transformations\":{\"type\":\"long\"},\"decoding\":{\"properties\":{\"count\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}},\"validation\":{\"properties\":{\"count\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}}}},\"transaction\":{\"properties\":{\"stacktraces\":{\"type\":\"long\"},\"spans\":{\"type\":\"long\"},\"frames\":{\"type\":\"long\"},\"transformations\":{\"type\":\"long\"},\"transactions\":{\"type\":\"long\"},\"decoding\":{\"properties\":{\"count\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}},\"validation\":{\"properties\":{\"count\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}}}},\"span\":{\"properties\":{\"transformations\":{\"type\":\"long\"}}}}}}},\"system\":{\"properties\":{\"load\":{\"properties\":{\"1\":{\"type\":\"double\"},\"15\":{\"type\":\"double\"},\"5\":{\"type\":\"double\"},\"norm\":{\"properties\":{\"1\":{\"type\":\"double\"},\"15\":{\"type\":\"double\"},\"5\":{\"type\":\"double\"}}}}},\"cpu\":{\"properties\":{\"cores\":{\"type\":\"long\"}}}}},\"beat\":{\"properties\":{\"host\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"cpu\":{\"properties\":{\"total\":{\"properties\":{\"ticks\":{\"type\":\"long\"},\"time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"value\":{\"type\":\"long\"}}},\"system\":{\"properties\":{\"ticks\":{\"type\":\"long\"},\"time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"user\":{\"properties\":{\"ticks\":{\"type\":\"long\"},\"time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}}}},\"runtime\":{\"properties\":{\"goroutines\":{\"type\":\"long\"}}},\"memstats\":{\"properties\":{\"memory\":{\"properties\":{\"total\":{\"type\":\"long\"},\"alloc\":{\"type\":\"long\"}}},\"rss\":{\"type\":\"long\"},\"gc_next\":{\"type\":\"long\"}}},\"libbeat\":{\"properties\":{\"output\":{\"properties\":{\"read\":{\"properties\":{\"bytes\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"write\":{\"properties\":{\"bytes\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}},\"events\":{\"properties\":{\"batches\":{\"type\":\"long\"},\"duplicates\":{\"type\":\"long\"},\"total\":{\"type\":\"long\"},\"dropped\":{\"type\":\"long\"},\"toomany\":{\"type\":\"long\"},\"active\":{\"type\":\"long\"},\"failed\":{\"type\":\"long\"},\"acked\":{\"type\":\"long\"}}}}},\"pipeline\":{\"properties\":{\"clients\":{\"type\":\"long\"},\"events\":{\"properties\":{\"total\":{\"type\":\"long\"},\"filtered\":{\"type\":\"long\"},\"dropped\":{\"type\":\"long\"},\"active\":{\"type\":\"long\"},\"failed\":{\"type\":\"long\"},\"published\":{\"type\":\"long\"},\"retry\":{\"type\":\"long\"}}},\"queue\":{\"properties\":{\"acked\":{\"type\":\"long\"}}}}},\"config\":{\"properties\":{\"running\":{\"type\":\"long\"},\"stops\":{\"type\":\"long\"},\"starts\":{\"type\":\"long\"},\"reloads\":{\"type\":\"long\"}}}}},\"cgroup\":{\"properties\":{\"memory\":{\"properties\":{\"mem\":{\"properties\":{\"usage\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"limit\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"cpu\":{\"properties\":{\"cfs\":{\"properties\":{\"period\":{\"properties\":{\"us\":{\"type\":\"long\"}}},\"quota\":{\"properties\":{\"us\":{\"type\":\"long\"}}}}},\"stats\":{\"properties\":{\"periods\":{\"type\":\"long\"},\"throttled\":{\"properties\":{\"ns\":{\"type\":\"long\"},\"periods\":{\"type\":\"long\"}}}}},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"cpuacct\":{\"properties\":{\"total\":{\"properties\":{\"ns\":{\"type\":\"long\"}}},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"info\":{\"properties\":{\"host\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"uptime\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"uptime\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"properties\":{\"output\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"cluster\":{\"properties\":{\"uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"input\":{\"properties\":{\"names\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"count\":{\"type\":\"long\"}}},\"management\":{\"properties\":{\"enabled\":{\"type\":\"boolean\"}}},\"service\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"module\":{\"properties\":{\"names\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"count\":{\"type\":\"long\"}}},\"beat\":{\"properties\":{\"host\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"host\":{\"properties\":{\"os\":{\"properties\":{\"kernel\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"platform\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"containerized\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"queue\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"host\":{\"properties\":{\"hostname\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"architecture\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"metricset\":{\"properties\":{\"period\":{\"type\":\"long\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"event\":{\"properties\":{\"reason\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"code\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"timezone\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"agent_id_status\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"duration\":{\"type\":\"long\"},\"reference\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ingested\":{\"type\":\"date\"},\"provider\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"action\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"end\":{\"type\":\"date\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"outcome\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"severity\":{\"type\":\"long\"},\"original\":{\"ignore_above\":1024,\"index\":false,\"type\":\"keyword\",\"doc_values\":false},\"risk_score\":{\"type\":\"float\"},\"created\":{\"type\":\"date\"},\"kind\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"module\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"start\":{\"type\":\"date\"},\"url\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"sequence\":{\"type\":\"long\"},\"risk_score_norm\":{\"type\":\"float\"},\"category\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"dataset\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"hash\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"beats_stats\":{\"properties\":{\"beat\":{\"properties\":{\"host\":{\"path\":\"beat.stats.beat.host\",\"type\":\"alias\"},\"name\":{\"path\":\"beat.stats.beat.name\",\"type\":\"alias\"},\"type\":{\"path\":\"beat.stats.beat.type\",\"type\":\"alias\"},\"uuid\":{\"path\":\"beat.stats.beat.uuid\",\"type\":\"alias\"},\"version\":{\"path\":\"beat.stats.beat.version\",\"type\":\"alias\"}}},\"metrics\":{\"properties\":{\"system\":{\"properties\":{\"load\":{\"properties\":{\"1\":{\"path\":\"beat.stats.system.load.1\",\"type\":\"alias\"},\"15\":{\"path\":\"beat.stats.system.load.15\",\"type\":\"alias\"},\"5\":{\"path\":\"beat.stats.system.load.5\",\"type\":\"alias\"},\"norm\":{\"properties\":{\"1\":{\"path\":\"beat.stats.system.load.norm.1\",\"type\":\"alias\"},\"15\":{\"path\":\"beat.stats.system.load.norm.15\",\"type\":\"alias\"},\"5\":{\"path\":\"beat.stats.system.load.norm.5\",\"type\":\"alias\"}}}}},\"cpu\":{\"properties\":{\"cores\":{\"path\":\"beat.stats.system.cpu.cores\",\"type\":\"alias\"}}}}},\"beat\":{\"properties\":{\"handles\":{\"properties\":{\"limit\":{\"properties\":{\"hard\":{\"path\":\"beat.stats.handles.limit.hard\",\"type\":\"alias\"},\"soft\":{\"path\":\"beat.stats.handles.limit.soft\",\"type\":\"alias\"}}},\"open\":{\"path\":\"beat.stats.handles.open\",\"type\":\"alias\"}}},\"cpu\":{\"properties\":{\"total\":{\"properties\":{\"ticks\":{\"path\":\"beat.stats.cpu.total.ticks\",\"type\":\"alias\"},\"time\":{\"properties\":{\"ms\":{\"path\":\"beat.stats.cpu.total.time.ms\",\"type\":\"alias\"}}},\"value\":{\"path\":\"beat.stats.cpu.total.value\",\"type\":\"alias\"}}},\"system\":{\"properties\":{\"ticks\":{\"path\":\"beat.stats.cpu.system.ticks\",\"type\":\"alias\"},\"time\":{\"properties\":{\"ms\":{\"path\":\"beat.stats.cpu.system.time.ms\",\"type\":\"alias\"}}}}},\"user\":{\"properties\":{\"ticks\":{\"path\":\"beat.stats.cpu.user.ticks\",\"type\":\"alias\"},\"time\":{\"properties\":{\"ms\":{\"path\":\"beat.stats.cpu.user.time.ms\",\"type\":\"alias\"}}}}}}},\"memstats\":{\"properties\":{\"rss\":{\"path\":\"beat.stats.memstats.rss\",\"type\":\"alias\"},\"gc_next\":{\"path\":\"beat.stats.memstats.gc_next\",\"type\":\"alias\"},\"memory_alloc\":{\"path\":\"beat.stats.memstats.memory.alloc\",\"type\":\"alias\"},\"memory_total\":{\"path\":\"beat.stats.memstats.memory.total\",\"type\":\"alias\"}}},\"cgroup\":{\"properties\":{\"memory\":{\"properties\":{\"id\":{\"path\":\"beat.stats.cgroup.memory.id\",\"type\":\"alias\"}}},\"mem\":{\"properties\":{\"usage\":{\"properties\":{\"bytes\":{\"path\":\"beat.stats.cgroup.memory.mem.usage.bytes\",\"type\":\"alias\"}}},\"limit\":{\"properties\":{\"bytes\":{\"path\":\"beat.stats.cgroup.memory.mem.limit.bytes\",\"type\":\"alias\"}}}}},\"cpu\":{\"properties\":{\"cfs\":{\"properties\":{\"period\":{\"properties\":{\"us\":{\"path\":\"beat.stats.cgroup.cpu.cfs.period.us\",\"type\":\"alias\"}}},\"quota\":{\"properties\":{\"us\":{\"path\":\"beat.stats.cgroup.cpu.cfs.quota.us\",\"type\":\"alias\"}}}}},\"stats\":{\"properties\":{\"periods\":{\"path\":\"beat.stats.cgroup.cpu.stats.periods\",\"type\":\"alias\"},\"throttled\":{\"properties\":{\"ns\":{\"path\":\"beat.stats.cgroup.cpu.stats.throttled.ns\",\"type\":\"alias\"},\"periods\":{\"path\":\"beat.stats.cgroup.cpu.stats.throttled.periods\",\"type\":\"alias\"}}}}},\"id\":{\"path\":\"beat.stats.cgroup.cpu.id\",\"type\":\"alias\"}}},\"cpuacct\":{\"properties\":{\"total\":{\"properties\":{\"ns\":{\"path\":\"beat.stats.cgroup.cpuacct.total.ns\",\"type\":\"alias\"}}},\"id\":{\"path\":\"beat.stats.cgroup.cpuacct.id\",\"type\":\"alias\"}}}}},\"info\":{\"properties\":{\"ephemeral_id\":{\"path\":\"beat.stats.info.ephemeral_id\",\"type\":\"alias\"},\"uptime\":{\"properties\":{\"ms\":{\"path\":\"beat.stats.info.uptime.ms\",\"type\":\"alias\"}}}}}}},\"libbeat\":{\"properties\":{\"output\":{\"properties\":{\"read\":{\"properties\":{\"bytes\":{\"path\":\"beat.stats.libbeat.output.read.bytes\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.libbeat.output.read.errors\",\"type\":\"alias\"}}},\"type\":{\"path\":\"beat.stats.libbeat.output.type\",\"type\":\"alias\"},\"write\":{\"properties\":{\"bytes\":{\"path\":\"beat.stats.libbeat.output.write.bytes\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.libbeat.output.write.errors\",\"type\":\"alias\"}}},\"events\":{\"properties\":{\"duplicated\":{\"path\":\"beat.stats.libbeat.output.events.duplicates\",\"type\":\"alias\"},\"batches\":{\"path\":\"beat.stats.libbeat.output.events.batches\",\"type\":\"alias\"},\"total\":{\"path\":\"beat.stats.libbeat.output.events.total\",\"type\":\"alias\"},\"dropped\":{\"path\":\"beat.stats.libbeat.output.events.dropped\",\"type\":\"alias\"},\"toomany\":{\"path\":\"beat.stats.libbeat.output.events.toomany\",\"type\":\"alias\"},\"active\":{\"path\":\"beat.stats.libbeat.output.events.active\",\"type\":\"alias\"},\"failed\":{\"path\":\"beat.stats.libbeat.output.events.failed\",\"type\":\"alias\"},\"acked\":{\"path\":\"beat.stats.libbeat.output.events.acked\",\"type\":\"alias\"}}}}},\"pipeline\":{\"properties\":{\"clients\":{\"path\":\"beat.stats.libbeat.pipeline.clients\",\"type\":\"alias\"},\"events\":{\"properties\":{\"total\":{\"path\":\"beat.stats.libbeat.pipeline.events.total\",\"type\":\"alias\"},\"filtered\":{\"path\":\"beat.stats.libbeat.pipeline.events.filtered\",\"type\":\"alias\"},\"dropped\":{\"path\":\"beat.stats.libbeat.pipeline.events.dropped\",\"type\":\"alias\"},\"active\":{\"path\":\"beat.stats.libbeat.pipeline.events.active\",\"type\":\"alias\"},\"failed\":{\"path\":\"beat.stats.libbeat.pipeline.events.failed\",\"type\":\"alias\"},\"published\":{\"path\":\"beat.stats.libbeat.pipeline.events.published\",\"type\":\"alias\"},\"retry\":{\"path\":\"beat.stats.libbeat.pipeline.events.retry\",\"type\":\"alias\"}}},\"queue\":{\"properties\":{\"acked\":{\"path\":\"beat.stats.libbeat.pipeline.queue.acked\",\"type\":\"alias\"}}}}},\"config\":{\"properties\":{\"module\":{\"properties\":{\"running\":{\"path\":\"beat.stats.libbeat.config.running\",\"type\":\"alias\"},\"stops\":{\"path\":\"beat.stats.libbeat.config.stops\",\"type\":\"alias\"},\"starts\":{\"path\":\"beat.stats.libbeat.config.starts\",\"type\":\"alias\"}}},\"reloads\":{\"path\":\"beat.stats.libbeat.config.reloads\",\"type\":\"alias\"}}}}},\"apm-server\":{\"properties\":{\"server\":{\"properties\":{\"request\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.server.request.count\",\"type\":\"alias\"}}},\"response\":{\"properties\":{\"valid\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.server.response.valid.count\",\"type\":\"alias\"},\"accepted\":{\"path\":\"beat.stats.apm_server.server.response.valid.accepted\",\"type\":\"alias\"},\"ok\":{\"path\":\"beat.stats.apm_server.server.response.valid.ok\",\"type\":\"alias\"}}},\"count\":{\"path\":\"beat.stats.apm_server.server.response.count\",\"type\":\"alias\"},\"errors\":{\"properties\":{\"ratelimit\":{\"path\":\"beat.stats.apm_server.server.response.errors.ratelimit\",\"type\":\"alias\"},\"internal\":{\"path\":\"beat.stats.apm_server.server.response.errors.internal\",\"type\":\"alias\"},\"toolarge\":{\"path\":\"beat.stats.apm_server.server.response.errors.toolarge\",\"type\":\"alias\"},\"method\":{\"path\":\"beat.stats.apm_server.server.response.errors.method\",\"type\":\"alias\"},\"unauthorized\":{\"path\":\"beat.stats.apm_server.server.response.errors.unauthorized\",\"type\":\"alias\"},\"forbidden\":{\"path\":\"beat.stats.apm_server.server.response.errors.forbidden\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.stats.apm_server.server.response.errors.count\",\"type\":\"alias\"},\"closed\":{\"path\":\"beat.stats.apm_server.server.response.errors.closed\",\"type\":\"alias\"},\"decode\":{\"path\":\"beat.stats.apm_server.server.response.errors.decode\",\"type\":\"alias\"},\"queue\":{\"path\":\"beat.stats.apm_server.server.response.errors.queue\",\"type\":\"alias\"},\"concurrency\":{\"path\":\"beat.stats.apm_server.server.response.errors.concurrency\",\"type\":\"alias\"},\"validate\":{\"path\":\"beat.stats.apm_server.server.response.errors.validate\",\"type\":\"alias\"}}}}},\"concurrent\":{\"properties\":{\"wait\":{\"properties\":{\"ms\":{\"path\":\"beat.stats.apm_server.server.concurrent.wait.ms\",\"type\":\"alias\"}}}}}}},\"sampling\":{\"properties\":{\"tail\":{\"properties\":{\"dynamic_service_groups\":{\"type\":\"long\"},\"storage\":{\"properties\":{\"lsm_size\":{\"type\":\"long\"},\"value_log_size\":{\"type\":\"long\"}}},\"events\":{\"properties\":{\"processed\":{\"type\":\"long\"},\"head_unsampled\":{\"type\":\"long\"},\"stored\":{\"type\":\"long\"},\"dropped\":{\"type\":\"long\"},\"sampled\":{\"type\":\"long\"}}}}},\"transactions_dropped\":{\"type\":\"long\"}}},\"acm\":{\"properties\":{\"request\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.acm.request.count\",\"type\":\"alias\"}}},\"response\":{\"properties\":{\"valid\":{\"properties\":{\"notmodified\":{\"path\":\"beat.stats.apm_server.acm.response.valid.notmodified\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.stats.apm_server.acm.response.valid.count\",\"type\":\"alias\"},\"accepted\":{\"path\":\"beat.stats.apm_server.acm.response.valid.accepted\",\"type\":\"alias\"},\"ok\":{\"path\":\"beat.stats.apm_server.acm.response.valid.ok\",\"type\":\"alias\"}}},\"request\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.acm.response.request.count\",\"type\":\"alias\"}}},\"count\":{\"path\":\"beat.stats.apm_server.acm.response.count\",\"type\":\"alias\"},\"unset\":{\"path\":\"beat.stats.apm_server.acm.response.unset\",\"type\":\"alias\"},\"errors\":{\"properties\":{\"ratelimit\":{\"path\":\"beat.stats.apm_server.acm.response.errors.ratelimit\",\"type\":\"alias\"},\"internal\":{\"path\":\"beat.stats.apm_server.acm.response.errors.internal\",\"type\":\"alias\"},\"toolarge\":{\"path\":\"beat.stats.apm_server.acm.response.errors.toolarge\",\"type\":\"alias\"},\"method\":{\"path\":\"beat.stats.apm_server.acm.response.errors.method\",\"type\":\"alias\"},\"unavailable\":{\"path\":\"beat.stats.apm_server.acm.response.errors.unavailable\",\"type\":\"alias\"},\"forbidden\":{\"path\":\"beat.stats.apm_server.acm.response.errors.forbidden\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.stats.apm_server.acm.response.errors.count\",\"type\":\"alias\"},\"decode\":{\"path\":\"beat.stats.apm_server.acm.response.errors.decode\",\"type\":\"alias\"},\"unauthorized\":{\"path\":\"beat.stats.apm_server.acm.response.errors.unauthorized\",\"type\":\"alias\"},\"notfound\":{\"path\":\"beat.stats.apm_server.acm.response.errors.notfound\",\"type\":\"alias\"},\"closed\":{\"path\":\"beat.stats.apm_server.acm.response.errors.closed\",\"type\":\"alias\"},\"invalidquery\":{\"path\":\"beat.stats.apm_server.acm.response.errors.invalidquery\",\"type\":\"alias\"},\"queue\":{\"path\":\"beat.stats.apm_server.acm.response.errors.queue\",\"type\":\"alias\"},\"validate\":{\"path\":\"beat.stats.apm_server.acm.response.errors.validate\",\"type\":\"alias\"}}}}}}},\"decoder\":{\"properties\":{\"deflate\":{\"properties\":{\"content-length\":{\"path\":\"beat.stats.apm_server.decoder.deflate.content-length\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.stats.apm_server.decoder.deflate.count\",\"type\":\"alias\"}}},\"reader\":{\"properties\":{\"size\":{\"path\":\"beat.stats.apm_server.decoder.reader.size\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.stats.apm_server.decoder.reader.count\",\"type\":\"alias\"}}},\"gzip\":{\"properties\":{\"content-length\":{\"path\":\"beat.stats.apm_server.decoder.gzip.content-length\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.stats.apm_server.decoder.gzip.count\",\"type\":\"alias\"}}},\"uncompressed\":{\"properties\":{\"content-length\":{\"path\":\"beat.stats.apm_server.decoder.uncompressed.content-length\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.stats.apm_server.decoder.uncompressed.count\",\"type\":\"alias\"}}},\"missing-content-length\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.decoder.missing-content-length.count\",\"type\":\"alias\"}}}}},\"processor\":{\"properties\":{\"metric\":{\"properties\":{\"transformations\":{\"path\":\"beat.stats.apm_server.processor.metric.transformations\",\"type\":\"alias\"},\"decoding\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.processor.metric.decoding.count\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.apm_server.processor.metric.decoding.errors\",\"type\":\"alias\"}}},\"validation\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.processor.metric.validation.count\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.apm_server.processor.metric.validation.errors\",\"type\":\"alias\"}}}}},\"sourcemap\":{\"properties\":{\"counter\":{\"path\":\"beat.stats.apm_server.processor.sourcemap.counter\",\"type\":\"alias\"},\"decoding\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.processor.sourcemap.decoding.count\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.apm_server.processor.sourcemap.decoding.errors\",\"type\":\"alias\"}}},\"validation\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.processor.sourcemap.validation.count\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.apm_server.processor.sourcemap.validation.errors\",\"type\":\"alias\"}}}}},\"error\":{\"properties\":{\"stacktraces\":{\"path\":\"beat.stats.apm_server.processor.error.stacktraces\",\"type\":\"alias\"},\"spans\":{\"path\":\"beat.stats.apm_server.processor.error.spans\",\"type\":\"alias\"},\"frames\":{\"path\":\"beat.stats.apm_server.processor.error.frames\",\"type\":\"alias\"},\"transformations\":{\"path\":\"beat.stats.apm_server.processor.error.transformations\",\"type\":\"alias\"},\"decoding\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.processor.error.decoding.count\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.apm_server.processor.error.decoding.errors\",\"type\":\"alias\"}}},\"validation\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.processor.error.validation.count\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.apm_server.processor.error.validation.errors\",\"type\":\"alias\"}}}}},\"transaction\":{\"properties\":{\"stacktraces\":{\"path\":\"beat.stats.apm_server.processor.transaction.stacktraces\",\"type\":\"alias\"},\"spans\":{\"path\":\"beat.stats.apm_server.processor.transaction.spans\",\"type\":\"alias\"},\"frames\":{\"path\":\"beat.stats.apm_server.processor.transaction.frames\",\"type\":\"alias\"},\"transformations\":{\"path\":\"beat.stats.apm_server.processor.transaction.transformations\",\"type\":\"alias\"},\"transactions\":{\"path\":\"beat.stats.apm_server.processor.transaction.transactions\",\"type\":\"alias\"},\"decoding\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.processor.transaction.decoding.count\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.apm_server.processor.transaction.decoding.errors\",\"type\":\"alias\"}}},\"validation\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.processor.transaction.validation.count\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.apm_server.processor.transaction.validation.errors\",\"type\":\"alias\"}}}}},\"span\":{\"properties\":{\"transformations\":{\"path\":\"beat.stats.apm_server.processor.span.transformations\",\"type\":\"alias\"}}}}}}}}},\"timestamp\":{\"path\":\"@timestamp\",\"type\":\"alias\"}}},\"beats_state\":{\"properties\":{\"beat\":{\"properties\":{\"host\":{\"path\":\"beat.state.beat.host\",\"type\":\"alias\"},\"name\":{\"path\":\"beat.state.beat.name\",\"type\":\"alias\"},\"type\":{\"path\":\"beat.state.beat.type\",\"type\":\"alias\"},\"uuid\":{\"path\":\"beat.state.beat.uuid\",\"type\":\"alias\"},\"version\":{\"path\":\"beat.state.beat.version\",\"type\":\"alias\"}}},\"state\":{\"properties\":{\"output\":{\"properties\":{\"name\":{\"path\":\"beat.state.output.name\",\"type\":\"alias\"}}},\"input\":{\"properties\":{\"names\":{\"path\":\"beat.state.input.names\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.state.input.count\",\"type\":\"alias\"}}},\"service\":{\"properties\":{\"name\":{\"path\":\"beat.state.service.name\",\"type\":\"alias\"},\"id\":{\"path\":\"beat.state.service.id\",\"type\":\"alias\"},\"version\":{\"path\":\"beat.state.service.version\",\"type\":\"alias\"}}},\"module\":{\"properties\":{\"names\":{\"path\":\"beat.state.module.names\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.state.module.count\",\"type\":\"alias\"}}},\"beat\":{\"properties\":{\"name\":{\"path\":\"beat.state.beat.name\",\"type\":\"alias\"}}},\"host\":{\"properties\":{\"hostname\":{\"path\":\"host.hostname\",\"type\":\"alias\"},\"os\":{\"properties\":{\"version\":{\"path\":\"beat.state.host.os.version\",\"type\":\"alias\"},\"platform\":{\"path\":\"beat.state.host.os.platform\",\"type\":\"alias\"}}},\"name\":{\"path\":\"host.name\",\"type\":\"alias\"},\"architecture\":{\"path\":\"host.architecture\",\"type\":\"alias\"}}}}},\"timestamp\":{\"path\":\"@timestamp\",\"type\":\"alias\"}}},\"timestamp\":{\"path\":\"@timestamp\",\"type\":\"alias\"}}}},\"composed_of\":[],\"version\":8000103,\"data_stream\":{\"hidden\":false,\"allow_custom_routing\":false}},\".monitoring-kibana-mb\":{\"index_patterns\":[\".monitoring-kibana-8-*\"],\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\".monitoring-8-ilm-policy\"},\"mapping\":{\"total_fields\":{\"limit\":\"2000\"}}}},\"mappings\":{\"dynamic\":false,\"properties\":{\"process\":{\"properties\":{\"pid\":{\"type\":\"long\"}}},\"@timestamp\":{\"type\":\"date\"},\"ecs\":{\"properties\":{\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"cluster_uuid\":{\"path\":\"kibana.elasticsearch.cluster.id\",\"type\":\"alias\"},\"service\":{\"properties\":{\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"host\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"metricset\":{\"properties\":{\"period\":{\"type\":\"long\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"kibana\":{\"properties\":{\"elasticsearch\":{\"properties\":{\"cluster\":{\"properties\":{\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"stats\":{\"properties\":{\"request\":{\"properties\":{\"total\":{\"type\":\"long\"},\"disconnects\":{\"type\":\"long\"}}},\"process\":{\"properties\":{\"memory\":{\"properties\":{\"resident_set_size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"heap\":{\"properties\":{\"total\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"used\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"size_limit\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"uptime\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}}}},\"event_loop_delay\":{\"properties\":{\"ms\":{\"scaling_factor\":1000,\"type\":\"scaled_float\"}}},\"uptime\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"os\":{\"properties\":{\"distroRelease\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"distro\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"memory\":{\"properties\":{\"used_in_bytes\":{\"type\":\"long\"},\"total_in_bytes\":{\"type\":\"long\"},\"free_in_bytes\":{\"type\":\"long\"}}},\"load\":{\"properties\":{\"5m\":{\"type\":\"half_float\"},\"15m\":{\"type\":\"half_float\"},\"1m\":{\"type\":\"half_float\"}}},\"platformRelease\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"platform\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"usage\":{\"properties\":{\"index\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"host\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"index\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"response_time\":{\"properties\":{\"avg\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"max\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"kibana\":{\"properties\":{\"status\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"concurrent_connections\":{\"type\":\"long\"},\"snapshot\":{\"type\":\"boolean\"},\"status\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"node_rules\":{\"properties\":{\"failures\":{\"type\":\"long\"},\"executions\":{\"type\":\"long\"},\"timeouts\":{\"type\":\"long\"}}},\"cluster_actions\":{\"properties\":{\"overdue\":{\"properties\":{\"delay\":{\"properties\":{\"p99\":{\"type\":\"float\"},\"p50\":{\"type\":\"float\"}}},\"count\":{\"type\":\"long\"}}}}},\"cluster_rules\":{\"properties\":{\"overdue\":{\"properties\":{\"delay\":{\"properties\":{\"p99\":{\"type\":\"float\"},\"p50\":{\"type\":\"float\"}}},\"count\":{\"type\":\"long\"}}}}},\"node_actions\":{\"properties\":{\"failures\":{\"type\":\"long\"},\"executions\":{\"type\":\"long\"},\"timeouts\":{\"type\":\"long\"}}},\"status\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"metrics\":{\"properties\":{\"requests\":{\"properties\":{\"total\":{\"type\":\"long\"},\"disconnects\":{\"type\":\"long\"}}},\"concurrent_connections\":{\"type\":\"long\"}}},\"status\":{\"properties\":{\"overall\":{\"properties\":{\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}}}}}},\"event\":{\"properties\":{\"duration\":{\"type\":\"long\"},\"module\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"dataset\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"kibana_stats\":{\"properties\":{\"process\":{\"properties\":{\"memory\":{\"properties\":{\"resident_set_size_in_bytes\":{\"path\":\"kibana.stats.process.memory.resident_set_size.bytes\",\"type\":\"alias\"},\"heap\":{\"properties\":{\"size_limit\":{\"path\":\"kibana.stats.process.memory.heap.size_limit.bytes\",\"type\":\"alias\"}}}}},\"event_loop_delay\":{\"path\":\"kibana.stats.process.event_loop_delay.ms\",\"type\":\"alias\"},\"uptime_in_millis\":{\"path\":\"kibana.stats.process.uptime.ms\",\"type\":\"alias\"}}},\"os\":{\"properties\":{\"memory\":{\"properties\":{\"free_in_bytes\":{\"path\":\"kibana.stats.os.memory.free_in_bytes\",\"type\":\"alias\"}}},\"load\":{\"properties\":{\"5m\":{\"path\":\"kibana.stats.os.load.5m\",\"type\":\"alias\"},\"15m\":{\"path\":\"kibana.stats.os.load.15m\",\"type\":\"alias\"},\"1m\":{\"path\":\"kibana.stats.os.load.1m\",\"type\":\"alias\"}}}}},\"response_times\":{\"properties\":{\"average\":{\"path\":\"kibana.stats.response_time.avg.ms\",\"type\":\"alias\"},\"max\":{\"path\":\"kibana.stats.response_time.max.ms\",\"type\":\"alias\"}}},\"requests\":{\"properties\":{\"total\":{\"path\":\"kibana.stats.request.total\",\"type\":\"alias\"},\"disconnects\":{\"path\":\"kibana.stats.request.disconnects\",\"type\":\"alias\"}}},\"kibana\":{\"properties\":{\"response_time\":{\"properties\":{\"max\":{\"path\":\"kibana.stats.response_time.max.ms\",\"type\":\"alias\"}}},\"uuid\":{\"path\":\"service.id\",\"type\":\"alias\"},\"version\":{\"path\":\"service.version\",\"type\":\"alias\"},\"status\":{\"path\":\"kibana.stats.status\",\"type\":\"alias\"}}},\"concurrent_connections\":{\"path\":\"kibana.stats.concurrent_connections\",\"type\":\"alias\"},\"timestamp\":{\"path\":\"@timestamp\",\"type\":\"alias\"}}},\"timestamp\":{\"path\":\"@timestamp\",\"type\":\"alias\"}}}},\"composed_of\":[],\"version\":8000103,\"data_stream\":{\"hidden\":false,\"allow_custom_routing\":false}},\".monitoring-ent-search-mb\":{\"index_patterns\":[\".monitoring-ent-search-8-*\"],\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\".monitoring-8-ilm-policy\"},\"mapping\":{\"total_fields\":{\"limit\":\"2000\"}}}},\"mappings\":{\"dynamic\":false,\"properties\":{\"enterprisesearch\":{\"properties\":{\"cluster_uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"stats\":{\"properties\":{\"connectors\":{\"properties\":{\"pool\":{\"properties\":{\"publish_worker_pool\":{\"properties\":{\"total_completed\":{\"type\":\"long\"},\"size\":{\"type\":\"long\"},\"idle\":{\"type\":\"long\"},\"busy\":{\"type\":\"long\"},\"queue_depth\":{\"type\":\"long\"},\"total_scheduled\":{\"type\":\"long\"}}},\"extract_worker_pool\":{\"properties\":{\"total_completed\":{\"type\":\"long\"},\"size\":{\"type\":\"long\"},\"idle\":{\"type\":\"long\"},\"busy\":{\"type\":\"long\"},\"queue_depth\":{\"type\":\"long\"},\"total_scheduled\":{\"type\":\"long\"}}},\"subextract_worker_pool\":{\"properties\":{\"total_completed\":{\"type\":\"long\"},\"size\":{\"type\":\"long\"},\"idle\":{\"type\":\"long\"},\"busy\":{\"type\":\"long\"},\"queue_depth\":{\"type\":\"long\"},\"total_scheduled\":{\"type\":\"long\"}}}}},\"job_store\":{\"properties\":{\"waiting\":{\"type\":\"long\"},\"job_types\":{\"properties\":{\"permissions\":{\"type\":\"long\"},\"incremental\":{\"type\":\"long\"},\"delete\":{\"type\":\"long\"},\"full\":{\"type\":\"long\"}}},\"working\":{\"type\":\"long\"}}}}},\"queues\":{\"properties\":{\"engine_destroyer\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"process_crawl\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"failed\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"mailer\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"http\":{\"properties\":{\"request_duration\":{\"properties\":{\"max\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"std_dev\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"mean\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"responses\":{\"properties\":{\"1xx\":{\"type\":\"long\"},\"2xx\":{\"type\":\"long\"},\"3xx\":{\"type\":\"long\"},\"4xx\":{\"type\":\"long\"},\"5xx\":{\"type\":\"long\"}}},\"connections\":{\"properties\":{\"current\":{\"type\":\"long\"},\"total\":{\"type\":\"long\"},\"max\":{\"type\":\"long\"}}},\"network\":{\"properties\":{\"received\":{\"properties\":{\"bytes_per_sec\":{\"type\":\"long\"},\"bytes\":{\"type\":\"long\"}}},\"sent\":{\"properties\":{\"bytes_per_sec\":{\"type\":\"long\"},\"bytes\":{\"type\":\"long\"}}}}}}},\"product_usage\":{\"properties\":{\"app_search\":{\"properties\":{\"total_engines\":{\"type\":\"long\"}}},\"workplace_search\":{\"properties\":{\"total_org_sources\":{\"type\":\"long\"},\"total_private_sources\":{\"type\":\"long\"}}}}}}},\"health\":{\"properties\":{\"jvm\":{\"properties\":{\"memory_usage\":{\"properties\":{\"heap_committed\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"heap_used\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"non_heap_committed\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"non_heap_init\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"heap_init\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"heap_max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"object_pending_finalization_count\":{\"type\":\"long\"}}},\"threads\":{\"properties\":{\"current\":{\"type\":\"long\"},\"total_started\":{\"type\":\"long\"},\"max\":{\"type\":\"long\"},\"daemon\":{\"type\":\"long\"}}},\"gc\":{\"properties\":{\"collection_count\":{\"type\":\"long\"},\"collection_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"process\":{\"properties\":{\"pid\":{\"type\":\"long\"},\"filebeat\":{\"properties\":{\"restart_count\":{\"type\":\"long\"},\"pid\":{\"type\":\"long\"},\"time_since_last_restart\":{\"properties\":{\"sec\":{\"type\":\"long\"}}}}},\"uptime\":{\"properties\":{\"sec\":{\"type\":\"long\"}}}}},\"crawler\":{\"properties\":{\"workers\":{\"properties\":{\"pool_size\":{\"type\":\"long\"},\"available\":{\"type\":\"long\"},\"active\":{\"type\":\"long\"}}}}},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"properties\":{\"number\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"build_hash\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}}}},\"agent\":{\"properties\":{\"id\":{\"type\":\"keyword\"},\"ephemeral_id\":{\"type\":\"keyword\"},\"type\":{\"type\":\"keyword\"},\"version\":{\"type\":\"keyword\"}}},\"process\":{\"properties\":{\"pid\":{\"type\":\"long\"}}},\"@timestamp\":{\"type\":\"date\"},\"ecs\":{\"properties\":{\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"service\":{\"properties\":{\"node\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"hostname\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"environment\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"origin\":{\"properties\":{\"node\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"environment\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"target\":{\"properties\":{\"node\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"environment\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"host\":{\"properties\":{\"hostname\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"architecture\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"metricset\":{\"properties\":{\"period\":{\"type\":\"long\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"event\":{\"properties\":{\"reason\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"code\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"timezone\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"agent_id_status\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"duration\":{\"type\":\"long\"},\"reference\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ingested\":{\"type\":\"date\"},\"provider\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"action\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"end\":{\"type\":\"date\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"outcome\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"severity\":{\"type\":\"long\"},\"original\":{\"ignore_above\":1024,\"index\":false,\"type\":\"keyword\",\"doc_values\":false},\"risk_score\":{\"type\":\"float\"},\"created\":{\"type\":\"date\"},\"kind\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"module\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"start\":{\"type\":\"date\"},\"url\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"sequence\":{\"type\":\"long\"},\"risk_score_norm\":{\"type\":\"float\"},\"category\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"dataset\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"hash\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}}},\"composed_of\":[],\"version\":8000103,\"_meta\":{\"managed\":true,\"description\":\"Template used by Enterprise Search Metricbeat module monitoring information for Stack Monitoring\"},\"data_stream\":{\"hidden\":false,\"allow_custom_routing\":false}},\"synthetics\":{\"index_patterns\":[\"synthetics-*-*\"],\"composed_of\":[\"synthetics-mappings\",\"data-streams-mappings\",\"synthetics-settings\"],\"priority\":100,\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default synthetics template installed by x-pack\"},\"data_stream\":{\"hidden\":false,\"allow_custom_routing\":false},\"allow_auto_create\":true},\"ilm-history\":{\"index_patterns\":[\"ilm-history-5*\"],\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\"ilm-history-ilm-policy\"},\"number_of_shards\":\"1\",\"auto_expand_replicas\":\"0-1\",\"number_of_replicas\":\"0\"}},\"mappings\":{\"dynamic\":false,\"properties\":{\"index_age\":{\"type\":\"long\"},\"@timestamp\":{\"format\":\"epoch_millis\",\"type\":\"date\"},\"error_details\":{\"type\":\"text\"},\"success\":{\"type\":\"boolean\"},\"index\":{\"type\":\"keyword\"},\"state\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"phase\":{\"type\":\"keyword\"},\"failed_step\":{\"type\":\"keyword\"},\"phase_definition\":{\"type\":\"text\"},\"action_time\":{\"format\":\"epoch_millis\",\"type\":\"date\"},\"phase_time\":{\"format\":\"epoch_millis\",\"type\":\"date\"},\"step_info\":{\"type\":\"text\"},\"action\":{\"type\":\"keyword\"},\"step\":{\"type\":\"keyword\"},\"is_auto-retryable_error\":{\"type\":\"keyword\"},\"creation_date\":{\"format\":\"epoch_millis\",\"type\":\"date\"},\"step_time\":{\"format\":\"epoch_millis\",\"type\":\"date\"}}},\"policy\":{\"type\":\"keyword\"}}}},\"composed_of\":[],\"priority\":2147483647,\"version\":5,\"_meta\":{\"managed\":true,\"description\":\"index template for ILM history indices\"},\"data_stream\":{\"hidden\":true,\"allow_custom_routing\":false},\"allow_auto_create\":true},\".ml-state\":{\"index_patterns\":[\".ml-state*\"],\"template\":{\"settings\":{\"index\":{\"hidden\":\"true\",\"lifecycle\":{\"name\":\"ml-size-based-ilm-policy\",\"rollover_alias\":\".ml-state-write\"},\"auto_expand_replicas\":\"0-1\"}},\"mappings\":{\"_meta\":{\"version\":\"8060299\"},\"enabled\":false},\"aliases\":{}},\"composed_of\":[],\"priority\":2147483647,\"version\":8060299,\"_meta\":{\"managed\":true,\"description\":\"index template for ML state indices\"}},\".monitoring-es-mb\":{\"index_patterns\":[\".monitoring-es-8-*\"],\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\".monitoring-8-ilm-policy\"},\"mapping\":{\"total_fields\":{\"limit\":\"2000\"}}}},\"mappings\":{\"dynamic\":false,\"properties\":{\"ccr_auto_follow_stats\":{\"properties\":{\"number_of_failed_remote_cluster_state_requests\":{\"path\":\"elasticsearch.ccr.auto_follow.failed.remote_cluster_state_requests.count\",\"type\":\"alias\"},\"follower\":{\"properties\":{\"failed_read_requests\":{\"path\":\"elasticsearch.ccr.requests.failed.read.count\",\"type\":\"alias\"}}},\"number_of_failed_follow_indices\":{\"path\":\"elasticsearch.ccr.auto_follow.failed.follow_indices.count\",\"type\":\"alias\"},\"number_of_successful_follow_indices\":{\"path\":\"elasticsearch.ccr.auto_follow.success.follow_indices.count\",\"type\":\"alias\"}}},\"ccr_stats\":{\"properties\":{\"write_buffer_size_in_bytes\":{\"path\":\"elasticsearch.ccr.write_buffer.size.bytes\",\"type\":\"alias\"},\"leader_global_checkpoint\":{\"path\":\"elasticsearch.ccr.leader.global_checkpoint\",\"type\":\"alias\"},\"follower_index\":{\"path\":\"elasticsearch.ccr.follower.index\",\"type\":\"alias\"},\"leader_max_seq_no\":{\"path\":\"elasticsearch.ccr.leader.max_seq_no\",\"type\":\"alias\"},\"last_requested_seq_no\":{\"path\":\"elasticsearch.ccr.last_requested_seq_no\",\"type\":\"alias\"},\"follower_settings_version\":{\"path\":\"elasticsearch.ccr.follower.settings_version\",\"type\":\"alias\"},\"successful_write_requests\":{\"path\":\"elasticsearch.ccr.requests.successful.write.count\",\"type\":\"alias\"},\"remote_cluster\":{\"path\":\"elasticsearch.ccr.remote_cluster\",\"type\":\"alias\"},\"outstanding_write_requests\":{\"path\":\"elasticsearch.ccr.requests.outstanding.write.count\",\"type\":\"alias\"},\"total_read_time_millis\":{\"path\":\"elasticsearch.ccr.total_time.read.ms\",\"type\":\"alias\"},\"outstanding_read_requests\":{\"path\":\"elasticsearch.ccr.requests.outstanding.read.count\",\"type\":\"alias\"},\"total_write_time_millis\":{\"path\":\"elasticsearch.ccr.total_time.write.ms\",\"type\":\"alias\"},\"failed_write_requests\":{\"path\":\"elasticsearch.ccr.requests.failed.write.count\",\"type\":\"alias\"},\"failed_read_requests\":{\"path\":\"elasticsearch.ccr.requests.failed.read.count\",\"type\":\"alias\"},\"bytes_read\":{\"path\":\"elasticsearch.ccr.bytes_read\",\"type\":\"alias\"},\"leader_index\":{\"path\":\"elasticsearch.ccr.leader.index\",\"type\":\"alias\"},\"follower_max_seq_no\":{\"path\":\"elasticsearch.ccr.follower.max_seq_no\",\"type\":\"alias\"},\"operations_written\":{\"path\":\"elasticsearch.ccr.follower.operations_written\",\"type\":\"alias\"},\"write_buffer_operation_count\":{\"path\":\"elasticsearch.ccr.write_buffer.operation.count\",\"type\":\"alias\"},\"successful_read_requests\":{\"path\":\"elasticsearch.ccr.requests.successful.read.count\",\"type\":\"alias\"},\"shard_id\":{\"path\":\"elasticsearch.ccr.follower.shard.number\",\"type\":\"alias\"},\"follower_mapping_version\":{\"path\":\"elasticsearch.ccr.follower.mapping_version\",\"type\":\"alias\"},\"follower_aliases_version\":{\"path\":\"elasticsearch.ccr.follower.aliases_version\",\"type\":\"alias\"},\"follower_global_checkpoint\":{\"path\":\"elasticsearch.ccr.follower.global_checkpoint\",\"type\":\"alias\"},\"total_read_remote_exec_time_millis\":{\"path\":\"elasticsearch.ccr.total_time.read.remote_exec.ms\",\"type\":\"alias\"},\"operations_read\":{\"path\":\"elasticsearch.ccr.follower.operations.read.count\",\"type\":\"alias\"}}},\"cluster_state\":{\"properties\":{\"nodes_hash\":{\"path\":\"elasticsearch.cluster.stats.state.nodes_hash\",\"type\":\"alias\"},\"master_node\":{\"path\":\"elasticsearch.cluster.stats.state.master_node\",\"type\":\"alias\"},\"state_uuid\":{\"path\":\"elasticsearch.cluster.stats.state.state_uuid\",\"type\":\"alias\"},\"version\":{\"path\":\"elasticsearch.cluster.stats.state.version\",\"type\":\"alias\"},\"status\":{\"path\":\"elasticsearch.cluster.stats.status\",\"type\":\"alias\"}}},\"indices_stats\":{\"properties\":{\"_all\":{\"properties\":{\"primaries\":{\"properties\":{\"indexing\":{\"properties\":{\"index_time_in_millis\":{\"path\":\"elasticsearch.index.summary.primaries.indexing.index.time.ms\",\"type\":\"alias\"},\"index_total\":{\"path\":\"elasticsearch.index.summary.primaries.indexing.index.count\",\"type\":\"alias\"}}}}},\"total\":{\"properties\":{\"search\":{\"properties\":{\"query_total\":{\"path\":\"elasticsearch.index.summary.total.search.query.count\",\"type\":\"alias\"},\"query_time_in_millis\":{\"path\":\"elasticsearch.index.summary.total.search.query.time.ms\",\"type\":\"alias\"}}},\"indexing\":{\"properties\":{\"index_total\":{\"path\":\"elasticsearch.index.summary.total.indexing.index.count\",\"type\":\"alias\"}}}}}}}}},\"node_stats\":{\"properties\":{\"jvm\":{\"properties\":{\"mem\":{\"properties\":{\"heap_used_percent\":{\"path\":\"elasticsearch.node.stats.jvm.mem.heap.used.pct\",\"type\":\"alias\"},\"heap_max_in_bytes\":{\"path\":\"elasticsearch.node.stats.jvm.mem.heap.max.bytes\",\"type\":\"alias\"},\"heap_used_in_bytes\":{\"path\":\"elasticsearch.node.stats.jvm.mem.heap.used.bytes\",\"type\":\"alias\"}}},\"gc\":{\"properties\":{\"collectors\":{\"properties\":{\"young\":{\"properties\":{\"collection_count\":{\"path\":\"elasticsearch.node.stats.jvm.gc.collectors.young.collection.count\",\"type\":\"alias\"},\"collection_time_in_millis\":{\"path\":\"elasticsearch.node.stats.jvm.gc.collectors.young.collection.ms\",\"type\":\"alias\"}}},\"old\":{\"properties\":{\"collection_count\":{\"path\":\"elasticsearch.node.stats.jvm.gc.collectors.old.collection.count\",\"type\":\"alias\"},\"collection_time_in_millis\":{\"path\":\"elasticsearch.node.stats.jvm.gc.collectors.old.collection.ms\",\"type\":\"alias\"}}}}}}}}},\"indices\":{\"properties\":{\"search\":{\"properties\":{\"query_total\":{\"path\":\"elasticsearch.node.stats.indices.search.query_total.count\",\"type\":\"alias\"},\"query_time_in_millis\":{\"path\":\"elasticsearch.node.stats.indices.search.query_time.ms\",\"type\":\"alias\"}}},\"query_cache\":{\"properties\":{\"memory_size_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.query_cache.memory.bytes\",\"type\":\"alias\"}}},\"docs\":{\"properties\":{\"count\":{\"path\":\"elasticsearch.node.stats.indices.docs.count\",\"type\":\"alias\"}}},\"indexing\":{\"properties\":{\"throttle_time_in_millis\":{\"path\":\"elasticsearch.node.stats.indices.indexing.throttle_time.ms\",\"type\":\"alias\"},\"index_time_in_millis\":{\"path\":\"elasticsearch.node.stats.indices.indexing.index_time.ms\",\"type\":\"alias\"},\"index_total\":{\"path\":\"elasticsearch.node.stats.indices.indexing.index_total.count\",\"type\":\"alias\"}}},\"fielddata\":{\"properties\":{\"memory_size_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.fielddata.memory.bytes\",\"type\":\"alias\"}}},\"store\":{\"properties\":{\"size\":{\"properties\":{\"bytes\":{\"path\":\"elasticsearch.node.stats.indices.store.size.bytes\",\"type\":\"alias\"}}},\"size_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.store.size.bytes\",\"type\":\"alias\"}}},\"request_cache\":{\"properties\":{\"memory_size_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.request_cache.memory.bytes\",\"type\":\"alias\"}}},\"segments\":{\"properties\":{\"version_map_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.version_map.memory.bytes\",\"type\":\"alias\"},\"norms_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.norms.memory.bytes\",\"type\":\"alias\"},\"count\":{\"path\":\"elasticsearch.node.stats.indices.segments.count\",\"type\":\"alias\"},\"term_vectors_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.term_vectors.memory.bytes\",\"type\":\"alias\"},\"points_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.points.memory.bytes\",\"type\":\"alias\"},\"index_writer_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.index_writer.memory.bytes\",\"type\":\"alias\"},\"memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.memory.bytes\",\"type\":\"alias\"},\"doc_values_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.doc_values.memory.bytes\",\"type\":\"alias\"},\"terms_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.terms.memory.bytes\",\"type\":\"alias\"},\"fixed_bit_set_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.fixed_bit_set.memory.bytes\",\"type\":\"alias\"},\"stored_fields_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.stored_fields.memory.bytes\",\"type\":\"alias\"}}}}},\"process\":{\"properties\":{\"cpu\":{\"properties\":{\"percent\":{\"path\":\"elasticsearch.node.stats.process.cpu.pct\",\"type\":\"alias\"}}}}},\"os\":{\"properties\":{\"cpu\":{\"properties\":{\"load_average\":{\"properties\":{\"1m\":{\"path\":\"elasticsearch.node.stats.os.cpu.load_avg.1m\",\"type\":\"alias\"}}}}},\"cgroup\":{\"properties\":{\"memory\":{\"properties\":{\"usage_in_bytes\":{\"path\":\"elasticsearch.node.stats.os.cgroup.memory.usage.bytes\",\"type\":\"alias\"},\"control_group\":{\"path\":\"elasticsearch.node.stats.os.cgroup.memory.control_group\",\"type\":\"alias\"},\"limit_in_bytes\":{\"path\":\"elasticsearch.node.stats.os.cgroup.memory.limit.bytes\",\"type\":\"alias\"}}},\"cpu\":{\"properties\":{\"stat\":{\"properties\":{\"number_of_elapsed_periods\":{\"path\":\"elasticsearch.node.stats.os.cgroup.cpu.stat.elapsed_periods.count\",\"type\":\"alias\"},\"number_of_times_throttled\":{\"path\":\"elasticsearch.node.stats.os.cgroup.cpu.stat.times_throttled.count\",\"type\":\"alias\"},\"time_throttled_nanos\":{\"path\":\"elasticsearch.node.stats.os.cgroup.cpu.stat.time_throttled.ns\",\"type\":\"alias\"}}},\"cfs_quota_micros\":{\"path\":\"elasticsearch.node.stats.os.cgroup.cpu.cfs.quota.us\",\"type\":\"alias\"}}},\"cpuacct\":{\"properties\":{\"usage_nanos\":{\"path\":\"elasticsearch.node.stats.os.cgroup.cpuacct.usage.ns\",\"type\":\"alias\"}}}}}}},\"thread_pool\":{\"properties\":{\"search\":{\"properties\":{\"rejected\":{\"path\":\"elasticsearch.node.stats.thread_pool.search.rejected.count\",\"type\":\"alias\"},\"queue\":{\"path\":\"elasticsearch.node.stats.thread_pool.search.queue.count\",\"type\":\"alias\"}}},\"get\":{\"properties\":{\"rejected\":{\"path\":\"elasticsearch.node.stats.thread_pool.get.rejected.count\",\"type\":\"alias\"},\"queue\":{\"path\":\"elasticsearch.node.stats.thread_pool.get.queue.count\",\"type\":\"alias\"}}},\"index\":{\"properties\":{\"rejected\":{\"path\":\"elasticsearch.node.stats.thread_pool.index.rejected.count\",\"type\":\"alias\"},\"queue\":{\"path\":\"elasticsearch.node.stats.thread_pool.index.queue.count\",\"type\":\"alias\"}}},\"bulk\":{\"properties\":{\"rejected\":{\"path\":\"elasticsearch.node.stats.thread_pool.bulk.rejected.count\",\"type\":\"alias\"},\"queue\":{\"path\":\"elasticsearch.node.stats.thread_pool.bulk.queue.count\",\"type\":\"alias\"}}},\"write\":{\"properties\":{\"rejected\":{\"path\":\"elasticsearch.node.stats.thread_pool.write.rejected.count\",\"type\":\"alias\"},\"queue\":{\"path\":\"elasticsearch.node.stats.thread_pool.write.queue.count\",\"type\":\"alias\"}}}}},\"fs\":{\"properties\":{\"summary\":{\"properties\":{\"total\":{\"properties\":{\"bytes\":{\"path\":\"elasticsearch.node.stats.fs.summary.total.bytes\",\"type\":\"alias\"}}},\"available\":{\"properties\":{\"bytes\":{\"path\":\"elasticsearch.node.stats.fs.summary.available.bytes\",\"type\":\"alias\"}}}}},\"io_stats\":{\"properties\":{\"total\":{\"properties\":{\"write_operations\":{\"path\":\"elasticsearch.node.stats.fs.io_stats.total.write.operations.count\",\"type\":\"alias\"},\"operations\":{\"path\":\"elasticsearch.node.stats.fs.io_stats.total.operations.count\",\"type\":\"alias\"},\"read_operations\":{\"path\":\"elasticsearch.node.stats.fs.io_stats.total.read.operations.count\",\"type\":\"alias\"}}}}},\"total\":{\"properties\":{\"total_in_bytes\":{\"path\":\"elasticsearch.node.stats.fs.summary.total.bytes\",\"type\":\"alias\"},\"available_in_bytes\":{\"path\":\"elasticsearch.node.stats.fs.summary.available.bytes\",\"type\":\"alias\"}}}}},\"node_id\":{\"path\":\"elasticsearch.node.id\",\"type\":\"alias\"}}},\"stack_stats\":{\"properties\":{\"xpack\":{\"properties\":{\"ccr\":{\"properties\":{\"available\":{\"path\":\"elasticsearch.cluster.stats.stack.xpack.ccr.available\",\"type\":\"alias\"},\"enabled\":{\"path\":\"elasticsearch.cluster.stats.stack.xpack.ccr.enabled\",\"type\":\"alias\"}}}}},\"apm\":{\"properties\":{\"found\":{\"path\":\"elasticsearch.cluster.stats.stack.apm.found\",\"type\":\"alias\"}}}}},\"license\":{\"properties\":{\"type\":{\"path\":\"elasticsearch.cluster.stats.license.type\",\"type\":\"alias\"},\"status\":{\"path\":\"elasticsearch.cluster.stats.license.status\",\"type\":\"alias\"}}},\"index_stats\":{\"properties\":{\"primaries\":{\"properties\":{\"docs\":{\"properties\":{\"count\":{\"path\":\"elasticsearch.index.primaries.docs.count\",\"type\":\"alias\"}}},\"indexing\":{\"properties\":{\"throttle_time_in_millis\":{\"path\":\"elasticsearch.index.primaries.indexing.throttle_time_in_millis\",\"type\":\"alias\"},\"index_time_in_millis\":{\"path\":\"elasticsearch.index.primaries.indexing.index_time_in_millis\",\"type\":\"alias\"},\"index_total\":{\"path\":\"elasticsearch.index.primaries.indexing.index_total\",\"type\":\"alias\"}}},\"refresh\":{\"properties\":{\"total_time_in_millis\":{\"path\":\"elasticsearch.index.primaries.refresh.total_time_in_millis\",\"type\":\"alias\"}}},\"store\":{\"properties\":{\"size_in_bytes\":{\"path\":\"elasticsearch.index.primaries.store.size_in_bytes\",\"type\":\"alias\"}}},\"merges\":{\"properties\":{\"total_size_in_bytes\":{\"path\":\"elasticsearch.index.primaries.merges.total_size_in_bytes\",\"type\":\"alias\"}}},\"segments\":{\"properties\":{\"count\":{\"path\":\"elasticsearch.index.primaries.segments.count\",\"type\":\"alias\"}}}}},\"total\":{\"properties\":{\"search\":{\"properties\":{\"query_total\":{\"path\":\"elasticsearch.index.total.search.query_total\",\"type\":\"alias\"},\"query_time_in_millis\":{\"path\":\"elasticsearch.index.total.search.query_time_in_millis\",\"type\":\"alias\"}}},\"query_cache\":{\"properties\":{\"memory_size_in_bytes\":{\"path\":\"elasticsearch.index.total.query_cache.memory_size_in_bytes\",\"type\":\"alias\"}}},\"fielddata\":{\"properties\":{\"memory_size_in_bytes\":{\"path\":\"elasticsearch.index.total.fielddata.memory_size_in_bytes\",\"type\":\"alias\"}}},\"indexing\":{\"properties\":{\"throttle_time_in_millis\":{\"path\":\"elasticsearch.index.total.indexing.throttle_time_in_millis\",\"type\":\"alias\"},\"index_time_in_millis\":{\"path\":\"elasticsearch.index.total.indexing.index_time_in_millis\",\"type\":\"alias\"},\"index_total\":{\"path\":\"elasticsearch.index.total.indexing.index_total\",\"type\":\"alias\"}}},\"refresh\":{\"properties\":{\"total_time_in_millis\":{\"path\":\"elasticsearch.index.total.refresh.total_time_in_millis\",\"type\":\"alias\"}}},\"request_cache\":{\"properties\":{\"memory_size_in_bytes\":{\"path\":\"elasticsearch.index.total.request_cache.memory_size_in_bytes\",\"type\":\"alias\"}}},\"store\":{\"properties\":{\"size_in_bytes\":{\"path\":\"elasticsearch.index.total.store.size_in_bytes\",\"type\":\"alias\"}}},\"merges\":{\"properties\":{\"total_size_in_bytes\":{\"path\":\"elasticsearch.index.total.merges.total_size_in_bytes\",\"type\":\"alias\"}}},\"segments\":{\"properties\":{\"version_map_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.version_map_memory_in_bytes\",\"type\":\"alias\"},\"norms_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.norms_memory_in_bytes\",\"type\":\"alias\"},\"count\":{\"path\":\"elasticsearch.index.total.segments.count\",\"type\":\"alias\"},\"term_vectors_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.term_vectors_memory_in_bytes\",\"type\":\"alias\"},\"points_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.points_memory_in_bytes\",\"type\":\"alias\"},\"index_writer_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.index_writer_memory_in_bytes\",\"type\":\"alias\"},\"memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.memory_in_bytes\",\"type\":\"alias\"},\"doc_values_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.doc_values_memory_in_bytes\",\"type\":\"alias\"},\"terms_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.terms_memory_in_bytes\",\"type\":\"alias\"},\"fixed_bit_set_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.fixed_bit_set_memory_in_bytes\",\"type\":\"alias\"},\"stored_fields_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.stored_fields_memory_in_bytes\",\"type\":\"alias\"}}}}},\"index\":{\"path\":\"elasticsearch.index.name\",\"type\":\"alias\"}}},\"@timestamp\":{\"type\":\"date\"},\"elasticsearch\":{\"properties\":{\"cluster\":{\"properties\":{\"stats\":{\"properties\":{\"license\":{\"properties\":{\"expiry_date_in_millis\":{\"type\":\"long\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"status\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"indices\":{\"properties\":{\"shards\":{\"properties\":{\"primaries\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"total\":{\"type\":\"long\"},\"docs\":{\"properties\":{\"total\":{\"type\":\"long\"}}},\"fielddata\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"store\":{\"properties\":{\"size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}}}},\"stack\":{\"properties\":{\"xpack\":{\"properties\":{\"ccr\":{\"properties\":{\"available\":{\"type\":\"boolean\"},\"enabled\":{\"type\":\"boolean\"}}}}},\"apm\":{\"properties\":{\"found\":{\"type\":\"boolean\"}}}}},\"nodes\":{\"properties\":{\"jvm\":{\"properties\":{\"memory\":{\"properties\":{\"heap\":{\"properties\":{\"max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"used\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}}}},\"max_uptime\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"data\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"},\"fs\":{\"properties\":{\"total\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"available\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"master\":{\"type\":\"long\"}}},\"state\":{\"properties\":{\"nodes_hash\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"master_node\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state_uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"status\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"properties\":{\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"node\":{\"properties\":{\"jvm\":{\"properties\":{\"memory\":{\"properties\":{\"heap\":{\"properties\":{\"init\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"nonheap\":{\"properties\":{\"init\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}}}},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"process\":{\"properties\":{\"mlockall\":{\"type\":\"boolean\"}}},\"stats\":{\"properties\":{\"jvm\":{\"properties\":{\"mem\":{\"properties\":{\"pools\":{\"properties\":{\"young\":{\"properties\":{\"max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"peak\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"peak_max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"used\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"old\":{\"properties\":{\"max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"peak\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"peak_max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"used\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"survivor\":{\"properties\":{\"max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"peak\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"peak_max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"used\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}}}},\"heap\":{\"properties\":{\"max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"used\":{\"properties\":{\"pct\":{\"type\":\"double\"},\"bytes\":{\"type\":\"long\"}}}}}}},\"gc\":{\"properties\":{\"collectors\":{\"properties\":{\"young\":{\"properties\":{\"collection\":{\"properties\":{\"ms\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}}}},\"old\":{\"properties\":{\"collection\":{\"properties\":{\"ms\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}}}}}}}}}},\"indices\":{\"properties\":{\"search\":{\"properties\":{\"query_total\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"query_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"query_cache\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"docs\":{\"properties\":{\"deleted\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"fielddata\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"indexing\":{\"properties\":{\"index_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"index_total\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"throttle_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"store\":{\"properties\":{\"size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"request_cache\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"bulk\":{\"properties\":{\"operations\":{\"properties\":{\"total\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"total_size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"avg_size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"total_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"avg_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"segments\":{\"properties\":{\"version_map\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"norms\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"terms\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"stored_fields\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"index_writer\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"count\":{\"type\":\"long\"},\"fixed_bit_set\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"term_vectors\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"doc_values\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"points\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}}}}}},\"process\":{\"properties\":{\"cpu\":{\"properties\":{\"pct\":{\"type\":\"double\"}}}}},\"os\":{\"properties\":{\"cpu\":{\"properties\":{\"load_avg\":{\"properties\":{\"1m\":{\"type\":\"half_float\"}}}}},\"cgroup\":{\"properties\":{\"memory\":{\"properties\":{\"control_group\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"usage\":{\"properties\":{\"bytes\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"limit\":{\"properties\":{\"bytes\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"cpu\":{\"properties\":{\"cfs\":{\"properties\":{\"quota\":{\"properties\":{\"us\":{\"type\":\"long\"}}}}},\"stat\":{\"properties\":{\"elapsed_periods\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"times_throttled\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"time_throttled\":{\"properties\":{\"ns\":{\"type\":\"long\"}}}}}}},\"cpuacct\":{\"properties\":{\"usage\":{\"properties\":{\"ns\":{\"type\":\"long\"}}}}}}}}},\"thread_pool\":{\"properties\":{\"search\":{\"properties\":{\"rejected\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"queue\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"get\":{\"properties\":{\"rejected\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"queue\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"index\":{\"properties\":{\"rejected\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"queue\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"bulk\":{\"properties\":{\"rejected\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"queue\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"write\":{\"properties\":{\"rejected\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"queue\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}}}},\"fs\":{\"properties\":{\"summary\":{\"properties\":{\"total\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"available\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"free\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"io_stats\":{\"properties\":{\"total\":{\"properties\":{\"operations\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"read\":{\"properties\":{\"operations\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"write\":{\"properties\":{\"operations\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}}}}}},\"total\":{\"properties\":{\"total_in_bytes\":{\"type\":\"long\"},\"available_in_bytes\":{\"type\":\"long\"}}}}}}},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"mlockall\":{\"type\":\"boolean\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"master\":{\"type\":\"boolean\"}}},\"ccr\":{\"properties\":{\"leader\":{\"properties\":{\"index\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"max_seq_no\":{\"type\":\"long\"},\"global_checkpoint\":{\"type\":\"long\"}}},\"follower\":{\"properties\":{\"operations\":{\"properties\":{\"read\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"time_since_last_read\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"index\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"settings_version\":{\"type\":\"long\"},\"shard\":{\"properties\":{\"number\":{\"type\":\"long\"}}},\"max_seq_no\":{\"type\":\"long\"},\"mapping_version\":{\"type\":\"long\"},\"aliases_version\":{\"type\":\"long\"},\"operations_written\":{\"type\":\"long\"},\"global_checkpoint\":{\"type\":\"long\"}}},\"remote_cluster\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"read_exceptions\":{\"type\":\"nested\"},\"shard_id\":{\"type\":\"long\"},\"bytes_read\":{\"type\":\"long\"},\"requests\":{\"properties\":{\"outstanding\":{\"properties\":{\"read\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"write\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"failed\":{\"properties\":{\"read\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"write\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"successful\":{\"properties\":{\"read\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"write\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}}}},\"auto_follow\":{\"properties\":{\"success\":{\"properties\":{\"follow_indices\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"failed\":{\"properties\":{\"follow_indices\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"remote_cluster_state_requests\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}}}},\"total_time\":{\"properties\":{\"read\":{\"properties\":{\"ms\":{\"type\":\"long\"},\"remote_exec\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"write\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"last_requested_seq_no\":{\"type\":\"long\"},\"write_buffer\":{\"properties\":{\"size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"operation\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}}}},\"pending_tasks\":{\"properties\":{\"time_in_queue\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"source\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"priority\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"insert_order\":{\"type\":\"long\"}}},\"enrich\":{\"properties\":{\"executed_searches\":{\"properties\":{\"total\":{\"type\":\"long\"}}},\"remote_requests\":{\"properties\":{\"current\":{\"type\":\"long\"},\"total\":{\"type\":\"long\"}}},\"executing_policy\":{\"properties\":{\"task\":{\"properties\":{\"task\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"parent_task_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"action\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"type\":\"long\"},\"time\":{\"properties\":{\"running\":{\"properties\":{\"nano\":{\"type\":\"long\"}}},\"start\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"cancellable\":{\"type\":\"boolean\"}}},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"queue\":{\"properties\":{\"size\":{\"type\":\"long\"}}}}},\"index\":{\"properties\":{\"summary\":{\"properties\":{\"primaries\":{\"properties\":{\"search\":{\"properties\":{\"query\":{\"properties\":{\"count\":{\"type\":\"long\"},\"time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}}}},\"docs\":{\"properties\":{\"deleted\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"indexing\":{\"properties\":{\"index\":{\"properties\":{\"count\":{\"type\":\"long\"},\"time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}}}},\"store\":{\"properties\":{\"size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"bulk\":{\"properties\":{\"operations\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"time\":{\"properties\":{\"avg\":{\"properties\":{\"ms\":{\"type\":\"long\"},\"bytes\":{\"type\":\"long\"}}},\"count\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}}}},\"segments\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"count\":{\"type\":\"long\"}}}}},\"total\":{\"properties\":{\"search\":{\"properties\":{\"query\":{\"properties\":{\"count\":{\"type\":\"long\"},\"time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}}}},\"docs\":{\"properties\":{\"deleted\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"indexing\":{\"properties\":{\"index\":{\"properties\":{\"count\":{\"type\":\"long\"},\"time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"is_throttled\":{\"type\":\"boolean\"},\"throttle_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"store\":{\"properties\":{\"size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"bulk\":{\"properties\":{\"operations\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"time\":{\"properties\":{\"avg\":{\"properties\":{\"ms\":{\"type\":\"long\"},\"bytes\":{\"type\":\"long\"}}}}}}},\"segments\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"count\":{\"type\":\"long\"}}}}}}},\"shards\":{\"properties\":{\"primaries\":{\"type\":\"long\"},\"total\":{\"type\":\"long\"}}},\"primaries\":{\"properties\":{\"search\":{\"properties\":{\"query_total\":{\"type\":\"long\"},\"query_time_in_millis\":{\"type\":\"long\"}}},\"query_cache\":{\"properties\":{\"miss_count\":{\"type\":\"long\"},\"memory_size_in_bytes\":{\"type\":\"long\"},\"hit_count\":{\"type\":\"long\"}}},\"docs\":{\"properties\":{\"deleted\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"indexing\":{\"properties\":{\"throttle_time_in_millis\":{\"type\":\"long\"},\"index_time_in_millis\":{\"type\":\"long\"},\"index_total\":{\"type\":\"long\"}}},\"refresh\":{\"properties\":{\"external_total_time_in_millis\":{\"type\":\"long\"},\"total_time_in_millis\":{\"type\":\"long\"}}},\"request_cache\":{\"properties\":{\"miss_count\":{\"type\":\"long\"},\"memory_size_in_bytes\":{\"type\":\"long\"},\"evictions\":{\"type\":\"long\"},\"hit_count\":{\"type\":\"long\"}}},\"store\":{\"properties\":{\"size_in_bytes\":{\"type\":\"long\"}}},\"segments\":{\"properties\":{\"version_map_memory_in_bytes\":{\"type\":\"long\"},\"norms_memory_in_bytes\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"},\"term_vectors_memory_in_bytes\":{\"type\":\"long\"},\"points_memory_in_bytes\":{\"type\":\"long\"},\"index_writer_memory_in_bytes\":{\"type\":\"long\"},\"memory_in_bytes\":{\"type\":\"long\"},\"terms_memory_in_bytes\":{\"type\":\"long\"},\"doc_values_memory_in_bytes\":{\"type\":\"long\"},\"stored_fields_memory_in_bytes\":{\"type\":\"long\"},\"fixed_bit_set_memory_in_bytes\":{\"type\":\"long\"}}},\"merges\":{\"properties\":{\"total_size_in_bytes\":{\"type\":\"long\"}}}}},\"total\":{\"properties\":{\"search\":{\"properties\":{\"query_total\":{\"type\":\"long\"},\"query_time_in_millis\":{\"type\":\"long\"}}},\"query_cache\":{\"properties\":{\"miss_count\":{\"type\":\"long\"},\"memory_size_in_bytes\":{\"type\":\"long\"},\"evictions\":{\"type\":\"long\"},\"hit_count\":{\"type\":\"long\"}}},\"docs\":{\"properties\":{\"deleted\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"fielddata\":{\"properties\":{\"memory_size_in_bytes\":{\"type\":\"long\"},\"evictions\":{\"type\":\"long\"}}},\"indexing\":{\"properties\":{\"throttle_time_in_millis\":{\"type\":\"long\"},\"index_time_in_millis\":{\"type\":\"long\"},\"index_total\":{\"type\":\"long\"}}},\"refresh\":{\"properties\":{\"external_total_time_in_millis\":{\"type\":\"long\"},\"total_time_in_millis\":{\"type\":\"long\"}}},\"store\":{\"properties\":{\"size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"size_in_bytes\":{\"type\":\"long\"}}},\"request_cache\":{\"properties\":{\"miss_count\":{\"type\":\"long\"},\"memory_size_in_bytes\":{\"type\":\"long\"},\"evictions\":{\"type\":\"long\"},\"hit_count\":{\"type\":\"long\"}}},\"bulk\":{\"properties\":{\"avg_time_in_millis\":{\"type\":\"long\"},\"avg_size_in_bytes\":{\"type\":\"long\"},\"total_time_in_millis\":{\"type\":\"long\"},\"total_operations\":{\"type\":\"long\"},\"total_size_in_bytes\":{\"type\":\"long\"}}},\"merges\":{\"properties\":{\"total_size_in_bytes\":{\"type\":\"long\"}}},\"segments\":{\"properties\":{\"version_map_memory_in_bytes\":{\"type\":\"long\"},\"norms_memory_in_bytes\":{\"type\":\"long\"},\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"count\":{\"type\":\"long\"},\"term_vectors_memory_in_bytes\":{\"type\":\"long\"},\"points_memory_in_bytes\":{\"type\":\"long\"},\"index_writer_memory_in_bytes\":{\"type\":\"long\"},\"memory_in_bytes\":{\"type\":\"long\"},\"terms_memory_in_bytes\":{\"type\":\"long\"},\"doc_values_memory_in_bytes\":{\"type\":\"long\"},\"stored_fields_memory_in_bytes\":{\"type\":\"long\"},\"fixed_bit_set_memory_in_bytes\":{\"type\":\"long\"}}}}},\"hidden\":{\"type\":\"boolean\"},\"created\":{\"type\":\"long\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"recovery\":{\"properties\":{\"stop_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"translog\":{\"properties\":{\"total_on_start\":{\"type\":\"long\"},\"total\":{\"type\":\"long\"},\"percent\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"verify_index\":{\"properties\":{\"total_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"check_index_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"index\":{\"properties\":{\"size\":{\"properties\":{\"total_in_bytes\":{\"type\":\"long\"},\"reused_in_bytes\":{\"type\":\"long\"},\"recovered_in_bytes\":{\"type\":\"long\"}}},\"files\":{\"properties\":{\"recovered\":{\"type\":\"long\"},\"total\":{\"type\":\"long\"},\"percent\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"reused\":{\"type\":\"long\"}}}}},\"source\":{\"properties\":{\"transport_address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"host\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"target\":{\"properties\":{\"transport_address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"host\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"start_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"stage\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"type\":\"long\"},\"total_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"primary\":{\"type\":\"boolean\"}}},\"uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"status\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"shard\":{\"properties\":{\"number\":{\"type\":\"long\"},\"source_node\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"relocating_node\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"primary\":{\"type\":\"boolean\"}}},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ml\":{\"properties\":{\"job\":{\"properties\":{\"data\":{\"properties\":{\"invalid_date\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"data_counts\":{\"properties\":{\"invalid_date_count\":{\"type\":\"long\"},\"processed_record_count\":{\"type\":\"long\"}}},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"forecasts_stats\":{\"properties\":{\"total\":{\"type\":\"long\"}}},\"model_size\":{\"properties\":{\"memory_status\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}}}}}},\"cluster_uuid\":{\"path\":\"elasticsearch.cluster.id\",\"type\":\"alias\"},\"ecs\":{\"properties\":{\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"service\":{\"properties\":{\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"index_recovery\":{\"properties\":{\"shards\":{\"properties\":{\"stop_time_in_millis\":{\"path\":\"elasticsearch.index.recovery.stop_time.ms\",\"type\":\"alias\"},\"start_time_in_millis\":{\"path\":\"elasticsearch.index.recovery.start_time.ms\",\"type\":\"alias\"}}}}},\"source_node\":{\"properties\":{\"name\":{\"path\":\"elasticsearch.node.name\",\"type\":\"alias\"},\"uuid\":{\"path\":\"elasticsearch.node.id\",\"type\":\"alias\"}}},\"host\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"metricset\":{\"properties\":{\"period\":{\"type\":\"long\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"shard\":{\"properties\":{\"node\":{\"path\":\"elasticsearch.node.id\",\"type\":\"alias\"},\"index\":{\"path\":\"elasticsearch.index.name\",\"type\":\"alias\"},\"state\":{\"path\":\"elasticsearch.shard.state\",\"type\":\"alias\"},\"shard\":{\"path\":\"elasticsearch.shard.number\",\"type\":\"alias\"},\"primary\":{\"path\":\"elasticsearch.shard.primary\",\"type\":\"alias\"}}},\"event\":{\"properties\":{\"duration\":{\"type\":\"long\"},\"module\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"dataset\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"cluster_stats\":{\"properties\":{\"indices\":{\"properties\":{\"shards\":{\"properties\":{\"total\":{\"path\":\"elasticsearch.cluster.stats.indices.shards.count\",\"type\":\"alias\"}}},\"count\":{\"path\":\"elasticsearch.cluster.stats.indices.total\",\"type\":\"alias\"}}},\"nodes\":{\"properties\":{\"jvm\":{\"properties\":{\"max_uptime_in_millis\":{\"path\":\"elasticsearch.cluster.stats.nodes.jvm.max_uptime.ms\",\"type\":\"alias\"},\"mem\":{\"properties\":{\"heap_max_in_bytes\":{\"path\":\"elasticsearch.cluster.stats.nodes.jvm.memory.heap.max.bytes\",\"type\":\"alias\"},\"heap_used_in_bytes\":{\"path\":\"elasticsearch.cluster.stats.nodes.jvm.memory.heap.used.bytes\",\"type\":\"alias\"}}}}},\"count\":{\"properties\":{\"total\":{\"path\":\"elasticsearch.cluster.stats.nodes.count\",\"type\":\"alias\"}}}}}}},\"timestamp\":{\"path\":\"@timestamp\",\"type\":\"alias\"},\"job_stats\":{\"properties\":{\"job_id\":{\"path\":\"elasticsearch.ml.job.id\",\"type\":\"alias\"},\"forecasts_stats\":{\"properties\":{\"total\":{\"path\":\"elasticsearch.ml.job.forecasts_stats.total\",\"type\":\"alias\"}}}}}}}},\"composed_of\":[],\"version\":8000103,\"data_stream\":{\"hidden\":false,\"allow_custom_routing\":false}},\".slm-history\":{\"index_patterns\":[\".slm-history-5*\"],\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\"slm-history-ilm-policy\"},\"number_of_shards\":\"1\",\"auto_expand_replicas\":\"0-1\",\"number_of_replicas\":\"0\"}},\"mappings\":{\"dynamic\":false,\"properties\":{\"snapshot_name\":{\"type\":\"keyword\"},\"@timestamp\":{\"format\":\"epoch_millis\",\"type\":\"date\"},\"configuration\":{\"dynamic\":false,\"type\":\"object\",\"properties\":{\"indices\":{\"type\":\"keyword\"},\"include_global_state\":{\"type\":\"boolean\"},\"partial\":{\"type\":\"boolean\"}}},\"error_details\":{\"index\":false,\"type\":\"text\"},\"success\":{\"type\":\"boolean\"},\"repository\":{\"type\":\"keyword\"},\"operation\":{\"type\":\"keyword\"},\"policy\":{\"type\":\"keyword\"}}}},\"composed_of\":[],\"priority\":2147483647,\"version\":5,\"_meta\":{\"managed\":true,\"description\":\"index template for SLM history indices\"},\"data_stream\":{\"hidden\":true,\"allow_custom_routing\":false},\"allow_auto_create\":true},\".monitoring-logstash-mb\":{\"index_patterns\":[\".monitoring-logstash-8-*\"],\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\".monitoring-8-ilm-policy\"},\"mapping\":{\"total_fields\":{\"limit\":\"2000\"}}}},\"mappings\":{\"dynamic\":false,\"properties\":{\"logstash\":{\"properties\":{\"node\":{\"properties\":{\"jvm\":{\"properties\":{\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"stats\":{\"properties\":{\"jvm\":{\"properties\":{\"mem\":{\"properties\":{\"heap_max_in_bytes\":{\"type\":\"long\"},\"heap_used_in_bytes\":{\"type\":\"long\"}}},\"uptime_in_millis\":{\"type\":\"long\"}}},\"logstash\":{\"properties\":{\"uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"process\":{\"properties\":{\"cpu\":{\"properties\":{\"percent\":{\"type\":\"double\"}}}}},\"pipelines\":{\"type\":\"nested\",\"properties\":{\"vertices\":{\"type\":\"nested\",\"properties\":{\"events_out\":{\"type\":\"long\"},\"pipeline_ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"events_in\":{\"type\":\"long\"},\"queue_push_duration_in_millis\":{\"type\":\"long\"},\"duration_in_millis\":{\"type\":\"long\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"long_counters\":{\"type\":\"nested\",\"properties\":{\"name\":{\"type\":\"keyword\"},\"value\":{\"type\":\"long\"}}}}},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"type\":\"keyword\"},\"hash\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"queue\":{\"properties\":{\"events_count\":{\"type\":\"long\"},\"max_queue_size_in_bytes\":{\"type\":\"long\"},\"queue_size_in_bytes\":{\"type\":\"long\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"events\":{\"properties\":{\"filtered\":{\"type\":\"long\"},\"in\":{\"type\":\"long\"},\"queue_push_duration_in_millis\":{\"type\":\"long\"},\"duration_in_millis\":{\"type\":\"long\"},\"out\":{\"type\":\"long\"}}},\"reloads\":{\"properties\":{\"failures\":{\"type\":\"long\"},\"successes\":{\"type\":\"long\"}}}}},\"os\":{\"properties\":{\"cpu\":{\"properties\":{\"load_average\":{\"properties\":{\"5m\":{\"type\":\"half_float\"},\"15m\":{\"type\":\"half_float\"},\"1m\":{\"type\":\"half_float\"}}}}},\"cgroup\":{\"properties\":{\"cpu\":{\"properties\":{\"stat\":{\"properties\":{\"number_of_elapsed_periods\":{\"type\":\"long\"},\"number_of_times_throttled\":{\"type\":\"long\"},\"time_throttled_nanos\":{\"type\":\"long\"}}},\"cfs_quota_micros\":{\"type\":\"long\"}}},\"cpuacct\":{\"properties\":{\"usage_nanos\":{\"type\":\"long\"}}}}}}},\"events\":{\"properties\":{\"filtered\":{\"type\":\"long\"},\"in\":{\"type\":\"long\"},\"duration_in_millis\":{\"type\":\"long\"},\"out\":{\"type\":\"long\"}}},\"queue\":{\"properties\":{\"events_count\":{\"type\":\"long\"}}},\"timestamp\":{\"type\":\"date\"}}},\"host\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"properties\":{\"pipeline\":{\"properties\":{\"batch_size\":{\"type\":\"long\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"workers\":{\"type\":\"long\"},\"representation\":{\"properties\":{\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"hash\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"graph\":{\"properties\":{\"vertices\":{\"type\":\"object\"},\"edges\":{\"type\":\"object\"}}}}},\"hash\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"elasticsearch\":{\"properties\":{\"cluster\":{\"properties\":{\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}}}},\"logstash_state\":{\"properties\":{\"pipeline\":{\"properties\":{\"id\":{\"path\":\"logstash.node.state.pipeline.id\",\"type\":\"alias\"},\"hash\":{\"path\":\"logstash.node.state.pipeline.hash\",\"type\":\"alias\"}}}}},\"process\":{\"properties\":{\"pid\":{\"type\":\"long\"}}},\"@timestamp\":{\"type\":\"date\"},\"logstash_stats\":{\"properties\":{\"jvm\":{\"properties\":{\"mem\":{\"properties\":{\"heap_max_in_bytes\":{\"path\":\"logstash.node.stats.jvm.mem.heap_max_in_bytes\",\"type\":\"alias\"},\"heap_used_in_bytes\":{\"path\":\"logstash.node.stats.jvm.mem.heap_used_in_bytes\",\"type\":\"alias\"}}},\"uptime_in_millis\":{\"path\":\"logstash.node.stats.jvm.uptime_in_millis\",\"type\":\"alias\"}}},\"logstash\":{\"properties\":{\"uuid\":{\"path\":\"logstash.node.stats.logstash.uuid\",\"type\":\"alias\"},\"version\":{\"path\":\"logstash.node.stats.logstash.version\",\"type\":\"alias\"}}},\"process\":{\"properties\":{\"cpu\":{\"properties\":{\"percent\":{\"path\":\"logstash.node.stats.process.cpu.percent\",\"type\":\"alias\"}}}}},\"pipelines\":{\"type\":\"nested\"},\"os\":{\"properties\":{\"cpu\":{\"properties\":{\"load_average\":{\"properties\":{\"5m\":{\"path\":\"logstash.node.stats.os.cpu.load_average.5m\",\"type\":\"alias\"},\"15m\":{\"path\":\"logstash.node.stats.os.cpu.load_average.15m\",\"type\":\"alias\"},\"1m\":{\"path\":\"logstash.node.stats.os.cpu.load_average.1m\",\"type\":\"alias\"}}}}},\"cgroup\":{\"properties\":{\"cpu\":{\"properties\":{\"stat\":{\"properties\":{\"number_of_elapsed_periods\":{\"path\":\"logstash.node.stats.os.cgroup.cpu.stat.number_of_elapsed_periods\",\"type\":\"alias\"},\"number_of_times_throttled\":{\"path\":\"logstash.node.stats.os.cgroup.cpu.stat.number_of_times_throttled\",\"type\":\"alias\"},\"time_throttled_nanos\":{\"path\":\"logstash.node.stats.os.cgroup.cpu.stat.time_throttled_nanos\",\"type\":\"alias\"}}},\"cfs_quota_micros\":{\"path\":\"logstash.node.stats.os.cgroup.cpu.cfs_quota_micros\",\"type\":\"alias\"}}},\"cpuacct\":{\"properties\":{\"usage_nanos\":{\"path\":\"logstash.node.stats.os.cgroup.cpuacct.usage_nanos\",\"type\":\"alias\"}}}}}}},\"events\":{\"properties\":{\"in\":{\"path\":\"logstash.node.stats.events.in\",\"type\":\"alias\"},\"duration_in_millis\":{\"path\":\"logstash.node.stats.events.duration_in_millis\",\"type\":\"alias\"},\"out\":{\"path\":\"logstash.node.stats.events.out\",\"type\":\"alias\"}}},\"queue\":{\"properties\":{\"events_count\":{\"path\":\"logstash.node.stats.queue.events_count\",\"type\":\"alias\"}}},\"timestamp\":{\"path\":\"@timestamp\",\"type\":\"alias\"}}},\"ecs\":{\"properties\":{\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"cluster_uuid\":{\"path\":\"logstash.elasticsearch.cluster.id\",\"type\":\"alias\"},\"service\":{\"properties\":{\"hostname\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"host\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"metricset\":{\"properties\":{\"period\":{\"type\":\"long\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"event\":{\"properties\":{\"duration\":{\"type\":\"long\"},\"module\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"dataset\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"timestamp\":{\"path\":\"@timestamp\",\"type\":\"alias\"}}}},\"composed_of\":[],\"version\":8000103,\"data_stream\":{\"hidden\":false,\"allow_custom_routing\":false}},\".ml-anomalies-\":{\"index_patterns\":[\".ml-anomalies-*\"],\"template\":{\"settings\":{\"index\":{\"hidden\":\"true\",\"translog\":{\"durability\":\"async\"},\"auto_expand_replicas\":\"0-1\",\"query\":{\"default_field\":\"all_field_values\"}}},\"mappings\":{\"_meta\":{\"version\":\"8.6.2\"},\"dynamic_templates\":[{\"strings_as_keywords\":{\"mapping\":{\"type\":\"keyword\"},\"match\":\"*\"}}],\"properties\":{\"search_count\":{\"type\":\"long\"},\"bucket_count\":{\"type\":\"long\"},\"anomaly_score_explanation\":{\"properties\":{\"high_variance_penalty\":{\"type\":\"boolean\"},\"typical_value\":{\"type\":\"double\"},\"upper_confidence_bound\":{\"type\":\"double\"},\"lower_confidence_bound\":{\"type\":\"double\"},\"incomplete_bucket_penalty\":{\"type\":\"boolean\"},\"anomaly_type\":{\"type\":\"keyword\"},\"anomaly_characteristics_impact\":{\"type\":\"integer\"},\"multi_bucket_impact\":{\"type\":\"integer\"},\"anomaly_length\":{\"type\":\"integer\"},\"single_bucket_impact\":{\"type\":\"integer\"}}},\"terms\":{\"type\":\"text\"},\"record_score\":{\"type\":\"double\"},\"forecast_expiry_timestamp\":{\"type\":\"date\"},\"over_field_value\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"preferred_to_categories\":{\"type\":\"long\"},\"empty_bucket_count\":{\"type\":\"long\"},\"forecast_create_timestamp\":{\"type\":\"date\"},\"probability\":{\"type\":\"double\"},\"missing_field_count\":{\"type\":\"long\"},\"forecast_lower\":{\"type\":\"double\"},\"influencer_field_value\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"multi_bucket_impact\":{\"type\":\"double\"},\"forecast_progress\":{\"type\":\"double\"},\"max_matching_length\":{\"type\":\"long\"},\"latest_record_time_stamp\":{\"type\":\"date\"},\"regex\":{\"type\":\"keyword\"},\"examples\":{\"type\":\"text\"},\"snapshot_doc_count\":{\"type\":\"integer\"},\"average_bucket_processing_time_ms\":{\"type\":\"double\"},\"initial_anomaly_score\":{\"type\":\"double\"},\"initial_influencer_score\":{\"type\":\"double\"},\"partition_field_value\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"model_lower\":{\"type\":\"double\"},\"total_over_field_count\":{\"type\":\"long\"},\"retain\":{\"type\":\"boolean\"},\"forecast_upper\":{\"type\":\"double\"},\"latest_sparse_bucket_timestamp\":{\"type\":\"date\"},\"model_median\":{\"type\":\"double\"},\"category_id\":{\"type\":\"long\"},\"causes\":{\"type\":\"nested\",\"properties\":{\"actual\":{\"type\":\"double\"},\"partition_field_name\":{\"type\":\"keyword\"},\"partition_field_value\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"by_field_name\":{\"type\":\"keyword\"},\"probability\":{\"type\":\"double\"},\"geo_results\":{\"properties\":{\"actual_point\":{\"type\":\"geo_point\"},\"typical_point\":{\"type\":\"geo_point\"}}},\"field_name\":{\"type\":\"keyword\"},\"by_field_value\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"over_field_name\":{\"type\":\"keyword\"},\"correlated_by_field_value\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"function\":{\"type\":\"keyword\"},\"typical\":{\"type\":\"double\"},\"over_field_value\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"function_description\":{\"type\":\"keyword\"}}},\"all_field_values\":{\"analyzer\":\"whitespace\",\"type\":\"text\"},\"timestamp\":{\"type\":\"date\"},\"input_field_count\":{\"type\":\"long\"},\"model_bytes\":{\"type\":\"long\"},\"quantiles\":{\"type\":\"object\",\"enabled\":false},\"@timestamp\":{\"path\":\"timestamp\",\"type\":\"alias\"},\"function_description\":{\"type\":\"keyword\"},\"min_version\":{\"type\":\"keyword\"},\"raw_anomaly_score\":{\"type\":\"double\"},\"exponential_average_bucket_processing_time_ms\":{\"type\":\"double\"},\"invalid_date_count\":{\"type\":\"long\"},\"snapshot_id\":{\"type\":\"keyword\"},\"model_size_stats\":{\"properties\":{\"model_bytes\":{\"type\":\"long\"},\"result_type\":{\"type\":\"keyword\"},\"peak_model_bytes\":{\"type\":\"long\"},\"job_id\":{\"type\":\"keyword\"},\"total_over_field_count\":{\"type\":\"long\"},\"total_partition_field_count\":{\"type\":\"long\"},\"total_by_field_count\":{\"type\":\"long\"},\"assignment_memory_basis\":{\"type\":\"keyword\"},\"bucket_allocation_failures_count\":{\"type\":\"long\"},\"memory_status\":{\"type\":\"keyword\"},\"log_time\":{\"type\":\"date\"},\"timestamp\":{\"type\":\"date\"}}},\"total_search_time_ms\":{\"type\":\"double\"},\"latest_empty_bucket_timestamp\":{\"type\":\"date\"},\"anomaly_score\":{\"type\":\"double\"},\"over_field_name\":{\"type\":\"keyword\"},\"earliest_record_timestamp\":{\"type\":\"date\"},\"scheduled_events\":{\"type\":\"keyword\"},\"bucket_span\":{\"type\":\"long\"},\"maximum_bucket_processing_time_ms\":{\"type\":\"double\"},\"exponential_average_calculation_context\":{\"properties\":{\"incremental_metric_value_ms\":{\"type\":\"double\"},\"previous_exponential_average_ms\":{\"type\":\"double\"},\"latest_timestamp\":{\"type\":\"date\"}}},\"function\":{\"type\":\"keyword\"},\"influencers\":{\"type\":\"nested\",\"properties\":{\"influencer_field_name\":{\"type\":\"keyword\"},\"influencer_field_values\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"}}},\"input_record_count\":{\"type\":\"long\"},\"latest_result_time_stamp\":{\"type\":\"date\"},\"model_upper\":{\"type\":\"double\"},\"actual\":{\"type\":\"double\"},\"forecast_memory_bytes\":{\"type\":\"long\"},\"total_by_field_count\":{\"type\":\"long\"},\"geo_results\":{\"properties\":{\"actual_point\":{\"type\":\"geo_point\"},\"typical_point\":{\"type\":\"geo_point\"}}},\"detector_index\":{\"type\":\"integer\"},\"by_field_value\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"processed_record_count\":{\"type\":\"long\"},\"total_partition_field_count\":{\"type\":\"long\"},\"mlcategory\":{\"type\":\"keyword\"},\"assignment_memory_basis\":{\"type\":\"keyword\"},\"forecast_end_timestamp\":{\"type\":\"date\"},\"forecast_id\":{\"type\":\"keyword\"},\"partition_field_name\":{\"type\":\"keyword\"},\"by_field_name\":{\"type\":\"keyword\"},\"event_count\":{\"type\":\"long\"},\"description\":{\"type\":\"text\"},\"is_interim\":{\"type\":\"boolean\"},\"bucket_allocation_failures_count\":{\"type\":\"long\"},\"memory_status\":{\"type\":\"keyword\"},\"model_feature\":{\"type\":\"keyword\"},\"num_matches\":{\"type\":\"long\"},\"influencer_score\":{\"type\":\"double\"},\"out_of_order_timestamp_count\":{\"type\":\"long\"},\"result_type\":{\"type\":\"keyword\"},\"last_data_time\":{\"type\":\"date\"},\"latest_record_timestamp\":{\"type\":\"date\"},\"influencer_field_name\":{\"type\":\"keyword\"},\"forecast_messages\":{\"type\":\"keyword\"},\"sparse_bucket_count\":{\"type\":\"long\"},\"log_time\":{\"type\":\"date\"},\"field_name\":{\"type\":\"keyword\"},\"minimum_bucket_processing_time_ms\":{\"type\":\"double\"},\"bucket_influencers\":{\"type\":\"nested\",\"properties\":{\"anomaly_score\":{\"type\":\"double\"},\"initial_anomaly_score\":{\"type\":\"double\"},\"result_type\":{\"type\":\"keyword\"},\"raw_anomaly_score\":{\"type\":\"double\"},\"bucket_span\":{\"type\":\"long\"},\"job_id\":{\"type\":\"keyword\"},\"probability\":{\"type\":\"double\"},\"influencer_field_name\":{\"type\":\"keyword\"},\"is_interim\":{\"type\":\"boolean\"},\"timestamp\":{\"type\":\"date\"}}},\"processing_time_ms\":{\"type\":\"long\"},\"input_bytes\":{\"type\":\"long\"},\"initial_record_score\":{\"type\":\"double\"},\"job_id\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"processed_field_count\":{\"type\":\"long\"},\"forecast_status\":{\"type\":\"keyword\"},\"typical\":{\"type\":\"double\"},\"forecast_prediction\":{\"type\":\"double\"},\"forecast_start_timestamp\":{\"type\":\"date\"}}}},\"composed_of\":[],\"priority\":2147483647,\"version\":8060299,\"_meta\":{\"managed\":true,\"description\":\"index template for ML anomaly detection results indices\"}},\"metrics\":{\"index_patterns\":[\"metrics-*-*\"],\"composed_of\":[\"metrics-mappings\",\"data-streams-mappings\",\"metrics-settings\"],\"priority\":100,\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default metrics template installed by x-pack\"},\"data_stream\":{\"hidden\":false,\"allow_custom_routing\":false},\"allow_auto_create\":true},\".ml-notifications-000002\":{\"index_patterns\":[\".ml-notifications-000002\"],\"template\":{\"settings\":{\"index\":{\"hidden\":\"true\",\"number_of_shards\":\"1\",\"auto_expand_replicas\":\"0-1\"}},\"mappings\":{\"_meta\":{\"version\":\"8.6.2\"},\"dynamic\":\"false\",\"properties\":{\"job_type\":{\"type\":\"keyword\"},\"level\":{\"type\":\"keyword\"},\"job_id\":{\"type\":\"keyword\"},\"node_name\":{\"type\":\"keyword\"},\"message\":{\"type\":\"text\",\"fields\":{\"raw\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"cleared\":{\"type\":\"boolean\"},\"timestamp\":{\"type\":\"date\"}}}},\"composed_of\":[],\"priority\":2147483647,\"version\":8060299,\"_meta\":{\"managed\":true,\"description\":\"index template for ML notifications indices\"}},\".deprecation-indexing-template\":{\"index_patterns\":[\".logs-deprecation.*\"],\"composed_of\":[\".deprecation-indexing-mappings\",\".deprecation-indexing-settings\"],\"priority\":1000,\"version\":1,\"_meta\":{\"managed\":true,\"description\":\"default template for Stack deprecation logs index template installed by x-pack\"},\"data_stream\":{\"hidden\":true,\"allow_custom_routing\":false},\"allow_auto_create\":true},\"logs\":{\"index_patterns\":[\"logs-*-*\"],\"composed_of\":[\"logs-mappings\",\"data-streams-mappings\",\"logs-settings\"],\"priority\":100,\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default logs template installed by x-pack\"},\"data_stream\":{\"hidden\":false,\"allow_custom_routing\":false},\"allow_auto_create\":true},\".ml-stats\":{\"index_patterns\":[\".ml-stats-*\"],\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\"ml-size-based-ilm-policy\",\"rollover_alias\":\".ml-stats-write\"},\"hidden\":\"true\",\"number_of_shards\":\"1\",\"auto_expand_replicas\":\"0-1\"}},\"mappings\":{\"_meta\":{\"version\":\"8.6.2\"},\"dynamic\":false,\"properties\":{\"skipped_docs_count\":{\"type\":\"long\"},\"validation_loss\":{\"properties\":{\"fold_values\":{\"properties\":{\"fold\":{\"type\":\"integer\"},\"values\":{\"type\":\"double\"}}},\"loss_type\":{\"type\":\"keyword\"}}},\"cache_miss_count\":{\"type\":\"long\"},\"timing_stats\":{\"properties\":{\"iteration_time\":{\"type\":\"long\"},\"elapsed_time\":{\"type\":\"long\"}}},\"failure_count\":{\"type\":\"long\"},\"model_id\":{\"type\":\"keyword\"},\"type\":{\"type\":\"keyword\"},\"training_docs_count\":{\"type\":\"long\"},\"inference_count\":{\"type\":\"long\"},\"job_id\":{\"type\":\"keyword\"},\"missing_all_fields_count\":{\"type\":\"long\"},\"peak_usage_bytes\":{\"type\":\"long\"},\"iteration\":{\"type\":\"integer\"},\"hyperparameters\":{\"properties\":{\"max_attempts_to_add_tree\":{\"type\":\"integer\"},\"downsample_factor\":{\"type\":\"double\"},\"eta_growth_rate_per_tree\":{\"type\":\"double\"},\"soft_tree_depth_tolerance\":{\"type\":\"double\"},\"max_trees\":{\"type\":\"integer\"},\"lambda\":{\"type\":\"double\"},\"max_optimization_rounds_per_hyperparameter\":{\"type\":\"integer\"},\"eta\":{\"type\":\"double\"},\"soft_tree_depth_limit\":{\"type\":\"double\"},\"alpha\":{\"type\":\"double\"},\"class_assignment_objective\":{\"type\":\"keyword\"},\"feature_bag_fraction\":{\"type\":\"double\"},\"num_splits_per_feature\":{\"type\":\"integer\"},\"gamma\":{\"type\":\"double\"},\"num_folds\":{\"type\":\"integer\"}}},\"parameters\":{\"properties\":{\"compute_feature_influence\":{\"type\":\"boolean\"},\"feature_influence_threshold\":{\"type\":\"double\"},\"outlier_fraction\":{\"type\":\"double\"},\"method\":{\"type\":\"keyword\"},\"standardization_enabled\":{\"type\":\"boolean\"},\"n_neighbors\":{\"type\":\"integer\"}}},\"test_docs_count\":{\"type\":\"long\"},\"node_id\":{\"type\":\"keyword\"},\"timestamp\":{\"type\":\"date\"}}},\"aliases\":{}},\"composed_of\":[],\"priority\":2147483647,\"version\":8060299,\"_meta\":{\"managed\":true,\"description\":\"index template for ML stats indices\"}}}} index-graveyard: IndexGraveyard[[[index=[abacus_go/zV2aNzxVQxqMDQTD4A9TEQ], deleteDate=2023-03-28T03:56:57.282Z], [index=[access_platform_core/ZuN0_cUfSQKzXg_yG4Sv5g], deleteDate=2023-03-28T03:58:27.958Z], [index=[ad_bidder/6ihfVU6rRqOo4Gv4Djtetg], deleteDate=2023-03-28T03:59:11.628Z], [index=[apphealth/EthxrkUtTGCTP2mX-tS63Q], deleteDate=2023-03-28T03:59:36.596Z]]]\nnodes: \n {hostb9}{iKPGCkp9RVOKXOj20uOt4g}{cPRh9FqiSlGP9jClhm8Ugg}{hostb9}{127.0.0.1}{127.0.0.1:9300}{cdfhilmrstw}{ml.max_jvm_size=33285996544, ml.allocated_processors_double=16.0, xpack.installed=true, ml.machine_memory=135062048768, ml.allocated_processors=16}, local, master\nrouting_table (version 35):\n-- index [[abacus_go/cfvmY-Y_Su63-qSkPODDzg]]\n----shard_id [abacus_go][0]\n--------[abacus_go][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=AMhgJMlSTPOoKSsL7s3OKA], failed_attempts[0]\n--------[abacus_go][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=INDEX_CREATED], at[2023-03-28T03:57:31.683Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[web_wok/R1IULAXSQ4SC0pLFCU5oJA]]\n----shard_id [web_wok][0]\n--------[web_wok][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=JXruVMhdRVGvP1kQCbUNSQ], failed_attempts[0]\n--------[web_wok][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[access_platform_core/l4kdkOEQRaKfWGLpTZURCQ]]\n----shard_id [access_platform_core][0]\n--------[access_platform_core][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=6ajNSTQZSr6tP3YtmRaxgw], failed_attempts[0]\n--------[access_platform_core][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=INDEX_CREATED], at[2023-03-28T03:58:31.318Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[driverapp/cqbade6KQFqLEK-wPKlkBg]]\n----shard_id [driverapp][0]\n--------[driverapp][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=b5_Bhd3ITjOkl-xLF8H1jg], failed_attempts[0]\n--------[driverapp][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[k8s_kubelet/fQ9Iy7wlTKahsWLbs-oE_Q]]\n----shard_id [k8s_kubelet][0]\n--------[k8s_kubelet][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=Q4Xb11JATPaFZqmgTQHcVQ], failed_attempts[0]\n--------[k8s_kubelet][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.307Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[eats_store/hF-kN4XFRy--z_gUhYLwdA]]\n----shard_id [eats_store][0]\n--------[eats_store][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=WRNwRQ6gTNGfSNQok4iSyQ], failed_attempts[0]\n--------[eats_store][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[rider_product_cored/AdGiEBLBQbicsXDomsa3mw]]\n----shard_id [rider_product_cored][0]\n--------[rider_product_cored][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=GwRH_ZlFRtCV7m_n626z7g], failed_attempts[0]\n--------[rider_product_cored][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[finprod_rewards_eligibility/dJNRGPwrQS2bIPlpsXgUoQ]]\n----shard_id [finprod_rewards_eligibility][0]\n--------[finprod_rewards_eligibility][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=Hg4Dm9O8QEqXWKY7rH1X0A], failed_attempts[0]\n--------[finprod_rewards_eligibility][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[event_logs/DcomuHoGQzOjZq2bX67TFg]]\n----shard_id [event_logs][0]\n--------[event_logs][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=1uCl_LIDSBCWh6koVA0fjQ], failed_attempts[0]\n--------[event_logs][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[apphealth/rcPrX1aWRmeED9W5PU6A4g]]\n----shard_id [apphealth][0]\n--------[apphealth][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=Abip50RxQTmavqi4NTwSqQ], failed_attempts[0]\n--------[apphealth][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=INDEX_CREATED], at[2023-03-28T03:59:38.780Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[ad_bidder/tLHfw5r2QH-o7lG5tapKxw]]\n----shard_id [ad_bidder][0]\n--------[ad_bidder][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=YtEUwPgfQDGM0Cm_C4hXlw], failed_attempts[0]\n--------[ad_bidder][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=INDEX_CREATED], at[2023-03-28T03:59:13.333Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[invoice_gen/RPu5EzJVRfuqyI8MQ7iTjQ]]\n----shard_id [invoice_gen][0]\n--------[invoice_gen][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=onWbjhHuTD2C3AEayRJq1w], failed_attempts[0]\n--------[invoice_gen][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[fares_management/F1lH-QvyTkS5hnbDFwE2Xg]]\n----shard_id [fares_management][0]\n--------[fares_management][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=sOxfKX3nReynYN5HN88cuQ], failed_attempts[0]\n--------[fares_management][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[michelangelo_gateway/ZV2vZ8PmS0uysg-xzKO7QA]]\n----shard_id [michelangelo_gateway][0]\n--------[michelangelo_gateway][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=k5vDE54CSTSakhfgtGaTcA], failed_attempts[0]\n--------[michelangelo_gateway][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[.geoip_databases/5k1GHwKvQBKersi12DFh4A]]\n----shard_id [.geoip_databases][0]\n--------[.geoip_databases][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=UdvCKUGqTEmYFzZlnSu_tA], failed_attempts[0]\n\n-- index [[fares_intelligence/pSRJWxyXSqefhwLVhmT9nQ]]\n----shard_id [fares_intelligence][0]\n--------[fares_intelligence][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=-dD8ROsgS6qaYMoalciocA], failed_attempts[0]\n--------[fares_intelligence][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[fare_route/a4fjivWhQRee8JW7ZbXiXw]]\n----shard_id [fare_route][0]\n--------[fare_route][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=ltSLFgu4Rry0_0vXq28DbA], failed_attempts[0]\n--------[fare_route][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[k8s_apiserver/qQasynZgQ6e6AmD5ZWqh3w]]\n----shard_id [k8s_apiserver][0]\n--------[k8s_apiserver][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=Aaa0F0jiStCkteGDYR8yRQ], failed_attempts[0]\n--------[k8s_apiserver][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[fulfillment_compatibled/tMgrGBWeR3irgR5PyN8Oig]]\n----shard_id [fulfillment_compatibled][0]\n--------[fulfillment_compatibled][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=D_3C_3kMSNKa-XJk7pyoGg], failed_attempts[0]\n--------[fulfillment_compatibled][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\nrouting_nodes:\n-----node_id[iKPGCkp9RVOKXOj20uOt4g][V]\n--------[abacus_go][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=AMhgJMlSTPOoKSsL7s3OKA], failed_attempts[0]\n--------[web_wok][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=JXruVMhdRVGvP1kQCbUNSQ], failed_attempts[0]\n--------[access_platform_core][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=6ajNSTQZSr6tP3YtmRaxgw], failed_attempts[0]\n--------[driverapp][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=b5_Bhd3ITjOkl-xLF8H1jg], failed_attempts[0]\n--------[k8s_kubelet][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=Q4Xb11JATPaFZqmgTQHcVQ], failed_attempts[0]\n--------[eats_store][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=WRNwRQ6gTNGfSNQok4iSyQ], failed_attempts[0]\n--------[rider_product_cored][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=GwRH_ZlFRtCV7m_n626z7g], failed_attempts[0]\n--------[finprod_rewards_eligibility][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=Hg4Dm9O8QEqXWKY7rH1X0A], failed_attempts[0]\n--------[event_logs][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=1uCl_LIDSBCWh6koVA0fjQ], failed_attempts[0]\n--------[apphealth][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=Abip50RxQTmavqi4NTwSqQ], failed_attempts[0]\n--------[ad_bidder][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=YtEUwPgfQDGM0Cm_C4hXlw], failed_attempts[0]\n--------[invoice_gen][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=onWbjhHuTD2C3AEayRJq1w], failed_attempts[0]\n--------[fares_management][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=sOxfKX3nReynYN5HN88cuQ], failed_attempts[0]\n--------[michelangelo_gateway][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=k5vDE54CSTSakhfgtGaTcA], failed_attempts[0]\n--------[.geoip_databases][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=UdvCKUGqTEmYFzZlnSu_tA], failed_attempts[0]\n--------[fares_intelligence][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=-dD8ROsgS6qaYMoalciocA], failed_attempts[0]\n--------[fare_route][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=ltSLFgu4Rry0_0vXq28DbA], failed_attempts[0]\n--------[k8s_apiserver][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=Aaa0F0jiStCkteGDYR8yRQ], failed_attempts[0]\n--------[fulfillment_compatibled][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=D_3C_3kMSNKa-XJk7pyoGg], failed_attempts[0]\n---- unassigned\n--------[abacus_go][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=INDEX_CREATED], at[2023-03-28T03:57:31.683Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[web_wok][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[access_platform_core][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=INDEX_CREATED], at[2023-03-28T03:58:31.318Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[driverapp][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[k8s_kubelet][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.307Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[eats_store][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[rider_product_cored][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[finprod_rewards_eligibility][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[event_logs][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[apphealth][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=INDEX_CREATED], at[2023-03-28T03:59:38.780Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[ad_bidder][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=INDEX_CREATED], at[2023-03-28T03:59:13.333Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[invoice_gen][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[fares_management][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[michelangelo_gateway][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[fares_intelligence][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[fare_route][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[k8s_apiserver][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[fulfillment_compatibled][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\ncustoms:\n health: org.elasticsearch.health.metadata.HealthMetadata@66bc0723", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.service.MasterService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:02.058Z", "log.level":"DEBUG", "message":"publishing cluster state version [154]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.service.MasterService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:02.059Z", "log.level":"TRACE", "message":"handleClientValue: processing request for version [154] and term [6]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.coordination.CoordinationState","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:02.059Z", "log.level":"TRACE", "message":"setCurrentNodes: [{hostb9}{iKPGCkp9RVOKXOj20uOt4g}{cPRh9FqiSlGP9jClhm8Ugg}{hostb9}{127.0.0.1}{127.0.0.1:9300}{cdfhilmrstw}{ml.max_jvm_size=33285996544, ml.allocated_processors_double=16.0, xpack.installed=true, ml.machine_memory=135062048768, ml.allocated_processors=16}]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.coordination.LeaderChecker","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:02.064Z", "log.level":"TRACE", "message":"publishing PublishRequest{term=6, version=154, state=cluster uuid: bp-xkJD1S1iyBlj6I_JRHA [committed: true]\nversion: 154\nstate uuid: Xx549IAcSMCY30xGiugtFg\nfrom_diff: false\nmeta data version: 56\n coordination_metadata:\n term: 6\n last_committed_config: VotingConfiguration{iKPGCkp9RVOKXOj20uOt4g}\n last_accepted_config: VotingConfiguration{iKPGCkp9RVOKXOj20uOt4g}\n voting tombstones: []\n [k8s_kubelet/fQ9Iy7wlTKahsWLbs-oE_Q]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [Q4Xb11JATPaFZqmgTQHcVQ]\n [fares_intelligence/pSRJWxyXSqefhwLVhmT9nQ]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [-dD8ROsgS6qaYMoalciocA]\n [event_logs/DcomuHoGQzOjZq2bX67TFg]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [1uCl_LIDSBCWh6koVA0fjQ]\n [rider_product_cored/AdGiEBLBQbicsXDomsa3mw]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [GwRH_ZlFRtCV7m_n626z7g]\n [access_platform_core/l4kdkOEQRaKfWGLpTZURCQ]: v[6], mv[1], sv[2], av[1]\n 0: p_term [1], isa_ids [6ajNSTQZSr6tP3YtmRaxgw]\n [.geoip_databases/5k1GHwKvQBKersi12DFh4A]: v[6], mv[1], sv[1], av[1]\n 0: p_term [3], isa_ids [UdvCKUGqTEmYFzZlnSu_tA]\n [web_wok/R1IULAXSQ4SC0pLFCU5oJA]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [JXruVMhdRVGvP1kQCbUNSQ]\n [abacus_go/cfvmY-Y_Su63-qSkPODDzg]: v[6], mv[1], sv[2], av[1]\n 0: p_term [1], isa_ids [AMhgJMlSTPOoKSsL7s3OKA]\n [k8s_apiserver/qQasynZgQ6e6AmD5ZWqh3w]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [Aaa0F0jiStCkteGDYR8yRQ]\n [michelangelo_gateway/ZV2vZ8PmS0uysg-xzKO7QA]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [k5vDE54CSTSakhfgtGaTcA]\n [driverapp/cqbade6KQFqLEK-wPKlkBg]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [b5_Bhd3ITjOkl-xLF8H1jg]\n [apphealth/rcPrX1aWRmeED9W5PU6A4g]: v[4], mv[1], sv[1], av[1]\n 0: p_term [1], isa_ids [Abip50RxQTmavqi4NTwSqQ]\n [fares_management/F1lH-QvyTkS5hnbDFwE2Xg]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [sOxfKX3nReynYN5HN88cuQ]\n [finprod_rewards_eligibility/dJNRGPwrQS2bIPlpsXgUoQ]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [Hg4Dm9O8QEqXWKY7rH1X0A]\n [fare_route/a4fjivWhQRee8JW7ZbXiXw]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [ltSLFgu4Rry0_0vXq28DbA]\n [ad_bidder/tLHfw5r2QH-o7lG5tapKxw]: v[6], mv[1], sv[2], av[1]\n 0: p_term [1], isa_ids [YtEUwPgfQDGM0Cm_C4hXlw]\n [fulfillment_compatibled/tMgrGBWeR3irgR5PyN8Oig]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [D_3C_3kMSNKa-XJk7pyoGg]\n [eats_store/hF-kN4XFRy--z_gUhYLwdA]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [WRNwRQ6gTNGfSNQok4iSyQ]\n [invoice_gen/RPu5EzJVRfuqyI8MQ7iTjQ]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [onWbjhHuTD2C3AEayRJq1w]\nmetadata customs:\n licenses: LicensesMetadata{license={\"uid\":\"8e65ab16-3f3f-48cc-ab5f-e20097ceb81e\",\"type\":\"basic\",\"issue_date_in_millis\":1679968689560,\"max_nodes\":1000,\"max_resource_units\":null,\"issued_to\":\"elasticsearch\",\"issuer\":\"elasticsearch\",\"signature\":\"////+wAAAOBYLHp18OvjZrRTi0AmCZ6DOpkrd2cXJB/ffBB+AxaDsF+z7lkfdPbgYofMvUB7XbiD5j95mIfUcdlTJHVdk9NpPTFMKL5wC7BXn4CTS6kNox+Xi0+qSLpcgHzelOT63alJr1L5j/2pqffHnuMLG88Ye15eFhAb9H2A5MS3hX0HG9uTxyEidFCyu4gWV4qrZJDQKLpzC49MMnpP+ob2bNH/pA7dGVdWqEJpVNPe5/hAtGtCdVdUX6DvV0Q3dgwuejc0hRg/vl0YUtp45bQPBHDHECNKgxWmJCZrfstl/prRHA==\",\"start_date_in_millis\":-1}, trialVersion=null} component_template: {\"component_template\":{\"metrics-mappings\":{\"template\":{\"mappings\":{\"dynamic_templates\":[{\"long_metrics\":{\"mapping\":{\"index\":false,\"type\":\"long\"},\"match_mapping_type\":\"long\"}},{\"double_metrics\":{\"mapping\":{\"index\":false,\"type\":\"float\"},\"match_mapping_type\":\"double\"}}],\"properties\":{\"data_stream\":{\"properties\":{\"type\":{\"type\":\"constant_keyword\",\"value\":\"metrics\"}}}}}},\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default mappings for the metrics index template installed by x-pack\"}},\".deprecation-indexing-mappings\":{\"template\":{\"mappings\":{\"dynamic_templates\":[{\"strings_as_keyword\":{\"mapping\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"match_mapping_type\":\"string\"}}],\"date_detection\":false,\"properties\":{\"@timestamp\":{\"type\":\"date\"},\"ecs\":{\"properties\":{\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"data_stream\":{\"properties\":{\"namespace\":{\"type\":\"constant_keyword\"},\"type\":{\"type\":\"constant_keyword\",\"value\":\"logs\"},\"dataset\":{\"type\":\"constant_keyword\"}}},\"host\":{\"properties\":{\"ip\":{\"type\":\"ip\"}}},\"message\":{\"type\":\"text\"}}}},\"version\":1,\"_meta\":{\"managed\":true,\"description\":\"default mappings for Stack deprecation logs index template installed by x-pack\"}},\"synthetics-mappings\":{\"template\":{\"mappings\":{\"properties\":{\"data_stream\":{\"properties\":{\"type\":{\"type\":\"constant_keyword\",\"value\":\"synthetics\"}}}}}},\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default mappings for the synthetics index template installed by x-pack\"}},\"metrics-settings\":{\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\"metrics\"},\"codec\":\"best_compression\",\"query\":{\"default_field\":[\"message\"]}}}},\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default settings for the metrics index template installed by x-pack\"}},\"logs-settings\":{\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\"logs\"},\"codec\":\"best_compression\",\"query\":{\"default_field\":[\"message\"]}}}},\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default settings for the logs index template installed by x-pack\"}},\".deprecation-indexing-settings\":{\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\".deprecation-indexing-ilm-policy\"},\"codec\":\"best_compression\",\"hidden\":\"true\",\"auto_expand_replicas\":\"0-1\",\"query\":{\"default_field\":[\"message\"]}}}},\"version\":1,\"_meta\":{\"managed\":true,\"description\":\"default settings for Stack deprecation logs index template installed by x-pack\"}},\"synthetics-settings\":{\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\"synthetics\"},\"codec\":\"best_compression\"}}},\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default settings for the synthetics index template installed by x-pack\"}},\"logs-mappings\":{\"template\":{\"mappings\":{\"properties\":{\"data_stream\":{\"properties\":{\"type\":{\"type\":\"constant_keyword\",\"value\":\"logs\"}}}}}},\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default mappings for the logs index template installed by x-pack\"}},\"data-streams-mappings\":{\"template\":{\"mappings\":{\"dynamic_templates\":[{\"match_ip\":{\"mapping\":{\"type\":\"ip\"},\"match_mapping_type\":\"string\",\"match\":\"ip\"}},{\"match_message\":{\"mapping\":{\"type\":\"match_only_text\"},\"match_mapping_type\":\"string\",\"match\":\"message\"}},{\"strings_as_keyword\":{\"mapping\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"match_mapping_type\":\"string\"}}],\"date_detection\":false,\"properties\":{\"@timestamp\":{\"type\":\"date\"},\"ecs\":{\"properties\":{\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"data_stream\":{\"properties\":{\"namespace\":{\"type\":\"constant_keyword\"},\"dataset\":{\"type\":\"constant_keyword\"}}},\"host\":{\"type\":\"object\"}}}},\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"general mapping conventions for data streams\"}}}} persistent_tasks: {\"last_allocation_id\":6,\"tasks\":[{\"id\":\"health-node\",\"task\":{\"health-node\":{\"params\":{}}},\"allocation_id\":5,\"assignment\":{\"executor_node\":\"iKPGCkp9RVOKXOj20uOt4g\",\"explanation\":\"\"},\"allocation_id_on_last_status_update\":0},{\"id\":\"geoip-downloader\",\"task\":{\"geoip-downloader\":{\"params\":{},\"state\":{\"databases\":{\"GeoLite2-ASN.mmdb\":{\"last_update\":1679968651799,\"last_check\":1679968651799,\"first_chunk\":0,\"last_chunk\":4,\"md5\":\"33268a0d7cb3ba8d9d581bf1e5dc0980\"},\"GeoLite2-Country.mmdb\":{\"last_update\":1679968809323,\"last_check\":1679968809323,\"first_chunk\":0,\"last_chunk\":2,\"md5\":\"da6d05a7b044ea35910f20f756c057ad\"},\"GeoLite2-City.mmdb\":{\"last_update\":1679968693385,\"last_check\":1679968693385,\"first_chunk\":0,\"last_chunk\":33,\"md5\":\"2d91f3e42689a12c2f496199befdfccf\"}}}}},\"allocation_id\":6,\"assignment\":{\"executor_node\":\"iKPGCkp9RVOKXOj20uOt4g\",\"explanation\":\"\"},\"allocation_id_on_last_status_update\":0}]} index_lifecycle: {\n \"policies\" : {\n \"90-days-default\" : {\n \"policy\" : {\n \"phases\" : {\n \"warm\" : {\n \"min_age\" : \"2d\",\n \"actions\" : {\n \"shrink\" : {\n \"number_of_shards\" : 1\n },\n \"forcemerge\" : {\n \"max_num_segments\" : 1\n }\n }\n },\n \"cold\" : {\n \"min_age\" : \"30d\",\n \"actions\" : { }\n },\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"90d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"built-in ILM policy using the hot, warm, and cold phases with a retention of 90 days\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968678193,\n \"modified_date_string\" : \"2023-03-28T01:57:58.193Z\"\n },\n \"ilm-history-ilm-policy\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"90d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"default policy for the ILM history indices\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968685095,\n \"modified_date_string\" : \"2023-03-28T01:58:05.095Z\"\n },\n \"watch-history-ilm-policy-16\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"3d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"4d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"default policy for the watcher history indices\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968647327,\n \"modified_date_string\" : \"2023-03-28T01:57:27.327Z\"\n },\n \"30-days-default\" : {\n \"policy\" : {\n \"phases\" : {\n \"warm\" : {\n \"min_age\" : \"2d\",\n \"actions\" : {\n \"shrink\" : {\n \"number_of_shards\" : 1\n },\n \"forcemerge\" : {\n \"max_num_segments\" : 1\n }\n }\n },\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"30d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"built-in ILM policy using the hot and warm phases with a retention of 30 days\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968662496,\n \"modified_date_string\" : \"2023-03-28T01:57:42.496Z\"\n },\n \"synthetics\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"default policy for the synthetics index template installed by x-pack\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968682454,\n \"modified_date_string\" : \"2023-03-28T01:58:02.454Z\"\n },\n \".monitoring-8-ilm-policy\" : {\n \"policy\" : {\n \"phases\" : {\n \"warm\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"forcemerge\" : {\n \"max_num_segments\" : 1\n }\n }\n },\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"3d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"3d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"defaults\" : {\n \"delete_min_age\" : \"Using value of [3d] based on the monitoring plugin default\"\n },\n \"description\" : \"Index lifecycle policy generated for [monitoring-*-8] data streams\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968676492,\n \"modified_date_string\" : \"2023-03-28T01:57:56.492Z\"\n },\n \"slm-history-ilm-policy\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"90d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"default policy for the SLM history indices\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968644040,\n \"modified_date_string\" : \"2023-03-28T01:57:24.040Z\"\n },\n \".fleet-actions-results-ilm-policy\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_size\" : \"300gb\",\n \"max_age\" : \"30d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"90d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"default policy for fleet action results indices\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968686060,\n \"modified_date_string\" : \"2023-03-28T01:58:06.060Z\"\n },\n \"365-days-default\" : {\n \"policy\" : {\n \"phases\" : {\n \"warm\" : {\n \"min_age\" : \"2d\",\n \"actions\" : {\n \"shrink\" : {\n \"number_of_shards\" : 1\n },\n \"forcemerge\" : {\n \"max_num_segments\" : 1\n }\n }\n },\n \"cold\" : {\n \"min_age\" : \"30d\",\n \"actions\" : { }\n },\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"365d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"built-in ILM policy using the hot, warm, and cold phases with a retention of 365 days\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968675542,\n \"modified_date_string\" : \"2023-03-28T01:57:55.542Z\"\n },\n \"7-days-default\" : {\n \"policy\" : {\n \"phases\" : {\n \"warm\" : {\n \"min_age\" : \"2d\",\n \"actions\" : {\n \"shrink\" : {\n \"number_of_shards\" : 1\n },\n \"forcemerge\" : {\n \"max_num_segments\" : 1\n }\n }\n },\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"7d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"7d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"built-in ILM policy using the hot and warm phases with a retention of 7 days\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968672404,\n \"modified_date_string\" : \"2023-03-28T01:57:52.404Z\"\n },\n \".deprecation-indexing-ilm-policy\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"10gb\",\n \"max_age\" : \"30d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"30d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"ILM policy used for deprecation logs\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968670134,\n \"modified_date_string\" : \"2023-03-28T01:57:50.134Z\"\n },\n \"metrics\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"default policy for the metrics index template installed by x-pack\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968660707,\n \"modified_date_string\" : \"2023-03-28T01:57:40.707Z\"\n },\n \"ml-size-based-ilm-policy\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\"\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"default policy for machine learning state and stats indices\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968650499,\n \"modified_date_string\" : \"2023-03-28T01:57:30.499Z\"\n },\n \"180-days-default\" : {\n \"policy\" : {\n \"phases\" : {\n \"warm\" : {\n \"min_age\" : \"2d\",\n \"actions\" : {\n \"shrink\" : {\n \"number_of_shards\" : 1\n },\n \"forcemerge\" : {\n \"max_num_segments\" : 1\n }\n }\n },\n \"cold\" : {\n \"min_age\" : \"30d\",\n \"actions\" : { }\n },\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"180d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"built-in ILM policy using the hot, warm, and cold phases with a retention of 180 days\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968681386,\n \"modified_date_string\" : \"2023-03-28T01:58:01.386Z\"\n },\n \"logs\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"default policy for the logs index template installed by x-pack\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968679651,\n \"modified_date_string\" : \"2023-03-28T01:57:59.651Z\"\n }\n },\n \"operation_mode\" : \"RUNNING\"\n} index_template: {\"index_template\":{\".watch-history-16\":{\"index_patterns\":[\".watcher-history-16*\"],\"template\":{\"settings\":{\"index\":{\"format\":\"6\",\"lifecycle\":{\"name\":\"watch-history-ilm-policy-16\"},\"hidden\":\"true\",\"number_of_shards\":\"1\",\"auto_expand_replicas\":\"0-1\",\"number_of_replicas\":\"0\"}},\"mappings\":{\"_meta\":{\"watcher-history-version\":\"16\"},\"dynamic\":false,\"dynamic_templates\":[{\"disabled_payload_fields\":{\"match_pattern\":\"regex\",\"path_match\":\"result\\\\.(input(\\\\..+)*|(transform(\\\\..+)*)|(actions\\\\.transform(\\\\..+)*))\\\\.payload\",\"mapping\":{\"type\":\"object\",\"enabled\":false}}},{\"disabled_search_request_body_fields\":{\"match_pattern\":\"regex\",\"path_match\":\"result\\\\.(input(\\\\..+)*|(transform(\\\\..+)*)|(actions\\\\.transform(\\\\..+)*))\\\\.search\\\\.request\\\\.(body|template)\",\"mapping\":{\"type\":\"object\",\"enabled\":false}}},{\"disabled_exception_fields\":{\"match_pattern\":\"regex\",\"path_match\":\"result\\\\.(input(\\\\..+)*|(transform(\\\\..+)*)|(actions\\\\.transform(\\\\..+)*)|actions)\\\\.error\",\"mapping\":{\"type\":\"object\",\"enabled\":false}}},{\"disabled_jira_custom_fields\":{\"path_match\":\"result.actions.jira.fields.customfield_*\",\"mapping\":{\"type\":\"object\",\"enabled\":false}}}],\"properties\":{\"exception\":{\"type\":\"object\",\"enabled\":false},\"metadata\":{\"dynamic\":true,\"type\":\"object\"},\"trigger_event\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"schedule\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"scheduled_time\":{\"type\":\"date\"}}},\"triggered_time\":{\"type\":\"date\"},\"type\":{\"type\":\"keyword\"},\"manual\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"schedule\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"scheduled_time\":{\"type\":\"date\"}}}}}}},\"result\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"input\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"search\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"request\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"indices\":{\"type\":\"keyword\"},\"types\":{\"type\":\"keyword\"},\"search_type\":{\"type\":\"keyword\"}}}}},\"payload\":{\"type\":\"object\",\"enabled\":false},\"http\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"request\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"path\":{\"type\":\"keyword\"},\"host\":{\"type\":\"keyword\"}}}}},\"type\":{\"type\":\"keyword\"},\"status\":{\"type\":\"keyword\"}}},\"condition\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"compare\":{\"type\":\"object\",\"enabled\":false},\"array_compare\":{\"type\":\"object\",\"enabled\":false},\"type\":{\"type\":\"keyword\"},\"met\":{\"type\":\"boolean\"},\"script\":{\"type\":\"object\",\"enabled\":false},\"status\":{\"type\":\"keyword\"}}},\"transform\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"search\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"request\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"indices\":{\"type\":\"keyword\"},\"types\":{\"type\":\"keyword\"}}}}},\"type\":{\"type\":\"keyword\"}}},\"execution_duration\":{\"type\":\"long\"},\"actions\":{\"include_in_parent\":true,\"dynamic\":true,\"type\":\"nested\",\"properties\":{\"reason\":{\"type\":\"keyword\"},\"foreach\":{\"type\":\"object\",\"enabled\":false},\"webhook\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"request\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"path\":{\"type\":\"keyword\"},\"host\":{\"type\":\"keyword\"}}}}},\"number_of_actions_executed\":{\"type\":\"integer\"},\"slack\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"sent_messages\":{\"include_in_parent\":true,\"dynamic\":true,\"type\":\"nested\",\"properties\":{\"reason\":{\"type\":\"text\"},\"request\":{\"type\":\"object\",\"enabled\":false},\"response\":{\"type\":\"object\",\"enabled\":false},\"to\":{\"type\":\"keyword\"},\"message\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"attachments\":{\"include_in_parent\":true,\"dynamic\":true,\"type\":\"nested\",\"properties\":{\"color\":{\"type\":\"keyword\"},\"fields\":{\"properties\":{\"value\":{\"type\":\"text\"}}}}},\"icon\":{\"type\":\"keyword\"},\"from\":{\"type\":\"text\"},\"text\":{\"type\":\"text\"}}},\"status\":{\"type\":\"keyword\"}}},\"account\":{\"type\":\"keyword\"}}},\"index\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"response\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"index\":{\"type\":\"keyword\"},\"id\":{\"type\":\"keyword\"},\"type\":{\"type\":\"keyword\"}}}}},\"pagerduty\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"sent_event\":{\"include_in_parent\":true,\"dynamic\":true,\"type\":\"nested\",\"properties\":{\"reason\":{\"type\":\"text\"},\"request\":{\"type\":\"object\",\"enabled\":false},\"response\":{\"type\":\"object\",\"enabled\":false},\"event\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"client_url\":{\"type\":\"keyword\"},\"context\":{\"include_in_parent\":true,\"dynamic\":true,\"type\":\"nested\",\"properties\":{\"src\":{\"type\":\"keyword\"},\"alt\":{\"type\":\"text\"},\"href\":{\"type\":\"keyword\"},\"type\":{\"type\":\"keyword\"}}},\"client\":{\"type\":\"text\"},\"description\":{\"type\":\"text\"},\"attach_payload\":{\"type\":\"boolean\"},\"incident_key\":{\"type\":\"keyword\"},\"type\":{\"type\":\"keyword\"},\"account\":{\"type\":\"keyword\"}}}}},\"account\":{\"type\":\"keyword\"}}},\"id\":{\"type\":\"keyword\"},\"type\":{\"type\":\"keyword\"},\"email\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"message\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"cc\":{\"type\":\"keyword\"},\"bcc\":{\"type\":\"keyword\"},\"reply_to\":{\"type\":\"keyword\"},\"from\":{\"type\":\"keyword\"},\"id\":{\"type\":\"keyword\"},\"to\":{\"type\":\"keyword\"}}}}},\"status\":{\"type\":\"keyword\"},\"jira\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"result\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"self\":{\"type\":\"keyword\"},\"id\":{\"type\":\"keyword\"},\"key\":{\"type\":\"keyword\"}}},\"reason\":{\"type\":\"text\"},\"request\":{\"type\":\"object\",\"enabled\":false},\"response\":{\"type\":\"object\",\"enabled\":false},\"fields\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"summary\":{\"type\":\"text\"},\"issuetype\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"name\":{\"type\":\"keyword\"},\"id\":{\"type\":\"keyword\"}}},\"description\":{\"type\":\"text\"},\"project\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"id\":{\"type\":\"keyword\"},\"key\":{\"type\":\"keyword\"}}},\"labels\":{\"type\":\"text\"}}},\"account\":{\"type\":\"keyword\"}}}}},\"execution_time\":{\"type\":\"date\"}}},\"node\":{\"type\":\"keyword\"},\"input\":{\"type\":\"object\",\"enabled\":false},\"condition\":{\"type\":\"object\",\"enabled\":false},\"watch_id\":{\"type\":\"keyword\"},\"messages\":{\"type\":\"text\"},\"vars\":{\"type\":\"object\",\"enabled\":false},\"state\":{\"type\":\"keyword\"},\"user\":{\"type\":\"text\"},\"status\":{\"dynamic\":true,\"type\":\"object\",\"enabled\":false}}}},\"composed_of\":[],\"priority\":2147483647,\"version\":16,\"_meta\":{\"managed\":true,\"description\":\"index template for watcher history indices\"},\"data_stream\":{\"hidden\":true,\"allow_custom_routing\":false},\"allow_auto_create\":true},\".monitoring-beats-mb\":{\"index_patterns\":[\".monitoring-beats-8-*\"],\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\".monitoring-8-ilm-policy\"},\"mapping\":{\"total_fields\":{\"limit\":\"2000\"}}}},\"mappings\":{\"dynamic\":false,\"properties\":{\"@timestamp\":{\"type\":\"date\"},\"cluster_uuid\":{\"path\":\"beat.elasticsearch.cluster.id\",\"type\":\"alias\"},\"service\":{\"properties\":{\"node\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"environment\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"hostname\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"origin\":{\"properties\":{\"node\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"environment\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"target\":{\"properties\":{\"node\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"environment\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"beat\":{\"properties\":{\"elasticsearch\":{\"properties\":{\"cluster\":{\"properties\":{\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"stats\":{\"properties\":{\"handles\":{\"properties\":{\"limit\":{\"properties\":{\"hard\":{\"type\":\"long\"},\"soft\":{\"type\":\"long\"}}},\"open\":{\"type\":\"long\"}}},\"apm_server\":{\"properties\":{\"server\":{\"properties\":{\"request\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"response\":{\"properties\":{\"valid\":{\"properties\":{\"count\":{\"type\":\"long\"},\"accepted\":{\"type\":\"long\"},\"ok\":{\"type\":\"long\"}}},\"count\":{\"type\":\"long\"},\"errors\":{\"properties\":{\"ratelimit\":{\"type\":\"long\"},\"internal\":{\"type\":\"long\"},\"toolarge\":{\"type\":\"long\"},\"method\":{\"type\":\"long\"},\"unauthorized\":{\"type\":\"long\"},\"forbidden\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"},\"closed\":{\"type\":\"long\"},\"decode\":{\"type\":\"long\"},\"queue\":{\"type\":\"long\"},\"concurrency\":{\"type\":\"long\"},\"validate\":{\"type\":\"long\"}}}}},\"concurrent\":{\"properties\":{\"wait\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}}}},\"acm\":{\"properties\":{\"request\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"response\":{\"properties\":{\"valid\":{\"properties\":{\"notmodified\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"},\"accepted\":{\"type\":\"long\"},\"ok\":{\"type\":\"long\"}}},\"request\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"count\":{\"type\":\"long\"},\"unset\":{\"type\":\"long\"},\"errors\":{\"properties\":{\"ratelimit\":{\"type\":\"long\"},\"internal\":{\"type\":\"long\"},\"toolarge\":{\"type\":\"long\"},\"method\":{\"type\":\"long\"},\"unavailable\":{\"type\":\"long\"},\"forbidden\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"},\"decode\":{\"type\":\"long\"},\"unauthorized\":{\"type\":\"long\"},\"notfound\":{\"type\":\"long\"},\"closed\":{\"type\":\"long\"},\"invalidquery\":{\"type\":\"long\"},\"queue\":{\"type\":\"long\"},\"validate\":{\"type\":\"long\"}}}}}}},\"decoder\":{\"properties\":{\"deflate\":{\"properties\":{\"content-length\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"reader\":{\"properties\":{\"size\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"gzip\":{\"properties\":{\"content-length\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"uncompressed\":{\"properties\":{\"content-length\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"missing-content-length\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"processor\":{\"properties\":{\"metric\":{\"properties\":{\"transformations\":{\"type\":\"long\"},\"decoding\":{\"properties\":{\"count\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}},\"validation\":{\"properties\":{\"count\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}}}},\"sourcemap\":{\"properties\":{\"counter\":{\"type\":\"long\"},\"decoding\":{\"properties\":{\"count\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}},\"validation\":{\"properties\":{\"count\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}}}},\"error\":{\"properties\":{\"stacktraces\":{\"type\":\"long\"},\"spans\":{\"type\":\"long\"},\"frames\":{\"type\":\"long\"},\"transformations\":{\"type\":\"long\"},\"decoding\":{\"properties\":{\"count\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}},\"validation\":{\"properties\":{\"count\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}}}},\"transaction\":{\"properties\":{\"stacktraces\":{\"type\":\"long\"},\"spans\":{\"type\":\"long\"},\"frames\":{\"type\":\"long\"},\"transformations\":{\"type\":\"long\"},\"transactions\":{\"type\":\"long\"},\"decoding\":{\"properties\":{\"count\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}},\"validation\":{\"properties\":{\"count\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}}}},\"span\":{\"properties\":{\"transformations\":{\"type\":\"long\"}}}}}}},\"system\":{\"properties\":{\"load\":{\"properties\":{\"1\":{\"type\":\"double\"},\"15\":{\"type\":\"double\"},\"5\":{\"type\":\"double\"},\"norm\":{\"properties\":{\"1\":{\"type\":\"double\"},\"15\":{\"type\":\"double\"},\"5\":{\"type\":\"double\"}}}}},\"cpu\":{\"properties\":{\"cores\":{\"type\":\"long\"}}}}},\"beat\":{\"properties\":{\"host\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"cpu\":{\"properties\":{\"total\":{\"properties\":{\"ticks\":{\"type\":\"long\"},\"time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"value\":{\"type\":\"long\"}}},\"system\":{\"properties\":{\"ticks\":{\"type\":\"long\"},\"time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"user\":{\"properties\":{\"ticks\":{\"type\":\"long\"},\"time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}}}},\"runtime\":{\"properties\":{\"goroutines\":{\"type\":\"long\"}}},\"memstats\":{\"properties\":{\"memory\":{\"properties\":{\"total\":{\"type\":\"long\"},\"alloc\":{\"type\":\"long\"}}},\"rss\":{\"type\":\"long\"},\"gc_next\":{\"type\":\"long\"}}},\"libbeat\":{\"properties\":{\"output\":{\"properties\":{\"read\":{\"properties\":{\"bytes\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"write\":{\"properties\":{\"bytes\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}},\"events\":{\"properties\":{\"batches\":{\"type\":\"long\"},\"duplicates\":{\"type\":\"long\"},\"total\":{\"type\":\"long\"},\"dropped\":{\"type\":\"long\"},\"toomany\":{\"type\":\"long\"},\"active\":{\"type\":\"long\"},\"failed\":{\"type\":\"long\"},\"acked\":{\"type\":\"long\"}}}}},\"pipeline\":{\"properties\":{\"clients\":{\"type\":\"long\"},\"events\":{\"properties\":{\"total\":{\"type\":\"long\"},\"filtered\":{\"type\":\"long\"},\"dropped\":{\"type\":\"long\"},\"active\":{\"type\":\"long\"},\"failed\":{\"type\":\"long\"},\"published\":{\"type\":\"long\"},\"retry\":{\"type\":\"long\"}}},\"queue\":{\"properties\":{\"acked\":{\"type\":\"long\"}}}}},\"config\":{\"properties\":{\"running\":{\"type\":\"long\"},\"stops\":{\"type\":\"long\"},\"starts\":{\"type\":\"long\"},\"reloads\":{\"type\":\"long\"}}}}},\"cgroup\":{\"properties\":{\"memory\":{\"properties\":{\"mem\":{\"properties\":{\"usage\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"limit\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"cpu\":{\"properties\":{\"cfs\":{\"properties\":{\"period\":{\"properties\":{\"us\":{\"type\":\"long\"}}},\"quota\":{\"properties\":{\"us\":{\"type\":\"long\"}}}}},\"stats\":{\"properties\":{\"periods\":{\"type\":\"long\"},\"throttled\":{\"properties\":{\"ns\":{\"type\":\"long\"},\"periods\":{\"type\":\"long\"}}}}},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"cpuacct\":{\"properties\":{\"total\":{\"properties\":{\"ns\":{\"type\":\"long\"}}},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"info\":{\"properties\":{\"host\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"uptime\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"uptime\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"properties\":{\"output\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"cluster\":{\"properties\":{\"uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"input\":{\"properties\":{\"names\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"count\":{\"type\":\"long\"}}},\"management\":{\"properties\":{\"enabled\":{\"type\":\"boolean\"}}},\"service\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"module\":{\"properties\":{\"names\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"count\":{\"type\":\"long\"}}},\"beat\":{\"properties\":{\"host\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"host\":{\"properties\":{\"os\":{\"properties\":{\"kernel\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"platform\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"containerized\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"queue\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"host\":{\"properties\":{\"hostname\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"architecture\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"metricset\":{\"properties\":{\"period\":{\"type\":\"long\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"event\":{\"properties\":{\"reason\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"code\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"timezone\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"agent_id_status\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"duration\":{\"type\":\"long\"},\"reference\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ingested\":{\"type\":\"date\"},\"provider\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"action\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"end\":{\"type\":\"date\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"outcome\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"severity\":{\"type\":\"long\"},\"original\":{\"ignore_above\":1024,\"index\":false,\"type\":\"keyword\",\"doc_values\":false},\"risk_score\":{\"type\":\"float\"},\"created\":{\"type\":\"date\"},\"kind\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"module\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"start\":{\"type\":\"date\"},\"url\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"sequence\":{\"type\":\"long\"},\"risk_score_norm\":{\"type\":\"float\"},\"category\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"dataset\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"hash\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"beats_stats\":{\"properties\":{\"beat\":{\"properties\":{\"host\":{\"path\":\"beat.stats.beat.host\",\"type\":\"alias\"},\"name\":{\"path\":\"beat.stats.beat.name\",\"type\":\"alias\"},\"type\":{\"path\":\"beat.stats.beat.type\",\"type\":\"alias\"},\"uuid\":{\"path\":\"beat.stats.beat.uuid\",\"type\":\"alias\"},\"version\":{\"path\":\"beat.stats.beat.version\",\"type\":\"alias\"}}},\"metrics\":{\"properties\":{\"system\":{\"properties\":{\"load\":{\"properties\":{\"1\":{\"path\":\"beat.stats.system.load.1\",\"type\":\"alias\"},\"15\":{\"path\":\"beat.stats.system.load.15\",\"type\":\"alias\"},\"5\":{\"path\":\"beat.stats.system.load.5\",\"type\":\"alias\"},\"norm\":{\"properties\":{\"1\":{\"path\":\"beat.stats.system.load.norm.1\",\"type\":\"alias\"},\"15\":{\"path\":\"beat.stats.system.load.norm.15\",\"type\":\"alias\"},\"5\":{\"path\":\"beat.stats.system.load.norm.5\",\"type\":\"alias\"}}}}},\"cpu\":{\"properties\":{\"cores\":{\"path\":\"beat.stats.system.cpu.cores\",\"type\":\"alias\"}}}}},\"beat\":{\"properties\":{\"handles\":{\"properties\":{\"limit\":{\"properties\":{\"hard\":{\"path\":\"beat.stats.handles.limit.hard\",\"type\":\"alias\"},\"soft\":{\"path\":\"beat.stats.handles.limit.soft\",\"type\":\"alias\"}}},\"open\":{\"path\":\"beat.stats.handles.open\",\"type\":\"alias\"}}},\"cpu\":{\"properties\":{\"total\":{\"properties\":{\"ticks\":{\"path\":\"beat.stats.cpu.total.ticks\",\"type\":\"alias\"},\"time\":{\"properties\":{\"ms\":{\"path\":\"beat.stats.cpu.total.time.ms\",\"type\":\"alias\"}}},\"value\":{\"path\":\"beat.stats.cpu.total.value\",\"type\":\"alias\"}}},\"system\":{\"properties\":{\"ticks\":{\"path\":\"beat.stats.cpu.system.ticks\",\"type\":\"alias\"},\"time\":{\"properties\":{\"ms\":{\"path\":\"beat.stats.cpu.system.time.ms\",\"type\":\"alias\"}}}}},\"user\":{\"properties\":{\"ticks\":{\"path\":\"beat.stats.cpu.user.ticks\",\"type\":\"alias\"},\"time\":{\"properties\":{\"ms\":{\"path\":\"beat.stats.cpu.user.time.ms\",\"type\":\"alias\"}}}}}}},\"memstats\":{\"properties\":{\"rss\":{\"path\":\"beat.stats.memstats.rss\",\"type\":\"alias\"},\"gc_next\":{\"path\":\"beat.stats.memstats.gc_next\",\"type\":\"alias\"},\"memory_alloc\":{\"path\":\"beat.stats.memstats.memory.alloc\",\"type\":\"alias\"},\"memory_total\":{\"path\":\"beat.stats.memstats.memory.total\",\"type\":\"alias\"}}},\"cgroup\":{\"properties\":{\"memory\":{\"properties\":{\"id\":{\"path\":\"beat.stats.cgroup.memory.id\",\"type\":\"alias\"}}},\"mem\":{\"properties\":{\"usage\":{\"properties\":{\"bytes\":{\"path\":\"beat.stats.cgroup.memory.mem.usage.bytes\",\"type\":\"alias\"}}},\"limit\":{\"properties\":{\"bytes\":{\"path\":\"beat.stats.cgroup.memory.mem.limit.bytes\",\"type\":\"alias\"}}}}},\"cpu\":{\"properties\":{\"cfs\":{\"properties\":{\"period\":{\"properties\":{\"us\":{\"path\":\"beat.stats.cgroup.cpu.cfs.period.us\",\"type\":\"alias\"}}},\"quota\":{\"properties\":{\"us\":{\"path\":\"beat.stats.cgroup.cpu.cfs.quota.us\",\"type\":\"alias\"}}}}},\"stats\":{\"properties\":{\"periods\":{\"path\":\"beat.stats.cgroup.cpu.stats.periods\",\"type\":\"alias\"},\"throttled\":{\"properties\":{\"ns\":{\"path\":\"beat.stats.cgroup.cpu.stats.throttled.ns\",\"type\":\"alias\"},\"periods\":{\"path\":\"beat.stats.cgroup.cpu.stats.throttled.periods\",\"type\":\"alias\"}}}}},\"id\":{\"path\":\"beat.stats.cgroup.cpu.id\",\"type\":\"alias\"}}},\"cpuacct\":{\"properties\":{\"total\":{\"properties\":{\"ns\":{\"path\":\"beat.stats.cgroup.cpuacct.total.ns\",\"type\":\"alias\"}}},\"id\":{\"path\":\"beat.stats.cgroup.cpuacct.id\",\"type\":\"alias\"}}}}},\"info\":{\"properties\":{\"ephemeral_id\":{\"path\":\"beat.stats.info.ephemeral_id\",\"type\":\"alias\"},\"uptime\":{\"properties\":{\"ms\":{\"path\":\"beat.stats.info.uptime.ms\",\"type\":\"alias\"}}}}}}},\"libbeat\":{\"properties\":{\"output\":{\"properties\":{\"read\":{\"properties\":{\"bytes\":{\"path\":\"beat.stats.libbeat.output.read.bytes\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.libbeat.output.read.errors\",\"type\":\"alias\"}}},\"type\":{\"path\":\"beat.stats.libbeat.output.type\",\"type\":\"alias\"},\"write\":{\"properties\":{\"bytes\":{\"path\":\"beat.stats.libbeat.output.write.bytes\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.libbeat.output.write.errors\",\"type\":\"alias\"}}},\"events\":{\"properties\":{\"duplicated\":{\"path\":\"beat.stats.libbeat.output.events.duplicates\",\"type\":\"alias\"},\"batches\":{\"path\":\"beat.stats.libbeat.output.events.batches\",\"type\":\"alias\"},\"total\":{\"path\":\"beat.stats.libbeat.output.events.total\",\"type\":\"alias\"},\"dropped\":{\"path\":\"beat.stats.libbeat.output.events.dropped\",\"type\":\"alias\"},\"toomany\":{\"path\":\"beat.stats.libbeat.output.events.toomany\",\"type\":\"alias\"},\"active\":{\"path\":\"beat.stats.libbeat.output.events.active\",\"type\":\"alias\"},\"failed\":{\"path\":\"beat.stats.libbeat.output.events.failed\",\"type\":\"alias\"},\"acked\":{\"path\":\"beat.stats.libbeat.output.events.acked\",\"type\":\"alias\"}}}}},\"pipeline\":{\"properties\":{\"clients\":{\"path\":\"beat.stats.libbeat.pipeline.clients\",\"type\":\"alias\"},\"events\":{\"properties\":{\"total\":{\"path\":\"beat.stats.libbeat.pipeline.events.total\",\"type\":\"alias\"},\"filtered\":{\"path\":\"beat.stats.libbeat.pipeline.events.filtered\",\"type\":\"alias\"},\"dropped\":{\"path\":\"beat.stats.libbeat.pipeline.events.dropped\",\"type\":\"alias\"},\"active\":{\"path\":\"beat.stats.libbeat.pipeline.events.active\",\"type\":\"alias\"},\"failed\":{\"path\":\"beat.stats.libbeat.pipeline.events.failed\",\"type\":\"alias\"},\"published\":{\"path\":\"beat.stats.libbeat.pipeline.events.published\",\"type\":\"alias\"},\"retry\":{\"path\":\"beat.stats.libbeat.pipeline.events.retry\",\"type\":\"alias\"}}},\"queue\":{\"properties\":{\"acked\":{\"path\":\"beat.stats.libbeat.pipeline.queue.acked\",\"type\":\"alias\"}}}}},\"config\":{\"properties\":{\"module\":{\"properties\":{\"running\":{\"path\":\"beat.stats.libbeat.config.running\",\"type\":\"alias\"},\"stops\":{\"path\":\"beat.stats.libbeat.config.stops\",\"type\":\"alias\"},\"starts\":{\"path\":\"beat.stats.libbeat.config.starts\",\"type\":\"alias\"}}},\"reloads\":{\"path\":\"beat.stats.libbeat.config.reloads\",\"type\":\"alias\"}}}}},\"apm-server\":{\"properties\":{\"server\":{\"properties\":{\"request\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.server.request.count\",\"type\":\"alias\"}}},\"response\":{\"properties\":{\"valid\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.server.response.valid.count\",\"type\":\"alias\"},\"accepted\":{\"path\":\"beat.stats.apm_server.server.response.valid.accepted\",\"type\":\"alias\"},\"ok\":{\"path\":\"beat.stats.apm_server.server.response.valid.ok\",\"type\":\"alias\"}}},\"count\":{\"path\":\"beat.stats.apm_server.server.response.count\",\"type\":\"alias\"},\"errors\":{\"properties\":{\"ratelimit\":{\"path\":\"beat.stats.apm_server.server.response.errors.ratelimit\",\"type\":\"alias\"},\"internal\":{\"path\":\"beat.stats.apm_server.server.response.errors.internal\",\"type\":\"alias\"},\"toolarge\":{\"path\":\"beat.stats.apm_server.server.response.errors.toolarge\",\"type\":\"alias\"},\"method\":{\"path\":\"beat.stats.apm_server.server.response.errors.method\",\"type\":\"alias\"},\"unauthorized\":{\"path\":\"beat.stats.apm_server.server.response.errors.unauthorized\",\"type\":\"alias\"},\"forbidden\":{\"path\":\"beat.stats.apm_server.server.response.errors.forbidden\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.stats.apm_server.server.response.errors.count\",\"type\":\"alias\"},\"closed\":{\"path\":\"beat.stats.apm_server.server.response.errors.closed\",\"type\":\"alias\"},\"decode\":{\"path\":\"beat.stats.apm_server.server.response.errors.decode\",\"type\":\"alias\"},\"queue\":{\"path\":\"beat.stats.apm_server.server.response.errors.queue\",\"type\":\"alias\"},\"concurrency\":{\"path\":\"beat.stats.apm_server.server.response.errors.concurrency\",\"type\":\"alias\"},\"validate\":{\"path\":\"beat.stats.apm_server.server.response.errors.validate\",\"type\":\"alias\"}}}}},\"concurrent\":{\"properties\":{\"wait\":{\"properties\":{\"ms\":{\"path\":\"beat.stats.apm_server.server.concurrent.wait.ms\",\"type\":\"alias\"}}}}}}},\"sampling\":{\"properties\":{\"tail\":{\"properties\":{\"dynamic_service_groups\":{\"type\":\"long\"},\"storage\":{\"properties\":{\"lsm_size\":{\"type\":\"long\"},\"value_log_size\":{\"type\":\"long\"}}},\"events\":{\"properties\":{\"processed\":{\"type\":\"long\"},\"head_unsampled\":{\"type\":\"long\"},\"stored\":{\"type\":\"long\"},\"dropped\":{\"type\":\"long\"},\"sampled\":{\"type\":\"long\"}}}}},\"transactions_dropped\":{\"type\":\"long\"}}},\"acm\":{\"properties\":{\"request\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.acm.request.count\",\"type\":\"alias\"}}},\"response\":{\"properties\":{\"valid\":{\"properties\":{\"notmodified\":{\"path\":\"beat.stats.apm_server.acm.response.valid.notmodified\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.stats.apm_server.acm.response.valid.count\",\"type\":\"alias\"},\"accepted\":{\"path\":\"beat.stats.apm_server.acm.response.valid.accepted\",\"type\":\"alias\"},\"ok\":{\"path\":\"beat.stats.apm_server.acm.response.valid.ok\",\"type\":\"alias\"}}},\"request\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.acm.response.request.count\",\"type\":\"alias\"}}},\"count\":{\"path\":\"beat.stats.apm_server.acm.response.count\",\"type\":\"alias\"},\"unset\":{\"path\":\"beat.stats.apm_server.acm.response.unset\",\"type\":\"alias\"},\"errors\":{\"properties\":{\"ratelimit\":{\"path\":\"beat.stats.apm_server.acm.response.errors.ratelimit\",\"type\":\"alias\"},\"internal\":{\"path\":\"beat.stats.apm_server.acm.response.errors.internal\",\"type\":\"alias\"},\"toolarge\":{\"path\":\"beat.stats.apm_server.acm.response.errors.toolarge\",\"type\":\"alias\"},\"method\":{\"path\":\"beat.stats.apm_server.acm.response.errors.method\",\"type\":\"alias\"},\"unavailable\":{\"path\":\"beat.stats.apm_server.acm.response.errors.unavailable\",\"type\":\"alias\"},\"forbidden\":{\"path\":\"beat.stats.apm_server.acm.response.errors.forbidden\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.stats.apm_server.acm.response.errors.count\",\"type\":\"alias\"},\"decode\":{\"path\":\"beat.stats.apm_server.acm.response.errors.decode\",\"type\":\"alias\"},\"unauthorized\":{\"path\":\"beat.stats.apm_server.acm.response.errors.unauthorized\",\"type\":\"alias\"},\"notfound\":{\"path\":\"beat.stats.apm_server.acm.response.errors.notfound\",\"type\":\"alias\"},\"closed\":{\"path\":\"beat.stats.apm_server.acm.response.errors.closed\",\"type\":\"alias\"},\"invalidquery\":{\"path\":\"beat.stats.apm_server.acm.response.errors.invalidquery\",\"type\":\"alias\"},\"queue\":{\"path\":\"beat.stats.apm_server.acm.response.errors.queue\",\"type\":\"alias\"},\"validate\":{\"path\":\"beat.stats.apm_server.acm.response.errors.validate\",\"type\":\"alias\"}}}}}}},\"decoder\":{\"properties\":{\"deflate\":{\"properties\":{\"content-length\":{\"path\":\"beat.stats.apm_server.decoder.deflate.content-length\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.stats.apm_server.decoder.deflate.count\",\"type\":\"alias\"}}},\"reader\":{\"properties\":{\"size\":{\"path\":\"beat.stats.apm_server.decoder.reader.size\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.stats.apm_server.decoder.reader.count\",\"type\":\"alias\"}}},\"gzip\":{\"properties\":{\"content-length\":{\"path\":\"beat.stats.apm_server.decoder.gzip.content-length\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.stats.apm_server.decoder.gzip.count\",\"type\":\"alias\"}}},\"uncompressed\":{\"properties\":{\"content-length\":{\"path\":\"beat.stats.apm_server.decoder.uncompressed.content-length\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.stats.apm_server.decoder.uncompressed.count\",\"type\":\"alias\"}}},\"missing-content-length\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.decoder.missing-content-length.count\",\"type\":\"alias\"}}}}},\"processor\":{\"properties\":{\"metric\":{\"properties\":{\"transformations\":{\"path\":\"beat.stats.apm_server.processor.metric.transformations\",\"type\":\"alias\"},\"decoding\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.processor.metric.decoding.count\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.apm_server.processor.metric.decoding.errors\",\"type\":\"alias\"}}},\"validation\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.processor.metric.validation.count\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.apm_server.processor.metric.validation.errors\",\"type\":\"alias\"}}}}},\"sourcemap\":{\"properties\":{\"counter\":{\"path\":\"beat.stats.apm_server.processor.sourcemap.counter\",\"type\":\"alias\"},\"decoding\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.processor.sourcemap.decoding.count\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.apm_server.processor.sourcemap.decoding.errors\",\"type\":\"alias\"}}},\"validation\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.processor.sourcemap.validation.count\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.apm_server.processor.sourcemap.validation.errors\",\"type\":\"alias\"}}}}},\"error\":{\"properties\":{\"stacktraces\":{\"path\":\"beat.stats.apm_server.processor.error.stacktraces\",\"type\":\"alias\"},\"spans\":{\"path\":\"beat.stats.apm_server.processor.error.spans\",\"type\":\"alias\"},\"frames\":{\"path\":\"beat.stats.apm_server.processor.error.frames\",\"type\":\"alias\"},\"transformations\":{\"path\":\"beat.stats.apm_server.processor.error.transformations\",\"type\":\"alias\"},\"decoding\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.processor.error.decoding.count\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.apm_server.processor.error.decoding.errors\",\"type\":\"alias\"}}},\"validation\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.processor.error.validation.count\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.apm_server.processor.error.validation.errors\",\"type\":\"alias\"}}}}},\"transaction\":{\"properties\":{\"stacktraces\":{\"path\":\"beat.stats.apm_server.processor.transaction.stacktraces\",\"type\":\"alias\"},\"spans\":{\"path\":\"beat.stats.apm_server.processor.transaction.spans\",\"type\":\"alias\"},\"frames\":{\"path\":\"beat.stats.apm_server.processor.transaction.frames\",\"type\":\"alias\"},\"transformations\":{\"path\":\"beat.stats.apm_server.processor.transaction.transformations\",\"type\":\"alias\"},\"transactions\":{\"path\":\"beat.stats.apm_server.processor.transaction.transactions\",\"type\":\"alias\"},\"decoding\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.processor.transaction.decoding.count\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.apm_server.processor.transaction.decoding.errors\",\"type\":\"alias\"}}},\"validation\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.processor.transaction.validation.count\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.apm_server.processor.transaction.validation.errors\",\"type\":\"alias\"}}}}},\"span\":{\"properties\":{\"transformations\":{\"path\":\"beat.stats.apm_server.processor.span.transformations\",\"type\":\"alias\"}}}}}}}}},\"timestamp\":{\"path\":\"@timestamp\",\"type\":\"alias\"}}},\"beats_state\":{\"properties\":{\"beat\":{\"properties\":{\"host\":{\"path\":\"beat.state.beat.host\",\"type\":\"alias\"},\"name\":{\"path\":\"beat.state.beat.name\",\"type\":\"alias\"},\"type\":{\"path\":\"beat.state.beat.type\",\"type\":\"alias\"},\"uuid\":{\"path\":\"beat.state.beat.uuid\",\"type\":\"alias\"},\"version\":{\"path\":\"beat.state.beat.version\",\"type\":\"alias\"}}},\"state\":{\"properties\":{\"output\":{\"properties\":{\"name\":{\"path\":\"beat.state.output.name\",\"type\":\"alias\"}}},\"input\":{\"properties\":{\"names\":{\"path\":\"beat.state.input.names\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.state.input.count\",\"type\":\"alias\"}}},\"service\":{\"properties\":{\"name\":{\"path\":\"beat.state.service.name\",\"type\":\"alias\"},\"id\":{\"path\":\"beat.state.service.id\",\"type\":\"alias\"},\"version\":{\"path\":\"beat.state.service.version\",\"type\":\"alias\"}}},\"module\":{\"properties\":{\"names\":{\"path\":\"beat.state.module.names\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.state.module.count\",\"type\":\"alias\"}}},\"beat\":{\"properties\":{\"name\":{\"path\":\"beat.state.beat.name\",\"type\":\"alias\"}}},\"host\":{\"properties\":{\"hostname\":{\"path\":\"host.hostname\",\"type\":\"alias\"},\"os\":{\"properties\":{\"version\":{\"path\":\"beat.state.host.os.version\",\"type\":\"alias\"},\"platform\":{\"path\":\"beat.state.host.os.platform\",\"type\":\"alias\"}}},\"name\":{\"path\":\"host.name\",\"type\":\"alias\"},\"architecture\":{\"path\":\"host.architecture\",\"type\":\"alias\"}}}}},\"timestamp\":{\"path\":\"@timestamp\",\"type\":\"alias\"}}},\"timestamp\":{\"path\":\"@timestamp\",\"type\":\"alias\"}}}},\"composed_of\":[],\"version\":8000103,\"data_stream\":{\"hidden\":false,\"allow_custom_routing\":false}},\".monitoring-kibana-mb\":{\"index_patterns\":[\".monitoring-kibana-8-*\"],\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\".monitoring-8-ilm-policy\"},\"mapping\":{\"total_fields\":{\"limit\":\"2000\"}}}},\"mappings\":{\"dynamic\":false,\"properties\":{\"process\":{\"properties\":{\"pid\":{\"type\":\"long\"}}},\"@timestamp\":{\"type\":\"date\"},\"ecs\":{\"properties\":{\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"cluster_uuid\":{\"path\":\"kibana.elasticsearch.cluster.id\",\"type\":\"alias\"},\"service\":{\"properties\":{\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"host\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"metricset\":{\"properties\":{\"period\":{\"type\":\"long\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"kibana\":{\"properties\":{\"elasticsearch\":{\"properties\":{\"cluster\":{\"properties\":{\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"stats\":{\"properties\":{\"request\":{\"properties\":{\"total\":{\"type\":\"long\"},\"disconnects\":{\"type\":\"long\"}}},\"process\":{\"properties\":{\"memory\":{\"properties\":{\"resident_set_size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"heap\":{\"properties\":{\"total\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"used\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"size_limit\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"uptime\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}}}},\"event_loop_delay\":{\"properties\":{\"ms\":{\"scaling_factor\":1000,\"type\":\"scaled_float\"}}},\"uptime\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"os\":{\"properties\":{\"distroRelease\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"distro\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"memory\":{\"properties\":{\"used_in_bytes\":{\"type\":\"long\"},\"total_in_bytes\":{\"type\":\"long\"},\"free_in_bytes\":{\"type\":\"long\"}}},\"load\":{\"properties\":{\"5m\":{\"type\":\"half_float\"},\"15m\":{\"type\":\"half_float\"},\"1m\":{\"type\":\"half_float\"}}},\"platformRelease\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"platform\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"usage\":{\"properties\":{\"index\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"host\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"index\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"response_time\":{\"properties\":{\"avg\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"max\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"kibana\":{\"properties\":{\"status\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"concurrent_connections\":{\"type\":\"long\"},\"snapshot\":{\"type\":\"boolean\"},\"status\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"node_rules\":{\"properties\":{\"failures\":{\"type\":\"long\"},\"executions\":{\"type\":\"long\"},\"timeouts\":{\"type\":\"long\"}}},\"cluster_actions\":{\"properties\":{\"overdue\":{\"properties\":{\"delay\":{\"properties\":{\"p99\":{\"type\":\"float\"},\"p50\":{\"type\":\"float\"}}},\"count\":{\"type\":\"long\"}}}}},\"cluster_rules\":{\"properties\":{\"overdue\":{\"properties\":{\"delay\":{\"properties\":{\"p99\":{\"type\":\"float\"},\"p50\":{\"type\":\"float\"}}},\"count\":{\"type\":\"long\"}}}}},\"node_actions\":{\"properties\":{\"failures\":{\"type\":\"long\"},\"executions\":{\"type\":\"long\"},\"timeouts\":{\"type\":\"long\"}}},\"status\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"metrics\":{\"properties\":{\"requests\":{\"properties\":{\"total\":{\"type\":\"long\"},\"disconnects\":{\"type\":\"long\"}}},\"concurrent_connections\":{\"type\":\"long\"}}},\"status\":{\"properties\":{\"overall\":{\"properties\":{\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}}}}}},\"event\":{\"properties\":{\"duration\":{\"type\":\"long\"},\"module\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"dataset\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"kibana_stats\":{\"properties\":{\"process\":{\"properties\":{\"memory\":{\"properties\":{\"resident_set_size_in_bytes\":{\"path\":\"kibana.stats.process.memory.resident_set_size.bytes\",\"type\":\"alias\"},\"heap\":{\"properties\":{\"size_limit\":{\"path\":\"kibana.stats.process.memory.heap.size_limit.bytes\",\"type\":\"alias\"}}}}},\"event_loop_delay\":{\"path\":\"kibana.stats.process.event_loop_delay.ms\",\"type\":\"alias\"},\"uptime_in_millis\":{\"path\":\"kibana.stats.process.uptime.ms\",\"type\":\"alias\"}}},\"os\":{\"properties\":{\"memory\":{\"properties\":{\"free_in_bytes\":{\"path\":\"kibana.stats.os.memory.free_in_bytes\",\"type\":\"alias\"}}},\"load\":{\"properties\":{\"5m\":{\"path\":\"kibana.stats.os.load.5m\",\"type\":\"alias\"},\"15m\":{\"path\":\"kibana.stats.os.load.15m\",\"type\":\"alias\"},\"1m\":{\"path\":\"kibana.stats.os.load.1m\",\"type\":\"alias\"}}}}},\"response_times\":{\"properties\":{\"average\":{\"path\":\"kibana.stats.response_time.avg.ms\",\"type\":\"alias\"},\"max\":{\"path\":\"kibana.stats.response_time.max.ms\",\"type\":\"alias\"}}},\"requests\":{\"properties\":{\"total\":{\"path\":\"kibana.stats.request.total\",\"type\":\"alias\"},\"disconnects\":{\"path\":\"kibana.stats.request.disconnects\",\"type\":\"alias\"}}},\"kibana\":{\"properties\":{\"response_time\":{\"properties\":{\"max\":{\"path\":\"kibana.stats.response_time.max.ms\",\"type\":\"alias\"}}},\"uuid\":{\"path\":\"service.id\",\"type\":\"alias\"},\"version\":{\"path\":\"service.version\",\"type\":\"alias\"},\"status\":{\"path\":\"kibana.stats.status\",\"type\":\"alias\"}}},\"concurrent_connections\":{\"path\":\"kibana.stats.concurrent_connections\",\"type\":\"alias\"},\"timestamp\":{\"path\":\"@timestamp\",\"type\":\"alias\"}}},\"timestamp\":{\"path\":\"@timestamp\",\"type\":\"alias\"}}}},\"composed_of\":[],\"version\":8000103,\"data_stream\":{\"hidden\":false,\"allow_custom_routing\":false}},\".monitoring-ent-search-mb\":{\"index_patterns\":[\".monitoring-ent-search-8-*\"],\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\".monitoring-8-ilm-policy\"},\"mapping\":{\"total_fields\":{\"limit\":\"2000\"}}}},\"mappings\":{\"dynamic\":false,\"properties\":{\"enterprisesearch\":{\"properties\":{\"cluster_uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"stats\":{\"properties\":{\"connectors\":{\"properties\":{\"pool\":{\"properties\":{\"publish_worker_pool\":{\"properties\":{\"total_completed\":{\"type\":\"long\"},\"size\":{\"type\":\"long\"},\"idle\":{\"type\":\"long\"},\"busy\":{\"type\":\"long\"},\"queue_depth\":{\"type\":\"long\"},\"total_scheduled\":{\"type\":\"long\"}}},\"extract_worker_pool\":{\"properties\":{\"total_completed\":{\"type\":\"long\"},\"size\":{\"type\":\"long\"},\"idle\":{\"type\":\"long\"},\"busy\":{\"type\":\"long\"},\"queue_depth\":{\"type\":\"long\"},\"total_scheduled\":{\"type\":\"long\"}}},\"subextract_worker_pool\":{\"properties\":{\"total_completed\":{\"type\":\"long\"},\"size\":{\"type\":\"long\"},\"idle\":{\"type\":\"long\"},\"busy\":{\"type\":\"long\"},\"queue_depth\":{\"type\":\"long\"},\"total_scheduled\":{\"type\":\"long\"}}}}},\"job_store\":{\"properties\":{\"waiting\":{\"type\":\"long\"},\"job_types\":{\"properties\":{\"permissions\":{\"type\":\"long\"},\"incremental\":{\"type\":\"long\"},\"delete\":{\"type\":\"long\"},\"full\":{\"type\":\"long\"}}},\"working\":{\"type\":\"long\"}}}}},\"queues\":{\"properties\":{\"engine_destroyer\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"process_crawl\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"failed\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"mailer\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"http\":{\"properties\":{\"request_duration\":{\"properties\":{\"max\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"std_dev\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"mean\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"responses\":{\"properties\":{\"1xx\":{\"type\":\"long\"},\"2xx\":{\"type\":\"long\"},\"3xx\":{\"type\":\"long\"},\"4xx\":{\"type\":\"long\"},\"5xx\":{\"type\":\"long\"}}},\"connections\":{\"properties\":{\"current\":{\"type\":\"long\"},\"total\":{\"type\":\"long\"},\"max\":{\"type\":\"long\"}}},\"network\":{\"properties\":{\"received\":{\"properties\":{\"bytes_per_sec\":{\"type\":\"long\"},\"bytes\":{\"type\":\"long\"}}},\"sent\":{\"properties\":{\"bytes_per_sec\":{\"type\":\"long\"},\"bytes\":{\"type\":\"long\"}}}}}}},\"product_usage\":{\"properties\":{\"app_search\":{\"properties\":{\"total_engines\":{\"type\":\"long\"}}},\"workplace_search\":{\"properties\":{\"total_org_sources\":{\"type\":\"long\"},\"total_private_sources\":{\"type\":\"long\"}}}}}}},\"health\":{\"properties\":{\"jvm\":{\"properties\":{\"memory_usage\":{\"properties\":{\"heap_committed\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"heap_used\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"non_heap_committed\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"non_heap_init\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"heap_init\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"heap_max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"object_pending_finalization_count\":{\"type\":\"long\"}}},\"threads\":{\"properties\":{\"current\":{\"type\":\"long\"},\"total_started\":{\"type\":\"long\"},\"max\":{\"type\":\"long\"},\"daemon\":{\"type\":\"long\"}}},\"gc\":{\"properties\":{\"collection_count\":{\"type\":\"long\"},\"collection_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"process\":{\"properties\":{\"pid\":{\"type\":\"long\"},\"filebeat\":{\"properties\":{\"restart_count\":{\"type\":\"long\"},\"pid\":{\"type\":\"long\"},\"time_since_last_restart\":{\"properties\":{\"sec\":{\"type\":\"long\"}}}}},\"uptime\":{\"properties\":{\"sec\":{\"type\":\"long\"}}}}},\"crawler\":{\"properties\":{\"workers\":{\"properties\":{\"pool_size\":{\"type\":\"long\"},\"available\":{\"type\":\"long\"},\"active\":{\"type\":\"long\"}}}}},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"properties\":{\"number\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"build_hash\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}}}},\"agent\":{\"properties\":{\"id\":{\"type\":\"keyword\"},\"ephemeral_id\":{\"type\":\"keyword\"},\"type\":{\"type\":\"keyword\"},\"version\":{\"type\":\"keyword\"}}},\"process\":{\"properties\":{\"pid\":{\"type\":\"long\"}}},\"@timestamp\":{\"type\":\"date\"},\"ecs\":{\"properties\":{\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"service\":{\"properties\":{\"node\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"hostname\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"environment\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"origin\":{\"properties\":{\"node\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"environment\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"target\":{\"properties\":{\"node\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"environment\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"host\":{\"properties\":{\"hostname\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"architecture\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"metricset\":{\"properties\":{\"period\":{\"type\":\"long\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"event\":{\"properties\":{\"reason\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"code\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"timezone\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"agent_id_status\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"duration\":{\"type\":\"long\"},\"reference\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ingested\":{\"type\":\"date\"},\"provider\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"action\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"end\":{\"type\":\"date\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"outcome\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"severity\":{\"type\":\"long\"},\"original\":{\"ignore_above\":1024,\"index\":false,\"type\":\"keyword\",\"doc_values\":false},\"risk_score\":{\"type\":\"float\"},\"created\":{\"type\":\"date\"},\"kind\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"module\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"start\":{\"type\":\"date\"},\"url\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"sequence\":{\"type\":\"long\"},\"risk_score_norm\":{\"type\":\"float\"},\"category\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"dataset\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"hash\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}}},\"composed_of\":[],\"version\":8000103,\"_meta\":{\"managed\":true,\"description\":\"Template used by Enterprise Search Metricbeat module monitoring information for Stack Monitoring\"},\"data_stream\":{\"hidden\":false,\"allow_custom_routing\":false}},\"synthetics\":{\"index_patterns\":[\"synthetics-*-*\"],\"composed_of\":[\"synthetics-mappings\",\"data-streams-mappings\",\"synthetics-settings\"],\"priority\":100,\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default synthetics template installed by x-pack\"},\"data_stream\":{\"hidden\":false,\"allow_custom_routing\":false},\"allow_auto_create\":true},\"ilm-history\":{\"index_patterns\":[\"ilm-history-5*\"],\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\"ilm-history-ilm-policy\"},\"number_of_shards\":\"1\",\"auto_expand_replicas\":\"0-1\",\"number_of_replicas\":\"0\"}},\"mappings\":{\"dynamic\":false,\"properties\":{\"index_age\":{\"type\":\"long\"},\"@timestamp\":{\"format\":\"epoch_millis\",\"type\":\"date\"},\"error_details\":{\"type\":\"text\"},\"success\":{\"type\":\"boolean\"},\"index\":{\"type\":\"keyword\"},\"state\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"phase\":{\"type\":\"keyword\"},\"failed_step\":{\"type\":\"keyword\"},\"phase_definition\":{\"type\":\"text\"},\"action_time\":{\"format\":\"epoch_millis\",\"type\":\"date\"},\"phase_time\":{\"format\":\"epoch_millis\",\"type\":\"date\"},\"step_info\":{\"type\":\"text\"},\"action\":{\"type\":\"keyword\"},\"step\":{\"type\":\"keyword\"},\"is_auto-retryable_error\":{\"type\":\"keyword\"},\"creation_date\":{\"format\":\"epoch_millis\",\"type\":\"date\"},\"step_time\":{\"format\":\"epoch_millis\",\"type\":\"date\"}}},\"policy\":{\"type\":\"keyword\"}}}},\"composed_of\":[],\"priority\":2147483647,\"version\":5,\"_meta\":{\"managed\":true,\"description\":\"index template for ILM history indices\"},\"data_stream\":{\"hidden\":true,\"allow_custom_routing\":false},\"allow_auto_create\":true},\".ml-state\":{\"index_patterns\":[\".ml-state*\"],\"template\":{\"settings\":{\"index\":{\"hidden\":\"true\",\"lifecycle\":{\"name\":\"ml-size-based-ilm-policy\",\"rollover_alias\":\".ml-state-write\"},\"auto_expand_replicas\":\"0-1\"}},\"mappings\":{\"_meta\":{\"version\":\"8060299\"},\"enabled\":false},\"aliases\":{}},\"composed_of\":[],\"priority\":2147483647,\"version\":8060299,\"_meta\":{\"managed\":true,\"description\":\"index template for ML state indices\"}},\".monitoring-es-mb\":{\"index_patterns\":[\".monitoring-es-8-*\"],\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\".monitoring-8-ilm-policy\"},\"mapping\":{\"total_fields\":{\"limit\":\"2000\"}}}},\"mappings\":{\"dynamic\":false,\"properties\":{\"ccr_auto_follow_stats\":{\"properties\":{\"number_of_failed_remote_cluster_state_requests\":{\"path\":\"elasticsearch.ccr.auto_follow.failed.remote_cluster_state_requests.count\",\"type\":\"alias\"},\"follower\":{\"properties\":{\"failed_read_requests\":{\"path\":\"elasticsearch.ccr.requests.failed.read.count\",\"type\":\"alias\"}}},\"number_of_failed_follow_indices\":{\"path\":\"elasticsearch.ccr.auto_follow.failed.follow_indices.count\",\"type\":\"alias\"},\"number_of_successful_follow_indices\":{\"path\":\"elasticsearch.ccr.auto_follow.success.follow_indices.count\",\"type\":\"alias\"}}},\"ccr_stats\":{\"properties\":{\"write_buffer_size_in_bytes\":{\"path\":\"elasticsearch.ccr.write_buffer.size.bytes\",\"type\":\"alias\"},\"leader_global_checkpoint\":{\"path\":\"elasticsearch.ccr.leader.global_checkpoint\",\"type\":\"alias\"},\"follower_index\":{\"path\":\"elasticsearch.ccr.follower.index\",\"type\":\"alias\"},\"leader_max_seq_no\":{\"path\":\"elasticsearch.ccr.leader.max_seq_no\",\"type\":\"alias\"},\"last_requested_seq_no\":{\"path\":\"elasticsearch.ccr.last_requested_seq_no\",\"type\":\"alias\"},\"follower_settings_version\":{\"path\":\"elasticsearch.ccr.follower.settings_version\",\"type\":\"alias\"},\"successful_write_requests\":{\"path\":\"elasticsearch.ccr.requests.successful.write.count\",\"type\":\"alias\"},\"remote_cluster\":{\"path\":\"elasticsearch.ccr.remote_cluster\",\"type\":\"alias\"},\"outstanding_write_requests\":{\"path\":\"elasticsearch.ccr.requests.outstanding.write.count\",\"type\":\"alias\"},\"total_read_time_millis\":{\"path\":\"elasticsearch.ccr.total_time.read.ms\",\"type\":\"alias\"},\"outstanding_read_requests\":{\"path\":\"elasticsearch.ccr.requests.outstanding.read.count\",\"type\":\"alias\"},\"total_write_time_millis\":{\"path\":\"elasticsearch.ccr.total_time.write.ms\",\"type\":\"alias\"},\"failed_write_requests\":{\"path\":\"elasticsearch.ccr.requests.failed.write.count\",\"type\":\"alias\"},\"failed_read_requests\":{\"path\":\"elasticsearch.ccr.requests.failed.read.count\",\"type\":\"alias\"},\"bytes_read\":{\"path\":\"elasticsearch.ccr.bytes_read\",\"type\":\"alias\"},\"leader_index\":{\"path\":\"elasticsearch.ccr.leader.index\",\"type\":\"alias\"},\"follower_max_seq_no\":{\"path\":\"elasticsearch.ccr.follower.max_seq_no\",\"type\":\"alias\"},\"operations_written\":{\"path\":\"elasticsearch.ccr.follower.operations_written\",\"type\":\"alias\"},\"write_buffer_operation_count\":{\"path\":\"elasticsearch.ccr.write_buffer.operation.count\",\"type\":\"alias\"},\"successful_read_requests\":{\"path\":\"elasticsearch.ccr.requests.successful.read.count\",\"type\":\"alias\"},\"shard_id\":{\"path\":\"elasticsearch.ccr.follower.shard.number\",\"type\":\"alias\"},\"follower_mapping_version\":{\"path\":\"elasticsearch.ccr.follower.mapping_version\",\"type\":\"alias\"},\"follower_aliases_version\":{\"path\":\"elasticsearch.ccr.follower.aliases_version\",\"type\":\"alias\"},\"follower_global_checkpoint\":{\"path\":\"elasticsearch.ccr.follower.global_checkpoint\",\"type\":\"alias\"},\"total_read_remote_exec_time_millis\":{\"path\":\"elasticsearch.ccr.total_time.read.remote_exec.ms\",\"type\":\"alias\"},\"operations_read\":{\"path\":\"elasticsearch.ccr.follower.operations.read.count\",\"type\":\"alias\"}}},\"cluster_state\":{\"properties\":{\"nodes_hash\":{\"path\":\"elasticsearch.cluster.stats.state.nodes_hash\",\"type\":\"alias\"},\"master_node\":{\"path\":\"elasticsearch.cluster.stats.state.master_node\",\"type\":\"alias\"},\"state_uuid\":{\"path\":\"elasticsearch.cluster.stats.state.state_uuid\",\"type\":\"alias\"},\"version\":{\"path\":\"elasticsearch.cluster.stats.state.version\",\"type\":\"alias\"},\"status\":{\"path\":\"elasticsearch.cluster.stats.status\",\"type\":\"alias\"}}},\"indices_stats\":{\"properties\":{\"_all\":{\"properties\":{\"primaries\":{\"properties\":{\"indexing\":{\"properties\":{\"index_time_in_millis\":{\"path\":\"elasticsearch.index.summary.primaries.indexing.index.time.ms\",\"type\":\"alias\"},\"index_total\":{\"path\":\"elasticsearch.index.summary.primaries.indexing.index.count\",\"type\":\"alias\"}}}}},\"total\":{\"properties\":{\"search\":{\"properties\":{\"query_total\":{\"path\":\"elasticsearch.index.summary.total.search.query.count\",\"type\":\"alias\"},\"query_time_in_millis\":{\"path\":\"elasticsearch.index.summary.total.search.query.time.ms\",\"type\":\"alias\"}}},\"indexing\":{\"properties\":{\"index_total\":{\"path\":\"elasticsearch.index.summary.total.indexing.index.count\",\"type\":\"alias\"}}}}}}}}},\"node_stats\":{\"properties\":{\"jvm\":{\"properties\":{\"mem\":{\"properties\":{\"heap_used_percent\":{\"path\":\"elasticsearch.node.stats.jvm.mem.heap.used.pct\",\"type\":\"alias\"},\"heap_max_in_bytes\":{\"path\":\"elasticsearch.node.stats.jvm.mem.heap.max.bytes\",\"type\":\"alias\"},\"heap_used_in_bytes\":{\"path\":\"elasticsearch.node.stats.jvm.mem.heap.used.bytes\",\"type\":\"alias\"}}},\"gc\":{\"properties\":{\"collectors\":{\"properties\":{\"young\":{\"properties\":{\"collection_count\":{\"path\":\"elasticsearch.node.stats.jvm.gc.collectors.young.collection.count\",\"type\":\"alias\"},\"collection_time_in_millis\":{\"path\":\"elasticsearch.node.stats.jvm.gc.collectors.young.collection.ms\",\"type\":\"alias\"}}},\"old\":{\"properties\":{\"collection_count\":{\"path\":\"elasticsearch.node.stats.jvm.gc.collectors.old.collection.count\",\"type\":\"alias\"},\"collection_time_in_millis\":{\"path\":\"elasticsearch.node.stats.jvm.gc.collectors.old.collection.ms\",\"type\":\"alias\"}}}}}}}}},\"indices\":{\"properties\":{\"search\":{\"properties\":{\"query_total\":{\"path\":\"elasticsearch.node.stats.indices.search.query_total.count\",\"type\":\"alias\"},\"query_time_in_millis\":{\"path\":\"elasticsearch.node.stats.indices.search.query_time.ms\",\"type\":\"alias\"}}},\"query_cache\":{\"properties\":{\"memory_size_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.query_cache.memory.bytes\",\"type\":\"alias\"}}},\"docs\":{\"properties\":{\"count\":{\"path\":\"elasticsearch.node.stats.indices.docs.count\",\"type\":\"alias\"}}},\"indexing\":{\"properties\":{\"throttle_time_in_millis\":{\"path\":\"elasticsearch.node.stats.indices.indexing.throttle_time.ms\",\"type\":\"alias\"},\"index_time_in_millis\":{\"path\":\"elasticsearch.node.stats.indices.indexing.index_time.ms\",\"type\":\"alias\"},\"index_total\":{\"path\":\"elasticsearch.node.stats.indices.indexing.index_total.count\",\"type\":\"alias\"}}},\"fielddata\":{\"properties\":{\"memory_size_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.fielddata.memory.bytes\",\"type\":\"alias\"}}},\"store\":{\"properties\":{\"size\":{\"properties\":{\"bytes\":{\"path\":\"elasticsearch.node.stats.indices.store.size.bytes\",\"type\":\"alias\"}}},\"size_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.store.size.bytes\",\"type\":\"alias\"}}},\"request_cache\":{\"properties\":{\"memory_size_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.request_cache.memory.bytes\",\"type\":\"alias\"}}},\"segments\":{\"properties\":{\"version_map_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.version_map.memory.bytes\",\"type\":\"alias\"},\"norms_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.norms.memory.bytes\",\"type\":\"alias\"},\"count\":{\"path\":\"elasticsearch.node.stats.indices.segments.count\",\"type\":\"alias\"},\"term_vectors_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.term_vectors.memory.bytes\",\"type\":\"alias\"},\"points_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.points.memory.bytes\",\"type\":\"alias\"},\"index_writer_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.index_writer.memory.bytes\",\"type\":\"alias\"},\"memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.memory.bytes\",\"type\":\"alias\"},\"doc_values_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.doc_values.memory.bytes\",\"type\":\"alias\"},\"terms_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.terms.memory.bytes\",\"type\":\"alias\"},\"fixed_bit_set_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.fixed_bit_set.memory.bytes\",\"type\":\"alias\"},\"stored_fields_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.stored_fields.memory.bytes\",\"type\":\"alias\"}}}}},\"process\":{\"properties\":{\"cpu\":{\"properties\":{\"percent\":{\"path\":\"elasticsearch.node.stats.process.cpu.pct\",\"type\":\"alias\"}}}}},\"os\":{\"properties\":{\"cpu\":{\"properties\":{\"load_average\":{\"properties\":{\"1m\":{\"path\":\"elasticsearch.node.stats.os.cpu.load_avg.1m\",\"type\":\"alias\"}}}}},\"cgroup\":{\"properties\":{\"memory\":{\"properties\":{\"usage_in_bytes\":{\"path\":\"elasticsearch.node.stats.os.cgroup.memory.usage.bytes\",\"type\":\"alias\"},\"control_group\":{\"path\":\"elasticsearch.node.stats.os.cgroup.memory.control_group\",\"type\":\"alias\"},\"limit_in_bytes\":{\"path\":\"elasticsearch.node.stats.os.cgroup.memory.limit.bytes\",\"type\":\"alias\"}}},\"cpu\":{\"properties\":{\"stat\":{\"properties\":{\"number_of_elapsed_periods\":{\"path\":\"elasticsearch.node.stats.os.cgroup.cpu.stat.elapsed_periods.count\",\"type\":\"alias\"},\"number_of_times_throttled\":{\"path\":\"elasticsearch.node.stats.os.cgroup.cpu.stat.times_throttled.count\",\"type\":\"alias\"},\"time_throttled_nanos\":{\"path\":\"elasticsearch.node.stats.os.cgroup.cpu.stat.time_throttled.ns\",\"type\":\"alias\"}}},\"cfs_quota_micros\":{\"path\":\"elasticsearch.node.stats.os.cgroup.cpu.cfs.quota.us\",\"type\":\"alias\"}}},\"cpuacct\":{\"properties\":{\"usage_nanos\":{\"path\":\"elasticsearch.node.stats.os.cgroup.cpuacct.usage.ns\",\"type\":\"alias\"}}}}}}},\"thread_pool\":{\"properties\":{\"search\":{\"properties\":{\"rejected\":{\"path\":\"elasticsearch.node.stats.thread_pool.search.rejected.count\",\"type\":\"alias\"},\"queue\":{\"path\":\"elasticsearch.node.stats.thread_pool.search.queue.count\",\"type\":\"alias\"}}},\"get\":{\"properties\":{\"rejected\":{\"path\":\"elasticsearch.node.stats.thread_pool.get.rejected.count\",\"type\":\"alias\"},\"queue\":{\"path\":\"elasticsearch.node.stats.thread_pool.get.queue.count\",\"type\":\"alias\"}}},\"index\":{\"properties\":{\"rejected\":{\"path\":\"elasticsearch.node.stats.thread_pool.index.rejected.count\",\"type\":\"alias\"},\"queue\":{\"path\":\"elasticsearch.node.stats.thread_pool.index.queue.count\",\"type\":\"alias\"}}},\"bulk\":{\"properties\":{\"rejected\":{\"path\":\"elasticsearch.node.stats.thread_pool.bulk.rejected.count\",\"type\":\"alias\"},\"queue\":{\"path\":\"elasticsearch.node.stats.thread_pool.bulk.queue.count\",\"type\":\"alias\"}}},\"write\":{\"properties\":{\"rejected\":{\"path\":\"elasticsearch.node.stats.thread_pool.write.rejected.count\",\"type\":\"alias\"},\"queue\":{\"path\":\"elasticsearch.node.stats.thread_pool.write.queue.count\",\"type\":\"alias\"}}}}},\"fs\":{\"properties\":{\"summary\":{\"properties\":{\"total\":{\"properties\":{\"bytes\":{\"path\":\"elasticsearch.node.stats.fs.summary.total.bytes\",\"type\":\"alias\"}}},\"available\":{\"properties\":{\"bytes\":{\"path\":\"elasticsearch.node.stats.fs.summary.available.bytes\",\"type\":\"alias\"}}}}},\"io_stats\":{\"properties\":{\"total\":{\"properties\":{\"write_operations\":{\"path\":\"elasticsearch.node.stats.fs.io_stats.total.write.operations.count\",\"type\":\"alias\"},\"operations\":{\"path\":\"elasticsearch.node.stats.fs.io_stats.total.operations.count\",\"type\":\"alias\"},\"read_operations\":{\"path\":\"elasticsearch.node.stats.fs.io_stats.total.read.operations.count\",\"type\":\"alias\"}}}}},\"total\":{\"properties\":{\"total_in_bytes\":{\"path\":\"elasticsearch.node.stats.fs.summary.total.bytes\",\"type\":\"alias\"},\"available_in_bytes\":{\"path\":\"elasticsearch.node.stats.fs.summary.available.bytes\",\"type\":\"alias\"}}}}},\"node_id\":{\"path\":\"elasticsearch.node.id\",\"type\":\"alias\"}}},\"stack_stats\":{\"properties\":{\"xpack\":{\"properties\":{\"ccr\":{\"properties\":{\"available\":{\"path\":\"elasticsearch.cluster.stats.stack.xpack.ccr.available\",\"type\":\"alias\"},\"enabled\":{\"path\":\"elasticsearch.cluster.stats.stack.xpack.ccr.enabled\",\"type\":\"alias\"}}}}},\"apm\":{\"properties\":{\"found\":{\"path\":\"elasticsearch.cluster.stats.stack.apm.found\",\"type\":\"alias\"}}}}},\"license\":{\"properties\":{\"type\":{\"path\":\"elasticsearch.cluster.stats.license.type\",\"type\":\"alias\"},\"status\":{\"path\":\"elasticsearch.cluster.stats.license.status\",\"type\":\"alias\"}}},\"index_stats\":{\"properties\":{\"primaries\":{\"properties\":{\"docs\":{\"properties\":{\"count\":{\"path\":\"elasticsearch.index.primaries.docs.count\",\"type\":\"alias\"}}},\"indexing\":{\"properties\":{\"throttle_time_in_millis\":{\"path\":\"elasticsearch.index.primaries.indexing.throttle_time_in_millis\",\"type\":\"alias\"},\"index_time_in_millis\":{\"path\":\"elasticsearch.index.primaries.indexing.index_time_in_millis\",\"type\":\"alias\"},\"index_total\":{\"path\":\"elasticsearch.index.primaries.indexing.index_total\",\"type\":\"alias\"}}},\"refresh\":{\"properties\":{\"total_time_in_millis\":{\"path\":\"elasticsearch.index.primaries.refresh.total_time_in_millis\",\"type\":\"alias\"}}},\"store\":{\"properties\":{\"size_in_bytes\":{\"path\":\"elasticsearch.index.primaries.store.size_in_bytes\",\"type\":\"alias\"}}},\"merges\":{\"properties\":{\"total_size_in_bytes\":{\"path\":\"elasticsearch.index.primaries.merges.total_size_in_bytes\",\"type\":\"alias\"}}},\"segments\":{\"properties\":{\"count\":{\"path\":\"elasticsearch.index.primaries.segments.count\",\"type\":\"alias\"}}}}},\"total\":{\"properties\":{\"search\":{\"properties\":{\"query_total\":{\"path\":\"elasticsearch.index.total.search.query_total\",\"type\":\"alias\"},\"query_time_in_millis\":{\"path\":\"elasticsearch.index.total.search.query_time_in_millis\",\"type\":\"alias\"}}},\"query_cache\":{\"properties\":{\"memory_size_in_bytes\":{\"path\":\"elasticsearch.index.total.query_cache.memory_size_in_bytes\",\"type\":\"alias\"}}},\"fielddata\":{\"properties\":{\"memory_size_in_bytes\":{\"path\":\"elasticsearch.index.total.fielddata.memory_size_in_bytes\",\"type\":\"alias\"}}},\"indexing\":{\"properties\":{\"throttle_time_in_millis\":{\"path\":\"elasticsearch.index.total.indexing.throttle_time_in_millis\",\"type\":\"alias\"},\"index_time_in_millis\":{\"path\":\"elasticsearch.index.total.indexing.index_time_in_millis\",\"type\":\"alias\"},\"index_total\":{\"path\":\"elasticsearch.index.total.indexing.index_total\",\"type\":\"alias\"}}},\"refresh\":{\"properties\":{\"total_time_in_millis\":{\"path\":\"elasticsearch.index.total.refresh.total_time_in_millis\",\"type\":\"alias\"}}},\"request_cache\":{\"properties\":{\"memory_size_in_bytes\":{\"path\":\"elasticsearch.index.total.request_cache.memory_size_in_bytes\",\"type\":\"alias\"}}},\"store\":{\"properties\":{\"size_in_bytes\":{\"path\":\"elasticsearch.index.total.store.size_in_bytes\",\"type\":\"alias\"}}},\"merges\":{\"properties\":{\"total_size_in_bytes\":{\"path\":\"elasticsearch.index.total.merges.total_size_in_bytes\",\"type\":\"alias\"}}},\"segments\":{\"properties\":{\"version_map_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.version_map_memory_in_bytes\",\"type\":\"alias\"},\"norms_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.norms_memory_in_bytes\",\"type\":\"alias\"},\"count\":{\"path\":\"elasticsearch.index.total.segments.count\",\"type\":\"alias\"},\"term_vectors_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.term_vectors_memory_in_bytes\",\"type\":\"alias\"},\"points_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.points_memory_in_bytes\",\"type\":\"alias\"},\"index_writer_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.index_writer_memory_in_bytes\",\"type\":\"alias\"},\"memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.memory_in_bytes\",\"type\":\"alias\"},\"doc_values_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.doc_values_memory_in_bytes\",\"type\":\"alias\"},\"terms_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.terms_memory_in_bytes\",\"type\":\"alias\"},\"fixed_bit_set_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.fixed_bit_set_memory_in_bytes\",\"type\":\"alias\"},\"stored_fields_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.stored_fields_memory_in_bytes\",\"type\":\"alias\"}}}}},\"index\":{\"path\":\"elasticsearch.index.name\",\"type\":\"alias\"}}},\"@timestamp\":{\"type\":\"date\"},\"elasticsearch\":{\"properties\":{\"cluster\":{\"properties\":{\"stats\":{\"properties\":{\"license\":{\"properties\":{\"expiry_date_in_millis\":{\"type\":\"long\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"status\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"indices\":{\"properties\":{\"shards\":{\"properties\":{\"primaries\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"total\":{\"type\":\"long\"},\"docs\":{\"properties\":{\"total\":{\"type\":\"long\"}}},\"fielddata\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"store\":{\"properties\":{\"size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}}}},\"stack\":{\"properties\":{\"xpack\":{\"properties\":{\"ccr\":{\"properties\":{\"available\":{\"type\":\"boolean\"},\"enabled\":{\"type\":\"boolean\"}}}}},\"apm\":{\"properties\":{\"found\":{\"type\":\"boolean\"}}}}},\"nodes\":{\"properties\":{\"jvm\":{\"properties\":{\"memory\":{\"properties\":{\"heap\":{\"properties\":{\"max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"used\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}}}},\"max_uptime\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"data\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"},\"fs\":{\"properties\":{\"total\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"available\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"master\":{\"type\":\"long\"}}},\"state\":{\"properties\":{\"nodes_hash\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"master_node\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state_uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"status\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"properties\":{\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"node\":{\"properties\":{\"jvm\":{\"properties\":{\"memory\":{\"properties\":{\"heap\":{\"properties\":{\"init\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"nonheap\":{\"properties\":{\"init\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}}}},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"process\":{\"properties\":{\"mlockall\":{\"type\":\"boolean\"}}},\"stats\":{\"properties\":{\"jvm\":{\"properties\":{\"mem\":{\"properties\":{\"pools\":{\"properties\":{\"young\":{\"properties\":{\"max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"peak\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"peak_max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"used\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"old\":{\"properties\":{\"max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"peak\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"peak_max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"used\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"survivor\":{\"properties\":{\"max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"peak\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"peak_max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"used\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}}}},\"heap\":{\"properties\":{\"max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"used\":{\"properties\":{\"pct\":{\"type\":\"double\"},\"bytes\":{\"type\":\"long\"}}}}}}},\"gc\":{\"properties\":{\"collectors\":{\"properties\":{\"young\":{\"properties\":{\"collection\":{\"properties\":{\"ms\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}}}},\"old\":{\"properties\":{\"collection\":{\"properties\":{\"ms\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}}}}}}}}}},\"indices\":{\"properties\":{\"search\":{\"properties\":{\"query_total\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"query_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"query_cache\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"docs\":{\"properties\":{\"deleted\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"fielddata\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"indexing\":{\"properties\":{\"index_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"index_total\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"throttle_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"store\":{\"properties\":{\"size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"request_cache\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"bulk\":{\"properties\":{\"operations\":{\"properties\":{\"total\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"total_size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"avg_size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"total_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"avg_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"segments\":{\"properties\":{\"version_map\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"norms\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"terms\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"stored_fields\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"index_writer\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"count\":{\"type\":\"long\"},\"fixed_bit_set\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"term_vectors\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"doc_values\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"points\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}}}}}},\"process\":{\"properties\":{\"cpu\":{\"properties\":{\"pct\":{\"type\":\"double\"}}}}},\"os\":{\"properties\":{\"cpu\":{\"properties\":{\"load_avg\":{\"properties\":{\"1m\":{\"type\":\"half_float\"}}}}},\"cgroup\":{\"properties\":{\"memory\":{\"properties\":{\"control_group\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"usage\":{\"properties\":{\"bytes\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"limit\":{\"properties\":{\"bytes\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"cpu\":{\"properties\":{\"cfs\":{\"properties\":{\"quota\":{\"properties\":{\"us\":{\"type\":\"long\"}}}}},\"stat\":{\"properties\":{\"elapsed_periods\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"times_throttled\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"time_throttled\":{\"properties\":{\"ns\":{\"type\":\"long\"}}}}}}},\"cpuacct\":{\"properties\":{\"usage\":{\"properties\":{\"ns\":{\"type\":\"long\"}}}}}}}}},\"thread_pool\":{\"properties\":{\"search\":{\"properties\":{\"rejected\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"queue\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"get\":{\"properties\":{\"rejected\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"queue\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"index\":{\"properties\":{\"rejected\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"queue\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"bulk\":{\"properties\":{\"rejected\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"queue\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"write\":{\"properties\":{\"rejected\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"queue\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}}}},\"fs\":{\"properties\":{\"summary\":{\"properties\":{\"total\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"available\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"free\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"io_stats\":{\"properties\":{\"total\":{\"properties\":{\"operations\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"read\":{\"properties\":{\"operations\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"write\":{\"properties\":{\"operations\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}}}}}},\"total\":{\"properties\":{\"total_in_bytes\":{\"type\":\"long\"},\"available_in_bytes\":{\"type\":\"long\"}}}}}}},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"mlockall\":{\"type\":\"boolean\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"master\":{\"type\":\"boolean\"}}},\"ccr\":{\"properties\":{\"leader\":{\"properties\":{\"index\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"max_seq_no\":{\"type\":\"long\"},\"global_checkpoint\":{\"type\":\"long\"}}},\"follower\":{\"properties\":{\"operations\":{\"properties\":{\"read\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"time_since_last_read\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"index\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"settings_version\":{\"type\":\"long\"},\"shard\":{\"properties\":{\"number\":{\"type\":\"long\"}}},\"max_seq_no\":{\"type\":\"long\"},\"mapping_version\":{\"type\":\"long\"},\"aliases_version\":{\"type\":\"long\"},\"operations_written\":{\"type\":\"long\"},\"global_checkpoint\":{\"type\":\"long\"}}},\"remote_cluster\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"read_exceptions\":{\"type\":\"nested\"},\"shard_id\":{\"type\":\"long\"},\"bytes_read\":{\"type\":\"long\"},\"requests\":{\"properties\":{\"outstanding\":{\"properties\":{\"read\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"write\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"failed\":{\"properties\":{\"read\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"write\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"successful\":{\"properties\":{\"read\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"write\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}}}},\"auto_follow\":{\"properties\":{\"success\":{\"properties\":{\"follow_indices\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"failed\":{\"properties\":{\"follow_indices\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"remote_cluster_state_requests\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}}}},\"total_time\":{\"properties\":{\"read\":{\"properties\":{\"ms\":{\"type\":\"long\"},\"remote_exec\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"write\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"last_requested_seq_no\":{\"type\":\"long\"},\"write_buffer\":{\"properties\":{\"size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"operation\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}}}},\"pending_tasks\":{\"properties\":{\"time_in_queue\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"source\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"priority\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"insert_order\":{\"type\":\"long\"}}},\"enrich\":{\"properties\":{\"executed_searches\":{\"properties\":{\"total\":{\"type\":\"long\"}}},\"remote_requests\":{\"properties\":{\"current\":{\"type\":\"long\"},\"total\":{\"type\":\"long\"}}},\"executing_policy\":{\"properties\":{\"task\":{\"properties\":{\"task\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"parent_task_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"action\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"type\":\"long\"},\"time\":{\"properties\":{\"running\":{\"properties\":{\"nano\":{\"type\":\"long\"}}},\"start\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"cancellable\":{\"type\":\"boolean\"}}},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"queue\":{\"properties\":{\"size\":{\"type\":\"long\"}}}}},\"index\":{\"properties\":{\"summary\":{\"properties\":{\"primaries\":{\"properties\":{\"search\":{\"properties\":{\"query\":{\"properties\":{\"count\":{\"type\":\"long\"},\"time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}}}},\"docs\":{\"properties\":{\"deleted\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"indexing\":{\"properties\":{\"index\":{\"properties\":{\"count\":{\"type\":\"long\"},\"time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}}}},\"store\":{\"properties\":{\"size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"bulk\":{\"properties\":{\"operations\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"time\":{\"properties\":{\"avg\":{\"properties\":{\"ms\":{\"type\":\"long\"},\"bytes\":{\"type\":\"long\"}}},\"count\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}}}},\"segments\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"count\":{\"type\":\"long\"}}}}},\"total\":{\"properties\":{\"search\":{\"properties\":{\"query\":{\"properties\":{\"count\":{\"type\":\"long\"},\"time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}}}},\"docs\":{\"properties\":{\"deleted\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"indexing\":{\"properties\":{\"index\":{\"properties\":{\"count\":{\"type\":\"long\"},\"time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"is_throttled\":{\"type\":\"boolean\"},\"throttle_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"store\":{\"properties\":{\"size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"bulk\":{\"properties\":{\"operations\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"time\":{\"properties\":{\"avg\":{\"properties\":{\"ms\":{\"type\":\"long\"},\"bytes\":{\"type\":\"long\"}}}}}}},\"segments\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"count\":{\"type\":\"long\"}}}}}}},\"shards\":{\"properties\":{\"primaries\":{\"type\":\"long\"},\"total\":{\"type\":\"long\"}}},\"primaries\":{\"properties\":{\"search\":{\"properties\":{\"query_total\":{\"type\":\"long\"},\"query_time_in_millis\":{\"type\":\"long\"}}},\"query_cache\":{\"properties\":{\"miss_count\":{\"type\":\"long\"},\"memory_size_in_bytes\":{\"type\":\"long\"},\"hit_count\":{\"type\":\"long\"}}},\"docs\":{\"properties\":{\"deleted\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"indexing\":{\"properties\":{\"throttle_time_in_millis\":{\"type\":\"long\"},\"index_time_in_millis\":{\"type\":\"long\"},\"index_total\":{\"type\":\"long\"}}},\"refresh\":{\"properties\":{\"external_total_time_in_millis\":{\"type\":\"long\"},\"total_time_in_millis\":{\"type\":\"long\"}}},\"request_cache\":{\"properties\":{\"miss_count\":{\"type\":\"long\"},\"memory_size_in_bytes\":{\"type\":\"long\"},\"evictions\":{\"type\":\"long\"},\"hit_count\":{\"type\":\"long\"}}},\"store\":{\"properties\":{\"size_in_bytes\":{\"type\":\"long\"}}},\"segments\":{\"properties\":{\"version_map_memory_in_bytes\":{\"type\":\"long\"},\"norms_memory_in_bytes\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"},\"term_vectors_memory_in_bytes\":{\"type\":\"long\"},\"points_memory_in_bytes\":{\"type\":\"long\"},\"index_writer_memory_in_bytes\":{\"type\":\"long\"},\"memory_in_bytes\":{\"type\":\"long\"},\"terms_memory_in_bytes\":{\"type\":\"long\"},\"doc_values_memory_in_bytes\":{\"type\":\"long\"},\"stored_fields_memory_in_bytes\":{\"type\":\"long\"},\"fixed_bit_set_memory_in_bytes\":{\"type\":\"long\"}}},\"merges\":{\"properties\":{\"total_size_in_bytes\":{\"type\":\"long\"}}}}},\"total\":{\"properties\":{\"search\":{\"properties\":{\"query_total\":{\"type\":\"long\"},\"query_time_in_millis\":{\"type\":\"long\"}}},\"query_cache\":{\"properties\":{\"miss_count\":{\"type\":\"long\"},\"memory_size_in_bytes\":{\"type\":\"long\"},\"evictions\":{\"type\":\"long\"},\"hit_count\":{\"type\":\"long\"}}},\"docs\":{\"properties\":{\"deleted\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"fielddata\":{\"properties\":{\"memory_size_in_bytes\":{\"type\":\"long\"},\"evictions\":{\"type\":\"long\"}}},\"indexing\":{\"properties\":{\"throttle_time_in_millis\":{\"type\":\"long\"},\"index_time_in_millis\":{\"type\":\"long\"},\"index_total\":{\"type\":\"long\"}}},\"refresh\":{\"properties\":{\"external_total_time_in_millis\":{\"type\":\"long\"},\"total_time_in_millis\":{\"type\":\"long\"}}},\"store\":{\"properties\":{\"size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"size_in_bytes\":{\"type\":\"long\"}}},\"request_cache\":{\"properties\":{\"miss_count\":{\"type\":\"long\"},\"memory_size_in_bytes\":{\"type\":\"long\"},\"evictions\":{\"type\":\"long\"},\"hit_count\":{\"type\":\"long\"}}},\"bulk\":{\"properties\":{\"avg_time_in_millis\":{\"type\":\"long\"},\"avg_size_in_bytes\":{\"type\":\"long\"},\"total_time_in_millis\":{\"type\":\"long\"},\"total_operations\":{\"type\":\"long\"},\"total_size_in_bytes\":{\"type\":\"long\"}}},\"merges\":{\"properties\":{\"total_size_in_bytes\":{\"type\":\"long\"}}},\"segments\":{\"properties\":{\"version_map_memory_in_bytes\":{\"type\":\"long\"},\"norms_memory_in_bytes\":{\"type\":\"long\"},\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"count\":{\"type\":\"long\"},\"term_vectors_memory_in_bytes\":{\"type\":\"long\"},\"points_memory_in_bytes\":{\"type\":\"long\"},\"index_writer_memory_in_bytes\":{\"type\":\"long\"},\"memory_in_bytes\":{\"type\":\"long\"},\"terms_memory_in_bytes\":{\"type\":\"long\"},\"doc_values_memory_in_bytes\":{\"type\":\"long\"},\"stored_fields_memory_in_bytes\":{\"type\":\"long\"},\"fixed_bit_set_memory_in_bytes\":{\"type\":\"long\"}}}}},\"hidden\":{\"type\":\"boolean\"},\"created\":{\"type\":\"long\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"recovery\":{\"properties\":{\"stop_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"translog\":{\"properties\":{\"total_on_start\":{\"type\":\"long\"},\"total\":{\"type\":\"long\"},\"percent\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"verify_index\":{\"properties\":{\"total_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"check_index_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"index\":{\"properties\":{\"size\":{\"properties\":{\"total_in_bytes\":{\"type\":\"long\"},\"reused_in_bytes\":{\"type\":\"long\"},\"recovered_in_bytes\":{\"type\":\"long\"}}},\"files\":{\"properties\":{\"recovered\":{\"type\":\"long\"},\"total\":{\"type\":\"long\"},\"percent\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"reused\":{\"type\":\"long\"}}}}},\"source\":{\"properties\":{\"transport_address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"host\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"target\":{\"properties\":{\"transport_address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"host\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"start_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"stage\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"type\":\"long\"},\"total_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"primary\":{\"type\":\"boolean\"}}},\"uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"status\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"shard\":{\"properties\":{\"number\":{\"type\":\"long\"},\"source_node\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"relocating_node\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"primary\":{\"type\":\"boolean\"}}},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ml\":{\"properties\":{\"job\":{\"properties\":{\"data\":{\"properties\":{\"invalid_date\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"data_counts\":{\"properties\":{\"invalid_date_count\":{\"type\":\"long\"},\"processed_record_count\":{\"type\":\"long\"}}},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"forecasts_stats\":{\"properties\":{\"total\":{\"type\":\"long\"}}},\"model_size\":{\"properties\":{\"memory_status\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}}}}}},\"cluster_uuid\":{\"path\":\"elasticsearch.cluster.id\",\"type\":\"alias\"},\"ecs\":{\"properties\":{\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"service\":{\"properties\":{\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"index_recovery\":{\"properties\":{\"shards\":{\"properties\":{\"stop_time_in_millis\":{\"path\":\"elasticsearch.index.recovery.stop_time.ms\",\"type\":\"alias\"},\"start_time_in_millis\":{\"path\":\"elasticsearch.index.recovery.start_time.ms\",\"type\":\"alias\"}}}}},\"source_node\":{\"properties\":{\"name\":{\"path\":\"elasticsearch.node.name\",\"type\":\"alias\"},\"uuid\":{\"path\":\"elasticsearch.node.id\",\"type\":\"alias\"}}},\"host\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"metricset\":{\"properties\":{\"period\":{\"type\":\"long\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"shard\":{\"properties\":{\"node\":{\"path\":\"elasticsearch.node.id\",\"type\":\"alias\"},\"index\":{\"path\":\"elasticsearch.index.name\",\"type\":\"alias\"},\"state\":{\"path\":\"elasticsearch.shard.state\",\"type\":\"alias\"},\"shard\":{\"path\":\"elasticsearch.shard.number\",\"type\":\"alias\"},\"primary\":{\"path\":\"elasticsearch.shard.primary\",\"type\":\"alias\"}}},\"event\":{\"properties\":{\"duration\":{\"type\":\"long\"},\"module\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"dataset\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"cluster_stats\":{\"properties\":{\"indices\":{\"properties\":{\"shards\":{\"properties\":{\"total\":{\"path\":\"elasticsearch.cluster.stats.indices.shards.count\",\"type\":\"alias\"}}},\"count\":{\"path\":\"elasticsearch.cluster.stats.indices.total\",\"type\":\"alias\"}}},\"nodes\":{\"properties\":{\"jvm\":{\"properties\":{\"max_uptime_in_millis\":{\"path\":\"elasticsearch.cluster.stats.nodes.jvm.max_uptime.ms\",\"type\":\"alias\"},\"mem\":{\"properties\":{\"heap_max_in_bytes\":{\"path\":\"elasticsearch.cluster.stats.nodes.jvm.memory.heap.max.bytes\",\"type\":\"alias\"},\"heap_used_in_bytes\":{\"path\":\"elasticsearch.cluster.stats.nodes.jvm.memory.heap.used.bytes\",\"type\":\"alias\"}}}}},\"count\":{\"properties\":{\"total\":{\"path\":\"elasticsearch.cluster.stats.nodes.count\",\"type\":\"alias\"}}}}}}},\"timestamp\":{\"path\":\"@timestamp\",\"type\":\"alias\"},\"job_stats\":{\"properties\":{\"job_id\":{\"path\":\"elasticsearch.ml.job.id\",\"type\":\"alias\"},\"forecasts_stats\":{\"properties\":{\"total\":{\"path\":\"elasticsearch.ml.job.forecasts_stats.total\",\"type\":\"alias\"}}}}}}}},\"composed_of\":[],\"version\":8000103,\"data_stream\":{\"hidden\":false,\"allow_custom_routing\":false}},\".slm-history\":{\"index_patterns\":[\".slm-history-5*\"],\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\"slm-history-ilm-policy\"},\"number_of_shards\":\"1\",\"auto_expand_replicas\":\"0-1\",\"number_of_replicas\":\"0\"}},\"mappings\":{\"dynamic\":false,\"properties\":{\"snapshot_name\":{\"type\":\"keyword\"},\"@timestamp\":{\"format\":\"epoch_millis\",\"type\":\"date\"},\"configuration\":{\"dynamic\":false,\"type\":\"object\",\"properties\":{\"indices\":{\"type\":\"keyword\"},\"include_global_state\":{\"type\":\"boolean\"},\"partial\":{\"type\":\"boolean\"}}},\"error_details\":{\"index\":false,\"type\":\"text\"},\"success\":{\"type\":\"boolean\"},\"repository\":{\"type\":\"keyword\"},\"operation\":{\"type\":\"keyword\"},\"policy\":{\"type\":\"keyword\"}}}},\"composed_of\":[],\"priority\":2147483647,\"version\":5,\"_meta\":{\"managed\":true,\"description\":\"index template for SLM history indices\"},\"data_stream\":{\"hidden\":true,\"allow_custom_routing\":false},\"allow_auto_create\":true},\".monitoring-logstash-mb\":{\"index_patterns\":[\".monitoring-logstash-8-*\"],\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\".monitoring-8-ilm-policy\"},\"mapping\":{\"total_fields\":{\"limit\":\"2000\"}}}},\"mappings\":{\"dynamic\":false,\"properties\":{\"logstash\":{\"properties\":{\"node\":{\"properties\":{\"jvm\":{\"properties\":{\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"stats\":{\"properties\":{\"jvm\":{\"properties\":{\"mem\":{\"properties\":{\"heap_max_in_bytes\":{\"type\":\"long\"},\"heap_used_in_bytes\":{\"type\":\"long\"}}},\"uptime_in_millis\":{\"type\":\"long\"}}},\"logstash\":{\"properties\":{\"uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"process\":{\"properties\":{\"cpu\":{\"properties\":{\"percent\":{\"type\":\"double\"}}}}},\"pipelines\":{\"type\":\"nested\",\"properties\":{\"vertices\":{\"type\":\"nested\",\"properties\":{\"events_out\":{\"type\":\"long\"},\"pipeline_ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"events_in\":{\"type\":\"long\"},\"queue_push_duration_in_millis\":{\"type\":\"long\"},\"duration_in_millis\":{\"type\":\"long\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"long_counters\":{\"type\":\"nested\",\"properties\":{\"name\":{\"type\":\"keyword\"},\"value\":{\"type\":\"long\"}}}}},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"type\":\"keyword\"},\"hash\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"queue\":{\"properties\":{\"events_count\":{\"type\":\"long\"},\"max_queue_size_in_bytes\":{\"type\":\"long\"},\"queue_size_in_bytes\":{\"type\":\"long\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"events\":{\"properties\":{\"filtered\":{\"type\":\"long\"},\"in\":{\"type\":\"long\"},\"queue_push_duration_in_millis\":{\"type\":\"long\"},\"duration_in_millis\":{\"type\":\"long\"},\"out\":{\"type\":\"long\"}}},\"reloads\":{\"properties\":{\"failures\":{\"type\":\"long\"},\"successes\":{\"type\":\"long\"}}}}},\"os\":{\"properties\":{\"cpu\":{\"properties\":{\"load_average\":{\"properties\":{\"5m\":{\"type\":\"half_float\"},\"15m\":{\"type\":\"half_float\"},\"1m\":{\"type\":\"half_float\"}}}}},\"cgroup\":{\"properties\":{\"cpu\":{\"properties\":{\"stat\":{\"properties\":{\"number_of_elapsed_periods\":{\"type\":\"long\"},\"number_of_times_throttled\":{\"type\":\"long\"},\"time_throttled_nanos\":{\"type\":\"long\"}}},\"cfs_quota_micros\":{\"type\":\"long\"}}},\"cpuacct\":{\"properties\":{\"usage_nanos\":{\"type\":\"long\"}}}}}}},\"events\":{\"properties\":{\"filtered\":{\"type\":\"long\"},\"in\":{\"type\":\"long\"},\"duration_in_millis\":{\"type\":\"long\"},\"out\":{\"type\":\"long\"}}},\"queue\":{\"properties\":{\"events_count\":{\"type\":\"long\"}}},\"timestamp\":{\"type\":\"date\"}}},\"host\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"properties\":{\"pipeline\":{\"properties\":{\"batch_size\":{\"type\":\"long\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"workers\":{\"type\":\"long\"},\"representation\":{\"properties\":{\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"hash\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"graph\":{\"properties\":{\"vertices\":{\"type\":\"object\"},\"edges\":{\"type\":\"object\"}}}}},\"hash\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"elasticsearch\":{\"properties\":{\"cluster\":{\"properties\":{\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}}}},\"logstash_state\":{\"properties\":{\"pipeline\":{\"properties\":{\"id\":{\"path\":\"logstash.node.state.pipeline.id\",\"type\":\"alias\"},\"hash\":{\"path\":\"logstash.node.state.pipeline.hash\",\"type\":\"alias\"}}}}},\"process\":{\"properties\":{\"pid\":{\"type\":\"long\"}}},\"@timestamp\":{\"type\":\"date\"},\"logstash_stats\":{\"properties\":{\"jvm\":{\"properties\":{\"mem\":{\"properties\":{\"heap_max_in_bytes\":{\"path\":\"logstash.node.stats.jvm.mem.heap_max_in_bytes\",\"type\":\"alias\"},\"heap_used_in_bytes\":{\"path\":\"logstash.node.stats.jvm.mem.heap_used_in_bytes\",\"type\":\"alias\"}}},\"uptime_in_millis\":{\"path\":\"logstash.node.stats.jvm.uptime_in_millis\",\"type\":\"alias\"}}},\"logstash\":{\"properties\":{\"uuid\":{\"path\":\"logstash.node.stats.logstash.uuid\",\"type\":\"alias\"},\"version\":{\"path\":\"logstash.node.stats.logstash.version\",\"type\":\"alias\"}}},\"process\":{\"properties\":{\"cpu\":{\"properties\":{\"percent\":{\"path\":\"logstash.node.stats.process.cpu.percent\",\"type\":\"alias\"}}}}},\"pipelines\":{\"type\":\"nested\"},\"os\":{\"properties\":{\"cpu\":{\"properties\":{\"load_average\":{\"properties\":{\"5m\":{\"path\":\"logstash.node.stats.os.cpu.load_average.5m\",\"type\":\"alias\"},\"15m\":{\"path\":\"logstash.node.stats.os.cpu.load_average.15m\",\"type\":\"alias\"},\"1m\":{\"path\":\"logstash.node.stats.os.cpu.load_average.1m\",\"type\":\"alias\"}}}}},\"cgroup\":{\"properties\":{\"cpu\":{\"properties\":{\"stat\":{\"properties\":{\"number_of_elapsed_periods\":{\"path\":\"logstash.node.stats.os.cgroup.cpu.stat.number_of_elapsed_periods\",\"type\":\"alias\"},\"number_of_times_throttled\":{\"path\":\"logstash.node.stats.os.cgroup.cpu.stat.number_of_times_throttled\",\"type\":\"alias\"},\"time_throttled_nanos\":{\"path\":\"logstash.node.stats.os.cgroup.cpu.stat.time_throttled_nanos\",\"type\":\"alias\"}}},\"cfs_quota_micros\":{\"path\":\"logstash.node.stats.os.cgroup.cpu.cfs_quota_micros\",\"type\":\"alias\"}}},\"cpuacct\":{\"properties\":{\"usage_nanos\":{\"path\":\"logstash.node.stats.os.cgroup.cpuacct.usage_nanos\",\"type\":\"alias\"}}}}}}},\"events\":{\"properties\":{\"in\":{\"path\":\"logstash.node.stats.events.in\",\"type\":\"alias\"},\"duration_in_millis\":{\"path\":\"logstash.node.stats.events.duration_in_millis\",\"type\":\"alias\"},\"out\":{\"path\":\"logstash.node.stats.events.out\",\"type\":\"alias\"}}},\"queue\":{\"properties\":{\"events_count\":{\"path\":\"logstash.node.stats.queue.events_count\",\"type\":\"alias\"}}},\"timestamp\":{\"path\":\"@timestamp\",\"type\":\"alias\"}}},\"ecs\":{\"properties\":{\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"cluster_uuid\":{\"path\":\"logstash.elasticsearch.cluster.id\",\"type\":\"alias\"},\"service\":{\"properties\":{\"hostname\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"host\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"metricset\":{\"properties\":{\"period\":{\"type\":\"long\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"event\":{\"properties\":{\"duration\":{\"type\":\"long\"},\"module\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"dataset\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"timestamp\":{\"path\":\"@timestamp\",\"type\":\"alias\"}}}},\"composed_of\":[],\"version\":8000103,\"data_stream\":{\"hidden\":false,\"allow_custom_routing\":false}},\".ml-anomalies-\":{\"index_patterns\":[\".ml-anomalies-*\"],\"template\":{\"settings\":{\"index\":{\"hidden\":\"true\",\"translog\":{\"durability\":\"async\"},\"auto_expand_replicas\":\"0-1\",\"query\":{\"default_field\":\"all_field_values\"}}},\"mappings\":{\"_meta\":{\"version\":\"8.6.2\"},\"dynamic_templates\":[{\"strings_as_keywords\":{\"mapping\":{\"type\":\"keyword\"},\"match\":\"*\"}}],\"properties\":{\"search_count\":{\"type\":\"long\"},\"bucket_count\":{\"type\":\"long\"},\"anomaly_score_explanation\":{\"properties\":{\"high_variance_penalty\":{\"type\":\"boolean\"},\"typical_value\":{\"type\":\"double\"},\"upper_confidence_bound\":{\"type\":\"double\"},\"lower_confidence_bound\":{\"type\":\"double\"},\"incomplete_bucket_penalty\":{\"type\":\"boolean\"},\"anomaly_type\":{\"type\":\"keyword\"},\"anomaly_characteristics_impact\":{\"type\":\"integer\"},\"multi_bucket_impact\":{\"type\":\"integer\"},\"anomaly_length\":{\"type\":\"integer\"},\"single_bucket_impact\":{\"type\":\"integer\"}}},\"terms\":{\"type\":\"text\"},\"record_score\":{\"type\":\"double\"},\"forecast_expiry_timestamp\":{\"type\":\"date\"},\"over_field_value\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"preferred_to_categories\":{\"type\":\"long\"},\"empty_bucket_count\":{\"type\":\"long\"},\"forecast_create_timestamp\":{\"type\":\"date\"},\"probability\":{\"type\":\"double\"},\"missing_field_count\":{\"type\":\"long\"},\"forecast_lower\":{\"type\":\"double\"},\"influencer_field_value\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"multi_bucket_impact\":{\"type\":\"double\"},\"forecast_progress\":{\"type\":\"double\"},\"max_matching_length\":{\"type\":\"long\"},\"latest_record_time_stamp\":{\"type\":\"date\"},\"regex\":{\"type\":\"keyword\"},\"examples\":{\"type\":\"text\"},\"snapshot_doc_count\":{\"type\":\"integer\"},\"average_bucket_processing_time_ms\":{\"type\":\"double\"},\"initial_anomaly_score\":{\"type\":\"double\"},\"initial_influencer_score\":{\"type\":\"double\"},\"partition_field_value\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"model_lower\":{\"type\":\"double\"},\"total_over_field_count\":{\"type\":\"long\"},\"retain\":{\"type\":\"boolean\"},\"forecast_upper\":{\"type\":\"double\"},\"latest_sparse_bucket_timestamp\":{\"type\":\"date\"},\"model_median\":{\"type\":\"double\"},\"category_id\":{\"type\":\"long\"},\"causes\":{\"type\":\"nested\",\"properties\":{\"actual\":{\"type\":\"double\"},\"partition_field_name\":{\"type\":\"keyword\"},\"partition_field_value\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"by_field_name\":{\"type\":\"keyword\"},\"probability\":{\"type\":\"double\"},\"geo_results\":{\"properties\":{\"actual_point\":{\"type\":\"geo_point\"},\"typical_point\":{\"type\":\"geo_point\"}}},\"field_name\":{\"type\":\"keyword\"},\"by_field_value\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"over_field_name\":{\"type\":\"keyword\"},\"correlated_by_field_value\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"function\":{\"type\":\"keyword\"},\"typical\":{\"type\":\"double\"},\"over_field_value\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"function_description\":{\"type\":\"keyword\"}}},\"all_field_values\":{\"analyzer\":\"whitespace\",\"type\":\"text\"},\"timestamp\":{\"type\":\"date\"},\"input_field_count\":{\"type\":\"long\"},\"model_bytes\":{\"type\":\"long\"},\"quantiles\":{\"type\":\"object\",\"enabled\":false},\"@timestamp\":{\"path\":\"timestamp\",\"type\":\"alias\"},\"function_description\":{\"type\":\"keyword\"},\"min_version\":{\"type\":\"keyword\"},\"raw_anomaly_score\":{\"type\":\"double\"},\"exponential_average_bucket_processing_time_ms\":{\"type\":\"double\"},\"invalid_date_count\":{\"type\":\"long\"},\"snapshot_id\":{\"type\":\"keyword\"},\"model_size_stats\":{\"properties\":{\"model_bytes\":{\"type\":\"long\"},\"result_type\":{\"type\":\"keyword\"},\"peak_model_bytes\":{\"type\":\"long\"},\"job_id\":{\"type\":\"keyword\"},\"total_over_field_count\":{\"type\":\"long\"},\"total_partition_field_count\":{\"type\":\"long\"},\"total_by_field_count\":{\"type\":\"long\"},\"assignment_memory_basis\":{\"type\":\"keyword\"},\"bucket_allocation_failures_count\":{\"type\":\"long\"},\"memory_status\":{\"type\":\"keyword\"},\"log_time\":{\"type\":\"date\"},\"timestamp\":{\"type\":\"date\"}}},\"total_search_time_ms\":{\"type\":\"double\"},\"latest_empty_bucket_timestamp\":{\"type\":\"date\"},\"anomaly_score\":{\"type\":\"double\"},\"over_field_name\":{\"type\":\"keyword\"},\"earliest_record_timestamp\":{\"type\":\"date\"},\"scheduled_events\":{\"type\":\"keyword\"},\"bucket_span\":{\"type\":\"long\"},\"maximum_bucket_processing_time_ms\":{\"type\":\"double\"},\"exponential_average_calculation_context\":{\"properties\":{\"incremental_metric_value_ms\":{\"type\":\"double\"},\"previous_exponential_average_ms\":{\"type\":\"double\"},\"latest_timestamp\":{\"type\":\"date\"}}},\"function\":{\"type\":\"keyword\"},\"influencers\":{\"type\":\"nested\",\"properties\":{\"influencer_field_name\":{\"type\":\"keyword\"},\"influencer_field_values\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"}}},\"input_record_count\":{\"type\":\"long\"},\"latest_result_time_stamp\":{\"type\":\"date\"},\"model_upper\":{\"type\":\"double\"},\"actual\":{\"type\":\"double\"},\"forecast_memory_bytes\":{\"type\":\"long\"},\"total_by_field_count\":{\"type\":\"long\"},\"geo_results\":{\"properties\":{\"actual_point\":{\"type\":\"geo_point\"},\"typical_point\":{\"type\":\"geo_point\"}}},\"detector_index\":{\"type\":\"integer\"},\"by_field_value\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"processed_record_count\":{\"type\":\"long\"},\"total_partition_field_count\":{\"type\":\"long\"},\"mlcategory\":{\"type\":\"keyword\"},\"assignment_memory_basis\":{\"type\":\"keyword\"},\"forecast_end_timestamp\":{\"type\":\"date\"},\"forecast_id\":{\"type\":\"keyword\"},\"partition_field_name\":{\"type\":\"keyword\"},\"by_field_name\":{\"type\":\"keyword\"},\"event_count\":{\"type\":\"long\"},\"description\":{\"type\":\"text\"},\"is_interim\":{\"type\":\"boolean\"},\"bucket_allocation_failures_count\":{\"type\":\"long\"},\"memory_status\":{\"type\":\"keyword\"},\"model_feature\":{\"type\":\"keyword\"},\"num_matches\":{\"type\":\"long\"},\"influencer_score\":{\"type\":\"double\"},\"out_of_order_timestamp_count\":{\"type\":\"long\"},\"result_type\":{\"type\":\"keyword\"},\"last_data_time\":{\"type\":\"date\"},\"latest_record_timestamp\":{\"type\":\"date\"},\"influencer_field_name\":{\"type\":\"keyword\"},\"forecast_messages\":{\"type\":\"keyword\"},\"sparse_bucket_count\":{\"type\":\"long\"},\"log_time\":{\"type\":\"date\"},\"field_name\":{\"type\":\"keyword\"},\"minimum_bucket_processing_time_ms\":{\"type\":\"double\"},\"bucket_influencers\":{\"type\":\"nested\",\"properties\":{\"anomaly_score\":{\"type\":\"double\"},\"initial_anomaly_score\":{\"type\":\"double\"},\"result_type\":{\"type\":\"keyword\"},\"raw_anomaly_score\":{\"type\":\"double\"},\"bucket_span\":{\"type\":\"long\"},\"job_id\":{\"type\":\"keyword\"},\"probability\":{\"type\":\"double\"},\"influencer_field_name\":{\"type\":\"keyword\"},\"is_interim\":{\"type\":\"boolean\"},\"timestamp\":{\"type\":\"date\"}}},\"processing_time_ms\":{\"type\":\"long\"},\"input_bytes\":{\"type\":\"long\"},\"initial_record_score\":{\"type\":\"double\"},\"job_id\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"processed_field_count\":{\"type\":\"long\"},\"forecast_status\":{\"type\":\"keyword\"},\"typical\":{\"type\":\"double\"},\"forecast_prediction\":{\"type\":\"double\"},\"forecast_start_timestamp\":{\"type\":\"date\"}}}},\"composed_of\":[],\"priority\":2147483647,\"version\":8060299,\"_meta\":{\"managed\":true,\"description\":\"index template for ML anomaly detection results indices\"}},\"metrics\":{\"index_patterns\":[\"metrics-*-*\"],\"composed_of\":[\"metrics-mappings\",\"data-streams-mappings\",\"metrics-settings\"],\"priority\":100,\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default metrics template installed by x-pack\"},\"data_stream\":{\"hidden\":false,\"allow_custom_routing\":false},\"allow_auto_create\":true},\".ml-notifications-000002\":{\"index_patterns\":[\".ml-notifications-000002\"],\"template\":{\"settings\":{\"index\":{\"hidden\":\"true\",\"number_of_shards\":\"1\",\"auto_expand_replicas\":\"0-1\"}},\"mappings\":{\"_meta\":{\"version\":\"8.6.2\"},\"dynamic\":\"false\",\"properties\":{\"job_type\":{\"type\":\"keyword\"},\"level\":{\"type\":\"keyword\"},\"job_id\":{\"type\":\"keyword\"},\"node_name\":{\"type\":\"keyword\"},\"message\":{\"type\":\"text\",\"fields\":{\"raw\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"cleared\":{\"type\":\"boolean\"},\"timestamp\":{\"type\":\"date\"}}}},\"composed_of\":[],\"priority\":2147483647,\"version\":8060299,\"_meta\":{\"managed\":true,\"description\":\"index template for ML notifications indices\"}},\".deprecation-indexing-template\":{\"index_patterns\":[\".logs-deprecation.*\"],\"composed_of\":[\".deprecation-indexing-mappings\",\".deprecation-indexing-settings\"],\"priority\":1000,\"version\":1,\"_meta\":{\"managed\":true,\"description\":\"default template for Stack deprecation logs index template installed by x-pack\"},\"data_stream\":{\"hidden\":true,\"allow_custom_routing\":false},\"allow_auto_create\":true},\"logs\":{\"index_patterns\":[\"logs-*-*\"],\"composed_of\":[\"logs-mappings\",\"data-streams-mappings\",\"logs-settings\"],\"priority\":100,\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default logs template installed by x-pack\"},\"data_stream\":{\"hidden\":false,\"allow_custom_routing\":false},\"allow_auto_create\":true},\".ml-stats\":{\"index_patterns\":[\".ml-stats-*\"],\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\"ml-size-based-ilm-policy\",\"rollover_alias\":\".ml-stats-write\"},\"hidden\":\"true\",\"number_of_shards\":\"1\",\"auto_expand_replicas\":\"0-1\"}},\"mappings\":{\"_meta\":{\"version\":\"8.6.2\"},\"dynamic\":false,\"properties\":{\"skipped_docs_count\":{\"type\":\"long\"},\"validation_loss\":{\"properties\":{\"fold_values\":{\"properties\":{\"fold\":{\"type\":\"integer\"},\"values\":{\"type\":\"double\"}}},\"loss_type\":{\"type\":\"keyword\"}}},\"cache_miss_count\":{\"type\":\"long\"},\"timing_stats\":{\"properties\":{\"iteration_time\":{\"type\":\"long\"},\"elapsed_time\":{\"type\":\"long\"}}},\"failure_count\":{\"type\":\"long\"},\"model_id\":{\"type\":\"keyword\"},\"type\":{\"type\":\"keyword\"},\"training_docs_count\":{\"type\":\"long\"},\"inference_count\":{\"type\":\"long\"},\"job_id\":{\"type\":\"keyword\"},\"missing_all_fields_count\":{\"type\":\"long\"},\"peak_usage_bytes\":{\"type\":\"long\"},\"iteration\":{\"type\":\"integer\"},\"hyperparameters\":{\"properties\":{\"max_attempts_to_add_tree\":{\"type\":\"integer\"},\"downsample_factor\":{\"type\":\"double\"},\"eta_growth_rate_per_tree\":{\"type\":\"double\"},\"soft_tree_depth_tolerance\":{\"type\":\"double\"},\"max_trees\":{\"type\":\"integer\"},\"lambda\":{\"type\":\"double\"},\"max_optimization_rounds_per_hyperparameter\":{\"type\":\"integer\"},\"eta\":{\"type\":\"double\"},\"soft_tree_depth_limit\":{\"type\":\"double\"},\"alpha\":{\"type\":\"double\"},\"class_assignment_objective\":{\"type\":\"keyword\"},\"feature_bag_fraction\":{\"type\":\"double\"},\"num_splits_per_feature\":{\"type\":\"integer\"},\"gamma\":{\"type\":\"double\"},\"num_folds\":{\"type\":\"integer\"}}},\"parameters\":{\"properties\":{\"compute_feature_influence\":{\"type\":\"boolean\"},\"feature_influence_threshold\":{\"type\":\"double\"},\"outlier_fraction\":{\"type\":\"double\"},\"method\":{\"type\":\"keyword\"},\"standardization_enabled\":{\"type\":\"boolean\"},\"n_neighbors\":{\"type\":\"integer\"}}},\"test_docs_count\":{\"type\":\"long\"},\"node_id\":{\"type\":\"keyword\"},\"timestamp\":{\"type\":\"date\"}}},\"aliases\":{}},\"composed_of\":[],\"priority\":2147483647,\"version\":8060299,\"_meta\":{\"managed\":true,\"description\":\"index template for ML stats indices\"}}}} index-graveyard: IndexGraveyard[[[index=[abacus_go/zV2aNzxVQxqMDQTD4A9TEQ], deleteDate=2023-03-28T03:56:57.282Z], [index=[access_platform_core/ZuN0_cUfSQKzXg_yG4Sv5g], deleteDate=2023-03-28T03:58:27.958Z], [index=[ad_bidder/6ihfVU6rRqOo4Gv4Djtetg], deleteDate=2023-03-28T03:59:11.628Z], [index=[apphealth/EthxrkUtTGCTP2mX-tS63Q], deleteDate=2023-03-28T03:59:36.596Z]]]\nnodes: \n {hostb9}{iKPGCkp9RVOKXOj20uOt4g}{cPRh9FqiSlGP9jClhm8Ugg}{hostb9}{127.0.0.1}{127.0.0.1:9300}{cdfhilmrstw}{ml.max_jvm_size=33285996544, ml.allocated_processors_double=16.0, xpack.installed=true, ml.machine_memory=135062048768, ml.allocated_processors=16}, local, master\nrouting_table (version 35):\n-- index [[abacus_go/cfvmY-Y_Su63-qSkPODDzg]]\n----shard_id [abacus_go][0]\n--------[abacus_go][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=AMhgJMlSTPOoKSsL7s3OKA], failed_attempts[0]\n--------[abacus_go][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=INDEX_CREATED], at[2023-03-28T03:57:31.683Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[web_wok/R1IULAXSQ4SC0pLFCU5oJA]]\n----shard_id [web_wok][0]\n--------[web_wok][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=JXruVMhdRVGvP1kQCbUNSQ], failed_attempts[0]\n--------[web_wok][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[access_platform_core/l4kdkOEQRaKfWGLpTZURCQ]]\n----shard_id [access_platform_core][0]\n--------[access_platform_core][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=6ajNSTQZSr6tP3YtmRaxgw], failed_attempts[0]\n--------[access_platform_core][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=INDEX_CREATED], at[2023-03-28T03:58:31.318Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[driverapp/cqbade6KQFqLEK-wPKlkBg]]\n----shard_id [driverapp][0]\n--------[driverapp][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=b5_Bhd3ITjOkl-xLF8H1jg], failed_attempts[0]\n--------[driverapp][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[k8s_kubelet/fQ9Iy7wlTKahsWLbs-oE_Q]]\n----shard_id [k8s_kubelet][0]\n--------[k8s_kubelet][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=Q4Xb11JATPaFZqmgTQHcVQ], failed_attempts[0]\n--------[k8s_kubelet][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.307Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[eats_store/hF-kN4XFRy--z_gUhYLwdA]]\n----shard_id [eats_store][0]\n--------[eats_store][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=WRNwRQ6gTNGfSNQok4iSyQ], failed_attempts[0]\n--------[eats_store][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[rider_product_cored/AdGiEBLBQbicsXDomsa3mw]]\n----shard_id [rider_product_cored][0]\n--------[rider_product_cored][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=GwRH_ZlFRtCV7m_n626z7g], failed_attempts[0]\n--------[rider_product_cored][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[finprod_rewards_eligibility/dJNRGPwrQS2bIPlpsXgUoQ]]\n----shard_id [finprod_rewards_eligibility][0]\n--------[finprod_rewards_eligibility][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=Hg4Dm9O8QEqXWKY7rH1X0A], failed_attempts[0]\n--------[finprod_rewards_eligibility][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[event_logs/DcomuHoGQzOjZq2bX67TFg]]\n----shard_id [event_logs][0]\n--------[event_logs][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=1uCl_LIDSBCWh6koVA0fjQ], failed_attempts[0]\n--------[event_logs][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[apphealth/rcPrX1aWRmeED9W5PU6A4g]]\n----shard_id [apphealth][0]\n--------[apphealth][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=Abip50RxQTmavqi4NTwSqQ], failed_attempts[0]\n--------[apphealth][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=INDEX_CREATED], at[2023-03-28T03:59:38.780Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[ad_bidder/tLHfw5r2QH-o7lG5tapKxw]]\n----shard_id [ad_bidder][0]\n--------[ad_bidder][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=YtEUwPgfQDGM0Cm_C4hXlw], failed_attempts[0]\n--------[ad_bidder][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=INDEX_CREATED], at[2023-03-28T03:59:13.333Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[invoice_gen/RPu5EzJVRfuqyI8MQ7iTjQ]]\n----shard_id [invoice_gen][0]\n--------[invoice_gen][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=onWbjhHuTD2C3AEayRJq1w], failed_attempts[0]\n--------[invoice_gen][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[fares_management/F1lH-QvyTkS5hnbDFwE2Xg]]\n----shard_id [fares_management][0]\n--------[fares_management][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=sOxfKX3nReynYN5HN88cuQ], failed_attempts[0]\n--------[fares_management][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[michelangelo_gateway/ZV2vZ8PmS0uysg-xzKO7QA]]\n----shard_id [michelangelo_gateway][0]\n--------[michelangelo_gateway][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=k5vDE54CSTSakhfgtGaTcA], failed_attempts[0]\n--------[michelangelo_gateway][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[.geoip_databases/5k1GHwKvQBKersi12DFh4A]]\n----shard_id [.geoip_databases][0]\n--------[.geoip_databases][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=UdvCKUGqTEmYFzZlnSu_tA], failed_attempts[0]\n\n-- index [[fares_intelligence/pSRJWxyXSqefhwLVhmT9nQ]]\n----shard_id [fares_intelligence][0]\n--------[fares_intelligence][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=-dD8ROsgS6qaYMoalciocA], failed_attempts[0]\n--------[fares_intelligence][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[fare_route/a4fjivWhQRee8JW7ZbXiXw]]\n----shard_id [fare_route][0]\n--------[fare_route][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=ltSLFgu4Rry0_0vXq28DbA], failed_attempts[0]\n--------[fare_route][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[k8s_apiserver/qQasynZgQ6e6AmD5ZWqh3w]]\n----shard_id [k8s_apiserver][0]\n--------[k8s_apiserver][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=Aaa0F0jiStCkteGDYR8yRQ], failed_attempts[0]\n--------[k8s_apiserver][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[fulfillment_compatibled/tMgrGBWeR3irgR5PyN8Oig]]\n----shard_id [fulfillment_compatibled][0]\n--------[fulfillment_compatibled][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=D_3C_3kMSNKa-XJk7pyoGg], failed_attempts[0]\n--------[fulfillment_compatibled][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\nrouting_nodes:\n-----node_id[iKPGCkp9RVOKXOj20uOt4g][V]\n--------[abacus_go][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=AMhgJMlSTPOoKSsL7s3OKA], failed_attempts[0]\n--------[web_wok][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=JXruVMhdRVGvP1kQCbUNSQ], failed_attempts[0]\n--------[access_platform_core][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=6ajNSTQZSr6tP3YtmRaxgw], failed_attempts[0]\n--------[driverapp][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=b5_Bhd3ITjOkl-xLF8H1jg], failed_attempts[0]\n--------[k8s_kubelet][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=Q4Xb11JATPaFZqmgTQHcVQ], failed_attempts[0]\n--------[eats_store][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=WRNwRQ6gTNGfSNQok4iSyQ], failed_attempts[0]\n--------[rider_product_cored][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=GwRH_ZlFRtCV7m_n626z7g], failed_attempts[0]\n--------[finprod_rewards_eligibility][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=Hg4Dm9O8QEqXWKY7rH1X0A], failed_attempts[0]\n--------[event_logs][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=1uCl_LIDSBCWh6koVA0fjQ], failed_attempts[0]\n--------[apphealth][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=Abip50RxQTmavqi4NTwSqQ], failed_attempts[0]\n--------[ad_bidder][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=YtEUwPgfQDGM0Cm_C4hXlw], failed_attempts[0]\n--------[invoice_gen][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=onWbjhHuTD2C3AEayRJq1w], failed_attempts[0]\n--------[fares_management][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=sOxfKX3nReynYN5HN88cuQ], failed_attempts[0]\n--------[michelangelo_gateway][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=k5vDE54CSTSakhfgtGaTcA], failed_attempts[0]\n--------[.geoip_databases][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=UdvCKUGqTEmYFzZlnSu_tA], failed_attempts[0]\n--------[fares_intelligence][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=-dD8ROsgS6qaYMoalciocA], failed_attempts[0]\n--------[fare_route][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=ltSLFgu4Rry0_0vXq28DbA], failed_attempts[0]\n--------[k8s_apiserver][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=Aaa0F0jiStCkteGDYR8yRQ], failed_attempts[0]\n--------[fulfillment_compatibled][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=D_3C_3kMSNKa-XJk7pyoGg], failed_attempts[0]\n---- unassigned\n--------[abacus_go][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=INDEX_CREATED], at[2023-03-28T03:57:31.683Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[web_wok][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[access_platform_core][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=INDEX_CREATED], at[2023-03-28T03:58:31.318Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[driverapp][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[k8s_kubelet][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.307Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[eats_store][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[rider_product_cored][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[finprod_rewards_eligibility][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[event_logs][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[apphealth][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=INDEX_CREATED], at[2023-03-28T03:59:38.780Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[ad_bidder][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=INDEX_CREATED], at[2023-03-28T03:59:13.333Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[invoice_gen][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[fares_management][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[michelangelo_gateway][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[fares_intelligence][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[fare_route][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[k8s_apiserver][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[fulfillment_compatibled][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\ncustoms:\n health: org.elasticsearch.health.metadata.HealthMetadata@66bc0723} to [PublicationTarget{discoveryNode={hostb9}{iKPGCkp9RVOKXOj20uOt4g}{cPRh9FqiSlGP9jClhm8Ugg}{hostb9}{127.0.0.1}{127.0.0.1:9300}{cdfhilmrstw}{ml.max_jvm_size=33285996544, ml.allocated_processors_double=16.0, xpack.installed=true, ml.machine_memory=135062048768, ml.allocated_processors=16}, state=NOT_STARTED, ackIsPending=true}]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.coordination.Coordinator.CoordinatorPublication","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:02.067Z", "log.level":"TRACE", "message":"handling cluster state version [154] locally on [{hostb9}{iKPGCkp9RVOKXOj20uOt4g}{cPRh9FqiSlGP9jClhm8Ugg}{hostb9}{127.0.0.1}{127.0.0.1:9300}{cdfhilmrstw}{ml.max_jvm_size=33285996544, ml.allocated_processors_double=16.0, xpack.installed=true, ml.machine_memory=135062048768, ml.allocated_processors=16}]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.coordination.PublicationTransportHandler","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:02.074Z", "log.level":"TRACE", "message":"handlePublishRequest: handling [PublishRequest{term=6, version=154, state=cluster uuid: bp-xkJD1S1iyBlj6I_JRHA [committed: true]\nversion: 154\nstate uuid: Xx549IAcSMCY30xGiugtFg\nfrom_diff: false\nmeta data version: 56\n coordination_metadata:\n term: 6\n last_committed_config: VotingConfiguration{iKPGCkp9RVOKXOj20uOt4g}\n last_accepted_config: VotingConfiguration{iKPGCkp9RVOKXOj20uOt4g}\n voting tombstones: []\n [k8s_kubelet/fQ9Iy7wlTKahsWLbs-oE_Q]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [Q4Xb11JATPaFZqmgTQHcVQ]\n [fares_intelligence/pSRJWxyXSqefhwLVhmT9nQ]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [-dD8ROsgS6qaYMoalciocA]\n [event_logs/DcomuHoGQzOjZq2bX67TFg]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [1uCl_LIDSBCWh6koVA0fjQ]\n [rider_product_cored/AdGiEBLBQbicsXDomsa3mw]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [GwRH_ZlFRtCV7m_n626z7g]\n [access_platform_core/l4kdkOEQRaKfWGLpTZURCQ]: v[6], mv[1], sv[2], av[1]\n 0: p_term [1], isa_ids [6ajNSTQZSr6tP3YtmRaxgw]\n [.geoip_databases/5k1GHwKvQBKersi12DFh4A]: v[6], mv[1], sv[1], av[1]\n 0: p_term [3], isa_ids [UdvCKUGqTEmYFzZlnSu_tA]\n [web_wok/R1IULAXSQ4SC0pLFCU5oJA]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [JXruVMhdRVGvP1kQCbUNSQ]\n [abacus_go/cfvmY-Y_Su63-qSkPODDzg]: v[6], mv[1], sv[2], av[1]\n 0: p_term [1], isa_ids [AMhgJMlSTPOoKSsL7s3OKA]\n [k8s_apiserver/qQasynZgQ6e6AmD5ZWqh3w]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [Aaa0F0jiStCkteGDYR8yRQ]\n [michelangelo_gateway/ZV2vZ8PmS0uysg-xzKO7QA]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [k5vDE54CSTSakhfgtGaTcA]\n [driverapp/cqbade6KQFqLEK-wPKlkBg]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [b5_Bhd3ITjOkl-xLF8H1jg]\n [apphealth/rcPrX1aWRmeED9W5PU6A4g]: v[4], mv[1], sv[1], av[1]\n 0: p_term [1], isa_ids [Abip50RxQTmavqi4NTwSqQ]\n [fares_management/F1lH-QvyTkS5hnbDFwE2Xg]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [sOxfKX3nReynYN5HN88cuQ]\n [finprod_rewards_eligibility/dJNRGPwrQS2bIPlpsXgUoQ]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [Hg4Dm9O8QEqXWKY7rH1X0A]\n [fare_route/a4fjivWhQRee8JW7ZbXiXw]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [ltSLFgu4Rry0_0vXq28DbA]\n [ad_bidder/tLHfw5r2QH-o7lG5tapKxw]: v[6], mv[1], sv[2], av[1]\n 0: p_term [1], isa_ids [YtEUwPgfQDGM0Cm_C4hXlw]\n [fulfillment_compatibled/tMgrGBWeR3irgR5PyN8Oig]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [D_3C_3kMSNKa-XJk7pyoGg]\n [eats_store/hF-kN4XFRy--z_gUhYLwdA]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [WRNwRQ6gTNGfSNQok4iSyQ]\n [invoice_gen/RPu5EzJVRfuqyI8MQ7iTjQ]: v[11], mv[1], sv[4], av[1]\n 0: p_term [2], isa_ids [onWbjhHuTD2C3AEayRJq1w]\nmetadata customs:\n licenses: LicensesMetadata{license={\"uid\":\"8e65ab16-3f3f-48cc-ab5f-e20097ceb81e\",\"type\":\"basic\",\"issue_date_in_millis\":1679968689560,\"max_nodes\":1000,\"max_resource_units\":null,\"issued_to\":\"elasticsearch\",\"issuer\":\"elasticsearch\",\"signature\":\"////+wAAAOBYLHp18OvjZrRTi0AmCZ6DOpkrd2cXJB/ffBB+AxaDsF+z7lkfdPbgYofMvUB7XbiD5j95mIfUcdlTJHVdk9NpPTFMKL5wC7BXn4CTS6kNox+Xi0+qSLpcgHzelOT63alJr1L5j/2pqffHnuMLG88Ye15eFhAb9H2A5MS3hX0HG9uTxyEidFCyu4gWV4qrZJDQKLpzC49MMnpP+ob2bNH/pA7dGVdWqEJpVNPe5/hAtGtCdVdUX6DvV0Q3dgwuejc0hRg/vl0YUtp45bQPBHDHECNKgxWmJCZrfstl/prRHA==\",\"start_date_in_millis\":-1}, trialVersion=null} component_template: {\"component_template\":{\"metrics-mappings\":{\"template\":{\"mappings\":{\"dynamic_templates\":[{\"long_metrics\":{\"mapping\":{\"index\":false,\"type\":\"long\"},\"match_mapping_type\":\"long\"}},{\"double_metrics\":{\"mapping\":{\"index\":false,\"type\":\"float\"},\"match_mapping_type\":\"double\"}}],\"properties\":{\"data_stream\":{\"properties\":{\"type\":{\"type\":\"constant_keyword\",\"value\":\"metrics\"}}}}}},\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default mappings for the metrics index template installed by x-pack\"}},\".deprecation-indexing-mappings\":{\"template\":{\"mappings\":{\"dynamic_templates\":[{\"strings_as_keyword\":{\"mapping\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"match_mapping_type\":\"string\"}}],\"date_detection\":false,\"properties\":{\"@timestamp\":{\"type\":\"date\"},\"ecs\":{\"properties\":{\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"data_stream\":{\"properties\":{\"namespace\":{\"type\":\"constant_keyword\"},\"type\":{\"type\":\"constant_keyword\",\"value\":\"logs\"},\"dataset\":{\"type\":\"constant_keyword\"}}},\"host\":{\"properties\":{\"ip\":{\"type\":\"ip\"}}},\"message\":{\"type\":\"text\"}}}},\"version\":1,\"_meta\":{\"managed\":true,\"description\":\"default mappings for Stack deprecation logs index template installed by x-pack\"}},\"synthetics-mappings\":{\"template\":{\"mappings\":{\"properties\":{\"data_stream\":{\"properties\":{\"type\":{\"type\":\"constant_keyword\",\"value\":\"synthetics\"}}}}}},\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default mappings for the synthetics index template installed by x-pack\"}},\"metrics-settings\":{\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\"metrics\"},\"codec\":\"best_compression\",\"query\":{\"default_field\":[\"message\"]}}}},\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default settings for the metrics index template installed by x-pack\"}},\"logs-settings\":{\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\"logs\"},\"codec\":\"best_compression\",\"query\":{\"default_field\":[\"message\"]}}}},\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default settings for the logs index template installed by x-pack\"}},\".deprecation-indexing-settings\":{\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\".deprecation-indexing-ilm-policy\"},\"codec\":\"best_compression\",\"hidden\":\"true\",\"auto_expand_replicas\":\"0-1\",\"query\":{\"default_field\":[\"message\"]}}}},\"version\":1,\"_meta\":{\"managed\":true,\"description\":\"default settings for Stack deprecation logs index template installed by x-pack\"}},\"synthetics-settings\":{\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\"synthetics\"},\"codec\":\"best_compression\"}}},\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default settings for the synthetics index template installed by x-pack\"}},\"logs-mappings\":{\"template\":{\"mappings\":{\"properties\":{\"data_stream\":{\"properties\":{\"type\":{\"type\":\"constant_keyword\",\"value\":\"logs\"}}}}}},\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default mappings for the logs index template installed by x-pack\"}},\"data-streams-mappings\":{\"template\":{\"mappings\":{\"dynamic_templates\":[{\"match_ip\":{\"mapping\":{\"type\":\"ip\"},\"match_mapping_type\":\"string\",\"match\":\"ip\"}},{\"match_message\":{\"mapping\":{\"type\":\"match_only_text\"},\"match_mapping_type\":\"string\",\"match\":\"message\"}},{\"strings_as_keyword\":{\"mapping\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"match_mapping_type\":\"string\"}}],\"date_detection\":false,\"properties\":{\"@timestamp\":{\"type\":\"date\"},\"ecs\":{\"properties\":{\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"data_stream\":{\"properties\":{\"namespace\":{\"type\":\"constant_keyword\"},\"dataset\":{\"type\":\"constant_keyword\"}}},\"host\":{\"type\":\"object\"}}}},\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"general mapping conventions for data streams\"}}}} persistent_tasks: {\"last_allocation_id\":6,\"tasks\":[{\"id\":\"health-node\",\"task\":{\"health-node\":{\"params\":{}}},\"allocation_id\":5,\"assignment\":{\"executor_node\":\"iKPGCkp9RVOKXOj20uOt4g\",\"explanation\":\"\"},\"allocation_id_on_last_status_update\":0},{\"id\":\"geoip-downloader\",\"task\":{\"geoip-downloader\":{\"params\":{},\"state\":{\"databases\":{\"GeoLite2-ASN.mmdb\":{\"last_update\":1679968651799,\"last_check\":1679968651799,\"first_chunk\":0,\"last_chunk\":4,\"md5\":\"33268a0d7cb3ba8d9d581bf1e5dc0980\"},\"GeoLite2-Country.mmdb\":{\"last_update\":1679968809323,\"last_check\":1679968809323,\"first_chunk\":0,\"last_chunk\":2,\"md5\":\"da6d05a7b044ea35910f20f756c057ad\"},\"GeoLite2-City.mmdb\":{\"last_update\":1679968693385,\"last_check\":1679968693385,\"first_chunk\":0,\"last_chunk\":33,\"md5\":\"2d91f3e42689a12c2f496199befdfccf\"}}}}},\"allocation_id\":6,\"assignment\":{\"executor_node\":\"iKPGCkp9RVOKXOj20uOt4g\",\"explanation\":\"\"},\"allocation_id_on_last_status_update\":0}]} index_lifecycle: {\n \"policies\" : {\n \"90-days-default\" : {\n \"policy\" : {\n \"phases\" : {\n \"warm\" : {\n \"min_age\" : \"2d\",\n \"actions\" : {\n \"shrink\" : {\n \"number_of_shards\" : 1\n },\n \"forcemerge\" : {\n \"max_num_segments\" : 1\n }\n }\n },\n \"cold\" : {\n \"min_age\" : \"30d\",\n \"actions\" : { }\n },\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"90d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"built-in ILM policy using the hot, warm, and cold phases with a retention of 90 days\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968678193,\n \"modified_date_string\" : \"2023-03-28T01:57:58.193Z\"\n },\n \"ilm-history-ilm-policy\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"90d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"default policy for the ILM history indices\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968685095,\n \"modified_date_string\" : \"2023-03-28T01:58:05.095Z\"\n },\n \"watch-history-ilm-policy-16\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"3d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"4d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"default policy for the watcher history indices\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968647327,\n \"modified_date_string\" : \"2023-03-28T01:57:27.327Z\"\n },\n \"30-days-default\" : {\n \"policy\" : {\n \"phases\" : {\n \"warm\" : {\n \"min_age\" : \"2d\",\n \"actions\" : {\n \"shrink\" : {\n \"number_of_shards\" : 1\n },\n \"forcemerge\" : {\n \"max_num_segments\" : 1\n }\n }\n },\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"30d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"built-in ILM policy using the hot and warm phases with a retention of 30 days\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968662496,\n \"modified_date_string\" : \"2023-03-28T01:57:42.496Z\"\n },\n \"synthetics\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"default policy for the synthetics index template installed by x-pack\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968682454,\n \"modified_date_string\" : \"2023-03-28T01:58:02.454Z\"\n },\n \".monitoring-8-ilm-policy\" : {\n \"policy\" : {\n \"phases\" : {\n \"warm\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"forcemerge\" : {\n \"max_num_segments\" : 1\n }\n }\n },\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"3d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"3d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"defaults\" : {\n \"delete_min_age\" : \"Using value of [3d] based on the monitoring plugin default\"\n },\n \"description\" : \"Index lifecycle policy generated for [monitoring-*-8] data streams\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968676492,\n \"modified_date_string\" : \"2023-03-28T01:57:56.492Z\"\n },\n \"slm-history-ilm-policy\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"90d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"default policy for the SLM history indices\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968644040,\n \"modified_date_string\" : \"2023-03-28T01:57:24.040Z\"\n },\n \".fleet-actions-results-ilm-policy\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_size\" : \"300gb\",\n \"max_age\" : \"30d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"90d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"default policy for fleet action results indices\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968686060,\n \"modified_date_string\" : \"2023-03-28T01:58:06.060Z\"\n },\n \"365-days-default\" : {\n \"policy\" : {\n \"phases\" : {\n \"warm\" : {\n \"min_age\" : \"2d\",\n \"actions\" : {\n \"shrink\" : {\n \"number_of_shards\" : 1\n },\n \"forcemerge\" : {\n \"max_num_segments\" : 1\n }\n }\n },\n \"cold\" : {\n \"min_age\" : \"30d\",\n \"actions\" : { }\n },\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"365d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"built-in ILM policy using the hot, warm, and cold phases with a retention of 365 days\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968675542,\n \"modified_date_string\" : \"2023-03-28T01:57:55.542Z\"\n },\n \"7-days-default\" : {\n \"policy\" : {\n \"phases\" : {\n \"warm\" : {\n \"min_age\" : \"2d\",\n \"actions\" : {\n \"shrink\" : {\n \"number_of_shards\" : 1\n },\n \"forcemerge\" : {\n \"max_num_segments\" : 1\n }\n }\n },\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"7d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"7d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"built-in ILM policy using the hot and warm phases with a retention of 7 days\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968672404,\n \"modified_date_string\" : \"2023-03-28T01:57:52.404Z\"\n },\n \".deprecation-indexing-ilm-policy\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"10gb\",\n \"max_age\" : \"30d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"30d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"ILM policy used for deprecation logs\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968670134,\n \"modified_date_string\" : \"2023-03-28T01:57:50.134Z\"\n },\n \"metrics\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"default policy for the metrics index template installed by x-pack\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968660707,\n \"modified_date_string\" : \"2023-03-28T01:57:40.707Z\"\n },\n \"ml-size-based-ilm-policy\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\"\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"default policy for machine learning state and stats indices\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968650499,\n \"modified_date_string\" : \"2023-03-28T01:57:30.499Z\"\n },\n \"180-days-default\" : {\n \"policy\" : {\n \"phases\" : {\n \"warm\" : {\n \"min_age\" : \"2d\",\n \"actions\" : {\n \"shrink\" : {\n \"number_of_shards\" : 1\n },\n \"forcemerge\" : {\n \"max_num_segments\" : 1\n }\n }\n },\n \"cold\" : {\n \"min_age\" : \"30d\",\n \"actions\" : { }\n },\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n },\n \"delete\" : {\n \"min_age\" : \"180d\",\n \"actions\" : {\n \"delete\" : {\n \"delete_searchable_snapshot\" : true\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"built-in ILM policy using the hot, warm, and cold phases with a retention of 180 days\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968681386,\n \"modified_date_string\" : \"2023-03-28T01:58:01.386Z\"\n },\n \"logs\" : {\n \"policy\" : {\n \"phases\" : {\n \"hot\" : {\n \"min_age\" : \"0ms\",\n \"actions\" : {\n \"rollover\" : {\n \"max_primary_shard_size\" : \"50gb\",\n \"max_age\" : \"30d\"\n }\n }\n }\n },\n \"_meta\" : {\n \"managed\" : true,\n \"description\" : \"default policy for the logs index template installed by x-pack\"\n }\n },\n \"headers\" : {\n \"_xpack_security_authentication\" : \"i/vrAwEGX3hwYWNrBmJha2VyOQhfX2F0dGFjaAhfX2F0dGFjaAAABAoA\"\n },\n \"version\" : 1,\n \"modified_date\" : 1679968679651,\n \"modified_date_string\" : \"2023-03-28T01:57:59.651Z\"\n }\n },\n \"operation_mode\" : \"RUNNING\"\n} index_template: {\"index_template\":{\".watch-history-16\":{\"index_patterns\":[\".watcher-history-16*\"],\"template\":{\"settings\":{\"index\":{\"format\":\"6\",\"lifecycle\":{\"name\":\"watch-history-ilm-policy-16\"},\"hidden\":\"true\",\"number_of_shards\":\"1\",\"auto_expand_replicas\":\"0-1\",\"number_of_replicas\":\"0\"}},\"mappings\":{\"_meta\":{\"watcher-history-version\":\"16\"},\"dynamic\":false,\"dynamic_templates\":[{\"disabled_payload_fields\":{\"match_pattern\":\"regex\",\"path_match\":\"result\\\\.(input(\\\\..+)*|(transform(\\\\..+)*)|(actions\\\\.transform(\\\\..+)*))\\\\.payload\",\"mapping\":{\"type\":\"object\",\"enabled\":false}}},{\"disabled_search_request_body_fields\":{\"match_pattern\":\"regex\",\"path_match\":\"result\\\\.(input(\\\\..+)*|(transform(\\\\..+)*)|(actions\\\\.transform(\\\\..+)*))\\\\.search\\\\.request\\\\.(body|template)\",\"mapping\":{\"type\":\"object\",\"enabled\":false}}},{\"disabled_exception_fields\":{\"match_pattern\":\"regex\",\"path_match\":\"result\\\\.(input(\\\\..+)*|(transform(\\\\..+)*)|(actions\\\\.transform(\\\\..+)*)|actions)\\\\.error\",\"mapping\":{\"type\":\"object\",\"enabled\":false}}},{\"disabled_jira_custom_fields\":{\"path_match\":\"result.actions.jira.fields.customfield_*\",\"mapping\":{\"type\":\"object\",\"enabled\":false}}}],\"properties\":{\"exception\":{\"type\":\"object\",\"enabled\":false},\"metadata\":{\"dynamic\":true,\"type\":\"object\"},\"trigger_event\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"schedule\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"scheduled_time\":{\"type\":\"date\"}}},\"triggered_time\":{\"type\":\"date\"},\"type\":{\"type\":\"keyword\"},\"manual\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"schedule\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"scheduled_time\":{\"type\":\"date\"}}}}}}},\"result\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"input\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"search\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"request\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"indices\":{\"type\":\"keyword\"},\"types\":{\"type\":\"keyword\"},\"search_type\":{\"type\":\"keyword\"}}}}},\"payload\":{\"type\":\"object\",\"enabled\":false},\"http\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"request\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"path\":{\"type\":\"keyword\"},\"host\":{\"type\":\"keyword\"}}}}},\"type\":{\"type\":\"keyword\"},\"status\":{\"type\":\"keyword\"}}},\"condition\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"compare\":{\"type\":\"object\",\"enabled\":false},\"array_compare\":{\"type\":\"object\",\"enabled\":false},\"type\":{\"type\":\"keyword\"},\"met\":{\"type\":\"boolean\"},\"script\":{\"type\":\"object\",\"enabled\":false},\"status\":{\"type\":\"keyword\"}}},\"transform\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"search\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"request\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"indices\":{\"type\":\"keyword\"},\"types\":{\"type\":\"keyword\"}}}}},\"type\":{\"type\":\"keyword\"}}},\"execution_duration\":{\"type\":\"long\"},\"actions\":{\"include_in_parent\":true,\"dynamic\":true,\"type\":\"nested\",\"properties\":{\"reason\":{\"type\":\"keyword\"},\"foreach\":{\"type\":\"object\",\"enabled\":false},\"webhook\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"request\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"path\":{\"type\":\"keyword\"},\"host\":{\"type\":\"keyword\"}}}}},\"number_of_actions_executed\":{\"type\":\"integer\"},\"slack\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"sent_messages\":{\"include_in_parent\":true,\"dynamic\":true,\"type\":\"nested\",\"properties\":{\"reason\":{\"type\":\"text\"},\"request\":{\"type\":\"object\",\"enabled\":false},\"response\":{\"type\":\"object\",\"enabled\":false},\"to\":{\"type\":\"keyword\"},\"message\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"attachments\":{\"include_in_parent\":true,\"dynamic\":true,\"type\":\"nested\",\"properties\":{\"color\":{\"type\":\"keyword\"},\"fields\":{\"properties\":{\"value\":{\"type\":\"text\"}}}}},\"icon\":{\"type\":\"keyword\"},\"from\":{\"type\":\"text\"},\"text\":{\"type\":\"text\"}}},\"status\":{\"type\":\"keyword\"}}},\"account\":{\"type\":\"keyword\"}}},\"index\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"response\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"index\":{\"type\":\"keyword\"},\"id\":{\"type\":\"keyword\"},\"type\":{\"type\":\"keyword\"}}}}},\"pagerduty\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"sent_event\":{\"include_in_parent\":true,\"dynamic\":true,\"type\":\"nested\",\"properties\":{\"reason\":{\"type\":\"text\"},\"request\":{\"type\":\"object\",\"enabled\":false},\"response\":{\"type\":\"object\",\"enabled\":false},\"event\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"client_url\":{\"type\":\"keyword\"},\"context\":{\"include_in_parent\":true,\"dynamic\":true,\"type\":\"nested\",\"properties\":{\"src\":{\"type\":\"keyword\"},\"alt\":{\"type\":\"text\"},\"href\":{\"type\":\"keyword\"},\"type\":{\"type\":\"keyword\"}}},\"client\":{\"type\":\"text\"},\"description\":{\"type\":\"text\"},\"attach_payload\":{\"type\":\"boolean\"},\"incident_key\":{\"type\":\"keyword\"},\"type\":{\"type\":\"keyword\"},\"account\":{\"type\":\"keyword\"}}}}},\"account\":{\"type\":\"keyword\"}}},\"id\":{\"type\":\"keyword\"},\"type\":{\"type\":\"keyword\"},\"email\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"message\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"cc\":{\"type\":\"keyword\"},\"bcc\":{\"type\":\"keyword\"},\"reply_to\":{\"type\":\"keyword\"},\"from\":{\"type\":\"keyword\"},\"id\":{\"type\":\"keyword\"},\"to\":{\"type\":\"keyword\"}}}}},\"status\":{\"type\":\"keyword\"},\"jira\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"result\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"self\":{\"type\":\"keyword\"},\"id\":{\"type\":\"keyword\"},\"key\":{\"type\":\"keyword\"}}},\"reason\":{\"type\":\"text\"},\"request\":{\"type\":\"object\",\"enabled\":false},\"response\":{\"type\":\"object\",\"enabled\":false},\"fields\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"summary\":{\"type\":\"text\"},\"issuetype\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"name\":{\"type\":\"keyword\"},\"id\":{\"type\":\"keyword\"}}},\"description\":{\"type\":\"text\"},\"project\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"id\":{\"type\":\"keyword\"},\"key\":{\"type\":\"keyword\"}}},\"labels\":{\"type\":\"text\"}}},\"account\":{\"type\":\"keyword\"}}}}},\"execution_time\":{\"type\":\"date\"}}},\"node\":{\"type\":\"keyword\"},\"input\":{\"type\":\"object\",\"enabled\":false},\"condition\":{\"type\":\"object\",\"enabled\":false},\"watch_id\":{\"type\":\"keyword\"},\"messages\":{\"type\":\"text\"},\"vars\":{\"type\":\"object\",\"enabled\":false},\"state\":{\"type\":\"keyword\"},\"user\":{\"type\":\"text\"},\"status\":{\"dynamic\":true,\"type\":\"object\",\"enabled\":false}}}},\"composed_of\":[],\"priority\":2147483647,\"version\":16,\"_meta\":{\"managed\":true,\"description\":\"index template for watcher history indices\"},\"data_stream\":{\"hidden\":true,\"allow_custom_routing\":false},\"allow_auto_create\":true},\".monitoring-beats-mb\":{\"index_patterns\":[\".monitoring-beats-8-*\"],\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\".monitoring-8-ilm-policy\"},\"mapping\":{\"total_fields\":{\"limit\":\"2000\"}}}},\"mappings\":{\"dynamic\":false,\"properties\":{\"@timestamp\":{\"type\":\"date\"},\"cluster_uuid\":{\"path\":\"beat.elasticsearch.cluster.id\",\"type\":\"alias\"},\"service\":{\"properties\":{\"node\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"environment\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"hostname\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"origin\":{\"properties\":{\"node\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"environment\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"target\":{\"properties\":{\"node\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"environment\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"beat\":{\"properties\":{\"elasticsearch\":{\"properties\":{\"cluster\":{\"properties\":{\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"stats\":{\"properties\":{\"handles\":{\"properties\":{\"limit\":{\"properties\":{\"hard\":{\"type\":\"long\"},\"soft\":{\"type\":\"long\"}}},\"open\":{\"type\":\"long\"}}},\"apm_server\":{\"properties\":{\"server\":{\"properties\":{\"request\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"response\":{\"properties\":{\"valid\":{\"properties\":{\"count\":{\"type\":\"long\"},\"accepted\":{\"type\":\"long\"},\"ok\":{\"type\":\"long\"}}},\"count\":{\"type\":\"long\"},\"errors\":{\"properties\":{\"ratelimit\":{\"type\":\"long\"},\"internal\":{\"type\":\"long\"},\"toolarge\":{\"type\":\"long\"},\"method\":{\"type\":\"long\"},\"unauthorized\":{\"type\":\"long\"},\"forbidden\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"},\"closed\":{\"type\":\"long\"},\"decode\":{\"type\":\"long\"},\"queue\":{\"type\":\"long\"},\"concurrency\":{\"type\":\"long\"},\"validate\":{\"type\":\"long\"}}}}},\"concurrent\":{\"properties\":{\"wait\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}}}},\"acm\":{\"properties\":{\"request\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"response\":{\"properties\":{\"valid\":{\"properties\":{\"notmodified\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"},\"accepted\":{\"type\":\"long\"},\"ok\":{\"type\":\"long\"}}},\"request\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"count\":{\"type\":\"long\"},\"unset\":{\"type\":\"long\"},\"errors\":{\"properties\":{\"ratelimit\":{\"type\":\"long\"},\"internal\":{\"type\":\"long\"},\"toolarge\":{\"type\":\"long\"},\"method\":{\"type\":\"long\"},\"unavailable\":{\"type\":\"long\"},\"forbidden\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"},\"decode\":{\"type\":\"long\"},\"unauthorized\":{\"type\":\"long\"},\"notfound\":{\"type\":\"long\"},\"closed\":{\"type\":\"long\"},\"invalidquery\":{\"type\":\"long\"},\"queue\":{\"type\":\"long\"},\"validate\":{\"type\":\"long\"}}}}}}},\"decoder\":{\"properties\":{\"deflate\":{\"properties\":{\"content-length\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"reader\":{\"properties\":{\"size\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"gzip\":{\"properties\":{\"content-length\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"uncompressed\":{\"properties\":{\"content-length\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"missing-content-length\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"processor\":{\"properties\":{\"metric\":{\"properties\":{\"transformations\":{\"type\":\"long\"},\"decoding\":{\"properties\":{\"count\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}},\"validation\":{\"properties\":{\"count\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}}}},\"sourcemap\":{\"properties\":{\"counter\":{\"type\":\"long\"},\"decoding\":{\"properties\":{\"count\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}},\"validation\":{\"properties\":{\"count\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}}}},\"error\":{\"properties\":{\"stacktraces\":{\"type\":\"long\"},\"spans\":{\"type\":\"long\"},\"frames\":{\"type\":\"long\"},\"transformations\":{\"type\":\"long\"},\"decoding\":{\"properties\":{\"count\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}},\"validation\":{\"properties\":{\"count\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}}}},\"transaction\":{\"properties\":{\"stacktraces\":{\"type\":\"long\"},\"spans\":{\"type\":\"long\"},\"frames\":{\"type\":\"long\"},\"transformations\":{\"type\":\"long\"},\"transactions\":{\"type\":\"long\"},\"decoding\":{\"properties\":{\"count\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}},\"validation\":{\"properties\":{\"count\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}}}},\"span\":{\"properties\":{\"transformations\":{\"type\":\"long\"}}}}}}},\"system\":{\"properties\":{\"load\":{\"properties\":{\"1\":{\"type\":\"double\"},\"15\":{\"type\":\"double\"},\"5\":{\"type\":\"double\"},\"norm\":{\"properties\":{\"1\":{\"type\":\"double\"},\"15\":{\"type\":\"double\"},\"5\":{\"type\":\"double\"}}}}},\"cpu\":{\"properties\":{\"cores\":{\"type\":\"long\"}}}}},\"beat\":{\"properties\":{\"host\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"cpu\":{\"properties\":{\"total\":{\"properties\":{\"ticks\":{\"type\":\"long\"},\"time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"value\":{\"type\":\"long\"}}},\"system\":{\"properties\":{\"ticks\":{\"type\":\"long\"},\"time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"user\":{\"properties\":{\"ticks\":{\"type\":\"long\"},\"time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}}}},\"runtime\":{\"properties\":{\"goroutines\":{\"type\":\"long\"}}},\"memstats\":{\"properties\":{\"memory\":{\"properties\":{\"total\":{\"type\":\"long\"},\"alloc\":{\"type\":\"long\"}}},\"rss\":{\"type\":\"long\"},\"gc_next\":{\"type\":\"long\"}}},\"libbeat\":{\"properties\":{\"output\":{\"properties\":{\"read\":{\"properties\":{\"bytes\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"write\":{\"properties\":{\"bytes\":{\"type\":\"long\"},\"errors\":{\"type\":\"long\"}}},\"events\":{\"properties\":{\"batches\":{\"type\":\"long\"},\"duplicates\":{\"type\":\"long\"},\"total\":{\"type\":\"long\"},\"dropped\":{\"type\":\"long\"},\"toomany\":{\"type\":\"long\"},\"active\":{\"type\":\"long\"},\"failed\":{\"type\":\"long\"},\"acked\":{\"type\":\"long\"}}}}},\"pipeline\":{\"properties\":{\"clients\":{\"type\":\"long\"},\"events\":{\"properties\":{\"total\":{\"type\":\"long\"},\"filtered\":{\"type\":\"long\"},\"dropped\":{\"type\":\"long\"},\"active\":{\"type\":\"long\"},\"failed\":{\"type\":\"long\"},\"published\":{\"type\":\"long\"},\"retry\":{\"type\":\"long\"}}},\"queue\":{\"properties\":{\"acked\":{\"type\":\"long\"}}}}},\"config\":{\"properties\":{\"running\":{\"type\":\"long\"},\"stops\":{\"type\":\"long\"},\"starts\":{\"type\":\"long\"},\"reloads\":{\"type\":\"long\"}}}}},\"cgroup\":{\"properties\":{\"memory\":{\"properties\":{\"mem\":{\"properties\":{\"usage\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"limit\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"cpu\":{\"properties\":{\"cfs\":{\"properties\":{\"period\":{\"properties\":{\"us\":{\"type\":\"long\"}}},\"quota\":{\"properties\":{\"us\":{\"type\":\"long\"}}}}},\"stats\":{\"properties\":{\"periods\":{\"type\":\"long\"},\"throttled\":{\"properties\":{\"ns\":{\"type\":\"long\"},\"periods\":{\"type\":\"long\"}}}}},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"cpuacct\":{\"properties\":{\"total\":{\"properties\":{\"ns\":{\"type\":\"long\"}}},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"info\":{\"properties\":{\"host\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"uptime\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"uptime\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"properties\":{\"output\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"cluster\":{\"properties\":{\"uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"input\":{\"properties\":{\"names\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"count\":{\"type\":\"long\"}}},\"management\":{\"properties\":{\"enabled\":{\"type\":\"boolean\"}}},\"service\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"module\":{\"properties\":{\"names\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"count\":{\"type\":\"long\"}}},\"beat\":{\"properties\":{\"host\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"host\":{\"properties\":{\"os\":{\"properties\":{\"kernel\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"platform\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"containerized\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"queue\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"host\":{\"properties\":{\"hostname\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"architecture\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"metricset\":{\"properties\":{\"period\":{\"type\":\"long\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"event\":{\"properties\":{\"reason\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"code\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"timezone\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"agent_id_status\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"duration\":{\"type\":\"long\"},\"reference\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ingested\":{\"type\":\"date\"},\"provider\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"action\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"end\":{\"type\":\"date\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"outcome\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"severity\":{\"type\":\"long\"},\"original\":{\"ignore_above\":1024,\"index\":false,\"type\":\"keyword\",\"doc_values\":false},\"risk_score\":{\"type\":\"float\"},\"created\":{\"type\":\"date\"},\"kind\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"module\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"start\":{\"type\":\"date\"},\"url\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"sequence\":{\"type\":\"long\"},\"risk_score_norm\":{\"type\":\"float\"},\"category\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"dataset\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"hash\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"beats_stats\":{\"properties\":{\"beat\":{\"properties\":{\"host\":{\"path\":\"beat.stats.beat.host\",\"type\":\"alias\"},\"name\":{\"path\":\"beat.stats.beat.name\",\"type\":\"alias\"},\"type\":{\"path\":\"beat.stats.beat.type\",\"type\":\"alias\"},\"uuid\":{\"path\":\"beat.stats.beat.uuid\",\"type\":\"alias\"},\"version\":{\"path\":\"beat.stats.beat.version\",\"type\":\"alias\"}}},\"metrics\":{\"properties\":{\"system\":{\"properties\":{\"load\":{\"properties\":{\"1\":{\"path\":\"beat.stats.system.load.1\",\"type\":\"alias\"},\"15\":{\"path\":\"beat.stats.system.load.15\",\"type\":\"alias\"},\"5\":{\"path\":\"beat.stats.system.load.5\",\"type\":\"alias\"},\"norm\":{\"properties\":{\"1\":{\"path\":\"beat.stats.system.load.norm.1\",\"type\":\"alias\"},\"15\":{\"path\":\"beat.stats.system.load.norm.15\",\"type\":\"alias\"},\"5\":{\"path\":\"beat.stats.system.load.norm.5\",\"type\":\"alias\"}}}}},\"cpu\":{\"properties\":{\"cores\":{\"path\":\"beat.stats.system.cpu.cores\",\"type\":\"alias\"}}}}},\"beat\":{\"properties\":{\"handles\":{\"properties\":{\"limit\":{\"properties\":{\"hard\":{\"path\":\"beat.stats.handles.limit.hard\",\"type\":\"alias\"},\"soft\":{\"path\":\"beat.stats.handles.limit.soft\",\"type\":\"alias\"}}},\"open\":{\"path\":\"beat.stats.handles.open\",\"type\":\"alias\"}}},\"cpu\":{\"properties\":{\"total\":{\"properties\":{\"ticks\":{\"path\":\"beat.stats.cpu.total.ticks\",\"type\":\"alias\"},\"time\":{\"properties\":{\"ms\":{\"path\":\"beat.stats.cpu.total.time.ms\",\"type\":\"alias\"}}},\"value\":{\"path\":\"beat.stats.cpu.total.value\",\"type\":\"alias\"}}},\"system\":{\"properties\":{\"ticks\":{\"path\":\"beat.stats.cpu.system.ticks\",\"type\":\"alias\"},\"time\":{\"properties\":{\"ms\":{\"path\":\"beat.stats.cpu.system.time.ms\",\"type\":\"alias\"}}}}},\"user\":{\"properties\":{\"ticks\":{\"path\":\"beat.stats.cpu.user.ticks\",\"type\":\"alias\"},\"time\":{\"properties\":{\"ms\":{\"path\":\"beat.stats.cpu.user.time.ms\",\"type\":\"alias\"}}}}}}},\"memstats\":{\"properties\":{\"rss\":{\"path\":\"beat.stats.memstats.rss\",\"type\":\"alias\"},\"gc_next\":{\"path\":\"beat.stats.memstats.gc_next\",\"type\":\"alias\"},\"memory_alloc\":{\"path\":\"beat.stats.memstats.memory.alloc\",\"type\":\"alias\"},\"memory_total\":{\"path\":\"beat.stats.memstats.memory.total\",\"type\":\"alias\"}}},\"cgroup\":{\"properties\":{\"memory\":{\"properties\":{\"id\":{\"path\":\"beat.stats.cgroup.memory.id\",\"type\":\"alias\"}}},\"mem\":{\"properties\":{\"usage\":{\"properties\":{\"bytes\":{\"path\":\"beat.stats.cgroup.memory.mem.usage.bytes\",\"type\":\"alias\"}}},\"limit\":{\"properties\":{\"bytes\":{\"path\":\"beat.stats.cgroup.memory.mem.limit.bytes\",\"type\":\"alias\"}}}}},\"cpu\":{\"properties\":{\"cfs\":{\"properties\":{\"period\":{\"properties\":{\"us\":{\"path\":\"beat.stats.cgroup.cpu.cfs.period.us\",\"type\":\"alias\"}}},\"quota\":{\"properties\":{\"us\":{\"path\":\"beat.stats.cgroup.cpu.cfs.quota.us\",\"type\":\"alias\"}}}}},\"stats\":{\"properties\":{\"periods\":{\"path\":\"beat.stats.cgroup.cpu.stats.periods\",\"type\":\"alias\"},\"throttled\":{\"properties\":{\"ns\":{\"path\":\"beat.stats.cgroup.cpu.stats.throttled.ns\",\"type\":\"alias\"},\"periods\":{\"path\":\"beat.stats.cgroup.cpu.stats.throttled.periods\",\"type\":\"alias\"}}}}},\"id\":{\"path\":\"beat.stats.cgroup.cpu.id\",\"type\":\"alias\"}}},\"cpuacct\":{\"properties\":{\"total\":{\"properties\":{\"ns\":{\"path\":\"beat.stats.cgroup.cpuacct.total.ns\",\"type\":\"alias\"}}},\"id\":{\"path\":\"beat.stats.cgroup.cpuacct.id\",\"type\":\"alias\"}}}}},\"info\":{\"properties\":{\"ephemeral_id\":{\"path\":\"beat.stats.info.ephemeral_id\",\"type\":\"alias\"},\"uptime\":{\"properties\":{\"ms\":{\"path\":\"beat.stats.info.uptime.ms\",\"type\":\"alias\"}}}}}}},\"libbeat\":{\"properties\":{\"output\":{\"properties\":{\"read\":{\"properties\":{\"bytes\":{\"path\":\"beat.stats.libbeat.output.read.bytes\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.libbeat.output.read.errors\",\"type\":\"alias\"}}},\"type\":{\"path\":\"beat.stats.libbeat.output.type\",\"type\":\"alias\"},\"write\":{\"properties\":{\"bytes\":{\"path\":\"beat.stats.libbeat.output.write.bytes\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.libbeat.output.write.errors\",\"type\":\"alias\"}}},\"events\":{\"properties\":{\"duplicated\":{\"path\":\"beat.stats.libbeat.output.events.duplicates\",\"type\":\"alias\"},\"batches\":{\"path\":\"beat.stats.libbeat.output.events.batches\",\"type\":\"alias\"},\"total\":{\"path\":\"beat.stats.libbeat.output.events.total\",\"type\":\"alias\"},\"dropped\":{\"path\":\"beat.stats.libbeat.output.events.dropped\",\"type\":\"alias\"},\"toomany\":{\"path\":\"beat.stats.libbeat.output.events.toomany\",\"type\":\"alias\"},\"active\":{\"path\":\"beat.stats.libbeat.output.events.active\",\"type\":\"alias\"},\"failed\":{\"path\":\"beat.stats.libbeat.output.events.failed\",\"type\":\"alias\"},\"acked\":{\"path\":\"beat.stats.libbeat.output.events.acked\",\"type\":\"alias\"}}}}},\"pipeline\":{\"properties\":{\"clients\":{\"path\":\"beat.stats.libbeat.pipeline.clients\",\"type\":\"alias\"},\"events\":{\"properties\":{\"total\":{\"path\":\"beat.stats.libbeat.pipeline.events.total\",\"type\":\"alias\"},\"filtered\":{\"path\":\"beat.stats.libbeat.pipeline.events.filtered\",\"type\":\"alias\"},\"dropped\":{\"path\":\"beat.stats.libbeat.pipeline.events.dropped\",\"type\":\"alias\"},\"active\":{\"path\":\"beat.stats.libbeat.pipeline.events.active\",\"type\":\"alias\"},\"failed\":{\"path\":\"beat.stats.libbeat.pipeline.events.failed\",\"type\":\"alias\"},\"published\":{\"path\":\"beat.stats.libbeat.pipeline.events.published\",\"type\":\"alias\"},\"retry\":{\"path\":\"beat.stats.libbeat.pipeline.events.retry\",\"type\":\"alias\"}}},\"queue\":{\"properties\":{\"acked\":{\"path\":\"beat.stats.libbeat.pipeline.queue.acked\",\"type\":\"alias\"}}}}},\"config\":{\"properties\":{\"module\":{\"properties\":{\"running\":{\"path\":\"beat.stats.libbeat.config.running\",\"type\":\"alias\"},\"stops\":{\"path\":\"beat.stats.libbeat.config.stops\",\"type\":\"alias\"},\"starts\":{\"path\":\"beat.stats.libbeat.config.starts\",\"type\":\"alias\"}}},\"reloads\":{\"path\":\"beat.stats.libbeat.config.reloads\",\"type\":\"alias\"}}}}},\"apm-server\":{\"properties\":{\"server\":{\"properties\":{\"request\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.server.request.count\",\"type\":\"alias\"}}},\"response\":{\"properties\":{\"valid\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.server.response.valid.count\",\"type\":\"alias\"},\"accepted\":{\"path\":\"beat.stats.apm_server.server.response.valid.accepted\",\"type\":\"alias\"},\"ok\":{\"path\":\"beat.stats.apm_server.server.response.valid.ok\",\"type\":\"alias\"}}},\"count\":{\"path\":\"beat.stats.apm_server.server.response.count\",\"type\":\"alias\"},\"errors\":{\"properties\":{\"ratelimit\":{\"path\":\"beat.stats.apm_server.server.response.errors.ratelimit\",\"type\":\"alias\"},\"internal\":{\"path\":\"beat.stats.apm_server.server.response.errors.internal\",\"type\":\"alias\"},\"toolarge\":{\"path\":\"beat.stats.apm_server.server.response.errors.toolarge\",\"type\":\"alias\"},\"method\":{\"path\":\"beat.stats.apm_server.server.response.errors.method\",\"type\":\"alias\"},\"unauthorized\":{\"path\":\"beat.stats.apm_server.server.response.errors.unauthorized\",\"type\":\"alias\"},\"forbidden\":{\"path\":\"beat.stats.apm_server.server.response.errors.forbidden\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.stats.apm_server.server.response.errors.count\",\"type\":\"alias\"},\"closed\":{\"path\":\"beat.stats.apm_server.server.response.errors.closed\",\"type\":\"alias\"},\"decode\":{\"path\":\"beat.stats.apm_server.server.response.errors.decode\",\"type\":\"alias\"},\"queue\":{\"path\":\"beat.stats.apm_server.server.response.errors.queue\",\"type\":\"alias\"},\"concurrency\":{\"path\":\"beat.stats.apm_server.server.response.errors.concurrency\",\"type\":\"alias\"},\"validate\":{\"path\":\"beat.stats.apm_server.server.response.errors.validate\",\"type\":\"alias\"}}}}},\"concurrent\":{\"properties\":{\"wait\":{\"properties\":{\"ms\":{\"path\":\"beat.stats.apm_server.server.concurrent.wait.ms\",\"type\":\"alias\"}}}}}}},\"sampling\":{\"properties\":{\"tail\":{\"properties\":{\"dynamic_service_groups\":{\"type\":\"long\"},\"storage\":{\"properties\":{\"lsm_size\":{\"type\":\"long\"},\"value_log_size\":{\"type\":\"long\"}}},\"events\":{\"properties\":{\"processed\":{\"type\":\"long\"},\"head_unsampled\":{\"type\":\"long\"},\"stored\":{\"type\":\"long\"},\"dropped\":{\"type\":\"long\"},\"sampled\":{\"type\":\"long\"}}}}},\"transactions_dropped\":{\"type\":\"long\"}}},\"acm\":{\"properties\":{\"request\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.acm.request.count\",\"type\":\"alias\"}}},\"response\":{\"properties\":{\"valid\":{\"properties\":{\"notmodified\":{\"path\":\"beat.stats.apm_server.acm.response.valid.notmodified\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.stats.apm_server.acm.response.valid.count\",\"type\":\"alias\"},\"accepted\":{\"path\":\"beat.stats.apm_server.acm.response.valid.accepted\",\"type\":\"alias\"},\"ok\":{\"path\":\"beat.stats.apm_server.acm.response.valid.ok\",\"type\":\"alias\"}}},\"request\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.acm.response.request.count\",\"type\":\"alias\"}}},\"count\":{\"path\":\"beat.stats.apm_server.acm.response.count\",\"type\":\"alias\"},\"unset\":{\"path\":\"beat.stats.apm_server.acm.response.unset\",\"type\":\"alias\"},\"errors\":{\"properties\":{\"ratelimit\":{\"path\":\"beat.stats.apm_server.acm.response.errors.ratelimit\",\"type\":\"alias\"},\"internal\":{\"path\":\"beat.stats.apm_server.acm.response.errors.internal\",\"type\":\"alias\"},\"toolarge\":{\"path\":\"beat.stats.apm_server.acm.response.errors.toolarge\",\"type\":\"alias\"},\"method\":{\"path\":\"beat.stats.apm_server.acm.response.errors.method\",\"type\":\"alias\"},\"unavailable\":{\"path\":\"beat.stats.apm_server.acm.response.errors.unavailable\",\"type\":\"alias\"},\"forbidden\":{\"path\":\"beat.stats.apm_server.acm.response.errors.forbidden\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.stats.apm_server.acm.response.errors.count\",\"type\":\"alias\"},\"decode\":{\"path\":\"beat.stats.apm_server.acm.response.errors.decode\",\"type\":\"alias\"},\"unauthorized\":{\"path\":\"beat.stats.apm_server.acm.response.errors.unauthorized\",\"type\":\"alias\"},\"notfound\":{\"path\":\"beat.stats.apm_server.acm.response.errors.notfound\",\"type\":\"alias\"},\"closed\":{\"path\":\"beat.stats.apm_server.acm.response.errors.closed\",\"type\":\"alias\"},\"invalidquery\":{\"path\":\"beat.stats.apm_server.acm.response.errors.invalidquery\",\"type\":\"alias\"},\"queue\":{\"path\":\"beat.stats.apm_server.acm.response.errors.queue\",\"type\":\"alias\"},\"validate\":{\"path\":\"beat.stats.apm_server.acm.response.errors.validate\",\"type\":\"alias\"}}}}}}},\"decoder\":{\"properties\":{\"deflate\":{\"properties\":{\"content-length\":{\"path\":\"beat.stats.apm_server.decoder.deflate.content-length\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.stats.apm_server.decoder.deflate.count\",\"type\":\"alias\"}}},\"reader\":{\"properties\":{\"size\":{\"path\":\"beat.stats.apm_server.decoder.reader.size\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.stats.apm_server.decoder.reader.count\",\"type\":\"alias\"}}},\"gzip\":{\"properties\":{\"content-length\":{\"path\":\"beat.stats.apm_server.decoder.gzip.content-length\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.stats.apm_server.decoder.gzip.count\",\"type\":\"alias\"}}},\"uncompressed\":{\"properties\":{\"content-length\":{\"path\":\"beat.stats.apm_server.decoder.uncompressed.content-length\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.stats.apm_server.decoder.uncompressed.count\",\"type\":\"alias\"}}},\"missing-content-length\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.decoder.missing-content-length.count\",\"type\":\"alias\"}}}}},\"processor\":{\"properties\":{\"metric\":{\"properties\":{\"transformations\":{\"path\":\"beat.stats.apm_server.processor.metric.transformations\",\"type\":\"alias\"},\"decoding\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.processor.metric.decoding.count\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.apm_server.processor.metric.decoding.errors\",\"type\":\"alias\"}}},\"validation\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.processor.metric.validation.count\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.apm_server.processor.metric.validation.errors\",\"type\":\"alias\"}}}}},\"sourcemap\":{\"properties\":{\"counter\":{\"path\":\"beat.stats.apm_server.processor.sourcemap.counter\",\"type\":\"alias\"},\"decoding\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.processor.sourcemap.decoding.count\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.apm_server.processor.sourcemap.decoding.errors\",\"type\":\"alias\"}}},\"validation\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.processor.sourcemap.validation.count\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.apm_server.processor.sourcemap.validation.errors\",\"type\":\"alias\"}}}}},\"error\":{\"properties\":{\"stacktraces\":{\"path\":\"beat.stats.apm_server.processor.error.stacktraces\",\"type\":\"alias\"},\"spans\":{\"path\":\"beat.stats.apm_server.processor.error.spans\",\"type\":\"alias\"},\"frames\":{\"path\":\"beat.stats.apm_server.processor.error.frames\",\"type\":\"alias\"},\"transformations\":{\"path\":\"beat.stats.apm_server.processor.error.transformations\",\"type\":\"alias\"},\"decoding\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.processor.error.decoding.count\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.apm_server.processor.error.decoding.errors\",\"type\":\"alias\"}}},\"validation\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.processor.error.validation.count\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.apm_server.processor.error.validation.errors\",\"type\":\"alias\"}}}}},\"transaction\":{\"properties\":{\"stacktraces\":{\"path\":\"beat.stats.apm_server.processor.transaction.stacktraces\",\"type\":\"alias\"},\"spans\":{\"path\":\"beat.stats.apm_server.processor.transaction.spans\",\"type\":\"alias\"},\"frames\":{\"path\":\"beat.stats.apm_server.processor.transaction.frames\",\"type\":\"alias\"},\"transformations\":{\"path\":\"beat.stats.apm_server.processor.transaction.transformations\",\"type\":\"alias\"},\"transactions\":{\"path\":\"beat.stats.apm_server.processor.transaction.transactions\",\"type\":\"alias\"},\"decoding\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.processor.transaction.decoding.count\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.apm_server.processor.transaction.decoding.errors\",\"type\":\"alias\"}}},\"validation\":{\"properties\":{\"count\":{\"path\":\"beat.stats.apm_server.processor.transaction.validation.count\",\"type\":\"alias\"},\"errors\":{\"path\":\"beat.stats.apm_server.processor.transaction.validation.errors\",\"type\":\"alias\"}}}}},\"span\":{\"properties\":{\"transformations\":{\"path\":\"beat.stats.apm_server.processor.span.transformations\",\"type\":\"alias\"}}}}}}}}},\"timestamp\":{\"path\":\"@timestamp\",\"type\":\"alias\"}}},\"beats_state\":{\"properties\":{\"beat\":{\"properties\":{\"host\":{\"path\":\"beat.state.beat.host\",\"type\":\"alias\"},\"name\":{\"path\":\"beat.state.beat.name\",\"type\":\"alias\"},\"type\":{\"path\":\"beat.state.beat.type\",\"type\":\"alias\"},\"uuid\":{\"path\":\"beat.state.beat.uuid\",\"type\":\"alias\"},\"version\":{\"path\":\"beat.state.beat.version\",\"type\":\"alias\"}}},\"state\":{\"properties\":{\"output\":{\"properties\":{\"name\":{\"path\":\"beat.state.output.name\",\"type\":\"alias\"}}},\"input\":{\"properties\":{\"names\":{\"path\":\"beat.state.input.names\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.state.input.count\",\"type\":\"alias\"}}},\"service\":{\"properties\":{\"name\":{\"path\":\"beat.state.service.name\",\"type\":\"alias\"},\"id\":{\"path\":\"beat.state.service.id\",\"type\":\"alias\"},\"version\":{\"path\":\"beat.state.service.version\",\"type\":\"alias\"}}},\"module\":{\"properties\":{\"names\":{\"path\":\"beat.state.module.names\",\"type\":\"alias\"},\"count\":{\"path\":\"beat.state.module.count\",\"type\":\"alias\"}}},\"beat\":{\"properties\":{\"name\":{\"path\":\"beat.state.beat.name\",\"type\":\"alias\"}}},\"host\":{\"properties\":{\"hostname\":{\"path\":\"host.hostname\",\"type\":\"alias\"},\"os\":{\"properties\":{\"version\":{\"path\":\"beat.state.host.os.version\",\"type\":\"alias\"},\"platform\":{\"path\":\"beat.state.host.os.platform\",\"type\":\"alias\"}}},\"name\":{\"path\":\"host.name\",\"type\":\"alias\"},\"architecture\":{\"path\":\"host.architecture\",\"type\":\"alias\"}}}}},\"timestamp\":{\"path\":\"@timestamp\",\"type\":\"alias\"}}},\"timestamp\":{\"path\":\"@timestamp\",\"type\":\"alias\"}}}},\"composed_of\":[],\"version\":8000103,\"data_stream\":{\"hidden\":false,\"allow_custom_routing\":false}},\".monitoring-kibana-mb\":{\"index_patterns\":[\".monitoring-kibana-8-*\"],\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\".monitoring-8-ilm-policy\"},\"mapping\":{\"total_fields\":{\"limit\":\"2000\"}}}},\"mappings\":{\"dynamic\":false,\"properties\":{\"process\":{\"properties\":{\"pid\":{\"type\":\"long\"}}},\"@timestamp\":{\"type\":\"date\"},\"ecs\":{\"properties\":{\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"cluster_uuid\":{\"path\":\"kibana.elasticsearch.cluster.id\",\"type\":\"alias\"},\"service\":{\"properties\":{\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"host\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"metricset\":{\"properties\":{\"period\":{\"type\":\"long\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"kibana\":{\"properties\":{\"elasticsearch\":{\"properties\":{\"cluster\":{\"properties\":{\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"stats\":{\"properties\":{\"request\":{\"properties\":{\"total\":{\"type\":\"long\"},\"disconnects\":{\"type\":\"long\"}}},\"process\":{\"properties\":{\"memory\":{\"properties\":{\"resident_set_size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"heap\":{\"properties\":{\"total\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"used\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"size_limit\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"uptime\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}}}},\"event_loop_delay\":{\"properties\":{\"ms\":{\"scaling_factor\":1000,\"type\":\"scaled_float\"}}},\"uptime\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"os\":{\"properties\":{\"distroRelease\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"distro\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"memory\":{\"properties\":{\"used_in_bytes\":{\"type\":\"long\"},\"total_in_bytes\":{\"type\":\"long\"},\"free_in_bytes\":{\"type\":\"long\"}}},\"load\":{\"properties\":{\"5m\":{\"type\":\"half_float\"},\"15m\":{\"type\":\"half_float\"},\"1m\":{\"type\":\"half_float\"}}},\"platformRelease\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"platform\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"usage\":{\"properties\":{\"index\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"host\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"index\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"response_time\":{\"properties\":{\"avg\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"max\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"kibana\":{\"properties\":{\"status\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"concurrent_connections\":{\"type\":\"long\"},\"snapshot\":{\"type\":\"boolean\"},\"status\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"node_rules\":{\"properties\":{\"failures\":{\"type\":\"long\"},\"executions\":{\"type\":\"long\"},\"timeouts\":{\"type\":\"long\"}}},\"cluster_actions\":{\"properties\":{\"overdue\":{\"properties\":{\"delay\":{\"properties\":{\"p99\":{\"type\":\"float\"},\"p50\":{\"type\":\"float\"}}},\"count\":{\"type\":\"long\"}}}}},\"cluster_rules\":{\"properties\":{\"overdue\":{\"properties\":{\"delay\":{\"properties\":{\"p99\":{\"type\":\"float\"},\"p50\":{\"type\":\"float\"}}},\"count\":{\"type\":\"long\"}}}}},\"node_actions\":{\"properties\":{\"failures\":{\"type\":\"long\"},\"executions\":{\"type\":\"long\"},\"timeouts\":{\"type\":\"long\"}}},\"status\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"metrics\":{\"properties\":{\"requests\":{\"properties\":{\"total\":{\"type\":\"long\"},\"disconnects\":{\"type\":\"long\"}}},\"concurrent_connections\":{\"type\":\"long\"}}},\"status\":{\"properties\":{\"overall\":{\"properties\":{\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}}}}}},\"event\":{\"properties\":{\"duration\":{\"type\":\"long\"},\"module\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"dataset\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"kibana_stats\":{\"properties\":{\"process\":{\"properties\":{\"memory\":{\"properties\":{\"resident_set_size_in_bytes\":{\"path\":\"kibana.stats.process.memory.resident_set_size.bytes\",\"type\":\"alias\"},\"heap\":{\"properties\":{\"size_limit\":{\"path\":\"kibana.stats.process.memory.heap.size_limit.bytes\",\"type\":\"alias\"}}}}},\"event_loop_delay\":{\"path\":\"kibana.stats.process.event_loop_delay.ms\",\"type\":\"alias\"},\"uptime_in_millis\":{\"path\":\"kibana.stats.process.uptime.ms\",\"type\":\"alias\"}}},\"os\":{\"properties\":{\"memory\":{\"properties\":{\"free_in_bytes\":{\"path\":\"kibana.stats.os.memory.free_in_bytes\",\"type\":\"alias\"}}},\"load\":{\"properties\":{\"5m\":{\"path\":\"kibana.stats.os.load.5m\",\"type\":\"alias\"},\"15m\":{\"path\":\"kibana.stats.os.load.15m\",\"type\":\"alias\"},\"1m\":{\"path\":\"kibana.stats.os.load.1m\",\"type\":\"alias\"}}}}},\"response_times\":{\"properties\":{\"average\":{\"path\":\"kibana.stats.response_time.avg.ms\",\"type\":\"alias\"},\"max\":{\"path\":\"kibana.stats.response_time.max.ms\",\"type\":\"alias\"}}},\"requests\":{\"properties\":{\"total\":{\"path\":\"kibana.stats.request.total\",\"type\":\"alias\"},\"disconnects\":{\"path\":\"kibana.stats.request.disconnects\",\"type\":\"alias\"}}},\"kibana\":{\"properties\":{\"response_time\":{\"properties\":{\"max\":{\"path\":\"kibana.stats.response_time.max.ms\",\"type\":\"alias\"}}},\"uuid\":{\"path\":\"service.id\",\"type\":\"alias\"},\"version\":{\"path\":\"service.version\",\"type\":\"alias\"},\"status\":{\"path\":\"kibana.stats.status\",\"type\":\"alias\"}}},\"concurrent_connections\":{\"path\":\"kibana.stats.concurrent_connections\",\"type\":\"alias\"},\"timestamp\":{\"path\":\"@timestamp\",\"type\":\"alias\"}}},\"timestamp\":{\"path\":\"@timestamp\",\"type\":\"alias\"}}}},\"composed_of\":[],\"version\":8000103,\"data_stream\":{\"hidden\":false,\"allow_custom_routing\":false}},\".monitoring-ent-search-mb\":{\"index_patterns\":[\".monitoring-ent-search-8-*\"],\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\".monitoring-8-ilm-policy\"},\"mapping\":{\"total_fields\":{\"limit\":\"2000\"}}}},\"mappings\":{\"dynamic\":false,\"properties\":{\"enterprisesearch\":{\"properties\":{\"cluster_uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"stats\":{\"properties\":{\"connectors\":{\"properties\":{\"pool\":{\"properties\":{\"publish_worker_pool\":{\"properties\":{\"total_completed\":{\"type\":\"long\"},\"size\":{\"type\":\"long\"},\"idle\":{\"type\":\"long\"},\"busy\":{\"type\":\"long\"},\"queue_depth\":{\"type\":\"long\"},\"total_scheduled\":{\"type\":\"long\"}}},\"extract_worker_pool\":{\"properties\":{\"total_completed\":{\"type\":\"long\"},\"size\":{\"type\":\"long\"},\"idle\":{\"type\":\"long\"},\"busy\":{\"type\":\"long\"},\"queue_depth\":{\"type\":\"long\"},\"total_scheduled\":{\"type\":\"long\"}}},\"subextract_worker_pool\":{\"properties\":{\"total_completed\":{\"type\":\"long\"},\"size\":{\"type\":\"long\"},\"idle\":{\"type\":\"long\"},\"busy\":{\"type\":\"long\"},\"queue_depth\":{\"type\":\"long\"},\"total_scheduled\":{\"type\":\"long\"}}}}},\"job_store\":{\"properties\":{\"waiting\":{\"type\":\"long\"},\"job_types\":{\"properties\":{\"permissions\":{\"type\":\"long\"},\"incremental\":{\"type\":\"long\"},\"delete\":{\"type\":\"long\"},\"full\":{\"type\":\"long\"}}},\"working\":{\"type\":\"long\"}}}}},\"queues\":{\"properties\":{\"engine_destroyer\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"process_crawl\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"failed\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"mailer\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"http\":{\"properties\":{\"request_duration\":{\"properties\":{\"max\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"std_dev\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"mean\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"responses\":{\"properties\":{\"1xx\":{\"type\":\"long\"},\"2xx\":{\"type\":\"long\"},\"3xx\":{\"type\":\"long\"},\"4xx\":{\"type\":\"long\"},\"5xx\":{\"type\":\"long\"}}},\"connections\":{\"properties\":{\"current\":{\"type\":\"long\"},\"total\":{\"type\":\"long\"},\"max\":{\"type\":\"long\"}}},\"network\":{\"properties\":{\"received\":{\"properties\":{\"bytes_per_sec\":{\"type\":\"long\"},\"bytes\":{\"type\":\"long\"}}},\"sent\":{\"properties\":{\"bytes_per_sec\":{\"type\":\"long\"},\"bytes\":{\"type\":\"long\"}}}}}}},\"product_usage\":{\"properties\":{\"app_search\":{\"properties\":{\"total_engines\":{\"type\":\"long\"}}},\"workplace_search\":{\"properties\":{\"total_org_sources\":{\"type\":\"long\"},\"total_private_sources\":{\"type\":\"long\"}}}}}}},\"health\":{\"properties\":{\"jvm\":{\"properties\":{\"memory_usage\":{\"properties\":{\"heap_committed\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"heap_used\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"non_heap_committed\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"non_heap_init\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"heap_init\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"heap_max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"object_pending_finalization_count\":{\"type\":\"long\"}}},\"threads\":{\"properties\":{\"current\":{\"type\":\"long\"},\"total_started\":{\"type\":\"long\"},\"max\":{\"type\":\"long\"},\"daemon\":{\"type\":\"long\"}}},\"gc\":{\"properties\":{\"collection_count\":{\"type\":\"long\"},\"collection_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"process\":{\"properties\":{\"pid\":{\"type\":\"long\"},\"filebeat\":{\"properties\":{\"restart_count\":{\"type\":\"long\"},\"pid\":{\"type\":\"long\"},\"time_since_last_restart\":{\"properties\":{\"sec\":{\"type\":\"long\"}}}}},\"uptime\":{\"properties\":{\"sec\":{\"type\":\"long\"}}}}},\"crawler\":{\"properties\":{\"workers\":{\"properties\":{\"pool_size\":{\"type\":\"long\"},\"available\":{\"type\":\"long\"},\"active\":{\"type\":\"long\"}}}}},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"properties\":{\"number\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"build_hash\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}}}},\"agent\":{\"properties\":{\"id\":{\"type\":\"keyword\"},\"ephemeral_id\":{\"type\":\"keyword\"},\"type\":{\"type\":\"keyword\"},\"version\":{\"type\":\"keyword\"}}},\"process\":{\"properties\":{\"pid\":{\"type\":\"long\"}}},\"@timestamp\":{\"type\":\"date\"},\"ecs\":{\"properties\":{\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"service\":{\"properties\":{\"node\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"hostname\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"environment\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"origin\":{\"properties\":{\"node\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"environment\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"target\":{\"properties\":{\"node\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"environment\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"host\":{\"properties\":{\"hostname\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"architecture\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"metricset\":{\"properties\":{\"period\":{\"type\":\"long\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"event\":{\"properties\":{\"reason\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"code\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"timezone\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"agent_id_status\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"duration\":{\"type\":\"long\"},\"reference\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ingested\":{\"type\":\"date\"},\"provider\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"action\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"end\":{\"type\":\"date\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"outcome\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"severity\":{\"type\":\"long\"},\"original\":{\"ignore_above\":1024,\"index\":false,\"type\":\"keyword\",\"doc_values\":false},\"risk_score\":{\"type\":\"float\"},\"created\":{\"type\":\"date\"},\"kind\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"module\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"start\":{\"type\":\"date\"},\"url\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"sequence\":{\"type\":\"long\"},\"risk_score_norm\":{\"type\":\"float\"},\"category\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"dataset\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"hash\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}}},\"composed_of\":[],\"version\":8000103,\"_meta\":{\"managed\":true,\"description\":\"Template used by Enterprise Search Metricbeat module monitoring information for Stack Monitoring\"},\"data_stream\":{\"hidden\":false,\"allow_custom_routing\":false}},\"synthetics\":{\"index_patterns\":[\"synthetics-*-*\"],\"composed_of\":[\"synthetics-mappings\",\"data-streams-mappings\",\"synthetics-settings\"],\"priority\":100,\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default synthetics template installed by x-pack\"},\"data_stream\":{\"hidden\":false,\"allow_custom_routing\":false},\"allow_auto_create\":true},\"ilm-history\":{\"index_patterns\":[\"ilm-history-5*\"],\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\"ilm-history-ilm-policy\"},\"number_of_shards\":\"1\",\"auto_expand_replicas\":\"0-1\",\"number_of_replicas\":\"0\"}},\"mappings\":{\"dynamic\":false,\"properties\":{\"index_age\":{\"type\":\"long\"},\"@timestamp\":{\"format\":\"epoch_millis\",\"type\":\"date\"},\"error_details\":{\"type\":\"text\"},\"success\":{\"type\":\"boolean\"},\"index\":{\"type\":\"keyword\"},\"state\":{\"dynamic\":true,\"type\":\"object\",\"properties\":{\"phase\":{\"type\":\"keyword\"},\"failed_step\":{\"type\":\"keyword\"},\"phase_definition\":{\"type\":\"text\"},\"action_time\":{\"format\":\"epoch_millis\",\"type\":\"date\"},\"phase_time\":{\"format\":\"epoch_millis\",\"type\":\"date\"},\"step_info\":{\"type\":\"text\"},\"action\":{\"type\":\"keyword\"},\"step\":{\"type\":\"keyword\"},\"is_auto-retryable_error\":{\"type\":\"keyword\"},\"creation_date\":{\"format\":\"epoch_millis\",\"type\":\"date\"},\"step_time\":{\"format\":\"epoch_millis\",\"type\":\"date\"}}},\"policy\":{\"type\":\"keyword\"}}}},\"composed_of\":[],\"priority\":2147483647,\"version\":5,\"_meta\":{\"managed\":true,\"description\":\"index template for ILM history indices\"},\"data_stream\":{\"hidden\":true,\"allow_custom_routing\":false},\"allow_auto_create\":true},\".ml-state\":{\"index_patterns\":[\".ml-state*\"],\"template\":{\"settings\":{\"index\":{\"hidden\":\"true\",\"lifecycle\":{\"name\":\"ml-size-based-ilm-policy\",\"rollover_alias\":\".ml-state-write\"},\"auto_expand_replicas\":\"0-1\"}},\"mappings\":{\"_meta\":{\"version\":\"8060299\"},\"enabled\":false},\"aliases\":{}},\"composed_of\":[],\"priority\":2147483647,\"version\":8060299,\"_meta\":{\"managed\":true,\"description\":\"index template for ML state indices\"}},\".monitoring-es-mb\":{\"index_patterns\":[\".monitoring-es-8-*\"],\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\".monitoring-8-ilm-policy\"},\"mapping\":{\"total_fields\":{\"limit\":\"2000\"}}}},\"mappings\":{\"dynamic\":false,\"properties\":{\"ccr_auto_follow_stats\":{\"properties\":{\"number_of_failed_remote_cluster_state_requests\":{\"path\":\"elasticsearch.ccr.auto_follow.failed.remote_cluster_state_requests.count\",\"type\":\"alias\"},\"follower\":{\"properties\":{\"failed_read_requests\":{\"path\":\"elasticsearch.ccr.requests.failed.read.count\",\"type\":\"alias\"}}},\"number_of_failed_follow_indices\":{\"path\":\"elasticsearch.ccr.auto_follow.failed.follow_indices.count\",\"type\":\"alias\"},\"number_of_successful_follow_indices\":{\"path\":\"elasticsearch.ccr.auto_follow.success.follow_indices.count\",\"type\":\"alias\"}}},\"ccr_stats\":{\"properties\":{\"write_buffer_size_in_bytes\":{\"path\":\"elasticsearch.ccr.write_buffer.size.bytes\",\"type\":\"alias\"},\"leader_global_checkpoint\":{\"path\":\"elasticsearch.ccr.leader.global_checkpoint\",\"type\":\"alias\"},\"follower_index\":{\"path\":\"elasticsearch.ccr.follower.index\",\"type\":\"alias\"},\"leader_max_seq_no\":{\"path\":\"elasticsearch.ccr.leader.max_seq_no\",\"type\":\"alias\"},\"last_requested_seq_no\":{\"path\":\"elasticsearch.ccr.last_requested_seq_no\",\"type\":\"alias\"},\"follower_settings_version\":{\"path\":\"elasticsearch.ccr.follower.settings_version\",\"type\":\"alias\"},\"successful_write_requests\":{\"path\":\"elasticsearch.ccr.requests.successful.write.count\",\"type\":\"alias\"},\"remote_cluster\":{\"path\":\"elasticsearch.ccr.remote_cluster\",\"type\":\"alias\"},\"outstanding_write_requests\":{\"path\":\"elasticsearch.ccr.requests.outstanding.write.count\",\"type\":\"alias\"},\"total_read_time_millis\":{\"path\":\"elasticsearch.ccr.total_time.read.ms\",\"type\":\"alias\"},\"outstanding_read_requests\":{\"path\":\"elasticsearch.ccr.requests.outstanding.read.count\",\"type\":\"alias\"},\"total_write_time_millis\":{\"path\":\"elasticsearch.ccr.total_time.write.ms\",\"type\":\"alias\"},\"failed_write_requests\":{\"path\":\"elasticsearch.ccr.requests.failed.write.count\",\"type\":\"alias\"},\"failed_read_requests\":{\"path\":\"elasticsearch.ccr.requests.failed.read.count\",\"type\":\"alias\"},\"bytes_read\":{\"path\":\"elasticsearch.ccr.bytes_read\",\"type\":\"alias\"},\"leader_index\":{\"path\":\"elasticsearch.ccr.leader.index\",\"type\":\"alias\"},\"follower_max_seq_no\":{\"path\":\"elasticsearch.ccr.follower.max_seq_no\",\"type\":\"alias\"},\"operations_written\":{\"path\":\"elasticsearch.ccr.follower.operations_written\",\"type\":\"alias\"},\"write_buffer_operation_count\":{\"path\":\"elasticsearch.ccr.write_buffer.operation.count\",\"type\":\"alias\"},\"successful_read_requests\":{\"path\":\"elasticsearch.ccr.requests.successful.read.count\",\"type\":\"alias\"},\"shard_id\":{\"path\":\"elasticsearch.ccr.follower.shard.number\",\"type\":\"alias\"},\"follower_mapping_version\":{\"path\":\"elasticsearch.ccr.follower.mapping_version\",\"type\":\"alias\"},\"follower_aliases_version\":{\"path\":\"elasticsearch.ccr.follower.aliases_version\",\"type\":\"alias\"},\"follower_global_checkpoint\":{\"path\":\"elasticsearch.ccr.follower.global_checkpoint\",\"type\":\"alias\"},\"total_read_remote_exec_time_millis\":{\"path\":\"elasticsearch.ccr.total_time.read.remote_exec.ms\",\"type\":\"alias\"},\"operations_read\":{\"path\":\"elasticsearch.ccr.follower.operations.read.count\",\"type\":\"alias\"}}},\"cluster_state\":{\"properties\":{\"nodes_hash\":{\"path\":\"elasticsearch.cluster.stats.state.nodes_hash\",\"type\":\"alias\"},\"master_node\":{\"path\":\"elasticsearch.cluster.stats.state.master_node\",\"type\":\"alias\"},\"state_uuid\":{\"path\":\"elasticsearch.cluster.stats.state.state_uuid\",\"type\":\"alias\"},\"version\":{\"path\":\"elasticsearch.cluster.stats.state.version\",\"type\":\"alias\"},\"status\":{\"path\":\"elasticsearch.cluster.stats.status\",\"type\":\"alias\"}}},\"indices_stats\":{\"properties\":{\"_all\":{\"properties\":{\"primaries\":{\"properties\":{\"indexing\":{\"properties\":{\"index_time_in_millis\":{\"path\":\"elasticsearch.index.summary.primaries.indexing.index.time.ms\",\"type\":\"alias\"},\"index_total\":{\"path\":\"elasticsearch.index.summary.primaries.indexing.index.count\",\"type\":\"alias\"}}}}},\"total\":{\"properties\":{\"search\":{\"properties\":{\"query_total\":{\"path\":\"elasticsearch.index.summary.total.search.query.count\",\"type\":\"alias\"},\"query_time_in_millis\":{\"path\":\"elasticsearch.index.summary.total.search.query.time.ms\",\"type\":\"alias\"}}},\"indexing\":{\"properties\":{\"index_total\":{\"path\":\"elasticsearch.index.summary.total.indexing.index.count\",\"type\":\"alias\"}}}}}}}}},\"node_stats\":{\"properties\":{\"jvm\":{\"properties\":{\"mem\":{\"properties\":{\"heap_used_percent\":{\"path\":\"elasticsearch.node.stats.jvm.mem.heap.used.pct\",\"type\":\"alias\"},\"heap_max_in_bytes\":{\"path\":\"elasticsearch.node.stats.jvm.mem.heap.max.bytes\",\"type\":\"alias\"},\"heap_used_in_bytes\":{\"path\":\"elasticsearch.node.stats.jvm.mem.heap.used.bytes\",\"type\":\"alias\"}}},\"gc\":{\"properties\":{\"collectors\":{\"properties\":{\"young\":{\"properties\":{\"collection_count\":{\"path\":\"elasticsearch.node.stats.jvm.gc.collectors.young.collection.count\",\"type\":\"alias\"},\"collection_time_in_millis\":{\"path\":\"elasticsearch.node.stats.jvm.gc.collectors.young.collection.ms\",\"type\":\"alias\"}}},\"old\":{\"properties\":{\"collection_count\":{\"path\":\"elasticsearch.node.stats.jvm.gc.collectors.old.collection.count\",\"type\":\"alias\"},\"collection_time_in_millis\":{\"path\":\"elasticsearch.node.stats.jvm.gc.collectors.old.collection.ms\",\"type\":\"alias\"}}}}}}}}},\"indices\":{\"properties\":{\"search\":{\"properties\":{\"query_total\":{\"path\":\"elasticsearch.node.stats.indices.search.query_total.count\",\"type\":\"alias\"},\"query_time_in_millis\":{\"path\":\"elasticsearch.node.stats.indices.search.query_time.ms\",\"type\":\"alias\"}}},\"query_cache\":{\"properties\":{\"memory_size_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.query_cache.memory.bytes\",\"type\":\"alias\"}}},\"docs\":{\"properties\":{\"count\":{\"path\":\"elasticsearch.node.stats.indices.docs.count\",\"type\":\"alias\"}}},\"indexing\":{\"properties\":{\"throttle_time_in_millis\":{\"path\":\"elasticsearch.node.stats.indices.indexing.throttle_time.ms\",\"type\":\"alias\"},\"index_time_in_millis\":{\"path\":\"elasticsearch.node.stats.indices.indexing.index_time.ms\",\"type\":\"alias\"},\"index_total\":{\"path\":\"elasticsearch.node.stats.indices.indexing.index_total.count\",\"type\":\"alias\"}}},\"fielddata\":{\"properties\":{\"memory_size_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.fielddata.memory.bytes\",\"type\":\"alias\"}}},\"store\":{\"properties\":{\"size\":{\"properties\":{\"bytes\":{\"path\":\"elasticsearch.node.stats.indices.store.size.bytes\",\"type\":\"alias\"}}},\"size_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.store.size.bytes\",\"type\":\"alias\"}}},\"request_cache\":{\"properties\":{\"memory_size_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.request_cache.memory.bytes\",\"type\":\"alias\"}}},\"segments\":{\"properties\":{\"version_map_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.version_map.memory.bytes\",\"type\":\"alias\"},\"norms_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.norms.memory.bytes\",\"type\":\"alias\"},\"count\":{\"path\":\"elasticsearch.node.stats.indices.segments.count\",\"type\":\"alias\"},\"term_vectors_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.term_vectors.memory.bytes\",\"type\":\"alias\"},\"points_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.points.memory.bytes\",\"type\":\"alias\"},\"index_writer_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.index_writer.memory.bytes\",\"type\":\"alias\"},\"memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.memory.bytes\",\"type\":\"alias\"},\"doc_values_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.doc_values.memory.bytes\",\"type\":\"alias\"},\"terms_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.terms.memory.bytes\",\"type\":\"alias\"},\"fixed_bit_set_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.fixed_bit_set.memory.bytes\",\"type\":\"alias\"},\"stored_fields_memory_in_bytes\":{\"path\":\"elasticsearch.node.stats.indices.segments.stored_fields.memory.bytes\",\"type\":\"alias\"}}}}},\"process\":{\"properties\":{\"cpu\":{\"properties\":{\"percent\":{\"path\":\"elasticsearch.node.stats.process.cpu.pct\",\"type\":\"alias\"}}}}},\"os\":{\"properties\":{\"cpu\":{\"properties\":{\"load_average\":{\"properties\":{\"1m\":{\"path\":\"elasticsearch.node.stats.os.cpu.load_avg.1m\",\"type\":\"alias\"}}}}},\"cgroup\":{\"properties\":{\"memory\":{\"properties\":{\"usage_in_bytes\":{\"path\":\"elasticsearch.node.stats.os.cgroup.memory.usage.bytes\",\"type\":\"alias\"},\"control_group\":{\"path\":\"elasticsearch.node.stats.os.cgroup.memory.control_group\",\"type\":\"alias\"},\"limit_in_bytes\":{\"path\":\"elasticsearch.node.stats.os.cgroup.memory.limit.bytes\",\"type\":\"alias\"}}},\"cpu\":{\"properties\":{\"stat\":{\"properties\":{\"number_of_elapsed_periods\":{\"path\":\"elasticsearch.node.stats.os.cgroup.cpu.stat.elapsed_periods.count\",\"type\":\"alias\"},\"number_of_times_throttled\":{\"path\":\"elasticsearch.node.stats.os.cgroup.cpu.stat.times_throttled.count\",\"type\":\"alias\"},\"time_throttled_nanos\":{\"path\":\"elasticsearch.node.stats.os.cgroup.cpu.stat.time_throttled.ns\",\"type\":\"alias\"}}},\"cfs_quota_micros\":{\"path\":\"elasticsearch.node.stats.os.cgroup.cpu.cfs.quota.us\",\"type\":\"alias\"}}},\"cpuacct\":{\"properties\":{\"usage_nanos\":{\"path\":\"elasticsearch.node.stats.os.cgroup.cpuacct.usage.ns\",\"type\":\"alias\"}}}}}}},\"thread_pool\":{\"properties\":{\"search\":{\"properties\":{\"rejected\":{\"path\":\"elasticsearch.node.stats.thread_pool.search.rejected.count\",\"type\":\"alias\"},\"queue\":{\"path\":\"elasticsearch.node.stats.thread_pool.search.queue.count\",\"type\":\"alias\"}}},\"get\":{\"properties\":{\"rejected\":{\"path\":\"elasticsearch.node.stats.thread_pool.get.rejected.count\",\"type\":\"alias\"},\"queue\":{\"path\":\"elasticsearch.node.stats.thread_pool.get.queue.count\",\"type\":\"alias\"}}},\"index\":{\"properties\":{\"rejected\":{\"path\":\"elasticsearch.node.stats.thread_pool.index.rejected.count\",\"type\":\"alias\"},\"queue\":{\"path\":\"elasticsearch.node.stats.thread_pool.index.queue.count\",\"type\":\"alias\"}}},\"bulk\":{\"properties\":{\"rejected\":{\"path\":\"elasticsearch.node.stats.thread_pool.bulk.rejected.count\",\"type\":\"alias\"},\"queue\":{\"path\":\"elasticsearch.node.stats.thread_pool.bulk.queue.count\",\"type\":\"alias\"}}},\"write\":{\"properties\":{\"rejected\":{\"path\":\"elasticsearch.node.stats.thread_pool.write.rejected.count\",\"type\":\"alias\"},\"queue\":{\"path\":\"elasticsearch.node.stats.thread_pool.write.queue.count\",\"type\":\"alias\"}}}}},\"fs\":{\"properties\":{\"summary\":{\"properties\":{\"total\":{\"properties\":{\"bytes\":{\"path\":\"elasticsearch.node.stats.fs.summary.total.bytes\",\"type\":\"alias\"}}},\"available\":{\"properties\":{\"bytes\":{\"path\":\"elasticsearch.node.stats.fs.summary.available.bytes\",\"type\":\"alias\"}}}}},\"io_stats\":{\"properties\":{\"total\":{\"properties\":{\"write_operations\":{\"path\":\"elasticsearch.node.stats.fs.io_stats.total.write.operations.count\",\"type\":\"alias\"},\"operations\":{\"path\":\"elasticsearch.node.stats.fs.io_stats.total.operations.count\",\"type\":\"alias\"},\"read_operations\":{\"path\":\"elasticsearch.node.stats.fs.io_stats.total.read.operations.count\",\"type\":\"alias\"}}}}},\"total\":{\"properties\":{\"total_in_bytes\":{\"path\":\"elasticsearch.node.stats.fs.summary.total.bytes\",\"type\":\"alias\"},\"available_in_bytes\":{\"path\":\"elasticsearch.node.stats.fs.summary.available.bytes\",\"type\":\"alias\"}}}}},\"node_id\":{\"path\":\"elasticsearch.node.id\",\"type\":\"alias\"}}},\"stack_stats\":{\"properties\":{\"xpack\":{\"properties\":{\"ccr\":{\"properties\":{\"available\":{\"path\":\"elasticsearch.cluster.stats.stack.xpack.ccr.available\",\"type\":\"alias\"},\"enabled\":{\"path\":\"elasticsearch.cluster.stats.stack.xpack.ccr.enabled\",\"type\":\"alias\"}}}}},\"apm\":{\"properties\":{\"found\":{\"path\":\"elasticsearch.cluster.stats.stack.apm.found\",\"type\":\"alias\"}}}}},\"license\":{\"properties\":{\"type\":{\"path\":\"elasticsearch.cluster.stats.license.type\",\"type\":\"alias\"},\"status\":{\"path\":\"elasticsearch.cluster.stats.license.status\",\"type\":\"alias\"}}},\"index_stats\":{\"properties\":{\"primaries\":{\"properties\":{\"docs\":{\"properties\":{\"count\":{\"path\":\"elasticsearch.index.primaries.docs.count\",\"type\":\"alias\"}}},\"indexing\":{\"properties\":{\"throttle_time_in_millis\":{\"path\":\"elasticsearch.index.primaries.indexing.throttle_time_in_millis\",\"type\":\"alias\"},\"index_time_in_millis\":{\"path\":\"elasticsearch.index.primaries.indexing.index_time_in_millis\",\"type\":\"alias\"},\"index_total\":{\"path\":\"elasticsearch.index.primaries.indexing.index_total\",\"type\":\"alias\"}}},\"refresh\":{\"properties\":{\"total_time_in_millis\":{\"path\":\"elasticsearch.index.primaries.refresh.total_time_in_millis\",\"type\":\"alias\"}}},\"store\":{\"properties\":{\"size_in_bytes\":{\"path\":\"elasticsearch.index.primaries.store.size_in_bytes\",\"type\":\"alias\"}}},\"merges\":{\"properties\":{\"total_size_in_bytes\":{\"path\":\"elasticsearch.index.primaries.merges.total_size_in_bytes\",\"type\":\"alias\"}}},\"segments\":{\"properties\":{\"count\":{\"path\":\"elasticsearch.index.primaries.segments.count\",\"type\":\"alias\"}}}}},\"total\":{\"properties\":{\"search\":{\"properties\":{\"query_total\":{\"path\":\"elasticsearch.index.total.search.query_total\",\"type\":\"alias\"},\"query_time_in_millis\":{\"path\":\"elasticsearch.index.total.search.query_time_in_millis\",\"type\":\"alias\"}}},\"query_cache\":{\"properties\":{\"memory_size_in_bytes\":{\"path\":\"elasticsearch.index.total.query_cache.memory_size_in_bytes\",\"type\":\"alias\"}}},\"fielddata\":{\"properties\":{\"memory_size_in_bytes\":{\"path\":\"elasticsearch.index.total.fielddata.memory_size_in_bytes\",\"type\":\"alias\"}}},\"indexing\":{\"properties\":{\"throttle_time_in_millis\":{\"path\":\"elasticsearch.index.total.indexing.throttle_time_in_millis\",\"type\":\"alias\"},\"index_time_in_millis\":{\"path\":\"elasticsearch.index.total.indexing.index_time_in_millis\",\"type\":\"alias\"},\"index_total\":{\"path\":\"elasticsearch.index.total.indexing.index_total\",\"type\":\"alias\"}}},\"refresh\":{\"properties\":{\"total_time_in_millis\":{\"path\":\"elasticsearch.index.total.refresh.total_time_in_millis\",\"type\":\"alias\"}}},\"request_cache\":{\"properties\":{\"memory_size_in_bytes\":{\"path\":\"elasticsearch.index.total.request_cache.memory_size_in_bytes\",\"type\":\"alias\"}}},\"store\":{\"properties\":{\"size_in_bytes\":{\"path\":\"elasticsearch.index.total.store.size_in_bytes\",\"type\":\"alias\"}}},\"merges\":{\"properties\":{\"total_size_in_bytes\":{\"path\":\"elasticsearch.index.total.merges.total_size_in_bytes\",\"type\":\"alias\"}}},\"segments\":{\"properties\":{\"version_map_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.version_map_memory_in_bytes\",\"type\":\"alias\"},\"norms_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.norms_memory_in_bytes\",\"type\":\"alias\"},\"count\":{\"path\":\"elasticsearch.index.total.segments.count\",\"type\":\"alias\"},\"term_vectors_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.term_vectors_memory_in_bytes\",\"type\":\"alias\"},\"points_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.points_memory_in_bytes\",\"type\":\"alias\"},\"index_writer_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.index_writer_memory_in_bytes\",\"type\":\"alias\"},\"memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.memory_in_bytes\",\"type\":\"alias\"},\"doc_values_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.doc_values_memory_in_bytes\",\"type\":\"alias\"},\"terms_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.terms_memory_in_bytes\",\"type\":\"alias\"},\"fixed_bit_set_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.fixed_bit_set_memory_in_bytes\",\"type\":\"alias\"},\"stored_fields_memory_in_bytes\":{\"path\":\"elasticsearch.index.total.segments.stored_fields_memory_in_bytes\",\"type\":\"alias\"}}}}},\"index\":{\"path\":\"elasticsearch.index.name\",\"type\":\"alias\"}}},\"@timestamp\":{\"type\":\"date\"},\"elasticsearch\":{\"properties\":{\"cluster\":{\"properties\":{\"stats\":{\"properties\":{\"license\":{\"properties\":{\"expiry_date_in_millis\":{\"type\":\"long\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"status\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"indices\":{\"properties\":{\"shards\":{\"properties\":{\"primaries\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"total\":{\"type\":\"long\"},\"docs\":{\"properties\":{\"total\":{\"type\":\"long\"}}},\"fielddata\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"store\":{\"properties\":{\"size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}}}},\"stack\":{\"properties\":{\"xpack\":{\"properties\":{\"ccr\":{\"properties\":{\"available\":{\"type\":\"boolean\"},\"enabled\":{\"type\":\"boolean\"}}}}},\"apm\":{\"properties\":{\"found\":{\"type\":\"boolean\"}}}}},\"nodes\":{\"properties\":{\"jvm\":{\"properties\":{\"memory\":{\"properties\":{\"heap\":{\"properties\":{\"max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"used\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}}}},\"max_uptime\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"data\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"},\"fs\":{\"properties\":{\"total\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"available\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"master\":{\"type\":\"long\"}}},\"state\":{\"properties\":{\"nodes_hash\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"master_node\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state_uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"status\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"properties\":{\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"node\":{\"properties\":{\"jvm\":{\"properties\":{\"memory\":{\"properties\":{\"heap\":{\"properties\":{\"init\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"nonheap\":{\"properties\":{\"init\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}}}},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"process\":{\"properties\":{\"mlockall\":{\"type\":\"boolean\"}}},\"stats\":{\"properties\":{\"jvm\":{\"properties\":{\"mem\":{\"properties\":{\"pools\":{\"properties\":{\"young\":{\"properties\":{\"max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"peak\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"peak_max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"used\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"old\":{\"properties\":{\"max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"peak\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"peak_max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"used\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"survivor\":{\"properties\":{\"max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"peak\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"peak_max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"used\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}}}},\"heap\":{\"properties\":{\"max\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"used\":{\"properties\":{\"pct\":{\"type\":\"double\"},\"bytes\":{\"type\":\"long\"}}}}}}},\"gc\":{\"properties\":{\"collectors\":{\"properties\":{\"young\":{\"properties\":{\"collection\":{\"properties\":{\"ms\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}}}},\"old\":{\"properties\":{\"collection\":{\"properties\":{\"ms\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}}}}}}}}}},\"indices\":{\"properties\":{\"search\":{\"properties\":{\"query_total\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"query_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"query_cache\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"docs\":{\"properties\":{\"deleted\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"fielddata\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"indexing\":{\"properties\":{\"index_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"index_total\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"throttle_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"store\":{\"properties\":{\"size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"request_cache\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"bulk\":{\"properties\":{\"operations\":{\"properties\":{\"total\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"total_size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"avg_size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"total_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"avg_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"segments\":{\"properties\":{\"version_map\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"norms\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"terms\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"stored_fields\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"index_writer\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"count\":{\"type\":\"long\"},\"fixed_bit_set\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"term_vectors\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"doc_values\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"points\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}}}}}},\"process\":{\"properties\":{\"cpu\":{\"properties\":{\"pct\":{\"type\":\"double\"}}}}},\"os\":{\"properties\":{\"cpu\":{\"properties\":{\"load_avg\":{\"properties\":{\"1m\":{\"type\":\"half_float\"}}}}},\"cgroup\":{\"properties\":{\"memory\":{\"properties\":{\"control_group\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"usage\":{\"properties\":{\"bytes\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"limit\":{\"properties\":{\"bytes\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"cpu\":{\"properties\":{\"cfs\":{\"properties\":{\"quota\":{\"properties\":{\"us\":{\"type\":\"long\"}}}}},\"stat\":{\"properties\":{\"elapsed_periods\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"times_throttled\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"time_throttled\":{\"properties\":{\"ns\":{\"type\":\"long\"}}}}}}},\"cpuacct\":{\"properties\":{\"usage\":{\"properties\":{\"ns\":{\"type\":\"long\"}}}}}}}}},\"thread_pool\":{\"properties\":{\"search\":{\"properties\":{\"rejected\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"queue\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"get\":{\"properties\":{\"rejected\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"queue\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"index\":{\"properties\":{\"rejected\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"queue\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"bulk\":{\"properties\":{\"rejected\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"queue\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"write\":{\"properties\":{\"rejected\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"queue\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}}}},\"fs\":{\"properties\":{\"summary\":{\"properties\":{\"total\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"available\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"free\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"io_stats\":{\"properties\":{\"total\":{\"properties\":{\"operations\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"read\":{\"properties\":{\"operations\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"write\":{\"properties\":{\"operations\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}}}}}},\"total\":{\"properties\":{\"total_in_bytes\":{\"type\":\"long\"},\"available_in_bytes\":{\"type\":\"long\"}}}}}}},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"mlockall\":{\"type\":\"boolean\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"master\":{\"type\":\"boolean\"}}},\"ccr\":{\"properties\":{\"leader\":{\"properties\":{\"index\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"max_seq_no\":{\"type\":\"long\"},\"global_checkpoint\":{\"type\":\"long\"}}},\"follower\":{\"properties\":{\"operations\":{\"properties\":{\"read\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"time_since_last_read\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"index\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"settings_version\":{\"type\":\"long\"},\"shard\":{\"properties\":{\"number\":{\"type\":\"long\"}}},\"max_seq_no\":{\"type\":\"long\"},\"mapping_version\":{\"type\":\"long\"},\"aliases_version\":{\"type\":\"long\"},\"operations_written\":{\"type\":\"long\"},\"global_checkpoint\":{\"type\":\"long\"}}},\"remote_cluster\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"read_exceptions\":{\"type\":\"nested\"},\"shard_id\":{\"type\":\"long\"},\"bytes_read\":{\"type\":\"long\"},\"requests\":{\"properties\":{\"outstanding\":{\"properties\":{\"read\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"write\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"failed\":{\"properties\":{\"read\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"write\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"successful\":{\"properties\":{\"read\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"write\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}}}},\"auto_follow\":{\"properties\":{\"success\":{\"properties\":{\"follow_indices\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"failed\":{\"properties\":{\"follow_indices\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"remote_cluster_state_requests\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}}}},\"total_time\":{\"properties\":{\"read\":{\"properties\":{\"ms\":{\"type\":\"long\"},\"remote_exec\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"write\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"last_requested_seq_no\":{\"type\":\"long\"},\"write_buffer\":{\"properties\":{\"size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"operation\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}}}},\"pending_tasks\":{\"properties\":{\"time_in_queue\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"source\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"priority\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"insert_order\":{\"type\":\"long\"}}},\"enrich\":{\"properties\":{\"executed_searches\":{\"properties\":{\"total\":{\"type\":\"long\"}}},\"remote_requests\":{\"properties\":{\"current\":{\"type\":\"long\"},\"total\":{\"type\":\"long\"}}},\"executing_policy\":{\"properties\":{\"task\":{\"properties\":{\"task\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"parent_task_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"action\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"type\":\"long\"},\"time\":{\"properties\":{\"running\":{\"properties\":{\"nano\":{\"type\":\"long\"}}},\"start\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"cancellable\":{\"type\":\"boolean\"}}},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"queue\":{\"properties\":{\"size\":{\"type\":\"long\"}}}}},\"index\":{\"properties\":{\"summary\":{\"properties\":{\"primaries\":{\"properties\":{\"search\":{\"properties\":{\"query\":{\"properties\":{\"count\":{\"type\":\"long\"},\"time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}}}},\"docs\":{\"properties\":{\"deleted\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"indexing\":{\"properties\":{\"index\":{\"properties\":{\"count\":{\"type\":\"long\"},\"time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}}}},\"store\":{\"properties\":{\"size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"bulk\":{\"properties\":{\"operations\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"time\":{\"properties\":{\"avg\":{\"properties\":{\"ms\":{\"type\":\"long\"},\"bytes\":{\"type\":\"long\"}}},\"count\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}}}},\"segments\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"count\":{\"type\":\"long\"}}}}},\"total\":{\"properties\":{\"search\":{\"properties\":{\"query\":{\"properties\":{\"count\":{\"type\":\"long\"},\"time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}}}},\"docs\":{\"properties\":{\"deleted\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"indexing\":{\"properties\":{\"index\":{\"properties\":{\"count\":{\"type\":\"long\"},\"time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"is_throttled\":{\"type\":\"boolean\"},\"throttle_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"store\":{\"properties\":{\"size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}}}},\"bulk\":{\"properties\":{\"operations\":{\"properties\":{\"count\":{\"type\":\"long\"}}},\"size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"time\":{\"properties\":{\"avg\":{\"properties\":{\"ms\":{\"type\":\"long\"},\"bytes\":{\"type\":\"long\"}}}}}}},\"segments\":{\"properties\":{\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"count\":{\"type\":\"long\"}}}}}}},\"shards\":{\"properties\":{\"primaries\":{\"type\":\"long\"},\"total\":{\"type\":\"long\"}}},\"primaries\":{\"properties\":{\"search\":{\"properties\":{\"query_total\":{\"type\":\"long\"},\"query_time_in_millis\":{\"type\":\"long\"}}},\"query_cache\":{\"properties\":{\"miss_count\":{\"type\":\"long\"},\"memory_size_in_bytes\":{\"type\":\"long\"},\"hit_count\":{\"type\":\"long\"}}},\"docs\":{\"properties\":{\"deleted\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"indexing\":{\"properties\":{\"throttle_time_in_millis\":{\"type\":\"long\"},\"index_time_in_millis\":{\"type\":\"long\"},\"index_total\":{\"type\":\"long\"}}},\"refresh\":{\"properties\":{\"external_total_time_in_millis\":{\"type\":\"long\"},\"total_time_in_millis\":{\"type\":\"long\"}}},\"request_cache\":{\"properties\":{\"miss_count\":{\"type\":\"long\"},\"memory_size_in_bytes\":{\"type\":\"long\"},\"evictions\":{\"type\":\"long\"},\"hit_count\":{\"type\":\"long\"}}},\"store\":{\"properties\":{\"size_in_bytes\":{\"type\":\"long\"}}},\"segments\":{\"properties\":{\"version_map_memory_in_bytes\":{\"type\":\"long\"},\"norms_memory_in_bytes\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"},\"term_vectors_memory_in_bytes\":{\"type\":\"long\"},\"points_memory_in_bytes\":{\"type\":\"long\"},\"index_writer_memory_in_bytes\":{\"type\":\"long\"},\"memory_in_bytes\":{\"type\":\"long\"},\"terms_memory_in_bytes\":{\"type\":\"long\"},\"doc_values_memory_in_bytes\":{\"type\":\"long\"},\"stored_fields_memory_in_bytes\":{\"type\":\"long\"},\"fixed_bit_set_memory_in_bytes\":{\"type\":\"long\"}}},\"merges\":{\"properties\":{\"total_size_in_bytes\":{\"type\":\"long\"}}}}},\"total\":{\"properties\":{\"search\":{\"properties\":{\"query_total\":{\"type\":\"long\"},\"query_time_in_millis\":{\"type\":\"long\"}}},\"query_cache\":{\"properties\":{\"miss_count\":{\"type\":\"long\"},\"memory_size_in_bytes\":{\"type\":\"long\"},\"evictions\":{\"type\":\"long\"},\"hit_count\":{\"type\":\"long\"}}},\"docs\":{\"properties\":{\"deleted\":{\"type\":\"long\"},\"count\":{\"type\":\"long\"}}},\"fielddata\":{\"properties\":{\"memory_size_in_bytes\":{\"type\":\"long\"},\"evictions\":{\"type\":\"long\"}}},\"indexing\":{\"properties\":{\"throttle_time_in_millis\":{\"type\":\"long\"},\"index_time_in_millis\":{\"type\":\"long\"},\"index_total\":{\"type\":\"long\"}}},\"refresh\":{\"properties\":{\"external_total_time_in_millis\":{\"type\":\"long\"},\"total_time_in_millis\":{\"type\":\"long\"}}},\"store\":{\"properties\":{\"size\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"size_in_bytes\":{\"type\":\"long\"}}},\"request_cache\":{\"properties\":{\"miss_count\":{\"type\":\"long\"},\"memory_size_in_bytes\":{\"type\":\"long\"},\"evictions\":{\"type\":\"long\"},\"hit_count\":{\"type\":\"long\"}}},\"bulk\":{\"properties\":{\"avg_time_in_millis\":{\"type\":\"long\"},\"avg_size_in_bytes\":{\"type\":\"long\"},\"total_time_in_millis\":{\"type\":\"long\"},\"total_operations\":{\"type\":\"long\"},\"total_size_in_bytes\":{\"type\":\"long\"}}},\"merges\":{\"properties\":{\"total_size_in_bytes\":{\"type\":\"long\"}}},\"segments\":{\"properties\":{\"version_map_memory_in_bytes\":{\"type\":\"long\"},\"norms_memory_in_bytes\":{\"type\":\"long\"},\"memory\":{\"properties\":{\"bytes\":{\"type\":\"long\"}}},\"count\":{\"type\":\"long\"},\"term_vectors_memory_in_bytes\":{\"type\":\"long\"},\"points_memory_in_bytes\":{\"type\":\"long\"},\"index_writer_memory_in_bytes\":{\"type\":\"long\"},\"memory_in_bytes\":{\"type\":\"long\"},\"terms_memory_in_bytes\":{\"type\":\"long\"},\"doc_values_memory_in_bytes\":{\"type\":\"long\"},\"stored_fields_memory_in_bytes\":{\"type\":\"long\"},\"fixed_bit_set_memory_in_bytes\":{\"type\":\"long\"}}}}},\"hidden\":{\"type\":\"boolean\"},\"created\":{\"type\":\"long\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"recovery\":{\"properties\":{\"stop_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"translog\":{\"properties\":{\"total_on_start\":{\"type\":\"long\"},\"total\":{\"type\":\"long\"},\"percent\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"verify_index\":{\"properties\":{\"total_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"check_index_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}}}},\"index\":{\"properties\":{\"size\":{\"properties\":{\"total_in_bytes\":{\"type\":\"long\"},\"reused_in_bytes\":{\"type\":\"long\"},\"recovered_in_bytes\":{\"type\":\"long\"}}},\"files\":{\"properties\":{\"recovered\":{\"type\":\"long\"},\"total\":{\"type\":\"long\"},\"percent\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"reused\":{\"type\":\"long\"}}}}},\"source\":{\"properties\":{\"transport_address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"host\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"target\":{\"properties\":{\"transport_address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"host\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"start_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"stage\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"type\":\"long\"},\"total_time\":{\"properties\":{\"ms\":{\"type\":\"long\"}}},\"primary\":{\"type\":\"boolean\"}}},\"uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"status\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"shard\":{\"properties\":{\"number\":{\"type\":\"long\"},\"source_node\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"relocating_node\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"primary\":{\"type\":\"boolean\"}}},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ml\":{\"properties\":{\"job\":{\"properties\":{\"data\":{\"properties\":{\"invalid_date\":{\"properties\":{\"count\":{\"type\":\"long\"}}}}},\"data_counts\":{\"properties\":{\"invalid_date_count\":{\"type\":\"long\"},\"processed_record_count\":{\"type\":\"long\"}}},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"forecasts_stats\":{\"properties\":{\"total\":{\"type\":\"long\"}}},\"model_size\":{\"properties\":{\"memory_status\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}}}}}},\"cluster_uuid\":{\"path\":\"elasticsearch.cluster.id\",\"type\":\"alias\"},\"ecs\":{\"properties\":{\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"service\":{\"properties\":{\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"index_recovery\":{\"properties\":{\"shards\":{\"properties\":{\"stop_time_in_millis\":{\"path\":\"elasticsearch.index.recovery.stop_time.ms\",\"type\":\"alias\"},\"start_time_in_millis\":{\"path\":\"elasticsearch.index.recovery.start_time.ms\",\"type\":\"alias\"}}}}},\"source_node\":{\"properties\":{\"name\":{\"path\":\"elasticsearch.node.name\",\"type\":\"alias\"},\"uuid\":{\"path\":\"elasticsearch.node.id\",\"type\":\"alias\"}}},\"host\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"metricset\":{\"properties\":{\"period\":{\"type\":\"long\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"shard\":{\"properties\":{\"node\":{\"path\":\"elasticsearch.node.id\",\"type\":\"alias\"},\"index\":{\"path\":\"elasticsearch.index.name\",\"type\":\"alias\"},\"state\":{\"path\":\"elasticsearch.shard.state\",\"type\":\"alias\"},\"shard\":{\"path\":\"elasticsearch.shard.number\",\"type\":\"alias\"},\"primary\":{\"path\":\"elasticsearch.shard.primary\",\"type\":\"alias\"}}},\"event\":{\"properties\":{\"duration\":{\"type\":\"long\"},\"module\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"dataset\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"cluster_stats\":{\"properties\":{\"indices\":{\"properties\":{\"shards\":{\"properties\":{\"total\":{\"path\":\"elasticsearch.cluster.stats.indices.shards.count\",\"type\":\"alias\"}}},\"count\":{\"path\":\"elasticsearch.cluster.stats.indices.total\",\"type\":\"alias\"}}},\"nodes\":{\"properties\":{\"jvm\":{\"properties\":{\"max_uptime_in_millis\":{\"path\":\"elasticsearch.cluster.stats.nodes.jvm.max_uptime.ms\",\"type\":\"alias\"},\"mem\":{\"properties\":{\"heap_max_in_bytes\":{\"path\":\"elasticsearch.cluster.stats.nodes.jvm.memory.heap.max.bytes\",\"type\":\"alias\"},\"heap_used_in_bytes\":{\"path\":\"elasticsearch.cluster.stats.nodes.jvm.memory.heap.used.bytes\",\"type\":\"alias\"}}}}},\"count\":{\"properties\":{\"total\":{\"path\":\"elasticsearch.cluster.stats.nodes.count\",\"type\":\"alias\"}}}}}}},\"timestamp\":{\"path\":\"@timestamp\",\"type\":\"alias\"},\"job_stats\":{\"properties\":{\"job_id\":{\"path\":\"elasticsearch.ml.job.id\",\"type\":\"alias\"},\"forecasts_stats\":{\"properties\":{\"total\":{\"path\":\"elasticsearch.ml.job.forecasts_stats.total\",\"type\":\"alias\"}}}}}}}},\"composed_of\":[],\"version\":8000103,\"data_stream\":{\"hidden\":false,\"allow_custom_routing\":false}},\".slm-history\":{\"index_patterns\":[\".slm-history-5*\"],\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\"slm-history-ilm-policy\"},\"number_of_shards\":\"1\",\"auto_expand_replicas\":\"0-1\",\"number_of_replicas\":\"0\"}},\"mappings\":{\"dynamic\":false,\"properties\":{\"snapshot_name\":{\"type\":\"keyword\"},\"@timestamp\":{\"format\":\"epoch_millis\",\"type\":\"date\"},\"configuration\":{\"dynamic\":false,\"type\":\"object\",\"properties\":{\"indices\":{\"type\":\"keyword\"},\"include_global_state\":{\"type\":\"boolean\"},\"partial\":{\"type\":\"boolean\"}}},\"error_details\":{\"index\":false,\"type\":\"text\"},\"success\":{\"type\":\"boolean\"},\"repository\":{\"type\":\"keyword\"},\"operation\":{\"type\":\"keyword\"},\"policy\":{\"type\":\"keyword\"}}}},\"composed_of\":[],\"priority\":2147483647,\"version\":5,\"_meta\":{\"managed\":true,\"description\":\"index template for SLM history indices\"},\"data_stream\":{\"hidden\":true,\"allow_custom_routing\":false},\"allow_auto_create\":true},\".monitoring-logstash-mb\":{\"index_patterns\":[\".monitoring-logstash-8-*\"],\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\".monitoring-8-ilm-policy\"},\"mapping\":{\"total_fields\":{\"limit\":\"2000\"}}}},\"mappings\":{\"dynamic\":false,\"properties\":{\"logstash\":{\"properties\":{\"node\":{\"properties\":{\"jvm\":{\"properties\":{\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"stats\":{\"properties\":{\"jvm\":{\"properties\":{\"mem\":{\"properties\":{\"heap_max_in_bytes\":{\"type\":\"long\"},\"heap_used_in_bytes\":{\"type\":\"long\"}}},\"uptime_in_millis\":{\"type\":\"long\"}}},\"logstash\":{\"properties\":{\"uuid\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"process\":{\"properties\":{\"cpu\":{\"properties\":{\"percent\":{\"type\":\"double\"}}}}},\"pipelines\":{\"type\":\"nested\",\"properties\":{\"vertices\":{\"type\":\"nested\",\"properties\":{\"events_out\":{\"type\":\"long\"},\"pipeline_ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"events_in\":{\"type\":\"long\"},\"queue_push_duration_in_millis\":{\"type\":\"long\"},\"duration_in_millis\":{\"type\":\"long\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"long_counters\":{\"type\":\"nested\",\"properties\":{\"name\":{\"type\":\"keyword\"},\"value\":{\"type\":\"long\"}}}}},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"type\":\"keyword\"},\"hash\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"queue\":{\"properties\":{\"events_count\":{\"type\":\"long\"},\"max_queue_size_in_bytes\":{\"type\":\"long\"},\"queue_size_in_bytes\":{\"type\":\"long\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"events\":{\"properties\":{\"filtered\":{\"type\":\"long\"},\"in\":{\"type\":\"long\"},\"queue_push_duration_in_millis\":{\"type\":\"long\"},\"duration_in_millis\":{\"type\":\"long\"},\"out\":{\"type\":\"long\"}}},\"reloads\":{\"properties\":{\"failures\":{\"type\":\"long\"},\"successes\":{\"type\":\"long\"}}}}},\"os\":{\"properties\":{\"cpu\":{\"properties\":{\"load_average\":{\"properties\":{\"5m\":{\"type\":\"half_float\"},\"15m\":{\"type\":\"half_float\"},\"1m\":{\"type\":\"half_float\"}}}}},\"cgroup\":{\"properties\":{\"cpu\":{\"properties\":{\"stat\":{\"properties\":{\"number_of_elapsed_periods\":{\"type\":\"long\"},\"number_of_times_throttled\":{\"type\":\"long\"},\"time_throttled_nanos\":{\"type\":\"long\"}}},\"cfs_quota_micros\":{\"type\":\"long\"}}},\"cpuacct\":{\"properties\":{\"usage_nanos\":{\"type\":\"long\"}}}}}}},\"events\":{\"properties\":{\"filtered\":{\"type\":\"long\"},\"in\":{\"type\":\"long\"},\"duration_in_millis\":{\"type\":\"long\"},\"out\":{\"type\":\"long\"}}},\"queue\":{\"properties\":{\"events_count\":{\"type\":\"long\"}}},\"timestamp\":{\"type\":\"date\"}}},\"host\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"state\":{\"properties\":{\"pipeline\":{\"properties\":{\"batch_size\":{\"type\":\"long\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"ephemeral_id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"workers\":{\"type\":\"long\"},\"representation\":{\"properties\":{\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"hash\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"graph\":{\"properties\":{\"vertices\":{\"type\":\"object\"},\"edges\":{\"type\":\"object\"}}}}},\"hash\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"elasticsearch\":{\"properties\":{\"cluster\":{\"properties\":{\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"}}}}}}},\"logstash_state\":{\"properties\":{\"pipeline\":{\"properties\":{\"id\":{\"path\":\"logstash.node.state.pipeline.id\",\"type\":\"alias\"},\"hash\":{\"path\":\"logstash.node.state.pipeline.hash\",\"type\":\"alias\"}}}}},\"process\":{\"properties\":{\"pid\":{\"type\":\"long\"}}},\"@timestamp\":{\"type\":\"date\"},\"logstash_stats\":{\"properties\":{\"jvm\":{\"properties\":{\"mem\":{\"properties\":{\"heap_max_in_bytes\":{\"path\":\"logstash.node.stats.jvm.mem.heap_max_in_bytes\",\"type\":\"alias\"},\"heap_used_in_bytes\":{\"path\":\"logstash.node.stats.jvm.mem.heap_used_in_bytes\",\"type\":\"alias\"}}},\"uptime_in_millis\":{\"path\":\"logstash.node.stats.jvm.uptime_in_millis\",\"type\":\"alias\"}}},\"logstash\":{\"properties\":{\"uuid\":{\"path\":\"logstash.node.stats.logstash.uuid\",\"type\":\"alias\"},\"version\":{\"path\":\"logstash.node.stats.logstash.version\",\"type\":\"alias\"}}},\"process\":{\"properties\":{\"cpu\":{\"properties\":{\"percent\":{\"path\":\"logstash.node.stats.process.cpu.percent\",\"type\":\"alias\"}}}}},\"pipelines\":{\"type\":\"nested\"},\"os\":{\"properties\":{\"cpu\":{\"properties\":{\"load_average\":{\"properties\":{\"5m\":{\"path\":\"logstash.node.stats.os.cpu.load_average.5m\",\"type\":\"alias\"},\"15m\":{\"path\":\"logstash.node.stats.os.cpu.load_average.15m\",\"type\":\"alias\"},\"1m\":{\"path\":\"logstash.node.stats.os.cpu.load_average.1m\",\"type\":\"alias\"}}}}},\"cgroup\":{\"properties\":{\"cpu\":{\"properties\":{\"stat\":{\"properties\":{\"number_of_elapsed_periods\":{\"path\":\"logstash.node.stats.os.cgroup.cpu.stat.number_of_elapsed_periods\",\"type\":\"alias\"},\"number_of_times_throttled\":{\"path\":\"logstash.node.stats.os.cgroup.cpu.stat.number_of_times_throttled\",\"type\":\"alias\"},\"time_throttled_nanos\":{\"path\":\"logstash.node.stats.os.cgroup.cpu.stat.time_throttled_nanos\",\"type\":\"alias\"}}},\"cfs_quota_micros\":{\"path\":\"logstash.node.stats.os.cgroup.cpu.cfs_quota_micros\",\"type\":\"alias\"}}},\"cpuacct\":{\"properties\":{\"usage_nanos\":{\"path\":\"logstash.node.stats.os.cgroup.cpuacct.usage_nanos\",\"type\":\"alias\"}}}}}}},\"events\":{\"properties\":{\"in\":{\"path\":\"logstash.node.stats.events.in\",\"type\":\"alias\"},\"duration_in_millis\":{\"path\":\"logstash.node.stats.events.duration_in_millis\",\"type\":\"alias\"},\"out\":{\"path\":\"logstash.node.stats.events.out\",\"type\":\"alias\"}}},\"queue\":{\"properties\":{\"events_count\":{\"path\":\"logstash.node.stats.queue.events_count\",\"type\":\"alias\"}}},\"timestamp\":{\"path\":\"@timestamp\",\"type\":\"alias\"}}},\"ecs\":{\"properties\":{\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"cluster_uuid\":{\"path\":\"logstash.elasticsearch.cluster.id\",\"type\":\"alias\"},\"service\":{\"properties\":{\"hostname\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"address\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"id\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"type\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"version\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"host\":{\"properties\":{\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"metricset\":{\"properties\":{\"period\":{\"type\":\"long\"},\"name\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"event\":{\"properties\":{\"duration\":{\"type\":\"long\"},\"module\":{\"ignore_above\":1024,\"type\":\"keyword\"},\"dataset\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"timestamp\":{\"path\":\"@timestamp\",\"type\":\"alias\"}}}},\"composed_of\":[],\"version\":8000103,\"data_stream\":{\"hidden\":false,\"allow_custom_routing\":false}},\".ml-anomalies-\":{\"index_patterns\":[\".ml-anomalies-*\"],\"template\":{\"settings\":{\"index\":{\"hidden\":\"true\",\"translog\":{\"durability\":\"async\"},\"auto_expand_replicas\":\"0-1\",\"query\":{\"default_field\":\"all_field_values\"}}},\"mappings\":{\"_meta\":{\"version\":\"8.6.2\"},\"dynamic_templates\":[{\"strings_as_keywords\":{\"mapping\":{\"type\":\"keyword\"},\"match\":\"*\"}}],\"properties\":{\"search_count\":{\"type\":\"long\"},\"bucket_count\":{\"type\":\"long\"},\"anomaly_score_explanation\":{\"properties\":{\"high_variance_penalty\":{\"type\":\"boolean\"},\"typical_value\":{\"type\":\"double\"},\"upper_confidence_bound\":{\"type\":\"double\"},\"lower_confidence_bound\":{\"type\":\"double\"},\"incomplete_bucket_penalty\":{\"type\":\"boolean\"},\"anomaly_type\":{\"type\":\"keyword\"},\"anomaly_characteristics_impact\":{\"type\":\"integer\"},\"multi_bucket_impact\":{\"type\":\"integer\"},\"anomaly_length\":{\"type\":\"integer\"},\"single_bucket_impact\":{\"type\":\"integer\"}}},\"terms\":{\"type\":\"text\"},\"record_score\":{\"type\":\"double\"},\"forecast_expiry_timestamp\":{\"type\":\"date\"},\"over_field_value\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"preferred_to_categories\":{\"type\":\"long\"},\"empty_bucket_count\":{\"type\":\"long\"},\"forecast_create_timestamp\":{\"type\":\"date\"},\"probability\":{\"type\":\"double\"},\"missing_field_count\":{\"type\":\"long\"},\"forecast_lower\":{\"type\":\"double\"},\"influencer_field_value\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"multi_bucket_impact\":{\"type\":\"double\"},\"forecast_progress\":{\"type\":\"double\"},\"max_matching_length\":{\"type\":\"long\"},\"latest_record_time_stamp\":{\"type\":\"date\"},\"regex\":{\"type\":\"keyword\"},\"examples\":{\"type\":\"text\"},\"snapshot_doc_count\":{\"type\":\"integer\"},\"average_bucket_processing_time_ms\":{\"type\":\"double\"},\"initial_anomaly_score\":{\"type\":\"double\"},\"initial_influencer_score\":{\"type\":\"double\"},\"partition_field_value\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"model_lower\":{\"type\":\"double\"},\"total_over_field_count\":{\"type\":\"long\"},\"retain\":{\"type\":\"boolean\"},\"forecast_upper\":{\"type\":\"double\"},\"latest_sparse_bucket_timestamp\":{\"type\":\"date\"},\"model_median\":{\"type\":\"double\"},\"category_id\":{\"type\":\"long\"},\"causes\":{\"type\":\"nested\",\"properties\":{\"actual\":{\"type\":\"double\"},\"partition_field_name\":{\"type\":\"keyword\"},\"partition_field_value\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"by_field_name\":{\"type\":\"keyword\"},\"probability\":{\"type\":\"double\"},\"geo_results\":{\"properties\":{\"actual_point\":{\"type\":\"geo_point\"},\"typical_point\":{\"type\":\"geo_point\"}}},\"field_name\":{\"type\":\"keyword\"},\"by_field_value\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"over_field_name\":{\"type\":\"keyword\"},\"correlated_by_field_value\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"function\":{\"type\":\"keyword\"},\"typical\":{\"type\":\"double\"},\"over_field_value\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"function_description\":{\"type\":\"keyword\"}}},\"all_field_values\":{\"analyzer\":\"whitespace\",\"type\":\"text\"},\"timestamp\":{\"type\":\"date\"},\"input_field_count\":{\"type\":\"long\"},\"model_bytes\":{\"type\":\"long\"},\"quantiles\":{\"type\":\"object\",\"enabled\":false},\"@timestamp\":{\"path\":\"timestamp\",\"type\":\"alias\"},\"function_description\":{\"type\":\"keyword\"},\"min_version\":{\"type\":\"keyword\"},\"raw_anomaly_score\":{\"type\":\"double\"},\"exponential_average_bucket_processing_time_ms\":{\"type\":\"double\"},\"invalid_date_count\":{\"type\":\"long\"},\"snapshot_id\":{\"type\":\"keyword\"},\"model_size_stats\":{\"properties\":{\"model_bytes\":{\"type\":\"long\"},\"result_type\":{\"type\":\"keyword\"},\"peak_model_bytes\":{\"type\":\"long\"},\"job_id\":{\"type\":\"keyword\"},\"total_over_field_count\":{\"type\":\"long\"},\"total_partition_field_count\":{\"type\":\"long\"},\"total_by_field_count\":{\"type\":\"long\"},\"assignment_memory_basis\":{\"type\":\"keyword\"},\"bucket_allocation_failures_count\":{\"type\":\"long\"},\"memory_status\":{\"type\":\"keyword\"},\"log_time\":{\"type\":\"date\"},\"timestamp\":{\"type\":\"date\"}}},\"total_search_time_ms\":{\"type\":\"double\"},\"latest_empty_bucket_timestamp\":{\"type\":\"date\"},\"anomaly_score\":{\"type\":\"double\"},\"over_field_name\":{\"type\":\"keyword\"},\"earliest_record_timestamp\":{\"type\":\"date\"},\"scheduled_events\":{\"type\":\"keyword\"},\"bucket_span\":{\"type\":\"long\"},\"maximum_bucket_processing_time_ms\":{\"type\":\"double\"},\"exponential_average_calculation_context\":{\"properties\":{\"incremental_metric_value_ms\":{\"type\":\"double\"},\"previous_exponential_average_ms\":{\"type\":\"double\"},\"latest_timestamp\":{\"type\":\"date\"}}},\"function\":{\"type\":\"keyword\"},\"influencers\":{\"type\":\"nested\",\"properties\":{\"influencer_field_name\":{\"type\":\"keyword\"},\"influencer_field_values\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"}}},\"input_record_count\":{\"type\":\"long\"},\"latest_result_time_stamp\":{\"type\":\"date\"},\"model_upper\":{\"type\":\"double\"},\"actual\":{\"type\":\"double\"},\"forecast_memory_bytes\":{\"type\":\"long\"},\"total_by_field_count\":{\"type\":\"long\"},\"geo_results\":{\"properties\":{\"actual_point\":{\"type\":\"geo_point\"},\"typical_point\":{\"type\":\"geo_point\"}}},\"detector_index\":{\"type\":\"integer\"},\"by_field_value\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"processed_record_count\":{\"type\":\"long\"},\"total_partition_field_count\":{\"type\":\"long\"},\"mlcategory\":{\"type\":\"keyword\"},\"assignment_memory_basis\":{\"type\":\"keyword\"},\"forecast_end_timestamp\":{\"type\":\"date\"},\"forecast_id\":{\"type\":\"keyword\"},\"partition_field_name\":{\"type\":\"keyword\"},\"by_field_name\":{\"type\":\"keyword\"},\"event_count\":{\"type\":\"long\"},\"description\":{\"type\":\"text\"},\"is_interim\":{\"type\":\"boolean\"},\"bucket_allocation_failures_count\":{\"type\":\"long\"},\"memory_status\":{\"type\":\"keyword\"},\"model_feature\":{\"type\":\"keyword\"},\"num_matches\":{\"type\":\"long\"},\"influencer_score\":{\"type\":\"double\"},\"out_of_order_timestamp_count\":{\"type\":\"long\"},\"result_type\":{\"type\":\"keyword\"},\"last_data_time\":{\"type\":\"date\"},\"latest_record_timestamp\":{\"type\":\"date\"},\"influencer_field_name\":{\"type\":\"keyword\"},\"forecast_messages\":{\"type\":\"keyword\"},\"sparse_bucket_count\":{\"type\":\"long\"},\"log_time\":{\"type\":\"date\"},\"field_name\":{\"type\":\"keyword\"},\"minimum_bucket_processing_time_ms\":{\"type\":\"double\"},\"bucket_influencers\":{\"type\":\"nested\",\"properties\":{\"anomaly_score\":{\"type\":\"double\"},\"initial_anomaly_score\":{\"type\":\"double\"},\"result_type\":{\"type\":\"keyword\"},\"raw_anomaly_score\":{\"type\":\"double\"},\"bucket_span\":{\"type\":\"long\"},\"job_id\":{\"type\":\"keyword\"},\"probability\":{\"type\":\"double\"},\"influencer_field_name\":{\"type\":\"keyword\"},\"is_interim\":{\"type\":\"boolean\"},\"timestamp\":{\"type\":\"date\"}}},\"processing_time_ms\":{\"type\":\"long\"},\"input_bytes\":{\"type\":\"long\"},\"initial_record_score\":{\"type\":\"double\"},\"job_id\":{\"copy_to\":[\"all_field_values\"],\"type\":\"keyword\"},\"processed_field_count\":{\"type\":\"long\"},\"forecast_status\":{\"type\":\"keyword\"},\"typical\":{\"type\":\"double\"},\"forecast_prediction\":{\"type\":\"double\"},\"forecast_start_timestamp\":{\"type\":\"date\"}}}},\"composed_of\":[],\"priority\":2147483647,\"version\":8060299,\"_meta\":{\"managed\":true,\"description\":\"index template for ML anomaly detection results indices\"}},\"metrics\":{\"index_patterns\":[\"metrics-*-*\"],\"composed_of\":[\"metrics-mappings\",\"data-streams-mappings\",\"metrics-settings\"],\"priority\":100,\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default metrics template installed by x-pack\"},\"data_stream\":{\"hidden\":false,\"allow_custom_routing\":false},\"allow_auto_create\":true},\".ml-notifications-000002\":{\"index_patterns\":[\".ml-notifications-000002\"],\"template\":{\"settings\":{\"index\":{\"hidden\":\"true\",\"number_of_shards\":\"1\",\"auto_expand_replicas\":\"0-1\"}},\"mappings\":{\"_meta\":{\"version\":\"8.6.2\"},\"dynamic\":\"false\",\"properties\":{\"job_type\":{\"type\":\"keyword\"},\"level\":{\"type\":\"keyword\"},\"job_id\":{\"type\":\"keyword\"},\"node_name\":{\"type\":\"keyword\"},\"message\":{\"type\":\"text\",\"fields\":{\"raw\":{\"ignore_above\":1024,\"type\":\"keyword\"}}},\"cleared\":{\"type\":\"boolean\"},\"timestamp\":{\"type\":\"date\"}}}},\"composed_of\":[],\"priority\":2147483647,\"version\":8060299,\"_meta\":{\"managed\":true,\"description\":\"index template for ML notifications indices\"}},\".deprecation-indexing-template\":{\"index_patterns\":[\".logs-deprecation.*\"],\"composed_of\":[\".deprecation-indexing-mappings\",\".deprecation-indexing-settings\"],\"priority\":1000,\"version\":1,\"_meta\":{\"managed\":true,\"description\":\"default template for Stack deprecation logs index template installed by x-pack\"},\"data_stream\":{\"hidden\":true,\"allow_custom_routing\":false},\"allow_auto_create\":true},\"logs\":{\"index_patterns\":[\"logs-*-*\"],\"composed_of\":[\"logs-mappings\",\"data-streams-mappings\",\"logs-settings\"],\"priority\":100,\"version\":2,\"_meta\":{\"managed\":true,\"description\":\"default logs template installed by x-pack\"},\"data_stream\":{\"hidden\":false,\"allow_custom_routing\":false},\"allow_auto_create\":true},\".ml-stats\":{\"index_patterns\":[\".ml-stats-*\"],\"template\":{\"settings\":{\"index\":{\"lifecycle\":{\"name\":\"ml-size-based-ilm-policy\",\"rollover_alias\":\".ml-stats-write\"},\"hidden\":\"true\",\"number_of_shards\":\"1\",\"auto_expand_replicas\":\"0-1\"}},\"mappings\":{\"_meta\":{\"version\":\"8.6.2\"},\"dynamic\":false,\"properties\":{\"skipped_docs_count\":{\"type\":\"long\"},\"validation_loss\":{\"properties\":{\"fold_values\":{\"properties\":{\"fold\":{\"type\":\"integer\"},\"values\":{\"type\":\"double\"}}},\"loss_type\":{\"type\":\"keyword\"}}},\"cache_miss_count\":{\"type\":\"long\"},\"timing_stats\":{\"properties\":{\"iteration_time\":{\"type\":\"long\"},\"elapsed_time\":{\"type\":\"long\"}}},\"failure_count\":{\"type\":\"long\"},\"model_id\":{\"type\":\"keyword\"},\"type\":{\"type\":\"keyword\"},\"training_docs_count\":{\"type\":\"long\"},\"inference_count\":{\"type\":\"long\"},\"job_id\":{\"type\":\"keyword\"},\"missing_all_fields_count\":{\"type\":\"long\"},\"peak_usage_bytes\":{\"type\":\"long\"},\"iteration\":{\"type\":\"integer\"},\"hyperparameters\":{\"properties\":{\"max_attempts_to_add_tree\":{\"type\":\"integer\"},\"downsample_factor\":{\"type\":\"double\"},\"eta_growth_rate_per_tree\":{\"type\":\"double\"},\"soft_tree_depth_tolerance\":{\"type\":\"double\"},\"max_trees\":{\"type\":\"integer\"},\"lambda\":{\"type\":\"double\"},\"max_optimization_rounds_per_hyperparameter\":{\"type\":\"integer\"},\"eta\":{\"type\":\"double\"},\"soft_tree_depth_limit\":{\"type\":\"double\"},\"alpha\":{\"type\":\"double\"},\"class_assignment_objective\":{\"type\":\"keyword\"},\"feature_bag_fraction\":{\"type\":\"double\"},\"num_splits_per_feature\":{\"type\":\"integer\"},\"gamma\":{\"type\":\"double\"},\"num_folds\":{\"type\":\"integer\"}}},\"parameters\":{\"properties\":{\"compute_feature_influence\":{\"type\":\"boolean\"},\"feature_influence_threshold\":{\"type\":\"double\"},\"outlier_fraction\":{\"type\":\"double\"},\"method\":{\"type\":\"keyword\"},\"standardization_enabled\":{\"type\":\"boolean\"},\"n_neighbors\":{\"type\":\"integer\"}}},\"test_docs_count\":{\"type\":\"long\"},\"node_id\":{\"type\":\"keyword\"},\"timestamp\":{\"type\":\"date\"}}},\"aliases\":{}},\"composed_of\":[],\"priority\":2147483647,\"version\":8060299,\"_meta\":{\"managed\":true,\"description\":\"index template for ML stats indices\"}}}} index-graveyard: IndexGraveyard[[[index=[abacus_go/zV2aNzxVQxqMDQTD4A9TEQ], deleteDate=2023-03-28T03:56:57.282Z], [index=[access_platform_core/ZuN0_cUfSQKzXg_yG4Sv5g], deleteDate=2023-03-28T03:58:27.958Z], [index=[ad_bidder/6ihfVU6rRqOo4Gv4Djtetg], deleteDate=2023-03-28T03:59:11.628Z], [index=[apphealth/EthxrkUtTGCTP2mX-tS63Q], deleteDate=2023-03-28T03:59:36.596Z]]]\nnodes: \n {hostb9}{iKPGCkp9RVOKXOj20uOt4g}{cPRh9FqiSlGP9jClhm8Ugg}{hostb9}{127.0.0.1}{127.0.0.1:9300}{cdfhilmrstw}{ml.max_jvm_size=33285996544, ml.allocated_processors_double=16.0, xpack.installed=true, ml.machine_memory=135062048768, ml.allocated_processors=16}, local, master\nrouting_table (version 35):\n-- index [[abacus_go/cfvmY-Y_Su63-qSkPODDzg]]\n----shard_id [abacus_go][0]\n--------[abacus_go][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=AMhgJMlSTPOoKSsL7s3OKA], failed_attempts[0]\n--------[abacus_go][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=INDEX_CREATED], at[2023-03-28T03:57:31.683Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[web_wok/R1IULAXSQ4SC0pLFCU5oJA]]\n----shard_id [web_wok][0]\n--------[web_wok][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=JXruVMhdRVGvP1kQCbUNSQ], failed_attempts[0]\n--------[web_wok][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[access_platform_core/l4kdkOEQRaKfWGLpTZURCQ]]\n----shard_id [access_platform_core][0]\n--------[access_platform_core][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=6ajNSTQZSr6tP3YtmRaxgw], failed_attempts[0]\n--------[access_platform_core][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=INDEX_CREATED], at[2023-03-28T03:58:31.318Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[driverapp/cqbade6KQFqLEK-wPKlkBg]]\n----shard_id [driverapp][0]\n--------[driverapp][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=b5_Bhd3ITjOkl-xLF8H1jg], failed_attempts[0]\n--------[driverapp][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[k8s_kubelet/fQ9Iy7wlTKahsWLbs-oE_Q]]\n----shard_id [k8s_kubelet][0]\n--------[k8s_kubelet][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=Q4Xb11JATPaFZqmgTQHcVQ], failed_attempts[0]\n--------[k8s_kubelet][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.307Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[eats_store/hF-kN4XFRy--z_gUhYLwdA]]\n----shard_id [eats_store][0]\n--------[eats_store][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=WRNwRQ6gTNGfSNQok4iSyQ], failed_attempts[0]\n--------[eats_store][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[rider_product_cored/AdGiEBLBQbicsXDomsa3mw]]\n----shard_id [rider_product_cored][0]\n--------[rider_product_cored][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=GwRH_ZlFRtCV7m_n626z7g], failed_attempts[0]\n--------[rider_product_cored][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[finprod_rewards_eligibility/dJNRGPwrQS2bIPlpsXgUoQ]]\n----shard_id [finprod_rewards_eligibility][0]\n--------[finprod_rewards_eligibility][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=Hg4Dm9O8QEqXWKY7rH1X0A], failed_attempts[0]\n--------[finprod_rewards_eligibility][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[event_logs/DcomuHoGQzOjZq2bX67TFg]]\n----shard_id [event_logs][0]\n--------[event_logs][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=1uCl_LIDSBCWh6koVA0fjQ], failed_attempts[0]\n--------[event_logs][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[apphealth/rcPrX1aWRmeED9W5PU6A4g]]\n----shard_id [apphealth][0]\n--------[apphealth][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=Abip50RxQTmavqi4NTwSqQ], failed_attempts[0]\n--------[apphealth][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=INDEX_CREATED], at[2023-03-28T03:59:38.780Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[ad_bidder/tLHfw5r2QH-o7lG5tapKxw]]\n----shard_id [ad_bidder][0]\n--------[ad_bidder][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=YtEUwPgfQDGM0Cm_C4hXlw], failed_attempts[0]\n--------[ad_bidder][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=INDEX_CREATED], at[2023-03-28T03:59:13.333Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[invoice_gen/RPu5EzJVRfuqyI8MQ7iTjQ]]\n----shard_id [invoice_gen][0]\n--------[invoice_gen][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=onWbjhHuTD2C3AEayRJq1w], failed_attempts[0]\n--------[invoice_gen][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[fares_management/F1lH-QvyTkS5hnbDFwE2Xg]]\n----shard_id [fares_management][0]\n--------[fares_management][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=sOxfKX3nReynYN5HN88cuQ], failed_attempts[0]\n--------[fares_management][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[michelangelo_gateway/ZV2vZ8PmS0uysg-xzKO7QA]]\n----shard_id [michelangelo_gateway][0]\n--------[michelangelo_gateway][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=k5vDE54CSTSakhfgtGaTcA], failed_attempts[0]\n--------[michelangelo_gateway][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[.geoip_databases/5k1GHwKvQBKersi12DFh4A]]\n----shard_id [.geoip_databases][0]\n--------[.geoip_databases][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=UdvCKUGqTEmYFzZlnSu_tA], failed_attempts[0]\n\n-- index [[fares_intelligence/pSRJWxyXSqefhwLVhmT9nQ]]\n----shard_id [fares_intelligence][0]\n--------[fares_intelligence][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=-dD8ROsgS6qaYMoalciocA], failed_attempts[0]\n--------[fares_intelligence][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[fare_route/a4fjivWhQRee8JW7ZbXiXw]]\n----shard_id [fare_route][0]\n--------[fare_route][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=ltSLFgu4Rry0_0vXq28DbA], failed_attempts[0]\n--------[fare_route][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[k8s_apiserver/qQasynZgQ6e6AmD5ZWqh3w]]\n----shard_id [k8s_apiserver][0]\n--------[k8s_apiserver][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=Aaa0F0jiStCkteGDYR8yRQ], failed_attempts[0]\n--------[k8s_apiserver][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\n-- index [[fulfillment_compatibled/tMgrGBWeR3irgR5PyN8Oig]]\n----shard_id [fulfillment_compatibled][0]\n--------[fulfillment_compatibled][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=D_3C_3kMSNKa-XJk7pyoGg], failed_attempts[0]\n--------[fulfillment_compatibled][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n\nrouting_nodes:\n-----node_id[iKPGCkp9RVOKXOj20uOt4g][V]\n--------[abacus_go][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=AMhgJMlSTPOoKSsL7s3OKA], failed_attempts[0]\n--------[web_wok][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=JXruVMhdRVGvP1kQCbUNSQ], failed_attempts[0]\n--------[access_platform_core][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=6ajNSTQZSr6tP3YtmRaxgw], failed_attempts[0]\n--------[driverapp][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=b5_Bhd3ITjOkl-xLF8H1jg], failed_attempts[0]\n--------[k8s_kubelet][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=Q4Xb11JATPaFZqmgTQHcVQ], failed_attempts[0]\n--------[eats_store][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=WRNwRQ6gTNGfSNQok4iSyQ], failed_attempts[0]\n--------[rider_product_cored][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=GwRH_ZlFRtCV7m_n626z7g], failed_attempts[0]\n--------[finprod_rewards_eligibility][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=Hg4Dm9O8QEqXWKY7rH1X0A], failed_attempts[0]\n--------[event_logs][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=1uCl_LIDSBCWh6koVA0fjQ], failed_attempts[0]\n--------[apphealth][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=Abip50RxQTmavqi4NTwSqQ], failed_attempts[0]\n--------[ad_bidder][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=YtEUwPgfQDGM0Cm_C4hXlw], failed_attempts[0]\n--------[invoice_gen][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=onWbjhHuTD2C3AEayRJq1w], failed_attempts[0]\n--------[fares_management][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=sOxfKX3nReynYN5HN88cuQ], failed_attempts[0]\n--------[michelangelo_gateway][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=k5vDE54CSTSakhfgtGaTcA], failed_attempts[0]\n--------[.geoip_databases][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=UdvCKUGqTEmYFzZlnSu_tA], failed_attempts[0]\n--------[fares_intelligence][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=-dD8ROsgS6qaYMoalciocA], failed_attempts[0]\n--------[fare_route][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=ltSLFgu4Rry0_0vXq28DbA], failed_attempts[0]\n--------[k8s_apiserver][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=Aaa0F0jiStCkteGDYR8yRQ], failed_attempts[0]\n--------[fulfillment_compatibled][0], node[iKPGCkp9RVOKXOj20uOt4g], [P], s[STARTED], a[id=D_3C_3kMSNKa-XJk7pyoGg], failed_attempts[0]\n---- unassigned\n--------[abacus_go][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=INDEX_CREATED], at[2023-03-28T03:57:31.683Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[web_wok][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[access_platform_core][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=INDEX_CREATED], at[2023-03-28T03:58:31.318Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[driverapp][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[k8s_kubelet][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.307Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[eats_store][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[rider_product_cored][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[finprod_rewards_eligibility][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[event_logs][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[apphealth][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=INDEX_CREATED], at[2023-03-28T03:59:38.780Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[ad_bidder][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=INDEX_CREATED], at[2023-03-28T03:59:13.333Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[invoice_gen][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[fares_management][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[michelangelo_gateway][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[fares_intelligence][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[fare_route][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[k8s_apiserver][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\n--------[fulfillment_compatibled][0], node[null], [R], recovery_source[peer recovery], s[UNASSIGNED], unassigned_info[[reason=CLUSTER_RECOVERED], at[2023-03-28T03:54:57.311Z], delayed=false, allocation_status[no_attempt]], failed_attempts[0]\ncustoms:\n health: org.elasticsearch.health.metadata.HealthMetadata@66bc0723}] from [{hostb9}{iKPGCkp9RVOKXOj20uOt4g}{cPRh9FqiSlGP9jClhm8Ugg}{hostb9}{127.0.0.1}{127.0.0.1:9300}{cdfhilmrstw}{ml.max_jvm_size=33285996544, ml.allocated_processors_double=16.0, xpack.installed=true, ml.machine_memory=135062048768, ml.allocated_processors=16}]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][cluster_coordination][T#1]","log.logger":"org.elasticsearch.cluster.coordination.Coordinator","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:02.078Z", "log.level":"TRACE", "message":"handlePublishRequest: accepting publish request for version [154] and term [6]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][cluster_coordination][T#1]","log.logger":"org.elasticsearch.cluster.coordination.CoordinationState","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:02.078Z", "log.level":"TRACE", "message":"currentTerm [6] matches previous currentTerm, writing changes only", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][cluster_coordination][T#1]","log.logger":"org.elasticsearch.gateway.PersistedClusterStateService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:02.079Z", "log.level":"TRACE", "message":"no action required for mapping [FOi5Cox/V4KpXwFa2TtyrVRtJ6YqC9xfpc5SC5ChT0c=]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][cluster_coordination][T#1]","log.logger":"org.elasticsearch.gateway.PersistedClusterStateService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:02.079Z", "log.level":"TRACE", "message":"no action required for index [[k8s_kubelet/fQ9Iy7wlTKahsWLbs-oE_Q]]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][cluster_coordination][T#1]","log.logger":"org.elasticsearch.gateway.PersistedClusterStateService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:02.079Z", "log.level":"TRACE", "message":"no action required for index [[fares_intelligence/pSRJWxyXSqefhwLVhmT9nQ]]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][cluster_coordination][T#1]","log.logger":"org.elasticsearch.gateway.PersistedClusterStateService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:02.079Z", "log.level":"TRACE", "message":"no action required for index [[event_logs/DcomuHoGQzOjZq2bX67TFg]]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][cluster_coordination][T#1]","log.logger":"org.elasticsearch.gateway.PersistedClusterStateService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:02.079Z", "log.level":"TRACE", "message":"no action required for index [[rider_product_cored/AdGiEBLBQbicsXDomsa3mw]]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][cluster_coordination][T#1]","log.logger":"org.elasticsearch.gateway.PersistedClusterStateService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:02.079Z", "log.level":"TRACE", "message":"no action required for index [[access_platform_core/l4kdkOEQRaKfWGLpTZURCQ]]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][cluster_coordination][T#1]","log.logger":"org.elasticsearch.gateway.PersistedClusterStateService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:02.079Z", "log.level":"TRACE", "message":"no action required for index [[.geoip_databases/5k1GHwKvQBKersi12DFh4A]]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][cluster_coordination][T#1]","log.logger":"org.elasticsearch.gateway.PersistedClusterStateService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:02.080Z", "log.level":"TRACE", "message":"no action required for index [[web_wok/R1IULAXSQ4SC0pLFCU5oJA]]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][cluster_coordination][T#1]","log.logger":"org.elasticsearch.gateway.PersistedClusterStateService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:02.080Z", "log.level":"TRACE", "message":"no action required for index [[abacus_go/cfvmY-Y_Su63-qSkPODDzg]]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][cluster_coordination][T#1]","log.logger":"org.elasticsearch.gateway.PersistedClusterStateService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:02.080Z", "log.level":"TRACE", "message":"no action required for index [[k8s_apiserver/qQasynZgQ6e6AmD5ZWqh3w]]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][cluster_coordination][T#1]","log.logger":"org.elasticsearch.gateway.PersistedClusterStateService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:02.080Z", "log.level":"TRACE", "message":"no action required for index [[michelangelo_gateway/ZV2vZ8PmS0uysg-xzKO7QA]]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][cluster_coordination][T#1]","log.logger":"org.elasticsearch.gateway.PersistedClusterStateService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:02.080Z", "log.level":"TRACE", "message":"no action required for index [[driverapp/cqbade6KQFqLEK-wPKlkBg]]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][cluster_coordination][T#1]","log.logger":"org.elasticsearch.gateway.PersistedClusterStateService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:02.080Z", "log.level":"TRACE", "message":"updating metadata for [[apphealth/rcPrX1aWRmeED9W5PU6A4g]], changing version from [2] to [4]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][cluster_coordination][T#1]","log.logger":"org.elasticsearch.gateway.PersistedClusterStateService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:02.081Z", "log.level":"TRACE", "message":"removing metadata for [rcPrX1aWRmeED9W5PU6A4g]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][cluster_coordination][T#1]","log.logger":"org.elasticsearch.gateway.PersistedClusterStateService.MetadataIndexWriter","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":["[NIOFSDirectory@/var/lib/elasticsearch/_state lockFactory=org.apache.lucene.store.NativeFSLockFactory@e8d54fb]"]} -{"@timestamp":"2023-03-28T04:00:02.081Z", "log.level":"TRACE", "message":"updating metadata for [[apphealth/rcPrX1aWRmeED9W5PU6A4g]]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][cluster_coordination][T#1]","log.logger":"org.elasticsearch.gateway.PersistedClusterStateService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:02.082Z", "log.level":"TRACE", "message":"no action required for index [[fares_management/F1lH-QvyTkS5hnbDFwE2Xg]]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][cluster_coordination][T#1]","log.logger":"org.elasticsearch.gateway.PersistedClusterStateService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:02.083Z", "log.level":"TRACE", "message":"no action required for index [[finprod_rewards_eligibility/dJNRGPwrQS2bIPlpsXgUoQ]]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][cluster_coordination][T#1]","log.logger":"org.elasticsearch.gateway.PersistedClusterStateService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:02.083Z", "log.level":"TRACE", "message":"no action required for index [[fare_route/a4fjivWhQRee8JW7ZbXiXw]]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][cluster_coordination][T#1]","log.logger":"org.elasticsearch.gateway.PersistedClusterStateService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:02.083Z", "log.level":"TRACE", "message":"no action required for index [[ad_bidder/tLHfw5r2QH-o7lG5tapKxw]]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][cluster_coordination][T#1]","log.logger":"org.elasticsearch.gateway.PersistedClusterStateService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:02.083Z", "log.level":"TRACE", "message":"no action required for index [[fulfillment_compatibled/tMgrGBWeR3irgR5PyN8Oig]]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][cluster_coordination][T#1]","log.logger":"org.elasticsearch.gateway.PersistedClusterStateService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:02.083Z", "log.level":"TRACE", "message":"no action required for index [[eats_store/hF-kN4XFRy--z_gUhYLwdA]]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][cluster_coordination][T#1]","log.logger":"org.elasticsearch.gateway.PersistedClusterStateService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:02.083Z", "log.level":"TRACE", "message":"no action required for index [[invoice_gen/RPu5EzJVRfuqyI8MQ7iTjQ]]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][cluster_coordination][T#1]","log.logger":"org.elasticsearch.gateway.PersistedClusterStateService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:02.083Z", "log.level":"TRACE", "message":"flushing", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][cluster_coordination][T#1]","log.logger":"org.elasticsearch.gateway.PersistedClusterStateService.MetadataIndexWriter","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":["[NIOFSDirectory@/var/lib/elasticsearch/_state lockFactory=org.apache.lucene.store.NativeFSLockFactory@e8d54fb]"]} -{"@timestamp":"2023-03-28T04:00:02.119Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#6]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [abacus_go]"]} -{"@timestamp":"2023-03-28T04:00:02.119Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#8]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [event_logs]"]} -{"@timestamp":"2023-03-28T04:00:02.119Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#2]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [fulfillment_compatibled]"]} -{"@timestamp":"2023-03-28T04:00:02.119Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#5]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [k8s_kubelet]"]} -{"@timestamp":"2023-03-28T04:00:02.120Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#7]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [driverapp]"]} -{"@timestamp":"2023-03-28T04:00:02.394Z", "log.level":"TRACE", "message":"checking jobs [2023-03-28T04:00:02.394359200Z]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"ticker-schedule-trigger-engine","log.logger":"org.elasticsearch.xpack.watcher.trigger.schedule.engine.TickerScheduleTriggerEngine","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:02.443Z", "log.level":"TRACE", "message":"total indexing heap bytes used [0b] vs indices.memory.index_buffer_size [3gb], currently writing bytes [0b]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][scheduler][T#1]","log.logger":"org.elasticsearch.indices.IndexingMemoryController","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch"} -{"@timestamp":"2023-03-28T04:00:02.892Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#1]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [.geoip_databases]"]} -{"@timestamp":"2023-03-28T04:00:02.892Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#4]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [finprod_rewards_eligibility]"]} -{"@timestamp":"2023-03-28T04:00:02.893Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#3]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [rider_product_cored]"]} -{"@timestamp":"2023-03-28T04:00:02.893Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#6]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [fares_management]"]} -{"@timestamp":"2023-03-28T04:00:02.893Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#8]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [michelangelo_gateway]"]} -{"@timestamp":"2023-03-28T04:00:02.893Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#2]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [k8s_apiserver]"]} -{"@timestamp":"2023-03-28T04:00:02.894Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#5]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [access_platform_core]"]} -{"@timestamp":"2023-03-28T04:00:02.894Z", "log.level":"TRACE", "message":"scheduling refresh every 1s", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[hostb9][refresh][T#7]","log.logger":"org.elasticsearch.index.IndexService","elasticsearch.cluster.uuid":"bp-xkJD1S1iyBlj6I_JRHA","elasticsearch.node.id":"iKPGCkp9RVOKXOj20uOt4g","elasticsearch.node.name":"hostb9","elasticsearch.cluster.name":"elasticsearch","tags":[" [web_wok]"]} diff --git a/tests/test_ir/test_json/spark-event-logs.jsonl b/tests/test_ir/test_json/spark-event-logs.jsonl deleted file mode 100644 index 06f851f0..00000000 --- a/tests/test_ir/test_json/spark-event-logs.jsonl +++ /dev/null @@ -1,334 +0,0 @@ -{"Event":"SparkListenerLogStart","Spark Version":"3.1.2"} -{"Event":"SparkListenerResourceProfileAdded","Resource Profile Id":0,"Executor Resource Requests":{"cores":{"Resource Name":"cores","Amount":1,"Discovery Script":"","Vendor":""},"memory":{"Resource Name":"memory","Amount":4096,"Discovery Script":"","Vendor":""},"offHeap":{"Resource Name":"offHeap","Amount":0,"Discovery Script":"","Vendor":""}},"Task Resource Requests":{"cpus":{"Resource Name":"cpus","Amount":1.0}}} -{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"driver","Host":"hostb7.dsrg.utoronto.ca","Port":41823},"Maximum Memory":2101975449,"Timestamp":1633789164234,"Maximum Onheap Memory":2101975449,"Maximum Offheap Memory":0} -{"Event":"SparkListenerEnvironmentUpdate","JVM Information":{"Java Home":"/usr/lib/jvm/java-8-openjdk-amd64/jre","Java Version":"1.8.0_162 (Oracle Corporation)","Scala Version":"version 2.12.10"},"Spark Properties":{"spark.driver.host":"hostb7.dsrg.utoronto.ca","spark.eventLog.enabled":"true","spark.driver.port":"44011","spark.jars":"file:/home/usera112/projects/log-compressor/deployments/HiBench/sparkbench/assembly/target/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.app.name":"ScalaJoin","spark.scheduler.mode":"FIFO","spark.driver.memory":"4g","spark.submit.pyFiles":"","spark.default.parallelism":"8","spark.app.startTime":"1633789163045","spark.executor.id":"driver","spark.app.initial.jar.urls":"spark://hostb7.dsrg.utoronto.ca:44011/jars/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.submit.deployMode":"client","spark.master":"spark://hostb7:7077","spark.executor.memory":"4g","spark.eventLog.dir":"hdfs://hostb7:8020/directory","spark.app.id":"app-20211009101924-2782","spark.sql.shuffle.partitions":"8"},"Hadoop Properties":{"hadoop.service.shutdown.timeout":"30s","yarn.resourcemanager.amlauncher.thread-count":"50","yarn.sharedcache.enabled":"false","fs.s3a.connection.maximum":"15","yarn.nodemanager.numa-awareness.numactl.cmd":"/usr/bin/numactl","fs.s3a.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","yarn.app.mapreduce.am.scheduler.heartbeat.interval-ms":"1000","yarn.timeline-service.timeline-client.number-of-async-entities-to-merge":"10","hadoop.security.kms.client.timeout":"60","hadoop.http.authentication.kerberos.principal":"HTTP/_HOST@LOCALHOST","mapreduce.jobhistory.loadedjob.tasks.max":"-1","mapreduce.framework.name":"yarn","yarn.sharedcache.uploader.server.thread-count":"50","yarn.nodemanager.linux-container-executor.nonsecure-mode.user-pattern":"^[_.A-Za-z0-9][-@_.A-Za-z0-9]{0,255}?[$]?$","tfile.fs.output.buffer.size":"262144","yarn.app.mapreduce.am.job.task.listener.thread-count":"30","hadoop.security.groups.cache.background.reload.threads":"3","yarn.resourcemanager.webapp.cross-origin.enabled":"false","fs.AbstractFileSystem.ftp.impl":"org.apache.hadoop.fs.ftp.FtpFs","hadoop.registry.secure":"false","hadoop.shell.safely.delete.limit.num.files":"100","dfs.bytes-per-checksum":"512","mapreduce.job.acl-view-job":" ","fs.s3a.s3guard.ddb.background.sleep":"25ms","fs.s3a.retry.limit":"${fs.s3a.attempts.maximum}","mapreduce.jobhistory.loadedjobs.cache.size":"5","fs.s3a.s3guard.ddb.table.create":"false","yarn.nodemanager.amrmproxy.enabled":"false","yarn.timeline-service.entity-group-fs-store.with-user-dir":"false","mapreduce.input.fileinputformat.split.minsize":"0","yarn.resourcemanager.container.liveness-monitor.interval-ms":"600000","yarn.resourcemanager.client.thread-count":"50","io.seqfile.compress.blocksize":"1000000","yarn.sharedcache.checksum.algo.impl":"org.apache.hadoop.yarn.sharedcache.ChecksumSHA256Impl","yarn.nodemanager.amrmproxy.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.nodemanager.amrmproxy.DefaultRequestInterceptor","dfs.replication":"3","yarn.timeline-service.entity-group-fs-store.leveldb-cache-read-cache-size":"10485760","mapreduce.reduce.shuffle.fetch.retry.interval-ms":"1000","mapreduce.task.profile.maps":"0-2","yarn.scheduler.include-port-in-node-name":"false","yarn.nodemanager.admin-env":"MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX","yarn.resourcemanager.node-removal-untracked.timeout-ms":"60000","mapreduce.am.max-attempts":"2","hadoop.security.kms.client.failover.sleep.base.millis":"100","mapreduce.jobhistory.webapp.https.address":"0.0.0.0:19890","yarn.node-labels.fs-store.impl.class":"org.apache.hadoop.yarn.nodelabels.FileSystemNodeLabelsStore","yarn.nodemanager.collector-service.address":"${yarn.nodemanager.hostname}:8048","fs.trash.checkpoint.interval":"0","mapreduce.job.map.output.collector.class":"org.apache.hadoop.mapred.MapTask$MapOutputBuffer","yarn.resourcemanager.node-ip-cache.expiry-interval-secs":"-1","hadoop.http.authentication.signature.secret.file":"*********(redacted)","hadoop.jetty.logs.serve.aliases":"true","yarn.resourcemanager.placement-constraints.handler":"disabled","yarn.timeline-service.handler-thread-count":"10","yarn.resourcemanager.max-completed-applications":"1000","yarn.resourcemanager.system-metrics-publisher.enabled":"false","yarn.resourcemanager.placement-constraints.algorithm.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm.DefaultPlacementAlgorithm","yarn.sharedcache.webapp.address":"0.0.0.0:8788","yarn.resourcemanager.delegation.token.renew-interval":"*********(redacted)","yarn.sharedcache.nm.uploader.replication.factor":"10","hadoop.security.groups.negative-cache.secs":"30","yarn.app.mapreduce.task.container.log.backups":"0","mapreduce.reduce.skip.proc-count.auto-incr":"true","hadoop.security.group.mapping.ldap.posix.attr.gid.name":"gidNumber","rpc.engine.org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolPB":"org.apache.hadoop.ipc.ProtobufRpcEngine","ipc.client.fallback-to-simple-auth-allowed":"false","yarn.nodemanager.resource.memory.enforced":"true","yarn.client.failover-proxy-provider":"org.apache.hadoop.yarn.client.ConfiguredRMFailoverProxyProvider","yarn.timeline-service.http-authentication.simple.anonymous.allowed":"true","ha.health-monitor.check-interval.ms":"1000","yarn.acl.reservation-enable":"false","yarn.resourcemanager.store.class":"org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore","yarn.app.mapreduce.am.hard-kill-timeout-ms":"10000","fs.s3a.etag.checksum.enabled":"false","yarn.nodemanager.container-metrics.enable":"true","yarn.timeline-service.client.fd-clean-interval-secs":"60","yarn.resourcemanager.nodemanagers.heartbeat-interval-ms":"1000","hadoop.common.configuration.version":"3.0.0","fs.s3a.s3guard.ddb.table.capacity.read":"500","yarn.nodemanager.remote-app-log-dir-suffix":"logs","yarn.nodemanager.windows-container.cpu-limit.enabled":"false","yarn.nodemanager.runtime.linux.docker.privileged-containers.allowed":"false","file.blocksize":"67108864","hadoop.registry.zk.retry.ceiling.ms":"60000","yarn.scheduler.configuration.leveldb-store.path":"${hadoop.tmp.dir}/yarn/system/confstore","yarn.sharedcache.store.in-memory.initial-delay-mins":"10","mapreduce.jobhistory.principal":"jhs/_HOST@REALM.TLD","mapreduce.map.skip.proc-count.auto-incr":"true","fs.s3a.committer.name":"file","mapreduce.task.profile.reduces":"0-2","hadoop.zk.num-retries":"1000","yarn.webapp.xfs-filter.enabled":"true","seq.io.sort.mb":"100","yarn.scheduler.configuration.max.version":"100","yarn.timeline-service.webapp.https.address":"${yarn.timeline-service.hostname}:8190","yarn.resourcemanager.scheduler.address":"${yarn.resourcemanager.hostname}:8030","yarn.node-labels.enabled":"false","yarn.resourcemanager.webapp.ui-actions.enabled":"true","mapreduce.task.timeout":"600000","yarn.sharedcache.client-server.thread-count":"50","hadoop.security.groups.shell.command.timeout":"0s","hadoop.security.crypto.cipher.suite":"AES/CTR/NoPadding","yarn.nodemanager.elastic-memory-control.oom-handler":"org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.DefaultOOMHandler","yarn.resourcemanager.connect.max-wait.ms":"900000","fs.defaultFS":"hdfs://hostb7:8020","yarn.minicluster.use-rpc":"false","fs.har.impl.disable.cache":"true","yarn.webapp.ui2.enable":"false","io.compression.codec.bzip2.library":"system-native","yarn.nodemanager.distributed-scheduling.enabled":"false","mapreduce.shuffle.connection-keep-alive.timeout":"5","yarn.resourcemanager.webapp.https.address":"${yarn.resourcemanager.hostname}:8090","mapreduce.jobhistory.address":"0.0.0.0:10020","yarn.resourcemanager.nm-tokens.master-key-rolling-interval-secs":"*********(redacted)","yarn.is.minicluster":"false","yarn.nodemanager.address":"${yarn.nodemanager.hostname}:0","fs.abfss.impl":"org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","fs.AbstractFileSystem.s3a.impl":"org.apache.hadoop.fs.s3a.S3A","mapreduce.task.combine.progress.records":"10000","yarn.resourcemanager.epoch.range":"0","yarn.resourcemanager.am.max-attempts":"2","yarn.nodemanager.linux-container-executor.cgroups.hierarchy":"/hadoop-yarn","fs.AbstractFileSystem.wasbs.impl":"org.apache.hadoop.fs.azure.Wasbs","yarn.timeline-service.entity-group-fs-store.cache-store-class":"org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore","fs.ftp.transfer.mode":"BLOCK_TRANSFER_MODE","ipc.server.log.slow.rpc":"false","yarn.resourcemanager.node-labels.provider.fetch-interval-ms":"1800000","yarn.router.webapp.https.address":"0.0.0.0:8091","yarn.nodemanager.webapp.cross-origin.enabled":"false","fs.wasb.impl":"org.apache.hadoop.fs.azure.NativeAzureFileSystem","yarn.resourcemanager.auto-update.containers":"false","yarn.app.mapreduce.am.job.committer.cancel-timeout":"60000","yarn.scheduler.configuration.zk-store.parent-path":"/confstore","yarn.nodemanager.default-container-executor.log-dirs.permissions":"710","yarn.app.attempt.diagnostics.limit.kc":"64","ftp.bytes-per-checksum":"512","yarn.nodemanager.resource.memory-mb":"-1","fs.AbstractFileSystem.abfs.impl":"org.apache.hadoop.fs.azurebfs.Abfs","yarn.timeline-service.writer.flush-interval-seconds":"60","fs.s3a.fast.upload.active.blocks":"4","hadoop.security.credential.clear-text-fallback":"true","yarn.nodemanager.collector-service.thread-count":"5","fs.azure.secure.mode":"false","mapreduce.jobhistory.joblist.cache.size":"20000","fs.ftp.host":"0.0.0.0","yarn.resourcemanager.fs.state-store.num-retries":"0","yarn.resourcemanager.nodemanager-connect-retries":"10","yarn.nodemanager.log-aggregation.num-log-files-per-app":"30","hadoop.security.kms.client.encrypted.key.cache.low-watermark":"0.3f","fs.s3a.committer.magic.enabled":"false","yarn.timeline-service.client.max-retries":"30","dfs.ha.fencing.ssh.connect-timeout":"30000","yarn.log-aggregation-enable":"false","yarn.system-metrics-publisher.enabled":"false","mapreduce.reduce.markreset.buffer.percent":"0.0","fs.AbstractFileSystem.viewfs.impl":"org.apache.hadoop.fs.viewfs.ViewFs","mapreduce.task.io.sort.factor":"10","yarn.nodemanager.amrmproxy.client.thread-count":"25","ha.failover-controller.new-active.rpc-timeout.ms":"60000","yarn.nodemanager.container-localizer.java.opts":"-Xmx256m","mapreduce.jobhistory.datestring.cache.size":"200000","mapreduce.job.acl-modify-job":" ","yarn.nodemanager.windows-container.memory-limit.enabled":"false","yarn.timeline-service.webapp.address":"${yarn.timeline-service.hostname}:8188","yarn.app.mapreduce.am.job.committer.commit-window":"10000","yarn.nodemanager.container-manager.thread-count":"20","yarn.minicluster.fixed.ports":"false","hadoop.tags.system":"YARN,HDFS,NAMENODE,DATANODE,REQUIRED,SECURITY,KERBEROS,PERFORMANCE,CLIENT\n ,SERVER,DEBUG,DEPRECATED,COMMON,OPTIONAL","yarn.cluster.max-application-priority":"0","yarn.timeline-service.ttl-enable":"true","mapreduce.jobhistory.recovery.store.fs.uri":"${hadoop.tmp.dir}/mapred/history/recoverystore","hadoop.caller.context.signature.max.size":"40","yarn.client.load.resource-types.from-server":"false","ha.zookeeper.session-timeout.ms":"10000","tfile.io.chunk.size":"1048576","fs.s3a.s3guard.ddb.table.capacity.write":"100","mapreduce.job.speculative.slowtaskthreshold":"1.0","io.serializations":"org.apache.hadoop.io.serializer.WritableSerialization, org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization, org.apache.hadoop.io.serializer.avro.AvroReflectSerialization","hadoop.security.kms.client.failover.sleep.max.millis":"2000","hadoop.security.group.mapping.ldap.directory.search.timeout":"10000","yarn.scheduler.configuration.store.max-logs":"1000","yarn.nodemanager.node-attributes.provider.fetch-interval-ms":"600000","fs.swift.impl":"org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem","yarn.nodemanager.local-cache.max-files-per-directory":"8192","hadoop.http.cross-origin.enabled":"false","hadoop.zk.acl":"world:anyone:rwcda","mapreduce.map.sort.spill.percent":"0.80","yarn.timeline-service.entity-group-fs-store.scan-interval-seconds":"60","yarn.node-attribute.fs-store.impl.class":"org.apache.hadoop.yarn.server.resourcemanager.nodelabels.FileSystemNodeAttributeStore","fs.s3a.retry.interval":"500ms","yarn.timeline-service.client.best-effort":"false","yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled":"*********(redacted)","hadoop.security.group.mapping.ldap.posix.attr.uid.name":"uidNumber","fs.AbstractFileSystem.swebhdfs.impl":"org.apache.hadoop.fs.SWebHdfs","yarn.nodemanager.elastic-memory-control.timeout-sec":"5","mapreduce.ifile.readahead":"true","yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms":"300000","yarn.timeline-service.reader.webapp.address":"${yarn.timeline-service.webapp.address}","yarn.resourcemanager.placement-constraints.algorithm.pool-size":"1","yarn.timeline-service.hbase.coprocessor.jar.hdfs.location":"/hbase/coprocessor/hadoop-yarn-server-timelineservice.jar","hadoop.security.kms.client.encrypted.key.cache.num.refill.threads":"2","yarn.resourcemanager.scheduler.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler","yarn.app.mapreduce.am.command-opts":"-Xmx1024m","mapreduce.cluster.local.dir":"${hadoop.tmp.dir}/mapred/local","io.mapfile.bloom.error.rate":"0.005","fs.client.resolve.topology.enabled":"false","yarn.nodemanager.runtime.linux.allowed-runtimes":"default","yarn.sharedcache.store.class":"org.apache.hadoop.yarn.server.sharedcachemanager.store.InMemorySCMStore","ha.failover-controller.graceful-fence.rpc-timeout.ms":"5000","ftp.replication":"3","hadoop.security.uid.cache.secs":"14400","mapreduce.job.maxtaskfailures.per.tracker":"3","fs.s3a.metadatastore.impl":"org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore","io.skip.checksum.errors":"false","yarn.app.mapreduce.client-am.ipc.max-retries-on-timeouts":"3","yarn.timeline-service.webapp.xfs-filter.xframe-options":"SAMEORIGIN","fs.s3a.connection.timeout":"200000","mapreduce.job.max.split.locations":"15","yarn.resourcemanager.nm-container-queuing.max-queue-length":"15","hadoop.registry.zk.session.timeout.ms":"60000","yarn.federation.cache-ttl.secs":"300","mapreduce.jvm.system-properties-to-log":"os.name,os.version,java.home,java.runtime.version,java.vendor,java.version,java.vm.name,java.class.path,java.io.tmpdir,user.dir,user.name","yarn.resourcemanager.opportunistic-container-allocation.nodes-used":"10","yarn.timeline-service.entity-group-fs-store.active-dir":"/tmp/entity-file-history/active","mapreduce.shuffle.transfer.buffer.size":"131072","yarn.timeline-service.client.retry-interval-ms":"1000","yarn.http.policy":"HTTP_ONLY","fs.s3a.socket.send.buffer":"8192","fs.AbstractFileSystem.abfss.impl":"org.apache.hadoop.fs.azurebfs.Abfss","yarn.sharedcache.uploader.server.address":"0.0.0.0:8046","yarn.resourcemanager.delegation-token.max-conf-size-bytes":"*********(redacted)","hadoop.http.authentication.token.validity":"*********(redacted)","mapreduce.shuffle.max.connections":"0","yarn.minicluster.yarn.nodemanager.resource.memory-mb":"4096","mapreduce.job.emit-timeline-data":"false","yarn.nodemanager.resource.system-reserved-memory-mb":"-1","hadoop.kerberos.min.seconds.before.relogin":"60","mapreduce.jobhistory.move.thread-count":"3","yarn.resourcemanager.admin.client.thread-count":"1","yarn.dispatcher.drain-events.timeout":"300000","fs.s3a.buffer.dir":"${hadoop.tmp.dir}/s3a","hadoop.ssl.enabled.protocols":"TLSv1,SSLv2Hello,TLSv1.1,TLSv1.2","mapreduce.jobhistory.admin.address":"0.0.0.0:10033","yarn.log-aggregation-status.time-out.ms":"600000","fs.s3a.assumed.role.sts.endpoint.region":"us-west-1","mapreduce.shuffle.port":"13562","yarn.resourcemanager.max-log-aggregation-diagnostics-in-memory":"10","yarn.nodemanager.health-checker.interval-ms":"600000","yarn.router.clientrm.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.clientrm.DefaultClientRequestInterceptor","yarn.resourcemanager.zk-appid-node.split-index":"0","ftp.blocksize":"67108864","yarn.nodemanager.runtime.linux.sandbox-mode.local-dirs.permissions":"read","yarn.router.rmadmin.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.rmadmin.DefaultRMAdminRequestInterceptor","yarn.nodemanager.log-container-debug-info.enabled":"true","yarn.client.max-cached-nodemanagers-proxies":"0","yarn.nodemanager.linux-container-executor.cgroups.delete-delay-ms":"20","yarn.nodemanager.delete.debug-delay-sec":"0","yarn.nodemanager.pmem-check-enabled":"true","yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage":"90.0","mapreduce.app-submission.cross-platform":"false","yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms":"10000","yarn.nodemanager.container-retry-minimum-interval-ms":"1000","hadoop.security.groups.cache.secs":"300","yarn.federation.enabled":"false","fs.azure.local.sas.key.mode":"false","ipc.maximum.data.length":"67108864","mapreduce.shuffle.max.threads":"0","yarn.router.pipeline.cache-max-size":"25","yarn.resourcemanager.nm-container-queuing.load-comparator":"QUEUE_LENGTH","hadoop.security.authorization":"false","mapreduce.job.complete.cancel.delegation.tokens":"*********(redacted)","fs.s3a.paging.maximum":"5000","nfs.exports.allowed.hosts":"* rw","yarn.nodemanager.amrmproxy.ha.enable":"false","mapreduce.jobhistory.http.policy":"HTTP_ONLY","yarn.sharedcache.store.in-memory.check-period-mins":"720","hadoop.security.group.mapping.ldap.ssl":"false","yarn.client.application-client-protocol.poll-interval-ms":"200","yarn.scheduler.configuration.leveldb-store.compaction-interval-secs":"86400","yarn.timeline-service.writer.class":"org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl","ha.zookeeper.parent-znode":"/hadoop-ha","yarn.nodemanager.log-aggregation.policy.class":"org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.AllContainerLogAggregationPolicy","mapreduce.reduce.shuffle.merge.percent":"0.66","hadoop.security.group.mapping.ldap.search.filter.group":"(objectClass=group)","yarn.resourcemanager.placement-constraints.scheduler.pool-size":"1","yarn.nodemanager.resourcemanager.minimum.version":"NONE","mapreduce.job.speculative.speculative-cap-running-tasks":"0.1","yarn.admin.acl":"*","yarn.nodemanager.recovery.supervised":"false","yarn.sharedcache.admin.thread-count":"1","yarn.resourcemanager.ha.automatic-failover.enabled":"true","mapreduce.reduce.skip.maxgroups":"0","mapreduce.reduce.shuffle.connect.timeout":"180000","yarn.resourcemanager.address":"${yarn.resourcemanager.hostname}:8032","ipc.client.ping":"true","mapreduce.task.local-fs.write-limit.bytes":"-1","fs.adl.oauth2.access.token.provider.type":"*********(redacted)","mapreduce.shuffle.ssl.file.buffer.size":"65536","yarn.resourcemanager.ha.automatic-failover.embedded":"true","yarn.nodemanager.resource-plugins.gpu.docker-plugin":"nvidia-docker-v1","hadoop.ssl.enabled":"false","fs.s3a.multipart.purge":"false","yarn.scheduler.configuration.store.class":"file","yarn.resourcemanager.nm-container-queuing.queue-limit-stdev":"1.0f","mapreduce.job.end-notification.max.attempts":"5","mapreduce.output.fileoutputformat.compress.codec":"org.apache.hadoop.io.compress.DefaultCodec","yarn.nodemanager.container-monitor.procfs-tree.smaps-based-rss.enabled":"false","ipc.client.bind.wildcard.addr":"false","yarn.resourcemanager.webapp.rest-csrf.enabled":"false","ha.health-monitor.connect-retry-interval.ms":"1000","yarn.nodemanager.keytab":"/etc/krb5.keytab","mapreduce.jobhistory.keytab":"/etc/security/keytab/jhs.service.keytab","fs.s3a.threads.max":"10","mapreduce.reduce.shuffle.input.buffer.percent":"0.70","yarn.nodemanager.runtime.linux.docker.allowed-container-networks":"host,none,bridge","yarn.nodemanager.node-labels.resync-interval-ms":"120000","hadoop.tmp.dir":"/tmp/hadoop-${user.name}","mapreduce.job.maps":"2","mapreduce.jobhistory.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.job.end-notification.max.retry.interval":"5000","yarn.log-aggregation.retain-check-interval-seconds":"-1","yarn.resourcemanager.resource-tracker.client.thread-count":"50","yarn.rm.system-metrics-publisher.emit-container-events":"false","yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size":"10000","yarn.resourcemanager.ha.automatic-failover.zk-base-path":"/yarn-leader-election","io.seqfile.local.dir":"${hadoop.tmp.dir}/io/local","fs.s3a.s3guard.ddb.throttle.retry.interval":"100ms","fs.AbstractFileSystem.wasb.impl":"org.apache.hadoop.fs.azure.Wasb","mapreduce.client.submit.file.replication":"10","mapreduce.jobhistory.minicluster.fixed.ports":"false","fs.s3a.multipart.threshold":"2147483647","yarn.resourcemanager.webapp.xfs-filter.xframe-options":"SAMEORIGIN","mapreduce.jobhistory.done-dir":"${yarn.app.mapreduce.am.staging-dir}/history/done","ipc.client.idlethreshold":"4000","yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage":"false","mapreduce.reduce.input.buffer.percent":"0.0","yarn.nodemanager.runtime.linux.docker.userremapping-gid-threshold":"1","yarn.nodemanager.webapp.rest-csrf.enabled":"false","fs.ftp.host.port":"21","ipc.ping.interval":"60000","yarn.resourcemanager.history-writer.multi-threaded-dispatcher.pool-size":"10","yarn.resourcemanager.admin.address":"${yarn.resourcemanager.hostname}:8033","file.client-write-packet-size":"65536","ipc.client.kill.max":"10","mapreduce.reduce.speculative":"true","hadoop.security.key.default.bitlength":"128","mapreduce.job.reducer.unconditional-preempt.delay.sec":"300","yarn.nodemanager.disk-health-checker.interval-ms":"120000","yarn.nodemanager.log.deletion-threads-count":"4","yarn.webapp.filter-entity-list-by-user":"false","ipc.client.connection.maxidletime":"10000","mapreduce.task.io.sort.mb":"100","yarn.nodemanager.localizer.client.thread-count":"5","io.erasurecode.codec.rs.rawcoders":"rs_native,rs_java","io.erasurecode.codec.rs-legacy.rawcoders":"rs-legacy_java","yarn.sharedcache.admin.address":"0.0.0.0:8047","yarn.resourcemanager.placement-constraints.algorithm.iterator":"SERIAL","yarn.nodemanager.localizer.cache.cleanup.interval-ms":"600000","hadoop.security.crypto.codec.classes.aes.ctr.nopadding":"org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec, org.apache.hadoop.crypto.JceAesCtrCryptoCodec","mapreduce.job.cache.limit.max-resources-mb":"0","fs.s3a.connection.ssl.enabled":"true","yarn.nodemanager.process-kill-wait.ms":"5000","mapreduce.job.hdfs-servers":"${fs.defaultFS}","hadoop.workaround.non.threadsafe.getpwuid":"true","fs.df.interval":"60000","fs.s3a.multiobjectdelete.enable":"true","yarn.sharedcache.cleaner.resource-sleep-ms":"0","yarn.nodemanager.disk-health-checker.min-healthy-disks":"0.25","hadoop.shell.missing.defaultFs.warning":"false","io.file.buffer.size":"65536","hadoop.security.group.mapping.ldap.search.attr.member":"member","hadoop.security.random.device.file.path":"/dev/urandom","hadoop.security.sensitive-config-keys":"*********(redacted)","fs.s3a.s3guard.ddb.max.retries":"9","hadoop.rpc.socket.factory.class.default":"org.apache.hadoop.net.StandardSocketFactory","yarn.intermediate-data-encryption.enable":"false","yarn.resourcemanager.connect.retry-interval.ms":"30000","yarn.nodemanager.container.stderr.pattern":"{*stderr*,*STDERR*}","yarn.scheduler.minimum-allocation-mb":"1024","yarn.app.mapreduce.am.staging-dir":"/tmp/hadoop-yarn/staging","mapreduce.reduce.shuffle.read.timeout":"180000","hadoop.http.cross-origin.max-age":"1800","io.erasurecode.codec.xor.rawcoders":"xor_native,xor_java","fs.s3a.connection.establish.timeout":"5000","mapreduce.job.running.map.limit":"0","yarn.minicluster.control-resource-monitoring":"false","hadoop.ssl.require.client.cert":"false","hadoop.kerberos.kinit.command":"kinit","yarn.federation.state-store.class":"org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore","mapreduce.reduce.log.level":"INFO","hadoop.security.dns.log-slow-lookups.threshold.ms":"1000","mapreduce.job.ubertask.enable":"false","adl.http.timeout":"-1","yarn.resourcemanager.placement-constraints.retry-attempts":"3","hadoop.caller.context.enabled":"false","yarn.nodemanager.vmem-pmem-ratio":"2.1","hadoop.rpc.protection":"authentication","ha.health-monitor.rpc-timeout.ms":"45000","yarn.nodemanager.remote-app-log-dir":"/tmp/logs","hadoop.zk.timeout-ms":"10000","fs.s3a.s3guard.cli.prune.age":"86400000","yarn.nodemanager.resource.pcores-vcores-multiplier":"1.0","yarn.nodemanager.runtime.linux.sandbox-mode":"disabled","yarn.app.mapreduce.am.containerlauncher.threadpool-initial-size":"10","fs.s3a.committer.threads":"8","hadoop.zk.retry-interval-ms":"1000","hadoop.security.crypto.buffer.size":"8192","yarn.nodemanager.node-labels.provider.fetch-interval-ms":"600000","mapreduce.jobhistory.recovery.store.leveldb.path":"${hadoop.tmp.dir}/mapred/history/recoverystore","yarn.client.failover-retries-on-socket-timeouts":"0","yarn.nodemanager.resource.memory.enabled":"false","fs.azure.authorization.caching.enable":"true","hadoop.security.instrumentation.requires.admin":"false","yarn.nodemanager.delete.thread-count":"4","mapreduce.job.finish-when-all-reducers-done":"true","hadoop.registry.jaas.context":"Client","yarn.timeline-service.leveldb-timeline-store.path":"${hadoop.tmp.dir}/yarn/timeline","io.map.index.interval":"128","yarn.resourcemanager.nm-container-queuing.max-queue-wait-time-ms":"100","fs.abfs.impl":"org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","mapreduce.job.counters.max":"120","mapreduce.jobhistory.webapp.rest-csrf.enabled":"false","yarn.timeline-service.store-class":"org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore","mapreduce.jobhistory.move.interval-ms":"180000","yarn.nodemanager.localizer.fetch.thread-count":"4","yarn.resourcemanager.scheduler.client.thread-count":"50","hadoop.ssl.hostname.verifier":"DEFAULT","yarn.timeline-service.leveldb-state-store.path":"${hadoop.tmp.dir}/yarn/timeline","mapreduce.job.classloader":"false","mapreduce.task.profile.map.params":"${mapreduce.task.profile.params}","ipc.client.connect.timeout":"20000","hadoop.security.auth_to_local.mechanism":"hadoop","yarn.timeline-service.app-collector.linger-period.ms":"60000","yarn.nm.liveness-monitor.expiry-interval-ms":"600000","yarn.resourcemanager.reservation-system.planfollower.time-step":"1000","yarn.nodemanager.runtime.linux.docker.enable-userremapping.allowed":"true","yarn.webapp.api-service.enable":"false","yarn.nodemanager.recovery.enabled":"false","mapreduce.job.end-notification.retry.interval":"1000","fs.du.interval":"600000","fs.ftp.impl":"org.apache.hadoop.fs.ftp.FTPFileSystem","yarn.nodemanager.container.stderr.tail.bytes":"4096","hadoop.security.group.mapping.ldap.read.timeout.ms":"60000","hadoop.security.groups.cache.warn.after.ms":"5000","file.bytes-per-checksum":"512","mapreduce.outputcommitter.factory.scheme.s3a":"org.apache.hadoop.fs.s3a.commit.S3ACommitterFactory","hadoop.security.groups.cache.background.reload":"false","yarn.nodemanager.container-monitor.enabled":"true","yarn.nodemanager.elastic-memory-control.enabled":"false","net.topology.script.number.args":"100","mapreduce.task.merge.progress.records":"10000","yarn.nodemanager.localizer.address":"${yarn.nodemanager.hostname}:8040","yarn.timeline-service.keytab":"/etc/krb5.keytab","mapreduce.reduce.shuffle.fetch.retry.timeout-ms":"30000","yarn.resourcemanager.rm.container-allocation.expiry-interval-ms":"600000","mapreduce.fileoutputcommitter.algorithm.version":"1","yarn.resourcemanager.work-preserving-recovery.enabled":"true","mapreduce.map.skip.maxrecords":"0","yarn.sharedcache.root-dir":"/sharedcache","fs.s3a.retry.throttle.limit":"${fs.s3a.attempts.maximum}","hadoop.http.authentication.type":"simple","mapreduce.job.cache.limit.max-resources":"0","mapreduce.task.userlog.limit.kb":"0","yarn.resourcemanager.scheduler.monitor.enable":"false","ipc.client.connect.max.retries":"10","hadoop.registry.zk.retry.times":"5","yarn.nodemanager.resource-monitor.interval-ms":"3000","yarn.nodemanager.resource-plugins.gpu.allowed-gpu-devices":"auto","mapreduce.job.sharedcache.mode":"disabled","yarn.nodemanager.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.shuffle.listen.queue.size":"128","yarn.scheduler.configuration.mutation.acl-policy.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy","mapreduce.map.cpu.vcores":"1","yarn.log-aggregation.file-formats":"TFile","yarn.timeline-service.client.fd-retain-secs":"300","hadoop.user.group.static.mapping.overrides":"dr.who=;","fs.azure.sas.expiry.period":"90d","mapreduce.jobhistory.recovery.store.class":"org.apache.hadoop.mapreduce.v2.hs.HistoryServerFileSystemStateStoreService","yarn.resourcemanager.fail-fast":"${yarn.fail-fast}","yarn.resourcemanager.proxy-user-privileges.enabled":"false","yarn.router.webapp.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.webapp.DefaultRequestInterceptorREST","yarn.nodemanager.resource.memory.cgroups.soft-limit-percentage":"90.0","mapreduce.job.reducer.preempt.delay.sec":"0","hadoop.util.hash.type":"murmur","yarn.nodemanager.disk-validator":"basic","yarn.app.mapreduce.client.job.max-retries":"3","mapreduce.reduce.shuffle.retry-delay.max.ms":"60000","hadoop.security.group.mapping.ldap.connection.timeout.ms":"60000","mapreduce.task.profile.params":"-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,verbose=n,file=%s","yarn.app.mapreduce.shuffle.log.backups":"0","yarn.nodemanager.container-diagnostics-maximum-size":"10000","hadoop.registry.zk.retry.interval.ms":"1000","yarn.nodemanager.linux-container-executor.cgroups.delete-timeout-ms":"1000","fs.AbstractFileSystem.file.impl":"org.apache.hadoop.fs.local.LocalFs","yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds":"-1","mapreduce.jobhistory.cleaner.interval-ms":"86400000","hadoop.registry.zk.quorum":"localhost:2181","mapreduce.output.fileoutputformat.compress":"false","yarn.resourcemanager.am-rm-tokens.master-key-rolling-interval-secs":"*********(redacted)","fs.s3a.assumed.role.session.duration":"30m","hadoop.security.group.mapping.ldap.conversion.rule":"none","hadoop.ssl.server.conf":"ssl-server.xml","fs.s3a.retry.throttle.interval":"1000ms","seq.io.sort.factor":"100","yarn.sharedcache.cleaner.initial-delay-mins":"10","mapreduce.client.completion.pollinterval":"5000","hadoop.ssl.keystores.factory.class":"org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory","yarn.app.mapreduce.am.resource.cpu-vcores":"1","yarn.timeline-service.enabled":"false","yarn.nodemanager.runtime.linux.docker.capabilities":"CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE","yarn.acl.enable":"false","yarn.timeline-service.entity-group-fs-store.done-dir":"/tmp/entity-file-history/done/","mapreduce.task.profile":"false","yarn.resourcemanager.fs.state-store.uri":"${hadoop.tmp.dir}/yarn/system/rmstore","mapreduce.jobhistory.always-scan-user-dir":"false","yarn.nodemanager.opportunistic-containers-use-pause-for-preemption":"false","yarn.nodemanager.linux-container-executor.nonsecure-mode.local-user":"nobody","yarn.timeline-service.reader.class":"org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineReaderImpl","yarn.resourcemanager.configuration.provider-class":"org.apache.hadoop.yarn.LocalConfigurationProvider","yarn.nodemanager.runtime.linux.docker.userremapping-uid-threshold":"1","yarn.resourcemanager.configuration.file-system-based-store":"/yarn/conf","mapreduce.job.cache.limit.max-single-resource-mb":"0","yarn.nodemanager.runtime.linux.docker.stop.grace-period":"10","yarn.resourcemanager.resource-profiles.source-file":"resource-profiles.json","yarn.nodemanager.resource.percentage-physical-cpu-limit":"100","mapreduce.jobhistory.client.thread-count":"10","tfile.fs.input.buffer.size":"262144","mapreduce.client.progressmonitor.pollinterval":"1000","yarn.nodemanager.log-dirs":"${yarn.log.dir}/userlogs","fs.automatic.close":"true","yarn.nodemanager.hostname":"0.0.0.0","yarn.nodemanager.resource.memory.cgroups.swappiness":"0","ftp.stream-buffer-size":"4096","yarn.fail-fast":"false","yarn.timeline-service.app-aggregation-interval-secs":"15","hadoop.security.group.mapping.ldap.search.filter.user":"(&(objectClass=user)(sAMAccountName={0}))","yarn.nodemanager.container-localizer.log.level":"INFO","yarn.timeline-service.address":"${yarn.timeline-service.hostname}:10200","mapreduce.job.ubertask.maxmaps":"9","fs.s3a.threads.keepalivetime":"60","mapreduce.jobhistory.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","mapreduce.task.files.preserve.failedtasks":"false","yarn.app.mapreduce.client.job.retry-interval":"2000","ha.failover-controller.graceful-fence.connection.retries":"1","yarn.resourcemanager.delegation.token.max-lifetime":"*********(redacted)","yarn.timeline-service.client.drain-entities.timeout.ms":"2000","yarn.nodemanager.resource-plugins.fpga.vendor-plugin.class":"org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.IntelFpgaOpenclPlugin","yarn.timeline-service.entity-group-fs-store.summary-store":"org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore","mapreduce.reduce.cpu.vcores":"1","mapreduce.job.encrypted-intermediate-data.buffer.kb":"128","fs.client.resolve.remote.symlinks":"true","yarn.nodemanager.webapp.https.address":"0.0.0.0:8044","hadoop.http.cross-origin.allowed-origins":"*","mapreduce.job.encrypted-intermediate-data":"false","yarn.timeline-service.entity-group-fs-store.retain-seconds":"604800","yarn.resourcemanager.metrics.runtime.buckets":"60,300,1440","yarn.timeline-service.generic-application-history.max-applications":"10000","yarn.nodemanager.local-dirs":"${hadoop.tmp.dir}/nm-local-dir","mapreduce.shuffle.connection-keep-alive.enable":"false","yarn.node-labels.configuration-type":"centralized","fs.s3a.path.style.access":"false","yarn.nodemanager.aux-services.mapreduce_shuffle.class":"org.apache.hadoop.mapred.ShuffleHandler","yarn.sharedcache.store.in-memory.staleness-period-mins":"10080","fs.adl.impl":"org.apache.hadoop.fs.adl.AdlFileSystem","yarn.resourcemanager.nodemanager.minimum.version":"NONE","mapreduce.jobhistory.webapp.xfs-filter.xframe-options":"SAMEORIGIN","yarn.app.mapreduce.am.staging-dir.erasurecoding.enabled":"false","net.topology.impl":"org.apache.hadoop.net.NetworkTopology","io.map.index.skip":"0","yarn.timeline-service.reader.webapp.https.address":"${yarn.timeline-service.webapp.https.address}","fs.ftp.data.connection.mode":"ACTIVE_LOCAL_DATA_CONNECTION_MODE","mapreduce.job.local-fs.single-disk-limit.check.kill-limit-exceed":"true","yarn.scheduler.maximum-allocation-vcores":"4","hadoop.http.cross-origin.allowed-headers":"X-Requested-With,Content-Type,Accept,Origin","yarn.nodemanager.log-aggregation.compression-type":"none","yarn.timeline-service.version":"1.0f","yarn.ipc.rpc.class":"org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC","mapreduce.reduce.maxattempts":"4","hadoop.security.dns.log-slow-lookups.enabled":"false","mapreduce.job.committer.setup.cleanup.needed":"true","mapreduce.job.running.reduce.limit":"0","ipc.maximum.response.length":"134217728","yarn.resourcemanager.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","mapreduce.job.token.tracking.ids.enabled":"*********(redacted)","hadoop.caller.context.max.size":"128","yarn.nodemanager.runtime.linux.docker.host-pid-namespace.allowed":"false","yarn.nodemanager.runtime.linux.docker.delayed-removal.allowed":"false","hadoop.registry.system.acls":"sasl:yarn@, sasl:mapred@, sasl:hdfs@","yarn.nodemanager.recovery.dir":"${hadoop.tmp.dir}/yarn-nm-recovery","fs.s3a.fast.upload.buffer":"disk","mapreduce.jobhistory.intermediate-done-dir":"${yarn.app.mapreduce.am.staging-dir}/history/done_intermediate","yarn.app.mapreduce.shuffle.log.separate":"true","fs.s3a.max.total.tasks":"5","fs.s3a.readahead.range":"64K","hadoop.http.authentication.simple.anonymous.allowed":"true","fs.s3a.attempts.maximum":"20","hadoop.registry.zk.connection.timeout.ms":"15000","yarn.resourcemanager.delegation-token-renewer.thread-count":"*********(redacted)","yarn.nodemanager.health-checker.script.timeout-ms":"1200000","yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size":"10000","yarn.resourcemanager.resource-profiles.enabled":"false","yarn.timeline-service.hbase-schema.prefix":"prod.","fs.azure.authorization":"false","mapreduce.map.log.level":"INFO","yarn.resourcemanager.decommissioning-nodes-watcher.poll-interval-secs":"20","mapreduce.output.fileoutputformat.compress.type":"RECORD","yarn.resourcemanager.leveldb-state-store.path":"${hadoop.tmp.dir}/yarn/system/rmstore","yarn.timeline-service.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.ifile.readahead.bytes":"4194304","yarn.sharedcache.app-checker.class":"org.apache.hadoop.yarn.server.sharedcachemanager.RemoteAppChecker","yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users":"true","yarn.nodemanager.resource.detect-hardware-capabilities":"false","mapreduce.cluster.acls.enabled":"false","mapreduce.job.speculative.retry-after-no-speculate":"1000","hadoop.security.group.mapping.ldap.search.group.hierarchy.levels":"0","yarn.resourcemanager.fs.state-store.retry-interval-ms":"1000","file.stream-buffer-size":"4096","yarn.resourcemanager.application-timeouts.monitor.interval-ms":"3000","mapreduce.map.output.compress.codec":"org.apache.hadoop.io.compress.DefaultCodec","mapreduce.map.speculative":"true","mapreduce.job.speculative.retry-after-speculate":"15000","yarn.nodemanager.linux-container-executor.cgroups.mount":"false","yarn.app.mapreduce.am.container.log.backups":"0","yarn.app.mapreduce.am.log.level":"INFO","mapreduce.job.reduce.slowstart.completedmaps":"0.05","yarn.timeline-service.http-authentication.type":"simple","hadoop.security.group.mapping.ldap.search.attr.group.name":"cn","yarn.nodemanager.resource-plugins.fpga.allowed-fpga-devices":"0,1","yarn.timeline-service.client.internal-timers-ttl-secs":"420","hadoop.http.logs.enabled":"true","fs.s3a.block.size":"32M","yarn.sharedcache.client-server.address":"0.0.0.0:8045","yarn.nodemanager.logaggregation.threadpool-size-max":"100","yarn.resourcemanager.hostname":"hostb7","yarn.resourcemanager.delegation.key.update-interval":"86400000","mapreduce.reduce.shuffle.fetch.retry.enabled":"${yarn.nodemanager.recovery.enabled}","mapreduce.map.memory.mb":"-1","mapreduce.task.skip.start.attempts":"2","fs.AbstractFileSystem.hdfs.impl":"org.apache.hadoop.fs.Hdfs","yarn.nodemanager.disk-health-checker.enable":"true","ipc.client.tcpnodelay":"true","ipc.client.rpc-timeout.ms":"0","yarn.nodemanager.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","ipc.client.low-latency":"false","mapreduce.input.lineinputformat.linespermap":"1","yarn.router.interceptor.user.threadpool-size":"5","ipc.client.connect.max.retries.on.timeouts":"45","yarn.timeline-service.leveldb-timeline-store.read-cache-size":"104857600","fs.AbstractFileSystem.har.impl":"org.apache.hadoop.fs.HarFs","mapreduce.job.split.metainfo.maxsize":"10000000","yarn.am.liveness-monitor.expiry-interval-ms":"600000","yarn.resourcemanager.container-tokens.master-key-rolling-interval-secs":"*********(redacted)","yarn.timeline-service.entity-group-fs-store.app-cache-size":"10","fs.s3a.socket.recv.buffer":"8192","yarn.resourcemanager.resource-tracker.address":"${yarn.resourcemanager.hostname}:8031","yarn.nodemanager.node-labels.provider.fetch-timeout-ms":"1200000","mapreduce.job.heap.memory-mb.ratio":"0.8","yarn.resourcemanager.leveldb-state-store.compaction-interval-secs":"3600","yarn.resourcemanager.webapp.rest-csrf.custom-header":"X-XSRF-Header","yarn.scheduler.configuration.fs.path":"file://${hadoop.tmp.dir}/yarn/system/schedconf","mapreduce.client.output.filter":"FAILED","hadoop.http.filter.initializers":"org.apache.hadoop.http.lib.StaticUserWebFilter","mapreduce.reduce.memory.mb":"-1","yarn.timeline-service.hostname":"0.0.0.0","file.replication":"1","yarn.nodemanager.container-metrics.unregister-delay-ms":"10000","yarn.nodemanager.container-metrics.period-ms":"-1","mapreduce.fileoutputcommitter.task.cleanup.enabled":"false","yarn.nodemanager.log.retain-seconds":"10800","yarn.timeline-service.entity-group-fs-store.cleaner-interval-seconds":"3600","yarn.resourcemanager.keytab":"/etc/krb5.keytab","hadoop.security.group.mapping.providers.combined":"true","mapreduce.reduce.merge.inmem.threshold":"1000","yarn.timeline-service.recovery.enabled":"false","fs.azure.saskey.usecontainersaskeyforallaccess":"true","yarn.sharedcache.nm.uploader.thread-count":"20","yarn.resourcemanager.nodemanager-graceful-decommission-timeout-secs":"3600","mapreduce.shuffle.ssl.enabled":"false","yarn.timeline-service.hbase.coprocessor.app-final-value-retention-milliseconds":"259200000","fs.s3a.committer.staging.abort.pending.uploads":"true","yarn.nodemanager.opportunistic-containers-max-queue-length":"0","yarn.resourcemanager.state-store.max-completed-applications":"${yarn.resourcemanager.max-completed-applications}","mapreduce.job.speculative.minimum-allowed-tasks":"10","yarn.log-aggregation.retain-seconds":"-1","yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb":"0","mapreduce.jobhistory.max-age-ms":"604800000","hadoop.http.cross-origin.allowed-methods":"GET,POST,HEAD","yarn.resourcemanager.opportunistic-container-allocation.enabled":"false","mapreduce.jobhistory.webapp.address":"0.0.0.0:19888","hadoop.system.tags":"YARN,HDFS,NAMENODE,DATANODE,REQUIRED,SECURITY,KERBEROS,PERFORMANCE,CLIENT\n ,SERVER,DEBUG,DEPRECATED,COMMON,OPTIONAL","yarn.log-aggregation.file-controller.TFile.class":"org.apache.hadoop.yarn.logaggregation.filecontroller.tfile.LogAggregationTFileController","yarn.client.nodemanager-connect.max-wait-ms":"180000","yarn.resourcemanager.webapp.address":"${yarn.resourcemanager.hostname}:8088","mapreduce.jobhistory.recovery.enable":"false","mapreduce.reduce.shuffle.parallelcopies":"5","fs.AbstractFileSystem.webhdfs.impl":"org.apache.hadoop.fs.WebHdfs","fs.trash.interval":"0","yarn.app.mapreduce.client.max-retries":"3","hadoop.security.authentication":"simple","mapreduce.task.profile.reduce.params":"${mapreduce.task.profile.params}","yarn.app.mapreduce.am.resource.mb":"1536","mapreduce.input.fileinputformat.list-status.num-threads":"1","yarn.nodemanager.container-executor.class":"org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor","io.mapfile.bloom.size":"1048576","yarn.timeline-service.ttl-ms":"604800000","yarn.resourcemanager.nm-container-queuing.min-queue-length":"5","yarn.nodemanager.resource.cpu-vcores":"-1","mapreduce.job.reduces":"1","fs.s3a.multipart.size":"100M","yarn.scheduler.minimum-allocation-vcores":"1","mapreduce.job.speculative.speculative-cap-total-tasks":"0.01","hadoop.ssl.client.conf":"ssl-client.xml","mapreduce.job.queuename":"default","mapreduce.job.encrypted-intermediate-data-key-size-bits":"128","fs.s3a.metadatastore.authoritative":"false","yarn.nodemanager.webapp.xfs-filter.xframe-options":"SAMEORIGIN","ha.health-monitor.sleep-after-disconnect.ms":"1000","yarn.app.mapreduce.shuffle.log.limit.kb":"0","hadoop.security.group.mapping":"org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback","yarn.client.application-client-protocol.poll-timeout-ms":"-1","mapreduce.jobhistory.jhist.format":"binary","yarn.resourcemanager.ha.enabled":"false","hadoop.http.staticuser.user":"dr.who","mapreduce.task.exit.timeout.check-interval-ms":"20000","mapreduce.jobhistory.intermediate-user-done-dir.permissions":"770","mapreduce.task.exit.timeout":"60000","yarn.nodemanager.linux-container-executor.resources-handler.class":"org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler","mapreduce.reduce.shuffle.memory.limit.percent":"0.25","yarn.resourcemanager.reservation-system.enable":"false","mapreduce.map.output.compress":"false","ha.zookeeper.acl":"world:anyone:rwcda","ipc.server.max.connections":"0","yarn.nodemanager.aux-services":"mapreduce_shuffle","yarn.nodemanager.runtime.linux.docker.default-container-network":"host","yarn.router.webapp.address":"0.0.0.0:8089","yarn.scheduler.maximum-allocation-mb":"8192","yarn.resourcemanager.scheduler.monitor.policies":"org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy","yarn.sharedcache.cleaner.period-mins":"1440","yarn.nodemanager.resource-plugins.gpu.docker-plugin.nvidia-docker-v1.endpoint":"http://localhost:3476/v1.0/docker/cli","yarn.app.mapreduce.am.container.log.limit.kb":"0","ipc.client.connect.retry.interval":"1000","yarn.timeline-service.http-cross-origin.enabled":"false","fs.wasbs.impl":"org.apache.hadoop.fs.azure.NativeAzureFileSystem$Secure","yarn.federation.subcluster-resolver.class":"org.apache.hadoop.yarn.server.federation.resolver.DefaultSubClusterResolverImpl","yarn.resourcemanager.zk-state-store.parent-path":"/rmstore","mapreduce.jobhistory.cleaner.enable":"true","yarn.timeline-service.client.fd-flush-interval-secs":"10","hadoop.security.kms.client.encrypted.key.cache.expiry":"43200000","yarn.client.nodemanager-client-async.thread-pool-max-size":"500","mapreduce.map.maxattempts":"4","yarn.resourcemanager.nm-container-queuing.sorting-nodes-interval-ms":"1000","fs.s3a.committer.staging.tmp.path":"tmp/staging","yarn.nodemanager.sleep-delay-before-sigkill.ms":"250","yarn.resourcemanager.nm-container-queuing.min-queue-wait-time-ms":"10","mapreduce.job.end-notification.retry.attempts":"0","yarn.nodemanager.resource.count-logical-processors-as-cores":"false","hadoop.registry.zk.root":"/registry","adl.feature.ownerandgroup.enableupn":"false","yarn.resourcemanager.zk-max-znode-size.bytes":"1048576","mapreduce.job.reduce.shuffle.consumer.plugin.class":"org.apache.hadoop.mapreduce.task.reduce.Shuffle","yarn.resourcemanager.delayed.delegation-token.removal-interval-ms":"*********(redacted)","yarn.nodemanager.localizer.cache.target-size-mb":"10240","fs.s3a.committer.staging.conflict-mode":"fail","mapreduce.client.libjars.wildcard":"true","fs.s3a.committer.staging.unique-filenames":"true","yarn.nodemanager.node-attributes.provider.fetch-timeout-ms":"1200000","fs.s3a.list.version":"2","ftp.client-write-packet-size":"65536","fs.AbstractFileSystem.adl.impl":"org.apache.hadoop.fs.adl.Adl","hadoop.security.key.default.cipher":"AES/CTR/NoPadding","yarn.client.failover-retries":"0","fs.s3a.multipart.purge.age":"86400","mapreduce.job.local-fs.single-disk-limit.check.interval-ms":"5000","net.topology.node.switch.mapping.impl":"org.apache.hadoop.net.ScriptBasedMapping","yarn.nodemanager.amrmproxy.address":"0.0.0.0:8049","ipc.server.listen.queue.size":"128","map.sort.class":"org.apache.hadoop.util.QuickSort","fs.viewfs.rename.strategy":"SAME_MOUNTPOINT","hadoop.security.kms.client.authentication.retry-count":"1","fs.permissions.umask-mode":"022","fs.s3a.assumed.role.credentials.provider":"org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider","yarn.nodemanager.vmem-check-enabled":"true","yarn.nodemanager.numa-awareness.enabled":"false","yarn.nodemanager.recovery.compaction-interval-secs":"3600","yarn.app.mapreduce.client-am.ipc.max-retries":"3","yarn.federation.registry.base-dir":"yarnfederation/","mapreduce.job.max.map":"-1","mapreduce.job.local-fs.single-disk-limit.bytes":"-1","mapreduce.job.ubertask.maxreduces":"1","hadoop.security.kms.client.encrypted.key.cache.size":"500","hadoop.security.java.secure.random.algorithm":"SHA1PRNG","ha.failover-controller.cli-check.rpc-timeout.ms":"20000","mapreduce.jobhistory.jobname.limit":"50","mapreduce.application.classpath":"$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*:$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*","yarn.client.nodemanager-connect.retry-interval-ms":"10000","yarn.timeline-service.state-store-class":"org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore","yarn.nodemanager.env-whitelist":"JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,PATH,LANG,TZ,HADOOP_MAPRED_HOME","yarn.sharedcache.nested-level":"3","yarn.timeline-service.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","fs.azure.user.agent.prefix":"unknown","yarn.resourcemanager.zk-delegation-token-node.split-index":"*********(redacted)","yarn.nodemanager.numa-awareness.read-topology":"false","yarn.nodemanager.webapp.address":"${yarn.nodemanager.hostname}:8042","rpc.metrics.quantile.enable":"false","yarn.registry.class":"org.apache.hadoop.registry.client.impl.FSRegistryOperationsService","mapreduce.jobhistory.admin.acl":"*","yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size":"10","yarn.scheduler.queue-placement-rules":"user-group","hadoop.http.authentication.kerberos.keytab":"${user.home}/hadoop.keytab","yarn.resourcemanager.recovery.enabled":"false","yarn.timeline-service.webapp.rest-csrf.enabled":"false"},"System Properties":{"java.io.tmpdir":"/tmp","line.separator":"\n","path.separator":":","sun.management.compiler":"HotSpot 64-Bit Tiered Compilers","SPARK_SUBMIT":"true","sun.cpu.endian":"little","java.specification.version":"1.8","java.vm.specification.name":"Java Virtual Machine Specification","java.vendor":"Oracle Corporation","java.vm.specification.version":"1.8","user.home":"/home/usera112","file.encoding.pkg":"sun.io","sun.nio.ch.bugLevel":"","sun.arch.data.model":"64","sun.boot.library.path":"/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64","user.dir":"/home/usera112/projects/log-compressor/deployments/HiBench","java.library.path":"/usr/java/packages/lib/amd64:/usr/lib/x86_64-linux-gnu/jni:/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:/usr/lib/jni:/lib:/usr/lib","sun.cpu.isalist":"","os.arch":"amd64","java.vm.version":"25.162-b12","jetty.git.hash":"b881a572662e1943a14ae12e7e1207989f218b74","java.endorsed.dirs":"/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/endorsed","java.runtime.version":"1.8.0_162-8u162-b12-1-b12","java.vm.info":"mixed mode","java.ext.dirs":"/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/ext:/usr/java/packages/lib/ext","java.runtime.name":"OpenJDK Runtime Environment","file.separator":"/","java.class.version":"52.0","java.specification.name":"Java Platform API Specification","sun.boot.class.path":"/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/resources.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/rt.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/sunrsasign.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/jsse.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/jce.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/charsets.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/jfr.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/classes","file.encoding":"UTF-8","user.timezone":"America/Toronto","java.specification.vendor":"Oracle Corporation","sun.java.launcher":"SUN_STANDARD","os.version":"4.15.0-29-generic","sun.os.patch.level":"unknown","java.vm.specification.vendor":"Oracle Corporation","user.country":"US","sun.jnu.encoding":"UTF-8","user.language":"en","java.vendor.url":"http://java.oracle.com/","java.awt.printerjob":"sun.print.PSPrinterJob","java.awt.graphicsenv":"sun.awt.X11GraphicsEnvironment","awt.toolkit":"sun.awt.X11.XToolkit","os.name":"Linux","java.vm.vendor":"Oracle Corporation","java.vendor.url.bug":"http://bugreport.sun.com/bugreport/","user.name":"usera112","java.vm.name":"OpenJDK 64-Bit Server VM","sun.java.command":"org.apache.spark.deploy.SparkSubmit --master spark://hostb7:7077 --properties-file /home/usera112/projects/log-compressor/deployments/HiBench/report/join/spark/conf/sparkbench/spark.conf --class com.intel.hibench.sparkbench.sql.ScalaSparkSQLBench /home/usera112/projects/log-compressor/deployments/HiBench/sparkbench/assembly/target/sparkbench-assembly-8.0-SNAPSHOT-dist.jar ScalaJoin /home/usera112/projects/log-compressor/deployments/HiBench/report/join/spark/conf/../rankings_uservisits_join.hive","java.home":"/usr/lib/jvm/java-8-openjdk-amd64/jre","java.version":"1.8.0_162","sun.io.unicode.encoding":"UnicodeLittle"},"Classpath Entries":{"/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jackson-mapper-asl-1.9.13.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-yarn-api-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hk2-locator-2.6.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/nimbus-jose-jwt-4.41.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/htrace-core4-4.1.0-incubating.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/cats-kernel_2.12-2.0.0-M4.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/httpclient-4.5.6.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-collections-3.2.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-beeline-2.3.7.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/json-smart-2.3.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/curator-recipes-2.13.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/stream-2.9.6.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-graphx_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jline-2.14.6.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-admissionregistration-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/guava-14.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-sql_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/parquet-common-1.10.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/chill-java-0.9.5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-beanutils-1.9.4.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kryo-shaded-4.0.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hk2-api-2.6.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-io-2.5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/datanucleus-rdbms-4.1.19.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/metrics-graphite-4.1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/breeze_2.12-1.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jakarta.xml.bind-api-2.3.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jakarta.activation-api-1.2.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-lang3-3.10.jar":"System Classpath","spark://hostb7.dsrg.utoronto.ca:44011/jars/sparkbench-assembly-8.0-SNAPSHOT-dist.jar":"Added By User","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-mesos_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/avro-mapred-1.8.2-hadoop2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jsr305-3.0.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/HikariCP-2.5.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/ehcache-3.3.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-cli-1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-core-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/joda-time-2.10.5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/javassist-3.25.0-GA.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerby-config-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/token-provider-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/leveldbjni-all-1.8.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-sketch_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-mllib-local_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-apps-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/json4s-ast_2.12-3.7.0-M5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/json-1.8.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/datanucleus-core-4.1.17.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spire-macros_2.12-0.17.0-M1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/parquet-jackson-1.10.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-yarn-server-web-proxy-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-math3-3.4.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-vector-code-gen-2.3.7.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-network-shuffle_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/RoaringBitmap-0.9.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-mapreduce-client-jobclient-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/super-csv-2.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/pyrolite-4.30.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-yarn-common-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/slf4j-log4j12-1.7.30.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/macro-compat_2.12-1.1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-discovery-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-hive_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-metastore-2.3.7.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-streaming_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jackson-databind-2.10.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/snakeyaml-1.24.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-lang-2.6.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/libfb303-0.9.3.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/curator-framework-2.13.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerby-xdr-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/xz-1.5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-shims-common-2.3.7.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/mesos-1.4.0-shaded-protobuf.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerb-crypto-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-llap-common-2.3.7.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jersey-server-2.30.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/minlog-1.3.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jackson-module-scala_2.12-2.10.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/okhttp-2.7.5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/transaction-api-1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-exec-2.3.7-core.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-tags_2.12-3.1.2-tests.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/shims-0.9.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-jdbc-2.3.7.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/paranamer-2.8.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/woodstox-core-5.0.3.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/generex-1.0.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jackson-annotations-2.10.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spire-platform_2.12-0.17.0-M1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/chill_2.12-0.9.5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/JLargeArrays-1.5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-net-3.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-service-rpc-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/py4j-0.10.9.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/okhttp-3.12.12.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-crypto-1.1.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-core_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/zjsonpatch-0.3.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/guice-servlet-4.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/derby-10.12.1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-pool-1.5.4.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/parquet-encoding-1.10.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/scala-xml_2.12-1.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jersey-client-2.30.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerby-pkix-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerb-common-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-annotations-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-shims-2.3.7.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-compiler-3.0.16.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-kvstore_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-repl_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jcip-annotations-1.0-1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/avro-1.8.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/osgi-resource-locator-1.0.3.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/javolution-5.5.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/re2j-1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/orc-core-1.5.12.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/javax.jdo-3.2.0-m3.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/bonecp-0.8.0.RELEASE.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jakarta.inject-2.6.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/objenesis-2.6.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jersey-hk2-2.30.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spire-util_2.12-0.17.0-M1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/datanucleus-api-jdo-4.2.4.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-settings-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/aopalliance-1.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-kubernetes_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/httpcore-4.4.12.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/janino-3.0.16.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/scala-reflect-2.12.10.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/aopalliance-repackaged-2.6.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/libthrift-0.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/json4s-jackson_2.12-3.7.0-M5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-policy-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jackson-module-paranamer-2.10.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jaxb-runtime-2.3.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/orc-mapreduce-1.5.12.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-storageclass-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jpam-1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/gson-2.2.4.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/automaton-1.11-8.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/curator-client-2.13.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-codec-1.10.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-httpclient-3.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/flatbuffers-java-1.9.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-apiextensions-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/zookeeper-3.4.14.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/netty-all-4.1.51.Final.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-yarn-client-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jersey-container-servlet-2.30.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jackson-module-jaxb-annotations-2.10.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jakarta.ws.rs-api-2.1.6.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/xbean-asm7-shaded-4.15.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-auth-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-batch-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/arrow-vector-2.0.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jakarta.servlet-api-4.0.3.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-unsafe_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-client-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-cli-2.3.7.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-mllib_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/json4s-core_2.12-3.7.0-M5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/scala-parser-combinators_2.12-1.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/arrow-memory-core-2.0.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jersey-common-2.30.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerby-util-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jdo-api-3.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-metrics-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-storage-api-2.7.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jackson-jaxrs-base-2.9.5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/ivy-2.4.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-extensions-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-networking-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-events-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hk2-utils-2.6.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-common-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-rbac-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/protobuf-java-2.5.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/scala-compiler-2.12.10.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/lz4-java-1.7.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/threeten-extra-1.5.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/stax-api-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-client-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/parquet-hadoop-1.10.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/breeze-macros_2.12-1.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/ST4-4.0.4.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-serde-2.3.7.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/hadoop-3.3.1/etc/hadoop/":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/snappy-java-1.1.8.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/core-1.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerb-identity-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/orc-shims-1.5.12.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-configuration2-2.1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jaxb-api-2.2.11.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/arrow-format-2.0.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jsp-api-2.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/activation-1.1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jakarta.annotation-api-1.3.5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/oro-2.0.8.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/shapeless_2.12-2.3.3.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jackson-core-2.10.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-daemon-1.0.13.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-launcher_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/opencsv-2.3.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-tags_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-text-1.6.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/arrow-memory-netty-2.0.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerb-util-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerby-asn1-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jackson-core-asl-1.9.13.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-coordination-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jackson-dataformat-yaml-2.10.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerb-admin-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/avro-ipc-1.8.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-mapreduce-client-common-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/antlr4-runtime-4.8-1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jackson-jaxrs-json-provider-2.9.5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-common-2.3.7.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/arpack_combined_all-0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/geronimo-jcache_1.0_spec-1.0-alpha-1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/machinist_2.12-0.6.8.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/aircompressor-0.10.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/univocity-parsers-2.9.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerb-server-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-dbcp-1.4.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/stax2-api-3.1.4.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/istack-commons-runtime-3.0.8.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-yarn-server-common-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerb-simplekdc-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/antlr-runtime-3.5.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/JTransforms-3.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/guice-4.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-yarn_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/slf4j-api-1.7.30.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jackson-datatype-jsr310-2.11.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/parquet-format-2.4.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-yarn-registry-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/velocity-1.5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/javax.inject-1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-scheduling-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-autoscaling-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerb-client-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spire_2.12-0.17.0-M1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/log4j-1.2.17.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-network-common_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/logging-interceptor-3.12.12.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/dnsjava-2.1.7.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/audience-annotations-0.5.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/metrics-jmx-4.1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jta-1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-shims-scheduler-2.3.7.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/json4s-scalap_2.12-3.7.0-M5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/conf/":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/metrics-json-4.1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jersey-container-servlet-core-2.30.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jodd-core-3.5.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/compress-lzf-1.0.3.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/metrics-jvm-4.1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/scala-collection-compat_2.12-2.1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/parquet-column-1.10.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/metrics-core-4.1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/scala-library-2.12.10.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-common-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-hive-thriftserver_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/zstd-jni-1.4.8-1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jakarta.validation-api-2.0.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerb-core-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/okio-1.14.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-logging-1.1.3.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jcl-over-slf4j-1.7.30.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jersey-media-jaxb-2.30.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/algebra_2.12-2.0.0-M2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-compress-1.20.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-hdfs-client-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-shims-0.23-2.3.7.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jul-to-slf4j-1.7.30.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-certificates-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-mapreduce-client-core-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-catalyst_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/accessors-smart-1.2.jar":"System Classpath"}} -{"Event":"SparkListenerApplicationStart","App Name":"ScalaJoin","App ID":"app-20211009101924-2782","Timestamp":1633789163045,"User":"usera112"} -{"Event":"SparkListenerExecutorAdded","Timestamp":1633789166140,"Executor ID":"5","Executor Info":{"Host":"10.1.0.19","Total Cores":16,"Log Urls":{"stdout":"http://10.1.0.19:8081/logPage/?appId=app-20211009101924-2782&executorId=5&logType=stdout","stderr":"http://10.1.0.19:8081/logPage/?appId=app-20211009101924-2782&executorId=5&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0}} -{"Event":"SparkListenerExecutorAdded","Timestamp":1633789166162,"Executor ID":"10","Executor Info":{"Host":"10.1.0.16","Total Cores":16,"Log Urls":{"stdout":"http://10.1.0.16:8081/logPage/?appId=app-20211009101924-2782&executorId=10&logType=stdout","stderr":"http://10.1.0.16:8081/logPage/?appId=app-20211009101924-2782&executorId=10&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0}} -{"Event":"SparkListenerExecutorAdded","Timestamp":1633789166196,"Executor ID":"11","Executor Info":{"Host":"10.1.0.13","Total Cores":16,"Log Urls":{"stdout":"http://10.1.0.13:8081/logPage/?appId=app-20211009101924-2782&executorId=11&logType=stdout","stderr":"http://10.1.0.13:8081/logPage/?appId=app-20211009101924-2782&executorId=11&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0}} -{"Event":"SparkListenerExecutorAdded","Timestamp":1633789166205,"Executor ID":"1","Executor Info":{"Host":"10.1.0.17","Total Cores":16,"Log Urls":{"stdout":"http://10.1.0.17:8081/logPage/?appId=app-20211009101924-2782&executorId=1&logType=stdout","stderr":"http://10.1.0.17:8081/logPage/?appId=app-20211009101924-2782&executorId=1&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0}} -{"Event":"SparkListenerExecutorAdded","Timestamp":1633789166218,"Executor ID":"4","Executor Info":{"Host":"10.1.0.8","Total Cores":16,"Log Urls":{"stdout":"http://10.1.0.8:8081/logPage/?appId=app-20211009101924-2782&executorId=4&logType=stdout","stderr":"http://10.1.0.8:8081/logPage/?appId=app-20211009101924-2782&executorId=4&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0}} -{"Event":"SparkListenerExecutorAdded","Timestamp":1633789166224,"Executor ID":"6","Executor Info":{"Host":"10.1.0.18","Total Cores":16,"Log Urls":{"stdout":"http://10.1.0.18:8081/logPage/?appId=app-20211009101924-2782&executorId=6&logType=stdout","stderr":"http://10.1.0.18:8081/logPage/?appId=app-20211009101924-2782&executorId=6&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0}} -{"Event":"SparkListenerExecutorAdded","Timestamp":1633789166231,"Executor ID":"0","Executor Info":{"Host":"10.1.0.12","Total Cores":16,"Log Urls":{"stdout":"http://10.1.0.12:8081/logPage/?appId=app-20211009101924-2782&executorId=0&logType=stdout","stderr":"http://10.1.0.12:8081/logPage/?appId=app-20211009101924-2782&executorId=0&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0}} -{"Event":"SparkListenerExecutorAdded","Timestamp":1633789166233,"Executor ID":"2","Executor Info":{"Host":"10.1.0.10","Total Cores":16,"Log Urls":{"stdout":"http://10.1.0.10:8081/logPage/?appId=app-20211009101924-2782&executorId=2&logType=stdout","stderr":"http://10.1.0.10:8081/logPage/?appId=app-20211009101924-2782&executorId=2&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0}} -{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"5","Host":"10.1.0.19","Port":33541},"Maximum Memory":2101975449,"Timestamp":1633789166236,"Maximum Onheap Memory":2101975449,"Maximum Offheap Memory":0} -{"Event":"SparkListenerExecutorAdded","Timestamp":1633789166249,"Executor ID":"8","Executor Info":{"Host":"10.1.0.11","Total Cores":16,"Log Urls":{"stdout":"http://10.1.0.11:8081/logPage/?appId=app-20211009101924-2782&executorId=8&logType=stdout","stderr":"http://10.1.0.11:8081/logPage/?appId=app-20211009101924-2782&executorId=8&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0}} -{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"10","Host":"10.1.0.16","Port":32829},"Maximum Memory":2101975449,"Timestamp":1633789166253,"Maximum Onheap Memory":2101975449,"Maximum Offheap Memory":0} -{"Event":"SparkListenerExecutorAdded","Timestamp":1633789166258,"Executor ID":"9","Executor Info":{"Host":"10.1.0.14","Total Cores":16,"Log Urls":{"stdout":"http://10.1.0.14:8081/logPage/?appId=app-20211009101924-2782&executorId=9&logType=stdout","stderr":"http://10.1.0.14:8081/logPage/?appId=app-20211009101924-2782&executorId=9&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0}} -{"Event":"SparkListenerExecutorAdded","Timestamp":1633789166270,"Executor ID":"3","Executor Info":{"Host":"10.1.0.15","Total Cores":16,"Log Urls":{"stdout":"http://10.1.0.15:8081/logPage/?appId=app-20211009101924-2782&executorId=3&logType=stdout","stderr":"http://10.1.0.15:8081/logPage/?appId=app-20211009101924-2782&executorId=3&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0}} -{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"11","Host":"10.1.0.13","Port":37477},"Maximum Memory":2101975449,"Timestamp":1633789166302,"Maximum Onheap Memory":2101975449,"Maximum Offheap Memory":0} -{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"1","Host":"10.1.0.17","Port":36603},"Maximum Memory":2101975449,"Timestamp":1633789166304,"Maximum Onheap Memory":2101975449,"Maximum Offheap Memory":0} -{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"4","Host":"10.1.0.8","Port":42035},"Maximum Memory":2101975449,"Timestamp":1633789166315,"Maximum Onheap Memory":2101975449,"Maximum Offheap Memory":0} -{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"6","Host":"10.1.0.18","Port":42233},"Maximum Memory":2101975449,"Timestamp":1633789166325,"Maximum Onheap Memory":2101975449,"Maximum Offheap Memory":0} -{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"0","Host":"10.1.0.12","Port":44023},"Maximum Memory":2101975449,"Timestamp":1633789166327,"Maximum Onheap Memory":2101975449,"Maximum Offheap Memory":0} -{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"2","Host":"10.1.0.10","Port":34715},"Maximum Memory":2101975449,"Timestamp":1633789166331,"Maximum Onheap Memory":2101975449,"Maximum Offheap Memory":0} -{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"8","Host":"10.1.0.11","Port":41045},"Maximum Memory":2101975449,"Timestamp":1633789166342,"Maximum Onheap Memory":2101975449,"Maximum Offheap Memory":0} -{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"9","Host":"10.1.0.14","Port":35127},"Maximum Memory":2101975449,"Timestamp":1633789166357,"Maximum Onheap Memory":2101975449,"Maximum Offheap Memory":0} -{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"3","Host":"10.1.0.15","Port":34131},"Maximum Memory":2101975449,"Timestamp":1633789166373,"Maximum Onheap Memory":2101975449,"Maximum Offheap Memory":0} -{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":0,"description":"sql at ScalaSparkSQLBench.scala:43","details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nSetCatalogAndNamespace (1)\n\n\n(1) SetCatalogAndNamespace\nArguments: org.apache.spark.sql.connector.catalog.CatalogManager@5a6f6cac, spark_catalog, ArrayBuffer(DEFAULT)\n\n","sparkPlanInfo":{"nodeName":"SetCatalogAndNamespace","simpleString":"SetCatalogAndNamespace org.apache.spark.sql.connector.catalog.CatalogManager@5a6f6cac, spark_catalog, ArrayBuffer(DEFAULT)","children":[],"metadata":{},"metrics":[]},"time":1633789168029} -{"Event":"SparkListenerExecutorAdded","Timestamp":1633789171242,"Executor ID":"7","Executor Info":{"Host":"10.1.0.9","Total Cores":16,"Log Urls":{"stdout":"http://10.1.0.9:8081/logPage/?appId=app-20211009101924-2782&executorId=7&logType=stdout","stderr":"http://10.1.0.9:8081/logPage/?appId=app-20211009101924-2782&executorId=7&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0}} -{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"7","Host":"10.1.0.9","Port":45863},"Maximum Memory":2101975449,"Timestamp":1633789171347,"Maximum Onheap Memory":2101975449,"Maximum Offheap Memory":0} -{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":0,"time":1633789190680} -{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":1,"description":"sql at ScalaSparkSQLBench.scala:43","details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nExecute SetCommand (1)\n +- SetCommand (2)\n\n\n(1) Execute SetCommand\nOutput [2]: [key#8, value#9]\n\n(2) SetCommand\nArguments: (hive.input.format,Some(org.apache.hadoop.hive.ql.io.HiveInputFormat))\n\n","sparkPlanInfo":{"nodeName":"Execute SetCommand","simpleString":"Execute SetCommand","children":[],"metadata":{},"metrics":[]},"time":1633789190733} -{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":1,"time":1633789190738} -{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":2,"description":"sql at ScalaSparkSQLBench.scala:43","details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nExecute SetCommand (1)\n +- SetCommand (2)\n\n\n(1) Execute SetCommand\nOutput [2]: [key#18, value#19]\n\n(2) SetCommand\nArguments: (mapreduce.job.maps,Some(8))\n\n","sparkPlanInfo":{"nodeName":"Execute SetCommand","simpleString":"Execute SetCommand","children":[],"metadata":{},"metrics":[]},"time":1633789190754} -{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":2,"time":1633789190754} -{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":3,"description":"sql at ScalaSparkSQLBench.scala:43","details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nExecute SetCommand (1)\n +- SetCommand (2)\n\n\n(1) Execute SetCommand\nOutput [2]: [key#28, value#29]\n\n(2) SetCommand\nArguments: (mapreduce.job.reduces,Some(8))\n\n","sparkPlanInfo":{"nodeName":"Execute SetCommand","simpleString":"Execute SetCommand","children":[],"metadata":{},"metrics":[]},"time":1633789190767} -{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":3,"time":1633789190767} -{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":4,"description":"sql at ScalaSparkSQLBench.scala:43","details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nExecute SetCommand (1)\n +- SetCommand (2)\n\n\n(1) Execute SetCommand\nOutput [2]: [key#38, value#39]\n\n(2) SetCommand\nArguments: (hive.stats.autogather,Some(false))\n\n","sparkPlanInfo":{"nodeName":"Execute SetCommand","simpleString":"Execute SetCommand","children":[],"metadata":{},"metrics":[]},"time":1633789190779} -{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":4,"time":1633789190780} -{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":5,"description":"sql at ScalaSparkSQLBench.scala:43","details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nLocalTableScan (1)\n\n\n(1) LocalTableScan\nOutput: []\nArguments: \n\n","sparkPlanInfo":{"nodeName":"LocalTableScan","simpleString":"LocalTableScan ","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":0,"metricType":"sum"}]},"time":1633789190834} -{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":5,"time":1633789190835} -{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":6,"description":"sql at ScalaSparkSQLBench.scala:43","details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nExecute CreateTableCommand (1)\n +- CreateTableCommand (2)\n\n\n(1) Execute CreateTableCommand\nOutput: []\n\n(2) CreateTableCommand\nArguments: `default`.`rankings`, org.apache.hadoop.hive.serde2.OpenCSVSerde, false\n\n","sparkPlanInfo":{"nodeName":"Execute CreateTableCommand","simpleString":"Execute CreateTableCommand","children":[],"metadata":{},"metrics":[]},"time":1633789190897} -{"Event":"org.apache.spark.sql.catalyst.catalog.CreateTablePreEvent","database":"default","name":"rankings"} -{"Event":"org.apache.spark.sql.catalyst.catalog.CreateTableEvent","database":"default","name":"rankings"} -{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":6,"time":1633789191681} -{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":7,"description":"sql at ScalaSparkSQLBench.scala:43","details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nLocalTableScan (1)\n\n\n(1) LocalTableScan\nOutput: []\nArguments: \n\n","sparkPlanInfo":{"nodeName":"LocalTableScan","simpleString":"LocalTableScan ","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1,"metricType":"sum"}]},"time":1633789191704} -{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":7,"time":1633789191704} -{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":8,"description":"sql at ScalaSparkSQLBench.scala:43","details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nExecute CreateTableCommand (1)\n +- CreateTableCommand (2)\n\n\n(1) Execute CreateTableCommand\nOutput: []\n\n(2) CreateTableCommand\nArguments: `default`.`uservisits_copy`, org.apache.hadoop.hive.serde2.OpenCSVSerde, false\n\n","sparkPlanInfo":{"nodeName":"Execute CreateTableCommand","simpleString":"Execute CreateTableCommand","children":[],"metadata":{},"metrics":[]},"time":1633789191720} -{"Event":"org.apache.spark.sql.catalyst.catalog.CreateTablePreEvent","database":"default","name":"uservisits_copy"} -{"Event":"org.apache.spark.sql.catalyst.catalog.CreateTableEvent","database":"default","name":"uservisits_copy"} -{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":8,"time":1633789191791} -{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":9,"description":"sql at ScalaSparkSQLBench.scala:43","details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nLocalTableScan (1)\n\n\n(1) LocalTableScan\nOutput: []\nArguments: \n\n","sparkPlanInfo":{"nodeName":"LocalTableScan","simpleString":"LocalTableScan ","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2,"metricType":"sum"}]},"time":1633789191803} -{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":9,"time":1633789191803} -{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":10,"description":"sql at ScalaSparkSQLBench.scala:43","details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nExecute CreateTableCommand (1)\n +- CreateTableCommand (2)\n\n\n(1) Execute CreateTableCommand\nOutput: []\n\n(2) CreateTableCommand\nArguments: `default`.`rankings_uservisits_join`, org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, false\n\n","sparkPlanInfo":{"nodeName":"Execute CreateTableCommand","simpleString":"Execute CreateTableCommand","children":[],"metadata":{},"metrics":[]},"time":1633789191813} -{"Event":"org.apache.spark.sql.catalyst.catalog.CreateTablePreEvent","database":"default","name":"rankings_uservisits_join"} -{"Event":"org.apache.spark.sql.catalyst.catalog.CreateTableEvent","database":"default","name":"rankings_uservisits_join"} -{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":10,"time":1633789191990} -{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":11,"description":"sql at ScalaSparkSQLBench.scala:43","details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nExecute InsertIntoHiveTable (15)\n+- * Project (14)\n +- * Sort (13)\n +- Exchange (12)\n +- * HashAggregate (11)\n +- Exchange (10)\n +- * HashAggregate (9)\n +- * Project (8)\n +- * BroadcastHashJoin Inner BuildLeft (7)\n :- BroadcastExchange (3)\n : +- * Filter (2)\n : +- Scan hive default.rankings (1)\n +- * Project (6)\n +- * Filter (5)\n +- Scan hive default.uservisits_copy (4)\n\n\n(1) Scan hive default.rankings\nOutput [2]: [pageurl#41, pagerank#42]\nArguments: [pageurl#41, pagerank#42], HiveTableRelation [`default`.`rankings`, org.apache.hadoop.hive.serde2.OpenCSVSerde, Data Cols: [pageurl#41, pagerank#42, avgduration#43], Partition Cols: []]\n\n(2) Filter [codegen id : 1]\nInput [2]: [pageurl#41, pagerank#42]\nCondition : isnotnull(pageURL#41)\n\n(3) BroadcastExchange\nInput [2]: [pageurl#41, pagerank#42]\nArguments: HashedRelationBroadcastMode(List(input[0, string, false]),false), [id=#88]\n\n(4) Scan hive default.uservisits_copy\nOutput [4]: [adrevenue#47, desturl#45, sourceip#44, visitdate#46]\nArguments: [adRevenue#47, destURL#45, sourceIP#44, visitDate#46], HiveTableRelation [`default`.`uservisits_copy`, org.apache.hadoop.hive.serde2.OpenCSVSerde, Data Cols: [sourceip#44, desturl#45, visitdate#46, adrevenue#47, useragent#48, countrycode#49, languagecode#..., Partition Cols: []]\n\n(5) Filter\nInput [4]: [adrevenue#47, desturl#45, sourceip#44, visitdate#46]\nCondition : (((isnotnull(visitDate#46) AND (datediff(cast(visitDate#46 as date), 10592) >= 0)) AND (datediff(cast(visitDate#46 as date), 10957) <= 0)) AND isnotnull(destURL#45))\n\n(6) Project\nOutput [3]: [sourceIP#44, destURL#45, adRevenue#47]\nInput [4]: [adrevenue#47, desturl#45, sourceip#44, visitdate#46]\n\n(7) BroadcastHashJoin [codegen id : 2]\nLeft keys [1]: [pageURL#41]\nRight keys [1]: [destURL#45]\nJoin condition: None\n\n(8) Project [codegen id : 2]\nOutput [3]: [pagerank#42, sourceIP#44, adRevenue#47]\nInput [5]: [pageurl#41, pagerank#42, sourceIP#44, destURL#45, adRevenue#47]\n\n(9) HashAggregate [codegen id : 2]\nInput [3]: [pagerank#42, sourceIP#44, adRevenue#47]\nKeys [1]: [sourceIP#44]\nFunctions [2]: [partial_avg(cast(pageRank#42 as double)), partial_sum(cast(adRevenue#47 as double))]\nAggregate Attributes [3]: [sum#62, count#63L, sum#64]\nResults [4]: [sourceIP#44, sum#65, count#66L, sum#67]\n\n(10) Exchange\nInput [4]: [sourceIP#44, sum#65, count#66L, sum#67]\nArguments: hashpartitioning(sourceIP#44, 8), ENSURE_REQUIREMENTS, [id=#97]\n\n(11) HashAggregate [codegen id : 3]\nInput [4]: [sourceIP#44, sum#65, count#66L, sum#67]\nKeys [1]: [sourceIP#44]\nFunctions [2]: [avg(cast(pageRank#42 as double)), sum(cast(adRevenue#47 as double))]\nAggregate Attributes [2]: [avg(cast(pageRank#42 as double))#53, sum(cast(adRevenue#47 as double))#54]\nResults [3]: [sourceIP#44, avg(cast(pageRank#42 as double))#53 AS avg(CAST(pageRank AS DOUBLE))#55, sum(cast(adRevenue#47 as double))#54 AS totalRevenue#40]\n\n(12) Exchange\nInput [3]: [sourceIP#44, avg(CAST(pageRank AS DOUBLE))#55, totalRevenue#40]\nArguments: rangepartitioning(totalRevenue#40 DESC NULLS LAST, 8), ENSURE_REQUIREMENTS, [id=#101]\n\n(13) Sort [codegen id : 4]\nInput [3]: [sourceIP#44, avg(CAST(pageRank AS DOUBLE))#55, totalRevenue#40]\nArguments: [totalRevenue#40 DESC NULLS LAST], true, 0\n\n(14) Project [codegen id : 4]\nOutput [3]: [ansi_cast(sourceIP#44 as string) AS sourceIP#60, ansi_cast(avg(CAST(pageRank AS DOUBLE))#55 as double) AS avgPageRank#61, totalRevenue#40]\nInput [3]: [sourceIP#44, avg(CAST(pageRank AS DOUBLE))#55, totalRevenue#40]\n\n(15) Execute InsertIntoHiveTable\nInput [3]: [sourceIP#60, avgPageRank#61, totalRevenue#40]\nArguments: `default`.`rankings_uservisits_join`, org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, true, false, [sourceIP, avgPageRank, totalRevenue]\n\n","sparkPlanInfo":{"nodeName":"Execute InsertIntoHiveTable","simpleString":"Execute InsertIntoHiveTable `default`.`rankings_uservisits_join`, org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, true, false, [sourceIP, avgPageRank, totalRevenue]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"Project","simpleString":"Project [ansi_cast(sourceIP#44 as string) AS sourceIP#60, ansi_cast(avg(CAST(pageRank AS DOUBLE))#55 as double) AS avgPageRank#61, totalRevenue#40]","children":[{"nodeName":"Sort","simpleString":"Sort [totalRevenue#40 DESC NULLS LAST], true, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Exchange","simpleString":"Exchange rangepartitioning(totalRevenue#40 DESC NULLS LAST, 8), ENSURE_REQUIREMENTS, [id=#101]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[sourceIP#44], functions=[avg(cast(pageRank#42 as double)), sum(cast(adRevenue#47 as double))])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(sourceIP#44, 8), ENSURE_REQUIREMENTS, [id=#97]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[sourceIP#44], functions=[partial_avg(cast(pageRank#42 as double)), partial_sum(cast(adRevenue#47 as double))])","children":[{"nodeName":"Project","simpleString":"Project [pagerank#42, sourceIP#44, adRevenue#47]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [pageURL#41], [destURL#45], Inner, BuildLeft, false","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, false]),false), [id=#88]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Filter","simpleString":"Filter isnotnull(pageURL#41)","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan hive default.rankings","simpleString":"Scan hive default.rankings [pageurl#41, pagerank#42], HiveTableRelation [`default`.`rankings`, org.apache.hadoop.hive.serde2.OpenCSVSerde, Data Cols: [pageurl#41, pagerank#42, avgduration#43], Partition Cols: []]","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":75,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":74,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":73,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":72,"metricType":"timing"},{"name":"time to build","accumulatorId":71,"metricType":"timing"},{"name":"time to collect","accumulatorId":70,"metricType":"timing"},{"name":"number of output rows","accumulatorId":69,"metricType":"sum"},{"name":"data size","accumulatorId":68,"metricType":"size"}]}],"metadata":{},"metrics":[]},{"nodeName":"Project","simpleString":"Project [sourceIP#44, destURL#45, adRevenue#47]","children":[{"nodeName":"Filter","simpleString":"Filter (((isnotnull(visitDate#46) AND (datediff(cast(visitDate#46 as date), 10592) >= 0)) AND (datediff(cast(visitDate#46 as date), 10957) <= 0)) AND isnotnull(destURL#45))","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan hive default.uservisits_copy","simpleString":"Scan hive default.uservisits_copy [adRevenue#47, destURL#45, sourceIP#44, visitDate#46], HiveTableRelation [`default`.`uservisits_copy`, org.apache.hadoop.hive.serde2.OpenCSVSerde, Data Cols: [sourceip#44, desturl#45, visitdate#46, adrevenue#47, useragent#48, countrycode#49, languagecode#..., Partition Cols: []]","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":77,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":76,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":67,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":64,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":65,"metricType":"timing"},{"name":"peak memory","accumulatorId":63,"metricType":"size"},{"name":"number of output rows","accumulatorId":62,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":66,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":61,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":34,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":35,"metricType":"nsTiming"},{"name":"records read","accumulatorId":32,"metricType":"sum"},{"name":"local bytes read","accumulatorId":30,"metricType":"size"},{"name":"fetch wait time","accumulatorId":31,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":28,"metricType":"size"},{"name":"local blocks read","accumulatorId":27,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":26,"metricType":"sum"},{"name":"data size","accumulatorId":25,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":29,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":33,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":58,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":59,"metricType":"timing"},{"name":"peak memory","accumulatorId":57,"metricType":"size"},{"name":"number of output rows","accumulatorId":56,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":60,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":55,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":45,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":46,"metricType":"nsTiming"},{"name":"records read","accumulatorId":43,"metricType":"sum"},{"name":"local bytes read","accumulatorId":41,"metricType":"size"},{"name":"fetch wait time","accumulatorId":42,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":39,"metricType":"size"},{"name":"local blocks read","accumulatorId":38,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":37,"metricType":"sum"},{"name":"data size","accumulatorId":36,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":40,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":44,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":52,"metricType":"timing"},{"name":"peak memory","accumulatorId":53,"metricType":"size"},{"name":"spill size","accumulatorId":54,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":51,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"number of written files","accumulatorId":47,"metricType":"sum"},{"name":"written output","accumulatorId":48,"metricType":"size"},{"name":"number of output rows","accumulatorId":49,"metricType":"sum"},{"name":"number of dynamic part","accumulatorId":50,"metricType":"sum"}]},"time":1633789193359} -{"Event":"SparkListenerJobStart","Job ID":0,"Submission Time":1633789194271,"Stage Infos":[{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Number of Tasks":8,"RDD Info":[{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"2\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"Scan hive default.rankings\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"Scan hive default.rankings\"}","Callsite":"","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"HadoopRDD","Scope":"{\"id\":\"7\",\"name\":\"Scan hive default.rankings\"}","Callsite":"","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"Scan hive default.rankings\"}","Callsite":"","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:185)\njava.util.concurrent.FutureTask.run(FutureTask.java:266)\njava.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\njava.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[0],"Properties":{"spark.sql.warehouse.dir":"file:/home/usera112/projects/log-compressor/deployments/HiBench/spark-warehouse","spark.driver.host":"hostb7.dsrg.utoronto.ca","spark.eventLog.enabled":"true","spark.job.interruptOnCancel":"true","spark.driver.port":"44011","spark.jars":"file:/home/usera112/projects/log-compressor/deployments/HiBench/sparkbench/assembly/target/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.app.name":"ScalaJoin","spark.rdd.scope":"{\"id\":\"0\",\"name\":\"WholeStageCodegen (4)\"}","spark.driver.memory":"4g","spark.rdd.scope.noOverride":"true","spark.submit.pyFiles":"","spark.default.parallelism":"8","spark.job.description":"broadcast exchange (runId d47509dc-011f-41d9-9b59-eb259b8dacbe)","spark.app.startTime":"1633789163045","spark.executor.id":"driver","spark.jobGroup.id":"d47509dc-011f-41d9-9b59-eb259b8dacbe","spark.app.initial.jar.urls":"spark://hostb7.dsrg.utoronto.ca:44011/jars/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.submit.deployMode":"client","spark.master":"spark://hostb7:7077","spark.executor.memory":"4g","spark.eventLog.dir":"hdfs://hostb7:8020/directory","spark.sql.execution.id":"11","spark.sql.catalogImplementation":"hive","spark.app.id":"app-20211009101924-2782","spark.sql.shuffle.partitions":"8"}} -{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Number of Tasks":8,"RDD Info":[{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"2\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"Scan hive default.rankings\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"Scan hive default.rankings\"}","Callsite":"","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"HadoopRDD","Scope":"{\"id\":\"7\",\"name\":\"Scan hive default.rankings\"}","Callsite":"","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"Scan hive default.rankings\"}","Callsite":"","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:185)\njava.util.concurrent.FutureTask.run(FutureTask.java:266)\njava.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\njava.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1633789194289,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.sql.warehouse.dir":"file:/home/usera112/projects/log-compressor/deployments/HiBench/spark-warehouse","spark.driver.host":"hostb7.dsrg.utoronto.ca","spark.eventLog.enabled":"true","spark.job.interruptOnCancel":"true","spark.driver.port":"44011","spark.jars":"file:/home/usera112/projects/log-compressor/deployments/HiBench/sparkbench/assembly/target/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.app.name":"ScalaJoin","spark.rdd.scope":"{\"id\":\"0\",\"name\":\"WholeStageCodegen (4)\"}","spark.driver.memory":"4g","spark.rdd.scope.noOverride":"true","spark.submit.pyFiles":"","spark.default.parallelism":"8","spark.job.description":"broadcast exchange (runId d47509dc-011f-41d9-9b59-eb259b8dacbe)","spark.app.startTime":"1633789163045","spark.executor.id":"driver","spark.jobGroup.id":"d47509dc-011f-41d9-9b59-eb259b8dacbe","spark.app.initial.jar.urls":"spark://hostb7.dsrg.utoronto.ca:44011/jars/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.submit.deployMode":"client","spark.master":"spark://hostb7:7077","spark.executor.memory":"4g","spark.eventLog.dir":"hdfs://hostb7:8020/directory","spark.sql.execution.id":"11","spark.sql.catalogImplementation":"hive","spark.app.id":"app-20211009101924-2782","spark.sql.shuffle.partitions":"8"}} -{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Launch Time":1633789194341,"Executor ID":"5","Host":"10.1.0.19","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":1,"Index":1,"Attempt":0,"Launch Time":1633789194357,"Executor ID":"7","Host":"10.1.0.9","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":2,"Index":2,"Attempt":0,"Launch Time":1633789194357,"Executor ID":"1","Host":"10.1.0.17","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":3,"Index":3,"Attempt":0,"Launch Time":1633789194358,"Executor ID":"11","Host":"10.1.0.13","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":4,"Index":4,"Attempt":0,"Launch Time":1633789194358,"Executor ID":"9","Host":"10.1.0.14","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":5,"Index":5,"Attempt":0,"Launch Time":1633789194358,"Executor ID":"4","Host":"10.1.0.8","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":6,"Index":6,"Attempt":0,"Launch Time":1633789194359,"Executor ID":"10","Host":"10.1.0.16","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":7,"Index":7,"Attempt":0,"Launch Time":1633789194360,"Executor ID":"2","Host":"10.1.0.10","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskGettingResult","Task Info":{"Task ID":2,"Index":2,"Attempt":0,"Launch Time":1633789194357,"Executor ID":"1","Host":"10.1.0.17","Locality":"ANY","Speculative":false,"Getting Result Time":1633789196623,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskGettingResult","Task Info":{"Task ID":6,"Index":6,"Attempt":0,"Launch Time":1633789194359,"Executor ID":"10","Host":"10.1.0.16","Locality":"ANY","Speculative":false,"Getting Result Time":1633789196651,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskGettingResult","Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Launch Time":1633789194341,"Executor ID":"5","Host":"10.1.0.19","Locality":"ANY","Speculative":false,"Getting Result Time":1633789196662,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":2,"Index":2,"Attempt":0,"Launch Time":1633789194357,"Executor ID":"1","Host":"10.1.0.17","Locality":"ANY","Speculative":false,"Getting Result Time":1633789196623,"Finish Time":1633789196670,"Failed":false,"Killed":false,"Accumulables":[{"ID":73,"Name":"duration","Update":"202","Value":"202","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"number of output rows","Update":"15000","Value":"15000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":75,"Name":"number of output rows","Update":"15000","Value":"15000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":78,"Name":"internal.metrics.executorDeserializeTime","Update":671,"Value":671,"Internal":true,"Count Failed Values":true},{"ID":79,"Name":"internal.metrics.executorDeserializeCpuTime","Update":602637939,"Value":602637939,"Internal":true,"Count Failed Values":true},{"ID":80,"Name":"internal.metrics.executorRunTime","Update":1532,"Value":1532,"Internal":true,"Count Failed Values":true},{"ID":81,"Name":"internal.metrics.executorCpuTime","Update":1384098580,"Value":1384098580,"Internal":true,"Count Failed Values":true},{"ID":82,"Name":"internal.metrics.resultSize","Update":1064422,"Value":1064422,"Internal":true,"Count Failed Values":true},{"ID":83,"Name":"internal.metrics.jvmGCTime","Update":146,"Value":146,"Internal":true,"Count Failed Values":true},{"ID":84,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":99,"Name":"internal.metrics.input.bytesRead","Update":654511,"Value":654511,"Internal":true,"Count Failed Values":true},{"ID":100,"Name":"internal.metrics.input.recordsRead","Update":15000,"Value":15000,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":671,"Executor Deserialize CPU Time":602637939,"Executor Run Time":1532,"Executor CPU Time":1384098580,"Peak Execution Memory":0,"Result Size":1064422,"JVM GC Time":146,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":654511,"Records Read":15000},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":6,"Index":6,"Attempt":0,"Launch Time":1633789194359,"Executor ID":"10","Host":"10.1.0.16","Locality":"ANY","Speculative":false,"Getting Result Time":1633789196651,"Finish Time":1633789196675,"Failed":false,"Killed":false,"Accumulables":[{"ID":73,"Name":"duration","Update":"191","Value":"393","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"number of output rows","Update":"15000","Value":"30000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":75,"Name":"number of output rows","Update":"15000","Value":"30000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":78,"Name":"internal.metrics.executorDeserializeTime","Update":694,"Value":1365,"Internal":true,"Count Failed Values":true},{"ID":79,"Name":"internal.metrics.executorDeserializeCpuTime","Update":623857712,"Value":1226495651,"Internal":true,"Count Failed Values":true},{"ID":80,"Name":"internal.metrics.executorRunTime","Update":1542,"Value":3074,"Internal":true,"Count Failed Values":true},{"ID":81,"Name":"internal.metrics.executorCpuTime","Update":1387426225,"Value":2771524805,"Internal":true,"Count Failed Values":true},{"ID":82,"Name":"internal.metrics.resultSize","Update":1067500,"Value":2131922,"Internal":true,"Count Failed Values":true},{"ID":83,"Name":"internal.metrics.jvmGCTime","Update":147,"Value":293,"Internal":true,"Count Failed Values":true},{"ID":84,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":99,"Name":"internal.metrics.input.bytesRead","Update":655811,"Value":1310322,"Internal":true,"Count Failed Values":true},{"ID":100,"Name":"internal.metrics.input.recordsRead","Update":15000,"Value":30000,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":694,"Executor Deserialize CPU Time":623857712,"Executor Run Time":1542,"Executor CPU Time":1387426225,"Peak Execution Memory":0,"Result Size":1067500,"JVM GC Time":147,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":655811,"Records Read":15000},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskGettingResult","Task Info":{"Task ID":4,"Index":4,"Attempt":0,"Launch Time":1633789194358,"Executor ID":"9","Host":"10.1.0.14","Locality":"ANY","Speculative":false,"Getting Result Time":1633789196690,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Launch Time":1633789194341,"Executor ID":"5","Host":"10.1.0.19","Locality":"ANY","Speculative":false,"Getting Result Time":1633789196662,"Finish Time":1633789196691,"Failed":false,"Killed":false,"Accumulables":[{"ID":73,"Name":"duration","Update":"179","Value":"572","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"number of output rows","Update":"15000","Value":"45000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":75,"Name":"number of output rows","Update":"15000","Value":"45000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":78,"Name":"internal.metrics.executorDeserializeTime","Update":684,"Value":2049,"Internal":true,"Count Failed Values":true},{"ID":79,"Name":"internal.metrics.executorDeserializeCpuTime","Update":607279304,"Value":1833774955,"Internal":true,"Count Failed Values":true},{"ID":80,"Name":"internal.metrics.executorRunTime","Update":1570,"Value":4644,"Internal":true,"Count Failed Values":true},{"ID":81,"Name":"internal.metrics.executorCpuTime","Update":1419605504,"Value":4191130309,"Internal":true,"Count Failed Values":true},{"ID":82,"Name":"internal.metrics.resultSize","Update":1063975,"Value":3195897,"Internal":true,"Count Failed Values":true},{"ID":83,"Name":"internal.metrics.jvmGCTime","Update":136,"Value":429,"Internal":true,"Count Failed Values":true},{"ID":84,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":99,"Name":"internal.metrics.input.bytesRead","Update":652408,"Value":1962730,"Internal":true,"Count Failed Values":true},{"ID":100,"Name":"internal.metrics.input.recordsRead","Update":15000,"Value":45000,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":24095592,"JVMOffHeapMemory":49273008,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":9994,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":2,"MinorGCTime":18,"MajorGCCount":2,"MajorGCTime":48},"Task Metrics":{"Executor Deserialize Time":684,"Executor Deserialize CPU Time":607279304,"Executor Run Time":1570,"Executor CPU Time":1419605504,"Peak Execution Memory":0,"Result Size":1063975,"JVM GC Time":136,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":652408,"Records Read":15000},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":4,"Index":4,"Attempt":0,"Launch Time":1633789194358,"Executor ID":"9","Host":"10.1.0.14","Locality":"ANY","Speculative":false,"Getting Result Time":1633789196690,"Finish Time":1633789196716,"Failed":false,"Killed":false,"Accumulables":[{"ID":73,"Name":"duration","Update":"199","Value":"771","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"number of output rows","Update":"15000","Value":"60000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":75,"Name":"number of output rows","Update":"15000","Value":"60000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":78,"Name":"internal.metrics.executorDeserializeTime","Update":696,"Value":2745,"Internal":true,"Count Failed Values":true},{"ID":79,"Name":"internal.metrics.executorDeserializeCpuTime","Update":616746885,"Value":2450521840,"Internal":true,"Count Failed Values":true},{"ID":80,"Name":"internal.metrics.executorRunTime","Update":1583,"Value":6227,"Internal":true,"Count Failed Values":true},{"ID":81,"Name":"internal.metrics.executorCpuTime","Update":1433872773,"Value":5625003082,"Internal":true,"Count Failed Values":true},{"ID":82,"Name":"internal.metrics.resultSize","Update":1062242,"Value":4258139,"Internal":true,"Count Failed Values":true},{"ID":83,"Name":"internal.metrics.jvmGCTime","Update":145,"Value":574,"Internal":true,"Count Failed Values":true},{"ID":84,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":6,"Internal":true,"Count Failed Values":true},{"ID":99,"Name":"internal.metrics.input.bytesRead","Update":653437,"Value":2616167,"Internal":true,"Count Failed Values":true},{"ID":100,"Name":"internal.metrics.input.recordsRead","Update":15000,"Value":60000,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":237641824,"JVMOffHeapMemory":65001768,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":497850,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":497850,"OffHeapUnifiedMemory":0,"DirectPoolMemory":10117,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":2,"MinorGCTime":18,"MajorGCCount":2,"MajorGCTime":46},"Task Metrics":{"Executor Deserialize Time":696,"Executor Deserialize CPU Time":616746885,"Executor Run Time":1583,"Executor CPU Time":1433872773,"Peak Execution Memory":0,"Result Size":1062242,"JVM GC Time":145,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":653437,"Records Read":15000},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskGettingResult","Task Info":{"Task ID":3,"Index":3,"Attempt":0,"Launch Time":1633789194358,"Executor ID":"11","Host":"10.1.0.13","Locality":"ANY","Speculative":false,"Getting Result Time":1633789196752,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":3,"Index":3,"Attempt":0,"Launch Time":1633789194358,"Executor ID":"11","Host":"10.1.0.13","Locality":"ANY","Speculative":false,"Getting Result Time":1633789196752,"Finish Time":1633789196763,"Failed":false,"Killed":false,"Accumulables":[{"ID":73,"Name":"duration","Update":"186","Value":"957","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"number of output rows","Update":"15000","Value":"75000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":75,"Name":"number of output rows","Update":"15000","Value":"75000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":78,"Name":"internal.metrics.executorDeserializeTime","Update":700,"Value":3445,"Internal":true,"Count Failed Values":true},{"ID":79,"Name":"internal.metrics.executorDeserializeCpuTime","Update":618580776,"Value":3069102616,"Internal":true,"Count Failed Values":true},{"ID":80,"Name":"internal.metrics.executorRunTime","Update":1644,"Value":7871,"Internal":true,"Count Failed Values":true},{"ID":81,"Name":"internal.metrics.executorCpuTime","Update":1494597853,"Value":7119600935,"Internal":true,"Count Failed Values":true},{"ID":82,"Name":"internal.metrics.resultSize","Update":1057603,"Value":5315742,"Internal":true,"Count Failed Values":true},{"ID":83,"Name":"internal.metrics.jvmGCTime","Update":148,"Value":722,"Internal":true,"Count Failed Values":true},{"ID":99,"Name":"internal.metrics.input.bytesRead","Update":650282,"Value":3266449,"Internal":true,"Count Failed Values":true},{"ID":100,"Name":"internal.metrics.input.recordsRead","Update":15000,"Value":75000,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":285876616,"JVMOffHeapMemory":65073704,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":497850,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":497850,"OffHeapUnifiedMemory":0,"DirectPoolMemory":39200,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":2,"MinorGCTime":19,"MajorGCCount":2,"MajorGCTime":47},"Task Metrics":{"Executor Deserialize Time":700,"Executor Deserialize CPU Time":618580776,"Executor Run Time":1644,"Executor CPU Time":1494597853,"Peak Execution Memory":0,"Result Size":1057603,"JVM GC Time":148,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":650282,"Records Read":15000},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskGettingResult","Task Info":{"Task ID":7,"Index":7,"Attempt":0,"Launch Time":1633789194360,"Executor ID":"2","Host":"10.1.0.10","Locality":"ANY","Speculative":false,"Getting Result Time":1633789196773,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":7,"Index":7,"Attempt":0,"Launch Time":1633789194360,"Executor ID":"2","Host":"10.1.0.10","Locality":"ANY","Speculative":false,"Getting Result Time":1633789196773,"Finish Time":1633789196794,"Failed":false,"Killed":false,"Accumulables":[{"ID":73,"Name":"duration","Update":"200","Value":"1157","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"number of output rows","Update":"15000","Value":"90000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":75,"Name":"number of output rows","Update":"15000","Value":"90000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":78,"Name":"internal.metrics.executorDeserializeTime","Update":720,"Value":4165,"Internal":true,"Count Failed Values":true},{"ID":79,"Name":"internal.metrics.executorDeserializeCpuTime","Update":635126187,"Value":3704228803,"Internal":true,"Count Failed Values":true},{"ID":80,"Name":"internal.metrics.executorRunTime","Update":1630,"Value":9501,"Internal":true,"Count Failed Values":true},{"ID":81,"Name":"internal.metrics.executorCpuTime","Update":1486285416,"Value":8605886351,"Internal":true,"Count Failed Values":true},{"ID":82,"Name":"internal.metrics.resultSize","Update":1058522,"Value":6374264,"Internal":true,"Count Failed Values":true},{"ID":83,"Name":"internal.metrics.jvmGCTime","Update":144,"Value":866,"Internal":true,"Count Failed Values":true},{"ID":84,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":99,"Name":"internal.metrics.input.bytesRead","Update":650945,"Value":3917394,"Internal":true,"Count Failed Values":true},{"ID":100,"Name":"internal.metrics.input.recordsRead","Update":15000,"Value":90000,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":720,"Executor Deserialize CPU Time":635126187,"Executor Run Time":1630,"Executor CPU Time":1486285416,"Peak Execution Memory":0,"Result Size":1058522,"JVM GC Time":144,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":650945,"Records Read":15000},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskGettingResult","Task Info":{"Task ID":5,"Index":5,"Attempt":0,"Launch Time":1633789194358,"Executor ID":"4","Host":"10.1.0.8","Locality":"ANY","Speculative":false,"Getting Result Time":1633789196823,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":5,"Index":5,"Attempt":0,"Launch Time":1633789194358,"Executor ID":"4","Host":"10.1.0.8","Locality":"ANY","Speculative":false,"Getting Result Time":1633789196823,"Finish Time":1633789196844,"Failed":false,"Killed":false,"Accumulables":[{"ID":73,"Name":"duration","Update":"186","Value":"1343","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"number of output rows","Update":"15000","Value":"105000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":75,"Name":"number of output rows","Update":"15000","Value":"105000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":78,"Name":"internal.metrics.executorDeserializeTime","Update":744,"Value":4909,"Internal":true,"Count Failed Values":true},{"ID":79,"Name":"internal.metrics.executorDeserializeCpuTime","Update":668702485,"Value":4372931288,"Internal":true,"Count Failed Values":true},{"ID":80,"Name":"internal.metrics.executorRunTime","Update":1666,"Value":11167,"Internal":true,"Count Failed Values":true},{"ID":81,"Name":"internal.metrics.executorCpuTime","Update":1516220264,"Value":10122106615,"Internal":true,"Count Failed Values":true},{"ID":82,"Name":"internal.metrics.resultSize","Update":1059139,"Value":7433403,"Internal":true,"Count Failed Values":true},{"ID":83,"Name":"internal.metrics.jvmGCTime","Update":129,"Value":995,"Internal":true,"Count Failed Values":true},{"ID":84,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":8,"Internal":true,"Count Failed Values":true},{"ID":99,"Name":"internal.metrics.input.bytesRead","Update":651025,"Value":4568419,"Internal":true,"Count Failed Values":true},{"ID":100,"Name":"internal.metrics.input.recordsRead","Update":15000,"Value":105000,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":744,"Executor Deserialize CPU Time":668702485,"Executor Run Time":1666,"Executor CPU Time":1516220264,"Peak Execution Memory":0,"Result Size":1059139,"JVM GC Time":129,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":651025,"Records Read":15000},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskGettingResult","Task Info":{"Task ID":1,"Index":1,"Attempt":0,"Launch Time":1633789194357,"Executor ID":"7","Host":"10.1.0.9","Locality":"ANY","Speculative":false,"Getting Result Time":1633789196876,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":1,"Index":1,"Attempt":0,"Launch Time":1633789194357,"Executor ID":"7","Host":"10.1.0.9","Locality":"ANY","Speculative":false,"Getting Result Time":1633789196876,"Finish Time":1633789196886,"Failed":false,"Killed":false,"Accumulables":[{"ID":73,"Name":"duration","Update":"191","Value":"1534","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"number of output rows","Update":"15000","Value":"120000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":75,"Name":"number of output rows","Update":"15000","Value":"120000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":78,"Name":"internal.metrics.executorDeserializeTime","Update":711,"Value":5620,"Internal":true,"Count Failed Values":true},{"ID":79,"Name":"internal.metrics.executorDeserializeCpuTime","Update":627471483,"Value":5000402771,"Internal":true,"Count Failed Values":true},{"ID":80,"Name":"internal.metrics.executorRunTime","Update":1751,"Value":12918,"Internal":true,"Count Failed Values":true},{"ID":81,"Name":"internal.metrics.executorCpuTime","Update":1587465036,"Value":11709571651,"Internal":true,"Count Failed Values":true},{"ID":82,"Name":"internal.metrics.resultSize","Update":1060240,"Value":8493643,"Internal":true,"Count Failed Values":true},{"ID":83,"Name":"internal.metrics.jvmGCTime","Update":144,"Value":1139,"Internal":true,"Count Failed Values":true},{"ID":84,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":9,"Internal":true,"Count Failed Values":true},{"ID":99,"Name":"internal.metrics.input.bytesRead","Update":651866,"Value":5220285,"Internal":true,"Count Failed Values":true},{"ID":100,"Name":"internal.metrics.input.recordsRead","Update":15000,"Value":120000,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":711,"Executor Deserialize CPU Time":627471483,"Executor Run Time":1751,"Executor CPU Time":1587465036,"Peak Execution Memory":0,"Result Size":1060240,"JVM GC Time":144,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":651866,"Records Read":15000},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Number of Tasks":8,"RDD Info":[{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"2\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"Scan hive default.rankings\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"Scan hive default.rankings\"}","Callsite":"","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"HadoopRDD","Scope":"{\"id\":\"7\",\"name\":\"Scan hive default.rankings\"}","Callsite":"","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"Scan hive default.rankings\"}","Callsite":"","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:185)\njava.util.concurrent.FutureTask.run(FutureTask.java:266)\njava.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\njava.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1633789194289,"Completion Time":1633789196889,"Accumulables":[{"ID":73,"Name":"duration","Value":"1534","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"number of output rows","Value":"120000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":75,"Name":"number of output rows","Value":"120000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":78,"Name":"internal.metrics.executorDeserializeTime","Value":5620,"Internal":true,"Count Failed Values":true},{"ID":79,"Name":"internal.metrics.executorDeserializeCpuTime","Value":5000402771,"Internal":true,"Count Failed Values":true},{"ID":80,"Name":"internal.metrics.executorRunTime","Value":12918,"Internal":true,"Count Failed Values":true},{"ID":81,"Name":"internal.metrics.executorCpuTime","Value":11709571651,"Internal":true,"Count Failed Values":true},{"ID":82,"Name":"internal.metrics.resultSize","Value":8493643,"Internal":true,"Count Failed Values":true},{"ID":83,"Name":"internal.metrics.jvmGCTime","Value":1139,"Internal":true,"Count Failed Values":true},{"ID":84,"Name":"internal.metrics.resultSerializationTime","Value":9,"Internal":true,"Count Failed Values":true},{"ID":99,"Name":"internal.metrics.input.bytesRead","Value":5220285,"Internal":true,"Count Failed Values":true},{"ID":100,"Name":"internal.metrics.input.recordsRead","Value":120000,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} -{"Event":"SparkListenerJobEnd","Job ID":0,"Completion Time":1633789196894,"Job Result":{"Result":"JobSucceeded"}} -{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":11,"accumUpdates":[[72,118],[71,158],[70,3343],[69,120000],[68,29360128]]} -{"Event":"SparkListenerJobStart","Job ID":1,"Submission Time":1633789197481,"Stage Infos":[{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"sql at ScalaSparkSQLBench.scala:43","Number of Tasks":8,"RDD Info":[{"RDD ID":11,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"12\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[10],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":10,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"13\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[9],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":6,"Name":"HadoopRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":2,"Stage Attempt ID":0,"Stage Name":"sql at ScalaSparkSQLBench.scala:43","Number of Tasks":8,"RDD Info":[{"RDD ID":16,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[15],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":12,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"12\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[11],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":15,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[14],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":14,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[1],"Details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[1,2],"Properties":{"spark.sql.warehouse.dir":"file:/home/usera112/projects/log-compressor/deployments/HiBench/spark-warehouse","spark.driver.host":"hostb7.dsrg.utoronto.ca","spark.eventLog.enabled":"true","spark.driver.port":"44011","__fetch_continuous_blocks_in_batch_enabled":"true","spark.jars":"file:/home/usera112/projects/log-compressor/deployments/HiBench/sparkbench/assembly/target/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.app.name":"ScalaJoin","spark.rdd.scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","spark.driver.memory":"4g","spark.rdd.scope.noOverride":"true","spark.submit.pyFiles":"","spark.default.parallelism":"8","spark.app.startTime":"1633789163045","spark.executor.id":"driver","spark.app.initial.jar.urls":"spark://hostb7.dsrg.utoronto.ca:44011/jars/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.submit.deployMode":"client","spark.master":"spark://hostb7:7077","spark.executor.memory":"4g","spark.eventLog.dir":"hdfs://hostb7:8020/directory","spark.sql.execution.id":"11","spark.sql.catalogImplementation":"hive","spark.app.id":"app-20211009101924-2782","spark.sql.shuffle.partitions":"8"}} -{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"sql at ScalaSparkSQLBench.scala:43","Number of Tasks":8,"RDD Info":[{"RDD ID":11,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"12\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[10],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":10,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"13\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[9],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":6,"Name":"HadoopRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1633789197487,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.sql.warehouse.dir":"file:/home/usera112/projects/log-compressor/deployments/HiBench/spark-warehouse","spark.driver.host":"hostb7.dsrg.utoronto.ca","spark.eventLog.enabled":"true","spark.driver.port":"44011","__fetch_continuous_blocks_in_batch_enabled":"true","spark.jars":"file:/home/usera112/projects/log-compressor/deployments/HiBench/sparkbench/assembly/target/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.app.name":"ScalaJoin","spark.rdd.scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","spark.driver.memory":"4g","spark.rdd.scope.noOverride":"true","spark.submit.pyFiles":"","spark.default.parallelism":"8","spark.app.startTime":"1633789163045","spark.executor.id":"driver","spark.app.initial.jar.urls":"spark://hostb7.dsrg.utoronto.ca:44011/jars/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.submit.deployMode":"client","spark.master":"spark://hostb7:7077","spark.executor.memory":"4g","spark.eventLog.dir":"hdfs://hostb7:8020/directory","spark.sql.execution.id":"11","spark.sql.catalogImplementation":"hive","spark.app.id":"app-20211009101924-2782","spark.sql.shuffle.partitions":"8"}} -{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":8,"Index":0,"Attempt":0,"Launch Time":1633789197498,"Executor ID":"9","Host":"10.1.0.14","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":9,"Index":1,"Attempt":0,"Launch Time":1633789197499,"Executor ID":"1","Host":"10.1.0.17","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":10,"Index":2,"Attempt":0,"Launch Time":1633789197499,"Executor ID":"11","Host":"10.1.0.13","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":11,"Index":3,"Attempt":0,"Launch Time":1633789197500,"Executor ID":"8","Host":"10.1.0.11","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":12,"Index":4,"Attempt":0,"Launch Time":1633789197501,"Executor ID":"6","Host":"10.1.0.18","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":13,"Index":5,"Attempt":0,"Launch Time":1633789197501,"Executor ID":"2","Host":"10.1.0.10","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":14,"Index":6,"Attempt":0,"Launch Time":1633789197502,"Executor ID":"10","Host":"10.1.0.16","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":15,"Index":7,"Attempt":0,"Launch Time":1633789197502,"Executor ID":"4","Host":"10.1.0.8","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":9,"Index":1,"Attempt":0,"Launch Time":1633789197499,"Executor ID":"1","Host":"10.1.0.17","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":1633789199211,"Failed":false,"Killed":false,"Accumulables":[{"ID":25,"Name":"data size","Update":"165920","Value":"165920","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"shuffle bytes written","Update":"83472","Value":"83472","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"shuffle records written","Update":"2963","Value":"2963","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":35,"Name":"shuffle write time","Update":"8415526","Value":"8415526","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":61,"Name":"duration","Update":"1280","Value":"1280","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":62,"Name":"number of output rows","Update":"2963","Value":"2963","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":63,"Name":"peak memory","Update":"4456448","Value":"4456448","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":65,"Name":"time in aggregation build","Update":"972","Value":"972","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":66,"Name":"avg hash probe bucket list iters","Update":"10","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":67,"Name":"number of output rows","Update":"2963","Value":"2963","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":76,"Name":"number of output rows","Update":"2963","Value":"2963","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":77,"Name":"number of output rows","Update":"125061","Value":"125061","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":103,"Name":"internal.metrics.executorDeserializeTime","Update":196,"Value":196,"Internal":true,"Count Failed Values":true},{"ID":104,"Name":"internal.metrics.executorDeserializeCpuTime","Update":189573155,"Value":189573155,"Internal":true,"Count Failed Values":true},{"ID":105,"Name":"internal.metrics.executorRunTime","Update":1499,"Value":1499,"Internal":true,"Count Failed Values":true},{"ID":106,"Name":"internal.metrics.executorCpuTime","Update":1390424789,"Value":1390424789,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.resultSize","Update":2904,"Value":2904,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.jvmGCTime","Update":36,"Value":36,"Internal":true,"Count Failed Values":true},{"ID":112,"Name":"internal.metrics.peakExecutionMemory","Update":33816576,"Value":33816576,"Internal":true,"Count Failed Values":true},{"ID":121,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":83472,"Value":83472,"Internal":true,"Count Failed Values":true},{"ID":122,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":2963,"Value":2963,"Internal":true,"Count Failed Values":true},{"ID":123,"Name":"internal.metrics.shuffle.write.writeTime","Update":8415526,"Value":8415526,"Internal":true,"Count Failed Values":true},{"ID":124,"Name":"internal.metrics.input.bytesRead","Update":23125676,"Value":23125676,"Internal":true,"Count Failed Values":true},{"ID":125,"Name":"internal.metrics.input.recordsRead","Update":125061,"Value":125061,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":196,"Executor Deserialize CPU Time":189573155,"Executor Run Time":1499,"Executor CPU Time":1390424789,"Peak Execution Memory":33816576,"Result Size":2904,"JVM GC Time":36,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":83472,"Shuffle Write Time":8415526,"Shuffle Records Written":2963},"Input Metrics":{"Bytes Read":23125676,"Records Read":125061},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":14,"Index":6,"Attempt":0,"Launch Time":1633789197502,"Executor ID":"10","Host":"10.1.0.16","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":1633789199224,"Failed":false,"Killed":false,"Accumulables":[{"ID":25,"Name":"data size","Update":"169456","Value":"335376","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"shuffle bytes written","Update":"85405","Value":"168877","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"shuffle records written","Update":"3026","Value":"5989","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":35,"Name":"shuffle write time","Update":"8115759","Value":"16531285","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":61,"Name":"duration","Update":"1329","Value":"2609","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":62,"Name":"number of output rows","Update":"3026","Value":"5989","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":63,"Name":"peak memory","Update":"4456448","Value":"8912896","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":65,"Name":"time in aggregation build","Update":"1001","Value":"1973","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":66,"Name":"avg hash probe bucket list iters","Update":"10","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":67,"Name":"number of output rows","Update":"3026","Value":"5989","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":76,"Name":"number of output rows","Update":"3026","Value":"5989","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":77,"Name":"number of output rows","Update":"125342","Value":"250403","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":103,"Name":"internal.metrics.executorDeserializeTime","Update":186,"Value":382,"Internal":true,"Count Failed Values":true},{"ID":104,"Name":"internal.metrics.executorDeserializeCpuTime","Update":179340004,"Value":368913159,"Internal":true,"Count Failed Values":true},{"ID":105,"Name":"internal.metrics.executorRunTime","Update":1524,"Value":3023,"Internal":true,"Count Failed Values":true},{"ID":106,"Name":"internal.metrics.executorCpuTime","Update":1413829348,"Value":2804254137,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.resultSize","Update":2905,"Value":5809,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.jvmGCTime","Update":35,"Value":71,"Internal":true,"Count Failed Values":true},{"ID":112,"Name":"internal.metrics.peakExecutionMemory","Update":33816576,"Value":67633152,"Internal":true,"Count Failed Values":true},{"ID":121,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":85405,"Value":168877,"Internal":true,"Count Failed Values":true},{"ID":122,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":3026,"Value":5989,"Internal":true,"Count Failed Values":true},{"ID":123,"Name":"internal.metrics.shuffle.write.writeTime","Update":8115759,"Value":16531285,"Internal":true,"Count Failed Values":true},{"ID":124,"Name":"internal.metrics.input.bytesRead","Update":23247031,"Value":46372707,"Internal":true,"Count Failed Values":true},{"ID":125,"Name":"internal.metrics.input.recordsRead","Update":125342,"Value":250403,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":488649352,"JVMOffHeapMemory":84231984,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":11504640,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":11504640,"OffHeapUnifiedMemory":0,"DirectPoolMemory":5312351,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":3,"MinorGCTime":28,"MajorGCCount":3,"MajorGCTime":149},"Task Metrics":{"Executor Deserialize Time":186,"Executor Deserialize CPU Time":179340004,"Executor Run Time":1524,"Executor CPU Time":1413829348,"Peak Execution Memory":33816576,"Result Size":2905,"JVM GC Time":35,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":85405,"Shuffle Write Time":8115759,"Shuffle Records Written":3026},"Input Metrics":{"Bytes Read":23247031,"Records Read":125342},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":10,"Index":2,"Attempt":0,"Launch Time":1633789197499,"Executor ID":"11","Host":"10.1.0.13","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":1633789199230,"Failed":false,"Killed":false,"Accumulables":[{"ID":25,"Name":"data size","Update":"165872","Value":"501248","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"shuffle bytes written","Update":"83597","Value":"252474","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"shuffle records written","Update":"2962","Value":"8951","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":35,"Name":"shuffle write time","Update":"8919410","Value":"25450695","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":61,"Name":"duration","Update":"1323","Value":"3932","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":62,"Name":"number of output rows","Update":"2962","Value":"8951","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":63,"Name":"peak memory","Update":"4456448","Value":"13369344","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":65,"Name":"time in aggregation build","Update":"1001","Value":"2974","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":66,"Name":"avg hash probe bucket list iters","Update":"10","Value":"30","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":67,"Name":"number of output rows","Update":"2962","Value":"8951","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":76,"Name":"number of output rows","Update":"2962","Value":"8951","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":77,"Name":"number of output rows","Update":"125003","Value":"375406","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":103,"Name":"internal.metrics.executorDeserializeTime","Update":186,"Value":568,"Internal":true,"Count Failed Values":true},{"ID":104,"Name":"internal.metrics.executorDeserializeCpuTime","Update":179512290,"Value":548425449,"Internal":true,"Count Failed Values":true},{"ID":105,"Name":"internal.metrics.executorRunTime","Update":1530,"Value":4553,"Internal":true,"Count Failed Values":true},{"ID":106,"Name":"internal.metrics.executorCpuTime","Update":1423658983,"Value":4227913120,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.resultSize","Update":2905,"Value":8714,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.jvmGCTime","Update":37,"Value":108,"Internal":true,"Count Failed Values":true},{"ID":112,"Name":"internal.metrics.peakExecutionMemory","Update":33816576,"Value":101449728,"Internal":true,"Count Failed Values":true},{"ID":121,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":83597,"Value":252474,"Internal":true,"Count Failed Values":true},{"ID":122,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":2962,"Value":8951,"Internal":true,"Count Failed Values":true},{"ID":123,"Name":"internal.metrics.shuffle.write.writeTime","Update":8919410,"Value":25450695,"Internal":true,"Count Failed Values":true},{"ID":124,"Name":"internal.metrics.input.bytesRead","Update":23143498,"Value":69516205,"Internal":true,"Count Failed Values":true},{"ID":125,"Name":"internal.metrics.input.recordsRead","Update":125003,"Value":375406,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":186,"Executor Deserialize CPU Time":179512290,"Executor Run Time":1530,"Executor CPU Time":1423658983,"Peak Execution Memory":33816576,"Result Size":2905,"JVM GC Time":37,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":83597,"Shuffle Write Time":8919410,"Shuffle Records Written":2962},"Input Metrics":{"Bytes Read":23143498,"Records Read":125003},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":13,"Index":5,"Attempt":0,"Launch Time":1633789197501,"Executor ID":"2","Host":"10.1.0.10","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":1633789199236,"Failed":false,"Killed":false,"Accumulables":[{"ID":25,"Name":"data size","Update":"170848","Value":"672096","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"shuffle bytes written","Update":"85986","Value":"338460","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"shuffle records written","Update":"3051","Value":"12002","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":35,"Name":"shuffle write time","Update":"8490760","Value":"33941455","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":61,"Name":"duration","Update":"1328","Value":"5260","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":62,"Name":"number of output rows","Update":"3051","Value":"12002","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":63,"Name":"peak memory","Update":"4456448","Value":"17825792","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":65,"Name":"time in aggregation build","Update":"1012","Value":"3986","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":66,"Name":"avg hash probe bucket list iters","Update":"10","Value":"40","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":67,"Name":"number of output rows","Update":"3051","Value":"12002","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":76,"Name":"number of output rows","Update":"3051","Value":"12002","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":77,"Name":"number of output rows","Update":"124533","Value":"499939","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":103,"Name":"internal.metrics.executorDeserializeTime","Update":188,"Value":756,"Internal":true,"Count Failed Values":true},{"ID":104,"Name":"internal.metrics.executorDeserializeCpuTime","Update":181300121,"Value":729725570,"Internal":true,"Count Failed Values":true},{"ID":105,"Name":"internal.metrics.executorRunTime","Update":1533,"Value":6086,"Internal":true,"Count Failed Values":true},{"ID":106,"Name":"internal.metrics.executorCpuTime","Update":1426032738,"Value":5653945858,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.resultSize","Update":2904,"Value":11618,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.jvmGCTime","Update":38,"Value":146,"Internal":true,"Count Failed Values":true},{"ID":112,"Name":"internal.metrics.peakExecutionMemory","Update":33816576,"Value":135266304,"Internal":true,"Count Failed Values":true},{"ID":121,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":85986,"Value":338460,"Internal":true,"Count Failed Values":true},{"ID":122,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":3051,"Value":12002,"Internal":true,"Count Failed Values":true},{"ID":123,"Name":"internal.metrics.shuffle.write.writeTime","Update":8490760,"Value":33941455,"Internal":true,"Count Failed Values":true},{"ID":124,"Name":"internal.metrics.input.bytesRead","Update":23014586,"Value":92530791,"Internal":true,"Count Failed Values":true},{"ID":125,"Name":"internal.metrics.input.recordsRead","Update":124533,"Value":499939,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":188,"Executor Deserialize CPU Time":181300121,"Executor Run Time":1533,"Executor CPU Time":1426032738,"Peak Execution Memory":33816576,"Result Size":2904,"JVM GC Time":38,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":85986,"Shuffle Write Time":8490760,"Shuffle Records Written":3051},"Input Metrics":{"Bytes Read":23014586,"Records Read":124533},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":15,"Index":7,"Attempt":0,"Launch Time":1633789197502,"Executor ID":"4","Host":"10.1.0.8","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":1633789199295,"Failed":false,"Killed":false,"Accumulables":[{"ID":25,"Name":"data size","Update":"167832","Value":"839928","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"shuffle bytes written","Update":"84637","Value":"423097","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"shuffle records written","Update":"2997","Value":"14999","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":35,"Name":"shuffle write time","Update":"8947328","Value":"42888783","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":61,"Name":"duration","Update":"1382","Value":"6642","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":62,"Name":"number of output rows","Update":"2997","Value":"14999","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":63,"Name":"peak memory","Update":"4456448","Value":"22282240","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":65,"Name":"time in aggregation build","Update":"1042","Value":"5028","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":66,"Name":"avg hash probe bucket list iters","Update":"10","Value":"50","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":67,"Name":"number of output rows","Update":"2997","Value":"14999","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":76,"Name":"number of output rows","Update":"2997","Value":"14999","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":77,"Name":"number of output rows","Update":"125488","Value":"625427","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":103,"Name":"internal.metrics.executorDeserializeTime","Update":195,"Value":951,"Internal":true,"Count Failed Values":true},{"ID":104,"Name":"internal.metrics.executorDeserializeCpuTime","Update":188608128,"Value":918333698,"Internal":true,"Count Failed Values":true},{"ID":105,"Name":"internal.metrics.executorRunTime","Update":1585,"Value":7671,"Internal":true,"Count Failed Values":true},{"ID":106,"Name":"internal.metrics.executorCpuTime","Update":1465371554,"Value":7119317412,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.resultSize","Update":2903,"Value":14521,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.jvmGCTime","Update":35,"Value":181,"Internal":true,"Count Failed Values":true},{"ID":112,"Name":"internal.metrics.peakExecutionMemory","Update":33816576,"Value":169082880,"Internal":true,"Count Failed Values":true},{"ID":121,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":84637,"Value":423097,"Internal":true,"Count Failed Values":true},{"ID":122,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":2997,"Value":14999,"Internal":true,"Count Failed Values":true},{"ID":123,"Name":"internal.metrics.shuffle.write.writeTime","Update":8947328,"Value":42888783,"Internal":true,"Count Failed Values":true},{"ID":124,"Name":"internal.metrics.input.bytesRead","Update":23178313,"Value":115709104,"Internal":true,"Count Failed Values":true},{"ID":125,"Name":"internal.metrics.input.recordsRead","Update":125488,"Value":625427,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":195,"Executor Deserialize CPU Time":188608128,"Executor Run Time":1585,"Executor CPU Time":1465371554,"Peak Execution Memory":33816576,"Result Size":2903,"JVM GC Time":35,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":84637,"Shuffle Write Time":8947328,"Shuffle Records Written":2997},"Input Metrics":{"Bytes Read":23178313,"Records Read":125488},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":8,"Index":0,"Attempt":0,"Launch Time":1633789197498,"Executor ID":"9","Host":"10.1.0.14","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":1633789199320,"Failed":false,"Killed":false,"Accumulables":[{"ID":25,"Name":"data size","Update":"158088","Value":"998016","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"shuffle bytes written","Update":"79832","Value":"502929","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"shuffle records written","Update":"2823","Value":"17822","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":35,"Name":"shuffle write time","Update":"9288526","Value":"52177309","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":61,"Name":"duration","Update":"1408","Value":"8050","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":62,"Name":"number of output rows","Update":"2823","Value":"17822","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":63,"Name":"peak memory","Update":"262144","Value":"22544384","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":65,"Name":"time in aggregation build","Update":"1070","Value":"6098","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":67,"Name":"number of output rows","Update":"2823","Value":"17822","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":76,"Name":"number of output rows","Update":"2823","Value":"17822","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":77,"Name":"number of output rows","Update":"124858","Value":"750285","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":103,"Name":"internal.metrics.executorDeserializeTime","Update":196,"Value":1147,"Internal":true,"Count Failed Values":true},{"ID":104,"Name":"internal.metrics.executorDeserializeCpuTime","Update":188541690,"Value":1106875388,"Internal":true,"Count Failed Values":true},{"ID":105,"Name":"internal.metrics.executorRunTime","Update":1610,"Value":9281,"Internal":true,"Count Failed Values":true},{"ID":106,"Name":"internal.metrics.executorCpuTime","Update":1487640781,"Value":8606958193,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.resultSize","Update":2904,"Value":17425,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.jvmGCTime","Update":35,"Value":216,"Internal":true,"Count Failed Values":true},{"ID":112,"Name":"internal.metrics.peakExecutionMemory","Update":29622272,"Value":198705152,"Internal":true,"Count Failed Values":true},{"ID":121,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":79832,"Value":502929,"Internal":true,"Count Failed Values":true},{"ID":122,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":2823,"Value":17822,"Internal":true,"Count Failed Values":true},{"ID":123,"Name":"internal.metrics.shuffle.write.writeTime","Update":9288526,"Value":52177309,"Internal":true,"Count Failed Values":true},{"ID":124,"Name":"internal.metrics.input.bytesRead","Update":23122020,"Value":138831124,"Internal":true,"Count Failed Values":true},{"ID":125,"Name":"internal.metrics.input.recordsRead","Update":124858,"Value":750285,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":196,"Executor Deserialize CPU Time":188541690,"Executor Run Time":1610,"Executor CPU Time":1487640781,"Peak Execution Memory":29622272,"Result Size":2904,"JVM GC Time":35,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":79832,"Shuffle Write Time":9288526,"Shuffle Records Written":2823},"Input Metrics":{"Bytes Read":23122020,"Records Read":124858},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":12,"Index":4,"Attempt":0,"Launch Time":1633789197501,"Executor ID":"6","Host":"10.1.0.18","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":1633789201142,"Failed":false,"Killed":false,"Accumulables":[{"ID":25,"Name":"data size","Update":"161944","Value":"1159960","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"shuffle bytes written","Update":"81819","Value":"584748","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"shuffle records written","Update":"2892","Value":"20714","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":35,"Name":"shuffle write time","Update":"9220560","Value":"61397869","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":61,"Name":"duration","Update":"1422","Value":"9472","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":62,"Name":"number of output rows","Update":"2892","Value":"20714","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":63,"Name":"peak memory","Update":"4456448","Value":"27000832","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":65,"Name":"time in aggregation build","Update":"1080","Value":"7178","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":66,"Name":"avg hash probe bucket list iters","Update":"10","Value":"60","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":67,"Name":"number of output rows","Update":"2892","Value":"20714","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":76,"Name":"number of output rows","Update":"2892","Value":"20714","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":77,"Name":"number of output rows","Update":"124603","Value":"874888","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":103,"Name":"internal.metrics.executorDeserializeTime","Update":856,"Value":2003,"Internal":true,"Count Failed Values":true},{"ID":104,"Name":"internal.metrics.executorDeserializeCpuTime","Update":792100442,"Value":1898975830,"Internal":true,"Count Failed Values":true},{"ID":105,"Name":"internal.metrics.executorRunTime","Update":2745,"Value":12026,"Internal":true,"Count Failed Values":true},{"ID":106,"Name":"internal.metrics.executorCpuTime","Update":2515745749,"Value":11122703942,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.resultSize","Update":2904,"Value":20329,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.jvmGCTime","Update":163,"Value":379,"Internal":true,"Count Failed Values":true},{"ID":112,"Name":"internal.metrics.peakExecutionMemory","Update":33816576,"Value":232521728,"Internal":true,"Count Failed Values":true},{"ID":121,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":81819,"Value":584748,"Internal":true,"Count Failed Values":true},{"ID":122,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":2892,"Value":20714,"Internal":true,"Count Failed Values":true},{"ID":123,"Name":"internal.metrics.shuffle.write.writeTime","Update":9220560,"Value":61397869,"Internal":true,"Count Failed Values":true},{"ID":124,"Name":"internal.metrics.input.bytesRead","Update":23064712,"Value":161895836,"Internal":true,"Count Failed Values":true},{"ID":125,"Name":"internal.metrics.input.recordsRead","Update":124603,"Value":874888,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":856,"Executor Deserialize CPU Time":792100442,"Executor Run Time":2745,"Executor CPU Time":2515745749,"Peak Execution Memory":33816576,"Result Size":2904,"JVM GC Time":163,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":81819,"Shuffle Write Time":9220560,"Shuffle Records Written":2892},"Input Metrics":{"Bytes Read":23064712,"Records Read":124603},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":11,"Index":3,"Attempt":0,"Launch Time":1633789197500,"Executor ID":"8","Host":"10.1.0.11","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":1633789201242,"Failed":false,"Killed":false,"Accumulables":[{"ID":25,"Name":"data size","Update":"161504","Value":"1321464","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"shuffle bytes written","Update":"81474","Value":"666222","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"shuffle records written","Update":"2884","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":35,"Name":"shuffle write time","Update":"9090562","Value":"70488431","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":61,"Name":"duration","Update":"1491","Value":"10963","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":62,"Name":"number of output rows","Update":"2884","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":63,"Name":"peak memory","Update":"4456448","Value":"31457280","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":65,"Name":"time in aggregation build","Update":"1137","Value":"8315","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":66,"Name":"avg hash probe bucket list iters","Update":"10","Value":"70","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":67,"Name":"number of output rows","Update":"2884","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":76,"Name":"number of output rows","Update":"2884","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":77,"Name":"number of output rows","Update":"125112","Value":"1000000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":103,"Name":"internal.metrics.executorDeserializeTime","Update":875,"Value":2878,"Internal":true,"Count Failed Values":true},{"ID":104,"Name":"internal.metrics.executorDeserializeCpuTime","Update":805744939,"Value":2704720769,"Internal":true,"Count Failed Values":true},{"ID":105,"Name":"internal.metrics.executorRunTime","Update":2827,"Value":14853,"Internal":true,"Count Failed Values":true},{"ID":106,"Name":"internal.metrics.executorCpuTime","Update":2598933099,"Value":13721637041,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.resultSize","Update":2904,"Value":23233,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.jvmGCTime","Update":169,"Value":548,"Internal":true,"Count Failed Values":true},{"ID":112,"Name":"internal.metrics.peakExecutionMemory","Update":33816576,"Value":266338304,"Internal":true,"Count Failed Values":true},{"ID":121,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":81474,"Value":666222,"Internal":true,"Count Failed Values":true},{"ID":122,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":2884,"Value":23598,"Internal":true,"Count Failed Values":true},{"ID":123,"Name":"internal.metrics.shuffle.write.writeTime","Update":9090562,"Value":70488431,"Internal":true,"Count Failed Values":true},{"ID":124,"Name":"internal.metrics.input.bytesRead","Update":23103793,"Value":184999629,"Internal":true,"Count Failed Values":true},{"ID":125,"Name":"internal.metrics.input.recordsRead","Update":125112,"Value":1000000,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":875,"Executor Deserialize CPU Time":805744939,"Executor Run Time":2827,"Executor CPU Time":2598933099,"Peak Execution Memory":33816576,"Result Size":2904,"JVM GC Time":169,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":81474,"Shuffle Write Time":9090562,"Shuffle Records Written":2884},"Input Metrics":{"Bytes Read":23103793,"Records Read":125112},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"sql at ScalaSparkSQLBench.scala:43","Number of Tasks":8,"RDD Info":[{"RDD ID":11,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"12\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[10],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":10,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"13\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[9],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":6,"Name":"HadoopRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1633789197487,"Completion Time":1633789201243,"Accumulables":[{"ID":25,"Name":"data size","Value":"1321464","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"shuffle bytes written","Value":"666222","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"shuffle records written","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":35,"Name":"shuffle write time","Value":"70488431","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":61,"Name":"duration","Value":"10963","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":62,"Name":"number of output rows","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":63,"Name":"peak memory","Value":"31457280","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":65,"Name":"time in aggregation build","Value":"8315","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":66,"Name":"avg hash probe bucket list iters","Value":"70","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":67,"Name":"number of output rows","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":76,"Name":"number of output rows","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":77,"Name":"number of output rows","Value":"1000000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":103,"Name":"internal.metrics.executorDeserializeTime","Value":2878,"Internal":true,"Count Failed Values":true},{"ID":104,"Name":"internal.metrics.executorDeserializeCpuTime","Value":2704720769,"Internal":true,"Count Failed Values":true},{"ID":105,"Name":"internal.metrics.executorRunTime","Value":14853,"Internal":true,"Count Failed Values":true},{"ID":106,"Name":"internal.metrics.executorCpuTime","Value":13721637041,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.resultSize","Value":23233,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.jvmGCTime","Value":548,"Internal":true,"Count Failed Values":true},{"ID":112,"Name":"internal.metrics.peakExecutionMemory","Value":266338304,"Internal":true,"Count Failed Values":true},{"ID":121,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":666222,"Internal":true,"Count Failed Values":true},{"ID":122,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":23598,"Internal":true,"Count Failed Values":true},{"ID":123,"Name":"internal.metrics.shuffle.write.writeTime","Value":70488431,"Internal":true,"Count Failed Values":true},{"ID":124,"Name":"internal.metrics.input.bytesRead","Value":184999629,"Internal":true,"Count Failed Values":true},{"ID":125,"Name":"internal.metrics.input.recordsRead","Value":1000000,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} -{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":2,"Stage Attempt ID":0,"Stage Name":"sql at ScalaSparkSQLBench.scala:43","Number of Tasks":8,"RDD Info":[{"RDD ID":16,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[15],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":12,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"12\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[11],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":15,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[14],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":14,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[1],"Details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1633789201258,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.sql.warehouse.dir":"file:/home/usera112/projects/log-compressor/deployments/HiBench/spark-warehouse","spark.driver.host":"hostb7.dsrg.utoronto.ca","spark.eventLog.enabled":"true","spark.driver.port":"44011","__fetch_continuous_blocks_in_batch_enabled":"true","spark.jars":"file:/home/usera112/projects/log-compressor/deployments/HiBench/sparkbench/assembly/target/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.app.name":"ScalaJoin","spark.rdd.scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","spark.driver.memory":"4g","spark.rdd.scope.noOverride":"true","spark.submit.pyFiles":"","spark.default.parallelism":"8","spark.app.startTime":"1633789163045","spark.executor.id":"driver","spark.app.initial.jar.urls":"spark://hostb7.dsrg.utoronto.ca:44011/jars/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.submit.deployMode":"client","spark.master":"spark://hostb7:7077","spark.executor.memory":"4g","spark.eventLog.dir":"hdfs://hostb7:8020/directory","spark.sql.execution.id":"11","spark.sql.catalogImplementation":"hive","spark.app.id":"app-20211009101924-2782","spark.sql.shuffle.partitions":"8"}} -{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":16,"Index":0,"Attempt":0,"Launch Time":1633789201269,"Executor ID":"10","Host":"10.1.0.16","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":17,"Index":1,"Attempt":0,"Launch Time":1633789201271,"Executor ID":"5","Host":"10.1.0.19","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":18,"Index":2,"Attempt":0,"Launch Time":1633789201271,"Executor ID":"4","Host":"10.1.0.8","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":19,"Index":3,"Attempt":0,"Launch Time":1633789201271,"Executor ID":"3","Host":"10.1.0.15","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":20,"Index":4,"Attempt":0,"Launch Time":1633789201272,"Executor ID":"2","Host":"10.1.0.10","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":21,"Index":5,"Attempt":0,"Launch Time":1633789201272,"Executor ID":"9","Host":"10.1.0.14","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":22,"Index":6,"Attempt":0,"Launch Time":1633789201273,"Executor ID":"1","Host":"10.1.0.17","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":23,"Index":7,"Attempt":0,"Launch Time":1633789201273,"Executor ID":"11","Host":"10.1.0.13","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":16,"Index":0,"Attempt":0,"Launch Time":1633789201269,"Executor ID":"10","Host":"10.1.0.16","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633789201495,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"7","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":27,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"71772","Value":"71772","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"local bytes read","Update":"10445","Value":"10445","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2905","Value":"2905","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"39","Value":"39","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2905","Value":"2905","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"4456448","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"16","Value":"16","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":128,"Name":"internal.metrics.executorDeserializeTime","Update":45,"Value":45,"Internal":true,"Count Failed Values":true},{"ID":129,"Name":"internal.metrics.executorDeserializeCpuTime","Update":40194173,"Value":40194173,"Internal":true,"Count Failed Values":true},{"ID":130,"Name":"internal.metrics.executorRunTime","Update":163,"Value":163,"Internal":true,"Count Failed Values":true},{"ID":131,"Name":"internal.metrics.executorCpuTime","Update":127454999,"Value":127454999,"Internal":true,"Count Failed Values":true},{"ID":132,"Name":"internal.metrics.resultSize","Update":14169,"Value":14169,"Internal":true,"Count Failed Values":true},{"ID":134,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":137,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":4456448,"Internal":true,"Count Failed Values":true},{"ID":139,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":140,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":141,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":71772,"Value":71772,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":10445,"Value":10445,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2905,"Value":2905,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":45,"Executor Deserialize CPU Time":40194173,"Executor Run Time":163,"Executor CPU Time":127454999,"Peak Execution Memory":4456448,"Result Size":14169,"JVM GC Time":0,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":71772,"Remote Bytes Read To Disk":0,"Local Bytes Read":10445,"Total Records Read":2905},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":18,"Index":2,"Attempt":0,"Launch Time":1633789201271,"Executor ID":"4","Host":"10.1.0.8","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633789201502,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"7","Value":"14","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":27,"Name":"local blocks read","Update":"1","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"75167","Value":"146939","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"local bytes read","Update":"10737","Value":"21182","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"3049","Value":"5954","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"40","Value":"79","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"3049","Value":"5954","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"8912896","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"16","Value":"32","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"22","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":128,"Name":"internal.metrics.executorDeserializeTime","Update":46,"Value":91,"Internal":true,"Count Failed Values":true},{"ID":129,"Name":"internal.metrics.executorDeserializeCpuTime","Update":39995158,"Value":80189331,"Internal":true,"Count Failed Values":true},{"ID":130,"Name":"internal.metrics.executorRunTime","Update":171,"Value":334,"Internal":true,"Count Failed Values":true},{"ID":131,"Name":"internal.metrics.executorCpuTime","Update":134762848,"Value":262217847,"Internal":true,"Count Failed Values":true},{"ID":132,"Name":"internal.metrics.resultSize","Update":14169,"Value":28338,"Internal":true,"Count Failed Values":true},{"ID":134,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":137,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":8912896,"Internal":true,"Count Failed Values":true},{"ID":139,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":14,"Internal":true,"Count Failed Values":true},{"ID":140,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":141,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":75167,"Value":146939,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":10737,"Value":21182,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.shuffle.read.recordsRead","Update":3049,"Value":5954,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":46,"Executor Deserialize CPU Time":39995158,"Executor Run Time":171,"Executor CPU Time":134762848,"Peak Execution Memory":4456448,"Result Size":14169,"JVM GC Time":0,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":75167,"Remote Bytes Read To Disk":0,"Local Bytes Read":10737,"Total Records Read":3049},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":22,"Index":6,"Attempt":0,"Launch Time":1633789201273,"Executor ID":"1","Host":"10.1.0.17","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633789201505,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"7","Value":"21","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":27,"Name":"local blocks read","Update":"1","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"71833","Value":"218772","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"local bytes read","Update":"10308","Value":"31490","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2910","Value":"8864","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"43","Value":"122","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2910","Value":"8864","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"13369344","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"18","Value":"50","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"33","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":128,"Name":"internal.metrics.executorDeserializeTime","Update":46,"Value":137,"Internal":true,"Count Failed Values":true},{"ID":129,"Name":"internal.metrics.executorDeserializeCpuTime","Update":40980094,"Value":121169425,"Internal":true,"Count Failed Values":true},{"ID":130,"Name":"internal.metrics.executorRunTime","Update":172,"Value":506,"Internal":true,"Count Failed Values":true},{"ID":131,"Name":"internal.metrics.executorCpuTime","Update":133541679,"Value":395759526,"Internal":true,"Count Failed Values":true},{"ID":132,"Name":"internal.metrics.resultSize","Update":14169,"Value":42507,"Internal":true,"Count Failed Values":true},{"ID":134,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":137,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":13369344,"Internal":true,"Count Failed Values":true},{"ID":139,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":21,"Internal":true,"Count Failed Values":true},{"ID":140,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":141,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":71833,"Value":218772,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":10308,"Value":31490,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2910,"Value":8864,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":46,"Executor Deserialize CPU Time":40980094,"Executor Run Time":172,"Executor CPU Time":133541679,"Peak Execution Memory":4456448,"Result Size":14169,"JVM GC Time":0,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":71833,"Remote Bytes Read To Disk":0,"Local Bytes Read":10308,"Total Records Read":2910},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":21,"Index":5,"Attempt":0,"Launch Time":1633789201272,"Executor ID":"9","Host":"10.1.0.14","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633789201505,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"7","Value":"28","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":27,"Name":"local blocks read","Update":"1","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"73503","Value":"292275","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"local bytes read","Update":"9706","Value":"41196","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2948","Value":"11812","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"42","Value":"164","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2948","Value":"11812","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"17825792","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"16","Value":"66","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"44","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":128,"Name":"internal.metrics.executorDeserializeTime","Update":48,"Value":185,"Internal":true,"Count Failed Values":true},{"ID":129,"Name":"internal.metrics.executorDeserializeCpuTime","Update":41014065,"Value":162183490,"Internal":true,"Count Failed Values":true},{"ID":130,"Name":"internal.metrics.executorRunTime","Update":170,"Value":676,"Internal":true,"Count Failed Values":true},{"ID":131,"Name":"internal.metrics.executorCpuTime","Update":134801322,"Value":530560848,"Internal":true,"Count Failed Values":true},{"ID":132,"Name":"internal.metrics.resultSize","Update":14169,"Value":56676,"Internal":true,"Count Failed Values":true},{"ID":134,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":137,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":17825792,"Internal":true,"Count Failed Values":true},{"ID":139,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":28,"Internal":true,"Count Failed Values":true},{"ID":140,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":141,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":73503,"Value":292275,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":9706,"Value":41196,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2948,"Value":11812,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":48,"Executor Deserialize CPU Time":41014065,"Executor Run Time":170,"Executor CPU Time":134801322,"Peak Execution Memory":4456448,"Result Size":14169,"JVM GC Time":0,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":73503,"Remote Bytes Read To Disk":0,"Local Bytes Read":9706,"Total Records Read":2948},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":20,"Index":4,"Attempt":0,"Launch Time":1633789201272,"Executor ID":"2","Host":"10.1.0.10","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633789201506,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"7","Value":"35","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":27,"Name":"local blocks read","Update":"1","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"71983","Value":"364258","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"local bytes read","Update":"11005","Value":"52201","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2937","Value":"14749","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"41","Value":"205","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2937","Value":"14749","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"22282240","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"17","Value":"83","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"55","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":128,"Name":"internal.metrics.executorDeserializeTime","Update":47,"Value":232,"Internal":true,"Count Failed Values":true},{"ID":129,"Name":"internal.metrics.executorDeserializeCpuTime","Update":41987190,"Value":204170680,"Internal":true,"Count Failed Values":true},{"ID":130,"Name":"internal.metrics.executorRunTime","Update":173,"Value":849,"Internal":true,"Count Failed Values":true},{"ID":131,"Name":"internal.metrics.executorCpuTime","Update":132010463,"Value":662571311,"Internal":true,"Count Failed Values":true},{"ID":132,"Name":"internal.metrics.resultSize","Update":14169,"Value":70845,"Internal":true,"Count Failed Values":true},{"ID":134,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":137,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":22282240,"Internal":true,"Count Failed Values":true},{"ID":139,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":35,"Internal":true,"Count Failed Values":true},{"ID":140,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":141,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":71983,"Value":364258,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":11005,"Value":52201,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2937,"Value":14749,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":47,"Executor Deserialize CPU Time":41987190,"Executor Run Time":173,"Executor CPU Time":132010463,"Peak Execution Memory":4456448,"Result Size":14169,"JVM GC Time":0,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":71983,"Remote Bytes Read To Disk":0,"Local Bytes Read":11005,"Total Records Read":2937},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":23,"Index":7,"Attempt":0,"Launch Time":1633789201273,"Executor ID":"11","Host":"10.1.0.13","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633789201506,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"7","Value":"42","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":27,"Name":"local blocks read","Update":"1","Value":"6","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"70073","Value":"434331","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"local bytes read","Update":"10623","Value":"62824","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2856","Value":"17605","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"45","Value":"250","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2856","Value":"17605","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"26738688","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"17","Value":"100","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"66","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":128,"Name":"internal.metrics.executorDeserializeTime","Update":49,"Value":281,"Internal":true,"Count Failed Values":true},{"ID":129,"Name":"internal.metrics.executorDeserializeCpuTime","Update":42961717,"Value":247132397,"Internal":true,"Count Failed Values":true},{"ID":130,"Name":"internal.metrics.executorRunTime","Update":171,"Value":1020,"Internal":true,"Count Failed Values":true},{"ID":131,"Name":"internal.metrics.executorCpuTime","Update":135033817,"Value":797605128,"Internal":true,"Count Failed Values":true},{"ID":132,"Name":"internal.metrics.resultSize","Update":14169,"Value":85014,"Internal":true,"Count Failed Values":true},{"ID":134,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":6,"Internal":true,"Count Failed Values":true},{"ID":137,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":26738688,"Internal":true,"Count Failed Values":true},{"ID":139,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":42,"Internal":true,"Count Failed Values":true},{"ID":140,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":6,"Internal":true,"Count Failed Values":true},{"ID":141,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":70073,"Value":434331,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":10623,"Value":62824,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2856,"Value":17605,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":49,"Executor Deserialize CPU Time":42961717,"Executor Run Time":171,"Executor CPU Time":135033817,"Peak Execution Memory":4456448,"Result Size":14169,"JVM GC Time":0,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":70073,"Remote Bytes Read To Disk":0,"Local Bytes Read":10623,"Total Records Read":2856},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":17,"Index":1,"Attempt":0,"Launch Time":1633789201271,"Executor ID":"5","Host":"10.1.0.19","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633789201776,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"8","Value":"50","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"84612","Value":"518943","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2999","Value":"20604","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"127","Value":"377","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2999","Value":"20604","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"31195136","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"35","Value":"135","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"77","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":128,"Name":"internal.metrics.executorDeserializeTime","Update":218,"Value":499,"Internal":true,"Count Failed Values":true},{"ID":129,"Name":"internal.metrics.executorDeserializeCpuTime","Update":208729295,"Value":455861692,"Internal":true,"Count Failed Values":true},{"ID":130,"Name":"internal.metrics.executorRunTime","Update":270,"Value":1290,"Internal":true,"Count Failed Values":true},{"ID":131,"Name":"internal.metrics.executorCpuTime","Update":253112853,"Value":1050717981,"Internal":true,"Count Failed Values":true},{"ID":132,"Name":"internal.metrics.resultSize","Update":14169,"Value":99183,"Internal":true,"Count Failed Values":true},{"ID":134,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":137,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":31195136,"Internal":true,"Count Failed Values":true},{"ID":139,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":8,"Value":50,"Internal":true,"Count Failed Values":true},{"ID":140,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":6,"Internal":true,"Count Failed Values":true},{"ID":141,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":84612,"Value":518943,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":62824,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2999,"Value":20604,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":218,"Executor Deserialize CPU Time":208729295,"Executor Run Time":270,"Executor CPU Time":253112853,"Peak Execution Memory":4456448,"Result Size":14169,"JVM GC Time":0,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":8,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":84612,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":2999},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":19,"Index":3,"Attempt":0,"Launch Time":1633789201271,"Executor ID":"3","Host":"10.1.0.15","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633789202924,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"8","Value":"58","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"84455","Value":"603398","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2994","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"194","Value":"571","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2994","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"35651584","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"41","Value":"176","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"88","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":128,"Name":"internal.metrics.executorDeserializeTime","Update":969,"Value":1468,"Internal":true,"Count Failed Values":true},{"ID":129,"Name":"internal.metrics.executorDeserializeCpuTime","Update":901040610,"Value":1356902302,"Internal":true,"Count Failed Values":true},{"ID":130,"Name":"internal.metrics.executorRunTime","Update":639,"Value":1929,"Internal":true,"Count Failed Values":true},{"ID":131,"Name":"internal.metrics.executorCpuTime","Update":615072661,"Value":1665790642,"Internal":true,"Count Failed Values":true},{"ID":132,"Name":"internal.metrics.resultSize","Update":14212,"Value":113395,"Internal":true,"Count Failed Values":true},{"ID":133,"Name":"internal.metrics.jvmGCTime","Update":33,"Value":33,"Internal":true,"Count Failed Values":true},{"ID":134,"Name":"internal.metrics.resultSerializationTime","Update":3,"Value":10,"Internal":true,"Count Failed Values":true},{"ID":137,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":35651584,"Internal":true,"Count Failed Values":true},{"ID":139,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":8,"Value":58,"Internal":true,"Count Failed Values":true},{"ID":140,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":6,"Internal":true,"Count Failed Values":true},{"ID":141,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":84455,"Value":603398,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":62824,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2994,"Value":23598,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":969,"Executor Deserialize CPU Time":901040610,"Executor Run Time":639,"Executor CPU Time":615072661,"Peak Execution Memory":4456448,"Result Size":14212,"JVM GC Time":33,"Result Serialization Time":3,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":8,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":84455,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":2994},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":2,"Stage Attempt ID":0,"Stage Name":"sql at ScalaSparkSQLBench.scala:43","Number of Tasks":8,"RDD Info":[{"RDD ID":16,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[15],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":12,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"12\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[11],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":15,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[14],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":14,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[1],"Details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1633789201258,"Completion Time":1633789202925,"Accumulables":[{"ID":26,"Name":"remote blocks read","Value":"58","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":27,"Name":"local blocks read","Value":"6","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Value":"603398","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"local bytes read","Value":"62824","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Value":"571","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Value":"35651584","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Value":"176","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Value":"88","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":128,"Name":"internal.metrics.executorDeserializeTime","Value":1468,"Internal":true,"Count Failed Values":true},{"ID":129,"Name":"internal.metrics.executorDeserializeCpuTime","Value":1356902302,"Internal":true,"Count Failed Values":true},{"ID":130,"Name":"internal.metrics.executorRunTime","Value":1929,"Internal":true,"Count Failed Values":true},{"ID":131,"Name":"internal.metrics.executorCpuTime","Value":1665790642,"Internal":true,"Count Failed Values":true},{"ID":132,"Name":"internal.metrics.resultSize","Value":113395,"Internal":true,"Count Failed Values":true},{"ID":133,"Name":"internal.metrics.jvmGCTime","Value":33,"Internal":true,"Count Failed Values":true},{"ID":134,"Name":"internal.metrics.resultSerializationTime","Value":10,"Internal":true,"Count Failed Values":true},{"ID":137,"Name":"internal.metrics.peakExecutionMemory","Value":35651584,"Internal":true,"Count Failed Values":true},{"ID":139,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":58,"Internal":true,"Count Failed Values":true},{"ID":140,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":6,"Internal":true,"Count Failed Values":true},{"ID":141,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":603398,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":62824,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.shuffle.read.recordsRead","Value":23598,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} -{"Event":"SparkListenerJobEnd","Job ID":1,"Completion Time":1633789202926,"Job Result":{"Result":"JobSucceeded"}} -{"Event":"SparkListenerJobStart","Job ID":2,"Submission Time":1633789202997,"Stage Infos":[{"Stage ID":5,"Stage Attempt ID":0,"Stage Name":"sql at ScalaSparkSQLBench.scala:43","Number of Tasks":8,"RDD Info":[{"RDD ID":19,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[18],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":18,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[17],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[4],"Details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":3,"Stage Attempt ID":0,"Stage Name":"sql at ScalaSparkSQLBench.scala:43","Number of Tasks":8,"RDD Info":[{"RDD ID":11,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"12\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[10],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":10,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"13\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[9],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":6,"Name":"HadoopRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":4,"Stage Attempt ID":0,"Stage Name":"sql at ScalaSparkSQLBench.scala:43","Number of Tasks":8,"RDD Info":[{"RDD ID":17,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":12,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"12\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[11],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[3],"Details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[5,3,4],"Properties":{"spark.sql.warehouse.dir":"file:/home/usera112/projects/log-compressor/deployments/HiBench/spark-warehouse","spark.driver.host":"hostb7.dsrg.utoronto.ca","spark.eventLog.enabled":"true","spark.driver.port":"44011","__fetch_continuous_blocks_in_batch_enabled":"true","spark.jars":"file:/home/usera112/projects/log-compressor/deployments/HiBench/sparkbench/assembly/target/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.app.name":"ScalaJoin","spark.driver.memory":"4g","spark.submit.pyFiles":"","spark.default.parallelism":"8","spark.app.startTime":"1633789163045","spark.executor.id":"driver","spark.app.initial.jar.urls":"spark://hostb7.dsrg.utoronto.ca:44011/jars/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.submit.deployMode":"client","spark.master":"spark://hostb7:7077","spark.executor.memory":"4g","spark.eventLog.dir":"hdfs://hostb7:8020/directory","spark.sql.execution.id":"11","spark.sql.catalogImplementation":"hive","spark.app.id":"app-20211009101924-2782","spark.sql.shuffle.partitions":"8"}} -{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":4,"Stage Attempt ID":0,"Stage Name":"sql at ScalaSparkSQLBench.scala:43","Number of Tasks":8,"RDD Info":[{"RDD ID":17,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":12,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"12\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[11],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[3],"Details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1633789203002,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.sql.warehouse.dir":"file:/home/usera112/projects/log-compressor/deployments/HiBench/spark-warehouse","spark.driver.host":"hostb7.dsrg.utoronto.ca","spark.eventLog.enabled":"true","spark.driver.port":"44011","__fetch_continuous_blocks_in_batch_enabled":"true","spark.jars":"file:/home/usera112/projects/log-compressor/deployments/HiBench/sparkbench/assembly/target/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.app.name":"ScalaJoin","spark.driver.memory":"4g","spark.submit.pyFiles":"","spark.default.parallelism":"8","spark.app.startTime":"1633789163045","spark.executor.id":"driver","spark.app.initial.jar.urls":"spark://hostb7.dsrg.utoronto.ca:44011/jars/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.submit.deployMode":"client","spark.master":"spark://hostb7:7077","spark.executor.memory":"4g","spark.eventLog.dir":"hdfs://hostb7:8020/directory","spark.sql.execution.id":"11","spark.sql.catalogImplementation":"hive","spark.app.id":"app-20211009101924-2782","spark.sql.shuffle.partitions":"8"}} -{"Event":"SparkListenerTaskStart","Stage ID":4,"Stage Attempt ID":0,"Task Info":{"Task ID":24,"Index":0,"Attempt":0,"Launch Time":1633789203014,"Executor ID":"0","Host":"10.1.0.12","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":4,"Stage Attempt ID":0,"Task Info":{"Task ID":25,"Index":1,"Attempt":0,"Launch Time":1633789203014,"Executor ID":"7","Host":"10.1.0.9","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":4,"Stage Attempt ID":0,"Task Info":{"Task ID":26,"Index":2,"Attempt":0,"Launch Time":1633789203015,"Executor ID":"10","Host":"10.1.0.16","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":4,"Stage Attempt ID":0,"Task Info":{"Task ID":27,"Index":3,"Attempt":0,"Launch Time":1633789203015,"Executor ID":"11","Host":"10.1.0.13","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":4,"Stage Attempt ID":0,"Task Info":{"Task ID":28,"Index":4,"Attempt":0,"Launch Time":1633789203015,"Executor ID":"1","Host":"10.1.0.17","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":4,"Stage Attempt ID":0,"Task Info":{"Task ID":29,"Index":5,"Attempt":0,"Launch Time":1633789203015,"Executor ID":"6","Host":"10.1.0.18","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":4,"Stage Attempt ID":0,"Task Info":{"Task ID":30,"Index":6,"Attempt":0,"Launch Time":1633789203016,"Executor ID":"5","Host":"10.1.0.19","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":4,"Stage Attempt ID":0,"Task Info":{"Task ID":31,"Index":7,"Attempt":0,"Launch Time":1633789203016,"Executor ID":"2","Host":"10.1.0.10","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":4,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":28,"Index":4,"Attempt":0,"Launch Time":1633789203015,"Executor ID":"1","Host":"10.1.0.17","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633789203110,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"7","Value":"65","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":27,"Name":"local blocks read","Update":"1","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"72296","Value":"675694","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"local bytes read","Update":"10692","Value":"73516","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2937","Value":"26535","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":36,"Name":"data size","Update":"140976","Value":"140976","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":44,"Name":"shuffle bytes written","Update":"79984","Value":"79984","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":45,"Name":"shuffle records written","Update":"2937","Value":"2937","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":46,"Name":"shuffle write time","Update":"3729687","Value":"3729687","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"27","Value":"598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2937","Value":"26535","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"40108032","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"6","Value":"182","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"99","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":153,"Name":"internal.metrics.executorDeserializeTime","Update":46,"Value":46,"Internal":true,"Count Failed Values":true},{"ID":154,"Name":"internal.metrics.executorDeserializeCpuTime","Update":42482569,"Value":42482569,"Internal":true,"Count Failed Values":true},{"ID":155,"Name":"internal.metrics.executorRunTime","Update":38,"Value":38,"Internal":true,"Count Failed Values":true},{"ID":156,"Name":"internal.metrics.executorCpuTime","Update":37030227,"Value":37030227,"Internal":true,"Count Failed Values":true},{"ID":157,"Name":"internal.metrics.resultSize","Update":4236,"Value":4236,"Internal":true,"Count Failed Values":true},{"ID":162,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":4456448,"Internal":true,"Count Failed Values":true},{"ID":164,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":165,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":166,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":72296,"Value":72296,"Internal":true,"Count Failed Values":true},{"ID":167,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":168,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":10692,"Value":10692,"Internal":true,"Count Failed Values":true},{"ID":169,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":170,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2937,"Value":2937,"Internal":true,"Count Failed Values":true},{"ID":171,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":79984,"Value":79984,"Internal":true,"Count Failed Values":true},{"ID":172,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":2937,"Value":2937,"Internal":true,"Count Failed Values":true},{"ID":173,"Name":"internal.metrics.shuffle.write.writeTime","Update":3729687,"Value":3729687,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":46,"Executor Deserialize CPU Time":42482569,"Executor Run Time":38,"Executor CPU Time":37030227,"Peak Execution Memory":4456448,"Result Size":4236,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":72296,"Remote Bytes Read To Disk":0,"Local Bytes Read":10692,"Total Records Read":2937},"Shuffle Write Metrics":{"Shuffle Bytes Written":79984,"Shuffle Write Time":3729687,"Shuffle Records Written":2937},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":4,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":31,"Index":7,"Attempt":0,"Launch Time":1633789203016,"Executor ID":"2","Host":"10.1.0.10","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633789203111,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"7","Value":"72","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":27,"Name":"local blocks read","Update":"1","Value":"8","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"70979","Value":"746673","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"local bytes read","Update":"9717","Value":"83233","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2856","Value":"29391","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":36,"Name":"data size","Update":"137080","Value":"278056","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":44,"Name":"shuffle bytes written","Update":"77865","Value":"157849","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":45,"Name":"shuffle records written","Update":"2856","Value":"5793","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":46,"Name":"shuffle write time","Update":"3713439","Value":"7443126","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"27","Value":"625","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2856","Value":"29391","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"44564480","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"6","Value":"188","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"110","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":153,"Name":"internal.metrics.executorDeserializeTime","Update":48,"Value":94,"Internal":true,"Count Failed Values":true},{"ID":154,"Name":"internal.metrics.executorDeserializeCpuTime","Update":43150044,"Value":85632613,"Internal":true,"Count Failed Values":true},{"ID":155,"Name":"internal.metrics.executorRunTime","Update":39,"Value":77,"Internal":true,"Count Failed Values":true},{"ID":156,"Name":"internal.metrics.executorCpuTime","Update":37363811,"Value":74394038,"Internal":true,"Count Failed Values":true},{"ID":157,"Name":"internal.metrics.resultSize","Update":4236,"Value":8472,"Internal":true,"Count Failed Values":true},{"ID":162,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":8912896,"Internal":true,"Count Failed Values":true},{"ID":164,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":14,"Internal":true,"Count Failed Values":true},{"ID":165,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":166,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":70979,"Value":143275,"Internal":true,"Count Failed Values":true},{"ID":167,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":168,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":9717,"Value":20409,"Internal":true,"Count Failed Values":true},{"ID":169,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":170,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2856,"Value":5793,"Internal":true,"Count Failed Values":true},{"ID":171,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":77865,"Value":157849,"Internal":true,"Count Failed Values":true},{"ID":172,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":2856,"Value":5793,"Internal":true,"Count Failed Values":true},{"ID":173,"Name":"internal.metrics.shuffle.write.writeTime","Update":3713439,"Value":7443126,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":48,"Executor Deserialize CPU Time":43150044,"Executor Run Time":39,"Executor CPU Time":37363811,"Peak Execution Memory":4456448,"Result Size":4236,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":70979,"Remote Bytes Read To Disk":0,"Local Bytes Read":9717,"Total Records Read":2856},"Shuffle Write Metrics":{"Shuffle Bytes Written":77865,"Shuffle Write Time":3713439,"Shuffle Records Written":2856},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":4,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":27,"Index":3,"Attempt":0,"Launch Time":1633789203015,"Executor ID":"11","Host":"10.1.0.13","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633789203113,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"7","Value":"79","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":27,"Name":"local blocks read","Update":"1","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"74394","Value":"821067","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"local bytes read","Update":"10061","Value":"93294","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2994","Value":"32385","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":36,"Name":"data size","Update":"143712","Value":"421768","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":44,"Name":"shuffle bytes written","Update":"81445","Value":"239294","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":45,"Name":"shuffle records written","Update":"2994","Value":"8787","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":46,"Name":"shuffle write time","Update":"2747305","Value":"10190431","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"28","Value":"653","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2994","Value":"32385","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"49020928","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"6","Value":"194","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"121","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":153,"Name":"internal.metrics.executorDeserializeTime","Update":51,"Value":145,"Internal":true,"Count Failed Values":true},{"ID":154,"Name":"internal.metrics.executorDeserializeCpuTime","Update":46947629,"Value":132580242,"Internal":true,"Count Failed Values":true},{"ID":155,"Name":"internal.metrics.executorRunTime","Update":38,"Value":115,"Internal":true,"Count Failed Values":true},{"ID":156,"Name":"internal.metrics.executorCpuTime","Update":36474407,"Value":110868445,"Internal":true,"Count Failed Values":true},{"ID":157,"Name":"internal.metrics.resultSize","Update":4237,"Value":12709,"Internal":true,"Count Failed Values":true},{"ID":162,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":13369344,"Internal":true,"Count Failed Values":true},{"ID":164,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":21,"Internal":true,"Count Failed Values":true},{"ID":165,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":166,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":74394,"Value":217669,"Internal":true,"Count Failed Values":true},{"ID":167,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":168,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":10061,"Value":30470,"Internal":true,"Count Failed Values":true},{"ID":169,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":170,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2994,"Value":8787,"Internal":true,"Count Failed Values":true},{"ID":171,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":81445,"Value":239294,"Internal":true,"Count Failed Values":true},{"ID":172,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":2994,"Value":8787,"Internal":true,"Count Failed Values":true},{"ID":173,"Name":"internal.metrics.shuffle.write.writeTime","Update":2747305,"Value":10190431,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":51,"Executor Deserialize CPU Time":46947629,"Executor Run Time":38,"Executor CPU Time":36474407,"Peak Execution Memory":4456448,"Result Size":4237,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":74394,"Remote Bytes Read To Disk":0,"Local Bytes Read":10061,"Total Records Read":2994},"Shuffle Write Metrics":{"Shuffle Bytes Written":81445,"Shuffle Write Time":2747305,"Shuffle Records Written":2994},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":4,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":26,"Index":2,"Attempt":0,"Launch Time":1633789203015,"Executor ID":"10","Host":"10.1.0.16","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633789203124,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"7","Value":"86","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":27,"Name":"local blocks read","Update":"1","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"74808","Value":"895875","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"local bytes read","Update":"11096","Value":"104390","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"3049","Value":"35434","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":36,"Name":"data size","Update":"146336","Value":"568104","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":44,"Name":"shuffle bytes written","Update":"82790","Value":"322084","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":45,"Name":"shuffle records written","Update":"3049","Value":"11836","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":46,"Name":"shuffle write time","Update":"5131148","Value":"15321579","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"37","Value":"690","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"3049","Value":"35434","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"53477376","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"8","Value":"202","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"132","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":153,"Name":"internal.metrics.executorDeserializeTime","Update":47,"Value":192,"Internal":true,"Count Failed Values":true},{"ID":154,"Name":"internal.metrics.executorDeserializeCpuTime","Update":42448641,"Value":175028883,"Internal":true,"Count Failed Values":true},{"ID":155,"Name":"internal.metrics.executorRunTime","Update":52,"Value":167,"Internal":true,"Count Failed Values":true},{"ID":156,"Name":"internal.metrics.executorCpuTime","Update":49767354,"Value":160635799,"Internal":true,"Count Failed Values":true},{"ID":157,"Name":"internal.metrics.resultSize","Update":4237,"Value":16946,"Internal":true,"Count Failed Values":true},{"ID":162,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":17825792,"Internal":true,"Count Failed Values":true},{"ID":164,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":28,"Internal":true,"Count Failed Values":true},{"ID":165,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":166,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":74808,"Value":292477,"Internal":true,"Count Failed Values":true},{"ID":167,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":168,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":11096,"Value":41566,"Internal":true,"Count Failed Values":true},{"ID":169,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":170,"Name":"internal.metrics.shuffle.read.recordsRead","Update":3049,"Value":11836,"Internal":true,"Count Failed Values":true},{"ID":171,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":82790,"Value":322084,"Internal":true,"Count Failed Values":true},{"ID":172,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":3049,"Value":11836,"Internal":true,"Count Failed Values":true},{"ID":173,"Name":"internal.metrics.shuffle.write.writeTime","Update":5131148,"Value":15321579,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":47,"Executor Deserialize CPU Time":42448641,"Executor Run Time":52,"Executor CPU Time":49767354,"Peak Execution Memory":4456448,"Result Size":4237,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":74808,"Remote Bytes Read To Disk":0,"Local Bytes Read":11096,"Total Records Read":3049},"Shuffle Write Metrics":{"Shuffle Bytes Written":82790,"Shuffle Write Time":5131148,"Shuffle Records Written":3049},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":4,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":30,"Index":6,"Attempt":0,"Launch Time":1633789203016,"Executor ID":"5","Host":"10.1.0.19","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633789203158,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"8","Value":"94","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"82141","Value":"978016","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2910","Value":"38344","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":36,"Name":"data size","Update":"139680","Value":"707784","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":44,"Name":"shuffle bytes written","Update":"79272","Value":"401356","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":45,"Name":"shuffle records written","Update":"2910","Value":"14746","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":46,"Name":"shuffle write time","Update":"9436045","Value":"24757624","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"41","Value":"731","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2910","Value":"38344","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"57933824","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"7","Value":"209","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"143","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":153,"Name":"internal.metrics.executorDeserializeTime","Update":63,"Value":255,"Internal":true,"Count Failed Values":true},{"ID":154,"Name":"internal.metrics.executorDeserializeCpuTime","Update":57910631,"Value":232939514,"Internal":true,"Count Failed Values":true},{"ID":155,"Name":"internal.metrics.executorRunTime","Update":70,"Value":237,"Internal":true,"Count Failed Values":true},{"ID":156,"Name":"internal.metrics.executorCpuTime","Update":66488500,"Value":227124299,"Internal":true,"Count Failed Values":true},{"ID":157,"Name":"internal.metrics.resultSize","Update":4236,"Value":21182,"Internal":true,"Count Failed Values":true},{"ID":162,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":22282240,"Internal":true,"Count Failed Values":true},{"ID":164,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":8,"Value":36,"Internal":true,"Count Failed Values":true},{"ID":165,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":166,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":82141,"Value":374618,"Internal":true,"Count Failed Values":true},{"ID":167,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":168,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":41566,"Internal":true,"Count Failed Values":true},{"ID":169,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":170,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2910,"Value":14746,"Internal":true,"Count Failed Values":true},{"ID":171,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":79272,"Value":401356,"Internal":true,"Count Failed Values":true},{"ID":172,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":2910,"Value":14746,"Internal":true,"Count Failed Values":true},{"ID":173,"Name":"internal.metrics.shuffle.write.writeTime","Update":9436045,"Value":24757624,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":63,"Executor Deserialize CPU Time":57910631,"Executor Run Time":70,"Executor CPU Time":66488500,"Peak Execution Memory":4456448,"Result Size":4236,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":8,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":82141,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":2910},"Shuffle Write Metrics":{"Shuffle Bytes Written":79272,"Shuffle Write Time":9436045,"Shuffle Records Written":2910},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":4,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":29,"Index":5,"Attempt":0,"Launch Time":1633789203015,"Executor ID":"6","Host":"10.1.0.18","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633789203254,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"7","Value":"101","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":27,"Name":"local blocks read","Update":"1","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"72828","Value":"1050844","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"local bytes read","Update":"10381","Value":"114771","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2948","Value":"41292","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":36,"Name":"data size","Update":"141504","Value":"849288","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":44,"Name":"shuffle bytes written","Update":"80279","Value":"481635","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":45,"Name":"shuffle records written","Update":"2948","Value":"17694","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":46,"Name":"shuffle write time","Update":"2955698","Value":"27713322","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"45","Value":"776","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2948","Value":"41292","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"62390272","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"16","Value":"225","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"154","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":153,"Name":"internal.metrics.executorDeserializeTime","Update":76,"Value":331,"Internal":true,"Count Failed Values":true},{"ID":154,"Name":"internal.metrics.executorDeserializeCpuTime","Update":69573431,"Value":302512945,"Internal":true,"Count Failed Values":true},{"ID":155,"Name":"internal.metrics.executorRunTime","Update":152,"Value":389,"Internal":true,"Count Failed Values":true},{"ID":156,"Name":"internal.metrics.executorCpuTime","Update":142832467,"Value":369956766,"Internal":true,"Count Failed Values":true},{"ID":157,"Name":"internal.metrics.resultSize","Update":4236,"Value":25418,"Internal":true,"Count Failed Values":true},{"ID":162,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":26738688,"Internal":true,"Count Failed Values":true},{"ID":164,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":43,"Internal":true,"Count Failed Values":true},{"ID":165,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":166,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":72828,"Value":447446,"Internal":true,"Count Failed Values":true},{"ID":167,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":168,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":10381,"Value":51947,"Internal":true,"Count Failed Values":true},{"ID":169,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":170,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2948,"Value":17694,"Internal":true,"Count Failed Values":true},{"ID":171,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":80279,"Value":481635,"Internal":true,"Count Failed Values":true},{"ID":172,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":2948,"Value":17694,"Internal":true,"Count Failed Values":true},{"ID":173,"Name":"internal.metrics.shuffle.write.writeTime","Update":2955698,"Value":27713322,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":76,"Executor Deserialize CPU Time":69573431,"Executor Run Time":152,"Executor CPU Time":142832467,"Peak Execution Memory":4456448,"Result Size":4236,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":72828,"Remote Bytes Read To Disk":0,"Local Bytes Read":10381,"Total Records Read":2948},"Shuffle Write Metrics":{"Shuffle Bytes Written":80279,"Shuffle Write Time":2955698,"Shuffle Records Written":2948},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":4,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":25,"Index":1,"Attempt":0,"Launch Time":1633789203014,"Executor ID":"7","Host":"10.1.0.9","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633789203559,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"8","Value":"109","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"84612","Value":"1135456","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2999","Value":"44291","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":36,"Name":"data size","Update":"143952","Value":"993240","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":44,"Name":"shuffle bytes written","Update":"81642","Value":"563277","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":45,"Name":"shuffle records written","Update":"2999","Value":"20693","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":46,"Name":"shuffle write time","Update":"25512377","Value":"53225699","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"156","Value":"932","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2999","Value":"44291","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"66846720","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"32","Value":"257","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"165","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":153,"Name":"internal.metrics.executorDeserializeTime","Update":230,"Value":561,"Internal":true,"Count Failed Values":true},{"ID":154,"Name":"internal.metrics.executorDeserializeCpuTime","Update":219370826,"Value":521883771,"Internal":true,"Count Failed Values":true},{"ID":155,"Name":"internal.metrics.executorRunTime","Update":302,"Value":691,"Internal":true,"Count Failed Values":true},{"ID":156,"Name":"internal.metrics.executorCpuTime","Update":269611352,"Value":639568118,"Internal":true,"Count Failed Values":true},{"ID":157,"Name":"internal.metrics.resultSize","Update":4235,"Value":29653,"Internal":true,"Count Failed Values":true},{"ID":162,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":31195136,"Internal":true,"Count Failed Values":true},{"ID":164,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":8,"Value":51,"Internal":true,"Count Failed Values":true},{"ID":165,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":166,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":84612,"Value":532058,"Internal":true,"Count Failed Values":true},{"ID":167,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":168,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":51947,"Internal":true,"Count Failed Values":true},{"ID":169,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":170,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2999,"Value":20693,"Internal":true,"Count Failed Values":true},{"ID":171,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":81642,"Value":563277,"Internal":true,"Count Failed Values":true},{"ID":172,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":2999,"Value":20693,"Internal":true,"Count Failed Values":true},{"ID":173,"Name":"internal.metrics.shuffle.write.writeTime","Update":25512377,"Value":53225699,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":472141848,"JVMOffHeapMemory":84463448,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":536366,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":536366,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1193118,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":3,"MinorGCTime":31,"MajorGCCount":3,"MajorGCTime":144},"Task Metrics":{"Executor Deserialize Time":230,"Executor Deserialize CPU Time":219370826,"Executor Run Time":302,"Executor CPU Time":269611352,"Peak Execution Memory":4456448,"Result Size":4235,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":8,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":84612,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":2999},"Shuffle Write Metrics":{"Shuffle Bytes Written":81642,"Shuffle Write Time":25512377,"Shuffle Records Written":2999},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":4,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":24,"Index":0,"Attempt":0,"Launch Time":1633789203014,"Executor ID":"0","Host":"10.1.0.12","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633789204642,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"8","Value":"117","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"82217","Value":"1217673","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2905","Value":"47196","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":36,"Name":"data size","Update":"139440","Value":"1132680","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":44,"Name":"shuffle bytes written","Update":"79213","Value":"642490","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":45,"Name":"shuffle records written","Update":"2905","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":46,"Name":"shuffle write time","Update":"12231018","Value":"65456717","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"178","Value":"1110","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2905","Value":"47196","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"71303168","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"41","Value":"298","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"176","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":153,"Name":"internal.metrics.executorDeserializeTime","Update":1178,"Value":1739,"Internal":true,"Count Failed Values":true},{"ID":154,"Name":"internal.metrics.executorDeserializeCpuTime","Update":1106395401,"Value":1628279172,"Internal":true,"Count Failed Values":true},{"ID":155,"Name":"internal.metrics.executorRunTime","Update":412,"Value":1103,"Internal":true,"Count Failed Values":true},{"ID":156,"Name":"internal.metrics.executorCpuTime","Update":388572482,"Value":1028140600,"Internal":true,"Count Failed Values":true},{"ID":157,"Name":"internal.metrics.resultSize","Update":4279,"Value":33932,"Internal":true,"Count Failed Values":true},{"ID":158,"Name":"internal.metrics.jvmGCTime","Update":34,"Value":34,"Internal":true,"Count Failed Values":true},{"ID":162,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":35651584,"Internal":true,"Count Failed Values":true},{"ID":164,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":8,"Value":59,"Internal":true,"Count Failed Values":true},{"ID":165,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":166,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":82217,"Value":614275,"Internal":true,"Count Failed Values":true},{"ID":167,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":168,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":51947,"Internal":true,"Count Failed Values":true},{"ID":169,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":170,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2905,"Value":23598,"Internal":true,"Count Failed Values":true},{"ID":171,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":79213,"Value":642490,"Internal":true,"Count Failed Values":true},{"ID":172,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":2905,"Value":23598,"Internal":true,"Count Failed Values":true},{"ID":173,"Name":"internal.metrics.shuffle.write.writeTime","Update":12231018,"Value":65456717,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":1178,"Executor Deserialize CPU Time":1106395401,"Executor Run Time":412,"Executor CPU Time":388572482,"Peak Execution Memory":4456448,"Result Size":4279,"JVM GC Time":34,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":8,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":82217,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":2905},"Shuffle Write Metrics":{"Shuffle Bytes Written":79213,"Shuffle Write Time":12231018,"Shuffle Records Written":2905},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":4,"Stage Attempt ID":0,"Stage Name":"sql at ScalaSparkSQLBench.scala:43","Number of Tasks":8,"RDD Info":[{"RDD ID":17,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":12,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"12\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[11],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[3],"Details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1633789203002,"Completion Time":1633789204642,"Accumulables":[{"ID":26,"Name":"remote blocks read","Value":"117","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":27,"Name":"local blocks read","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Value":"1217673","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"local bytes read","Value":"114771","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Value":"47196","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":36,"Name":"data size","Value":"1132680","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":44,"Name":"shuffle bytes written","Value":"642490","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":45,"Name":"shuffle records written","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":46,"Name":"shuffle write time","Value":"65456717","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Value":"1110","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Value":"47196","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Value":"71303168","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Value":"298","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Value":"176","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":153,"Name":"internal.metrics.executorDeserializeTime","Value":1739,"Internal":true,"Count Failed Values":true},{"ID":154,"Name":"internal.metrics.executorDeserializeCpuTime","Value":1628279172,"Internal":true,"Count Failed Values":true},{"ID":155,"Name":"internal.metrics.executorRunTime","Value":1103,"Internal":true,"Count Failed Values":true},{"ID":156,"Name":"internal.metrics.executorCpuTime","Value":1028140600,"Internal":true,"Count Failed Values":true},{"ID":157,"Name":"internal.metrics.resultSize","Value":33932,"Internal":true,"Count Failed Values":true},{"ID":158,"Name":"internal.metrics.jvmGCTime","Value":34,"Internal":true,"Count Failed Values":true},{"ID":162,"Name":"internal.metrics.peakExecutionMemory","Value":35651584,"Internal":true,"Count Failed Values":true},{"ID":164,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":59,"Internal":true,"Count Failed Values":true},{"ID":165,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":5,"Internal":true,"Count Failed Values":true},{"ID":166,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":614275,"Internal":true,"Count Failed Values":true},{"ID":167,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":168,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":51947,"Internal":true,"Count Failed Values":true},{"ID":169,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":170,"Name":"internal.metrics.shuffle.read.recordsRead","Value":23598,"Internal":true,"Count Failed Values":true},{"ID":171,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":642490,"Internal":true,"Count Failed Values":true},{"ID":172,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":23598,"Internal":true,"Count Failed Values":true},{"ID":173,"Name":"internal.metrics.shuffle.write.writeTime","Value":65456717,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} -{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":5,"Stage Attempt ID":0,"Stage Name":"sql at ScalaSparkSQLBench.scala:43","Number of Tasks":8,"RDD Info":[{"RDD ID":19,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[18],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":18,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[17],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[4],"Details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1633789204645,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.sql.warehouse.dir":"file:/home/usera112/projects/log-compressor/deployments/HiBench/spark-warehouse","spark.driver.host":"hostb7.dsrg.utoronto.ca","spark.eventLog.enabled":"true","spark.driver.port":"44011","__fetch_continuous_blocks_in_batch_enabled":"true","spark.jars":"file:/home/usera112/projects/log-compressor/deployments/HiBench/sparkbench/assembly/target/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.app.name":"ScalaJoin","spark.driver.memory":"4g","spark.submit.pyFiles":"","spark.default.parallelism":"8","spark.app.startTime":"1633789163045","spark.executor.id":"driver","spark.app.initial.jar.urls":"spark://hostb7.dsrg.utoronto.ca:44011/jars/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.submit.deployMode":"client","spark.master":"spark://hostb7:7077","spark.executor.memory":"4g","spark.eventLog.dir":"hdfs://hostb7:8020/directory","spark.sql.execution.id":"11","spark.sql.catalogImplementation":"hive","spark.app.id":"app-20211009101924-2782","spark.sql.shuffle.partitions":"8"}} -{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":32,"Index":0,"Attempt":0,"Launch Time":1633789204689,"Executor ID":"9","Host":"10.1.0.14","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":33,"Index":1,"Attempt":0,"Launch Time":1633789204689,"Executor ID":"8","Host":"10.1.0.11","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":34,"Index":2,"Attempt":0,"Launch Time":1633789204689,"Executor ID":"0","Host":"10.1.0.12","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":35,"Index":3,"Attempt":0,"Launch Time":1633789204689,"Executor ID":"4","Host":"10.1.0.8","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":36,"Index":4,"Attempt":0,"Launch Time":1633789204690,"Executor ID":"2","Host":"10.1.0.10","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":37,"Index":5,"Attempt":0,"Launch Time":1633789204690,"Executor ID":"3","Host":"10.1.0.15","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":38,"Index":6,"Attempt":0,"Launch Time":1633789204690,"Executor ID":"11","Host":"10.1.0.13","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":39,"Index":7,"Attempt":0,"Launch Time":1633789204690,"Executor ID":"10","Host":"10.1.0.16","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":38,"Index":6,"Attempt":0,"Launch Time":1633789204690,"Executor ID":"11","Host":"10.1.0.13","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633789205199,"Failed":false,"Killed":false,"Accumulables":[{"ID":37,"Name":"remote blocks read","Update":"7","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":38,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":39,"Name":"remote bytes read","Update":"68113","Value":"68113","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":41,"Name":"local bytes read","Update":"9554","Value":"9554","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":42,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":43,"Name":"records read","Update":"2846","Value":"2846","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":51,"Name":"duration","Update":"316","Value":"316","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":52,"Name":"sort time","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":53,"Name":"peak memory","Update":"4325376","Value":"4325376","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":54,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":178,"Name":"internal.metrics.executorDeserializeTime","Update":71,"Value":71,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.executorDeserializeCpuTime","Update":65347568,"Value":65347568,"Internal":true,"Count Failed Values":true},{"ID":180,"Name":"internal.metrics.executorRunTime","Update":421,"Value":421,"Internal":true,"Count Failed Values":true},{"ID":181,"Name":"internal.metrics.executorCpuTime","Update":334657806,"Value":334657806,"Internal":true,"Count Failed Values":true},{"ID":182,"Name":"internal.metrics.resultSize","Update":5952,"Value":5952,"Internal":true,"Count Failed Values":true},{"ID":184,"Name":"internal.metrics.resultSerializationTime","Update":3,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":187,"Name":"internal.metrics.peakExecutionMemory","Update":4325376,"Value":4325376,"Internal":true,"Count Failed Values":true},{"ID":189,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":190,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":191,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":68113,"Value":68113,"Internal":true,"Count Failed Values":true},{"ID":192,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":193,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":9554,"Value":9554,"Internal":true,"Count Failed Values":true},{"ID":194,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":195,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2846,"Value":2846,"Internal":true,"Count Failed Values":true},{"ID":201,"Name":"internal.metrics.output.bytesWritten","Update":120001,"Value":120001,"Internal":true,"Count Failed Values":true},{"ID":202,"Name":"internal.metrics.output.recordsWritten","Update":2846,"Value":2846,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":71,"Executor Deserialize CPU Time":65347568,"Executor Run Time":421,"Executor CPU Time":334657806,"Peak Execution Memory":4325376,"Result Size":5952,"JVM GC Time":0,"Result Serialization Time":3,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":68113,"Remote Bytes Read To Disk":0,"Local Bytes Read":9554,"Total Records Read":2846},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":120001,"Records Written":2846},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":35,"Index":3,"Attempt":0,"Launch Time":1633789204689,"Executor ID":"4","Host":"10.1.0.8","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633789205211,"Failed":false,"Killed":false,"Accumulables":[{"ID":37,"Name":"remote blocks read","Update":"8","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":39,"Name":"remote bytes read","Update":"83492","Value":"151605","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":42,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":43,"Name":"records read","Update":"3075","Value":"5921","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":51,"Name":"duration","Update":"327","Value":"643","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":52,"Name":"sort time","Update":"6","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":53,"Name":"peak memory","Update":"4325376","Value":"8650752","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":54,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":178,"Name":"internal.metrics.executorDeserializeTime","Update":77,"Value":148,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.executorDeserializeCpuTime","Update":71484546,"Value":136832114,"Internal":true,"Count Failed Values":true},{"ID":180,"Name":"internal.metrics.executorRunTime","Update":433,"Value":854,"Internal":true,"Count Failed Values":true},{"ID":181,"Name":"internal.metrics.executorCpuTime","Update":350450356,"Value":685108162,"Internal":true,"Count Failed Values":true},{"ID":182,"Name":"internal.metrics.resultSize","Update":5952,"Value":11904,"Internal":true,"Count Failed Values":true},{"ID":184,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":187,"Name":"internal.metrics.peakExecutionMemory","Update":4325376,"Value":8650752,"Internal":true,"Count Failed Values":true},{"ID":189,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":8,"Value":15,"Internal":true,"Count Failed Values":true},{"ID":190,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":191,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":83492,"Value":151605,"Internal":true,"Count Failed Values":true},{"ID":192,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":193,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":9554,"Internal":true,"Count Failed Values":true},{"ID":194,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":195,"Name":"internal.metrics.shuffle.read.recordsRead","Update":3075,"Value":5921,"Internal":true,"Count Failed Values":true},{"ID":201,"Name":"internal.metrics.output.bytesWritten","Update":128252,"Value":248253,"Internal":true,"Count Failed Values":true},{"ID":202,"Name":"internal.metrics.output.recordsWritten","Update":3075,"Value":5921,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":77,"Executor Deserialize CPU Time":71484546,"Executor Run Time":433,"Executor CPU Time":350450356,"Peak Execution Memory":4325376,"Result Size":5952,"JVM GC Time":0,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":8,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":83492,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":3075},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":128252,"Records Written":3075},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":39,"Index":7,"Attempt":0,"Launch Time":1633789204690,"Executor ID":"10","Host":"10.1.0.16","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633789205219,"Failed":false,"Killed":false,"Accumulables":[{"ID":37,"Name":"remote blocks read","Update":"7","Value":"22","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":38,"Name":"local blocks read","Update":"1","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":39,"Name":"remote bytes read","Update":"72297","Value":"223902","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":41,"Name":"local bytes read","Update":"10257","Value":"19811","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":42,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":43,"Name":"records read","Update":"3025","Value":"8946","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":51,"Name":"duration","Update":"296","Value":"939","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":52,"Name":"sort time","Update":"5","Value":"16","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":53,"Name":"peak memory","Update":"4325376","Value":"12976128","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":54,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":178,"Name":"internal.metrics.executorDeserializeTime","Update":92,"Value":240,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.executorDeserializeCpuTime","Update":85917179,"Value":222749293,"Internal":true,"Count Failed Values":true},{"ID":180,"Name":"internal.metrics.executorRunTime","Update":425,"Value":1279,"Internal":true,"Count Failed Values":true},{"ID":181,"Name":"internal.metrics.executorCpuTime","Update":318096729,"Value":1003204891,"Internal":true,"Count Failed Values":true},{"ID":182,"Name":"internal.metrics.resultSize","Update":5952,"Value":17856,"Internal":true,"Count Failed Values":true},{"ID":184,"Name":"internal.metrics.resultSerializationTime","Update":3,"Value":8,"Internal":true,"Count Failed Values":true},{"ID":187,"Name":"internal.metrics.peakExecutionMemory","Update":4325376,"Value":12976128,"Internal":true,"Count Failed Values":true},{"ID":189,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":22,"Internal":true,"Count Failed Values":true},{"ID":190,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":191,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":72297,"Value":223902,"Internal":true,"Count Failed Values":true},{"ID":192,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":193,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":10257,"Value":19811,"Internal":true,"Count Failed Values":true},{"ID":194,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":195,"Name":"internal.metrics.shuffle.read.recordsRead","Update":3025,"Value":8946,"Internal":true,"Count Failed Values":true},{"ID":201,"Name":"internal.metrics.output.bytesWritten","Update":129566,"Value":377819,"Internal":true,"Count Failed Values":true},{"ID":202,"Name":"internal.metrics.output.recordsWritten","Update":3025,"Value":8946,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":92,"Executor Deserialize CPU Time":85917179,"Executor Run Time":425,"Executor CPU Time":318096729,"Peak Execution Memory":4325376,"Result Size":5952,"JVM GC Time":0,"Result Serialization Time":3,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":72297,"Remote Bytes Read To Disk":0,"Local Bytes Read":10257,"Total Records Read":3025},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":129566,"Records Written":3025},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":36,"Index":4,"Attempt":0,"Launch Time":1633789204690,"Executor ID":"2","Host":"10.1.0.10","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633789205228,"Failed":false,"Killed":false,"Accumulables":[{"ID":37,"Name":"remote blocks read","Update":"7","Value":"29","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":38,"Name":"local blocks read","Update":"1","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":39,"Name":"remote bytes read","Update":"69831","Value":"293733","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":41,"Name":"local bytes read","Update":"9673","Value":"29484","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":42,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":43,"Name":"records read","Update":"2911","Value":"11857","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":51,"Name":"duration","Update":"312","Value":"1251","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":52,"Name":"sort time","Update":"6","Value":"22","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":53,"Name":"peak memory","Update":"4325376","Value":"17301504","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":54,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":178,"Name":"internal.metrics.executorDeserializeTime","Update":77,"Value":317,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.executorDeserializeCpuTime","Update":69361828,"Value":292111121,"Internal":true,"Count Failed Values":true},{"ID":180,"Name":"internal.metrics.executorRunTime","Update":449,"Value":1728,"Internal":true,"Count Failed Values":true},{"ID":181,"Name":"internal.metrics.executorCpuTime","Update":331558224,"Value":1334763115,"Internal":true,"Count Failed Values":true},{"ID":182,"Name":"internal.metrics.resultSize","Update":5952,"Value":23808,"Internal":true,"Count Failed Values":true},{"ID":184,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":10,"Internal":true,"Count Failed Values":true},{"ID":187,"Name":"internal.metrics.peakExecutionMemory","Update":4325376,"Value":17301504,"Internal":true,"Count Failed Values":true},{"ID":189,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":29,"Internal":true,"Count Failed Values":true},{"ID":190,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":191,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":69831,"Value":293733,"Internal":true,"Count Failed Values":true},{"ID":192,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":193,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":9673,"Value":29484,"Internal":true,"Count Failed Values":true},{"ID":194,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":195,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2911,"Value":11857,"Internal":true,"Count Failed Values":true},{"ID":201,"Name":"internal.metrics.output.bytesWritten","Update":122317,"Value":500136,"Internal":true,"Count Failed Values":true},{"ID":202,"Name":"internal.metrics.output.recordsWritten","Update":2911,"Value":11857,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":77,"Executor Deserialize CPU Time":69361828,"Executor Run Time":449,"Executor CPU Time":331558224,"Peak Execution Memory":4325376,"Result Size":5952,"JVM GC Time":0,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":69831,"Remote Bytes Read To Disk":0,"Local Bytes Read":9673,"Total Records Read":2911},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":122317,"Records Written":2911},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":32,"Index":0,"Attempt":0,"Launch Time":1633789204689,"Executor ID":"9","Host":"10.1.0.14","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633789205228,"Failed":false,"Killed":false,"Accumulables":[{"ID":37,"Name":"remote blocks read","Update":"8","Value":"37","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":39,"Name":"remote bytes read","Update":"78576","Value":"372309","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":42,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":43,"Name":"records read","Update":"2888","Value":"14745","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":51,"Name":"duration","Update":"314","Value":"1565","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":52,"Name":"sort time","Update":"6","Value":"28","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":53,"Name":"peak memory","Update":"4325376","Value":"21626880","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":54,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":178,"Name":"internal.metrics.executorDeserializeTime","Update":78,"Value":395,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.executorDeserializeCpuTime","Update":71457306,"Value":363568427,"Internal":true,"Count Failed Values":true},{"ID":180,"Name":"internal.metrics.executorRunTime","Update":448,"Value":2176,"Internal":true,"Count Failed Values":true},{"ID":181,"Name":"internal.metrics.executorCpuTime","Update":330505714,"Value":1665268829,"Internal":true,"Count Failed Values":true},{"ID":182,"Name":"internal.metrics.resultSize","Update":5952,"Value":29760,"Internal":true,"Count Failed Values":true},{"ID":184,"Name":"internal.metrics.resultSerializationTime","Update":3,"Value":13,"Internal":true,"Count Failed Values":true},{"ID":187,"Name":"internal.metrics.peakExecutionMemory","Update":4325376,"Value":21626880,"Internal":true,"Count Failed Values":true},{"ID":189,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":8,"Value":37,"Internal":true,"Count Failed Values":true},{"ID":190,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":191,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":78576,"Value":372309,"Internal":true,"Count Failed Values":true},{"ID":192,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":193,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":29484,"Internal":true,"Count Failed Values":true},{"ID":194,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":195,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2888,"Value":14745,"Internal":true,"Count Failed Values":true},{"ID":201,"Name":"internal.metrics.output.bytesWritten","Update":120473,"Value":620609,"Internal":true,"Count Failed Values":true},{"ID":202,"Name":"internal.metrics.output.recordsWritten","Update":2888,"Value":14745,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":78,"Executor Deserialize CPU Time":71457306,"Executor Run Time":448,"Executor CPU Time":330505714,"Peak Execution Memory":4325376,"Result Size":5952,"JVM GC Time":0,"Result Serialization Time":3,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":8,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":78576,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":2888},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":120473,"Records Written":2888},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":33,"Index":1,"Attempt":0,"Launch Time":1633789204689,"Executor ID":"8","Host":"10.1.0.11","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633789205294,"Failed":false,"Killed":false,"Accumulables":[{"ID":37,"Name":"remote blocks read","Update":"8","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":39,"Name":"remote bytes read","Update":"86048","Value":"458357","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":42,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":43,"Name":"records read","Update":"3175","Value":"17920","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":51,"Name":"duration","Update":"333","Value":"1898","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":52,"Name":"sort time","Update":"5","Value":"33","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":53,"Name":"peak memory","Update":"4325376","Value":"25952256","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":54,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":178,"Name":"internal.metrics.executorDeserializeTime","Update":89,"Value":484,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.executorDeserializeCpuTime","Update":82149783,"Value":445718210,"Internal":true,"Count Failed Values":true},{"ID":180,"Name":"internal.metrics.executorRunTime","Update":504,"Value":2680,"Internal":true,"Count Failed Values":true},{"ID":181,"Name":"internal.metrics.executorCpuTime","Update":417751654,"Value":2083020483,"Internal":true,"Count Failed Values":true},{"ID":182,"Name":"internal.metrics.resultSize","Update":5952,"Value":35712,"Internal":true,"Count Failed Values":true},{"ID":184,"Name":"internal.metrics.resultSerializationTime","Update":3,"Value":16,"Internal":true,"Count Failed Values":true},{"ID":187,"Name":"internal.metrics.peakExecutionMemory","Update":4325376,"Value":25952256,"Internal":true,"Count Failed Values":true},{"ID":189,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":8,"Value":45,"Internal":true,"Count Failed Values":true},{"ID":190,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":191,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":86048,"Value":458357,"Internal":true,"Count Failed Values":true},{"ID":192,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":193,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":29484,"Internal":true,"Count Failed Values":true},{"ID":194,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":195,"Name":"internal.metrics.shuffle.read.recordsRead","Update":3175,"Value":17920,"Internal":true,"Count Failed Values":true},{"ID":201,"Name":"internal.metrics.output.bytesWritten","Update":132377,"Value":752986,"Internal":true,"Count Failed Values":true},{"ID":202,"Name":"internal.metrics.output.recordsWritten","Update":3175,"Value":17920,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":89,"Executor Deserialize CPU Time":82149783,"Executor Run Time":504,"Executor CPU Time":417751654,"Peak Execution Memory":4325376,"Result Size":5952,"JVM GC Time":0,"Result Serialization Time":3,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":8,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":86048,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":3175},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":132377,"Records Written":3175},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":37,"Index":5,"Attempt":0,"Launch Time":1633789204690,"Executor ID":"3","Host":"10.1.0.15","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633789205962,"Failed":false,"Killed":false,"Accumulables":[{"ID":37,"Name":"remote blocks read","Update":"8","Value":"53","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":39,"Name":"remote bytes read","Update":"75179","Value":"533536","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":42,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":43,"Name":"records read","Update":"2747","Value":"20667","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":51,"Name":"duration","Update":"978","Value":"2876","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":52,"Name":"sort time","Update":"5","Value":"38","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":53,"Name":"peak memory","Update":"4325376","Value":"30277632","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":54,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":178,"Name":"internal.metrics.executorDeserializeTime","Update":130,"Value":614,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.executorDeserializeCpuTime","Update":121934949,"Value":567653159,"Internal":true,"Count Failed Values":true},{"ID":180,"Name":"internal.metrics.executorRunTime","Update":1128,"Value":3808,"Internal":true,"Count Failed Values":true},{"ID":181,"Name":"internal.metrics.executorCpuTime","Update":888495755,"Value":2971516238,"Internal":true,"Count Failed Values":true},{"ID":182,"Name":"internal.metrics.resultSize","Update":5995,"Value":41707,"Internal":true,"Count Failed Values":true},{"ID":183,"Name":"internal.metrics.jvmGCTime","Update":118,"Value":118,"Internal":true,"Count Failed Values":true},{"ID":184,"Name":"internal.metrics.resultSerializationTime","Update":3,"Value":19,"Internal":true,"Count Failed Values":true},{"ID":187,"Name":"internal.metrics.peakExecutionMemory","Update":4325376,"Value":30277632,"Internal":true,"Count Failed Values":true},{"ID":189,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":8,"Value":53,"Internal":true,"Count Failed Values":true},{"ID":190,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":191,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":75179,"Value":533536,"Internal":true,"Count Failed Values":true},{"ID":192,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":193,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":29484,"Internal":true,"Count Failed Values":true},{"ID":194,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":195,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2747,"Value":20667,"Internal":true,"Count Failed Values":true},{"ID":201,"Name":"internal.metrics.output.bytesWritten","Update":115563,"Value":868549,"Internal":true,"Count Failed Values":true},{"ID":202,"Name":"internal.metrics.output.recordsWritten","Update":2747,"Value":20667,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":130,"Executor Deserialize CPU Time":121934949,"Executor Run Time":1128,"Executor CPU Time":888495755,"Peak Execution Memory":4325376,"Result Size":5995,"JVM GC Time":118,"Result Serialization Time":3,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":8,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":75179,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":2747},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":115563,"Records Written":2747},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":34,"Index":2,"Attempt":0,"Launch Time":1633789204689,"Executor ID":"0","Host":"10.1.0.12","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633789206005,"Failed":false,"Killed":false,"Accumulables":[{"ID":37,"Name":"remote blocks read","Update":"7","Value":"60","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":38,"Name":"local blocks read","Update":"1","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":39,"Name":"remote bytes read","Update":"69341","Value":"602877","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":41,"Name":"local bytes read","Update":"10129","Value":"39613","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":42,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":43,"Name":"records read","Update":"2931","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":51,"Name":"duration","Update":"1025","Value":"3901","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":52,"Name":"sort time","Update":"7","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":53,"Name":"peak memory","Update":"4325376","Value":"34603008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":54,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":178,"Name":"internal.metrics.executorDeserializeTime","Update":119,"Value":733,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.executorDeserializeCpuTime","Update":107614273,"Value":675267432,"Internal":true,"Count Failed Values":true},{"ID":180,"Name":"internal.metrics.executorRunTime","Update":1181,"Value":4989,"Internal":true,"Count Failed Values":true},{"ID":181,"Name":"internal.metrics.executorCpuTime","Update":932909388,"Value":3904425626,"Internal":true,"Count Failed Values":true},{"ID":182,"Name":"internal.metrics.resultSize","Update":5995,"Value":47702,"Internal":true,"Count Failed Values":true},{"ID":183,"Name":"internal.metrics.jvmGCTime","Update":117,"Value":235,"Internal":true,"Count Failed Values":true},{"ID":184,"Name":"internal.metrics.resultSerializationTime","Update":3,"Value":22,"Internal":true,"Count Failed Values":true},{"ID":187,"Name":"internal.metrics.peakExecutionMemory","Update":4325376,"Value":34603008,"Internal":true,"Count Failed Values":true},{"ID":189,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":60,"Internal":true,"Count Failed Values":true},{"ID":190,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":191,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":69341,"Value":602877,"Internal":true,"Count Failed Values":true},{"ID":192,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":193,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":10129,"Value":39613,"Internal":true,"Count Failed Values":true},{"ID":194,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":195,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2931,"Value":23598,"Internal":true,"Count Failed Values":true},{"ID":201,"Name":"internal.metrics.output.bytesWritten","Update":122168,"Value":990717,"Internal":true,"Count Failed Values":true},{"ID":202,"Name":"internal.metrics.output.recordsWritten","Update":2931,"Value":23598,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":119,"Executor Deserialize CPU Time":107614273,"Executor Run Time":1181,"Executor CPU Time":932909388,"Peak Execution Memory":4325376,"Result Size":5995,"JVM GC Time":117,"Result Serialization Time":3,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":69341,"Remote Bytes Read To Disk":0,"Local Bytes Read":10129,"Total Records Read":2931},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":122168,"Records Written":2931},"Updated Blocks":[]}} -{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":5,"Stage Attempt ID":0,"Stage Name":"sql at ScalaSparkSQLBench.scala:43","Number of Tasks":8,"RDD Info":[{"RDD ID":19,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[18],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":18,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[17],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[4],"Details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1633789204645,"Completion Time":1633789206005,"Accumulables":[{"ID":37,"Name":"remote blocks read","Value":"60","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":38,"Name":"local blocks read","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":39,"Name":"remote bytes read","Value":"602877","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":41,"Name":"local bytes read","Value":"39613","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":42,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":43,"Name":"records read","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":51,"Name":"duration","Value":"3901","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":52,"Name":"sort time","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":53,"Name":"peak memory","Value":"34603008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":54,"Name":"spill size","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":178,"Name":"internal.metrics.executorDeserializeTime","Value":733,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.executorDeserializeCpuTime","Value":675267432,"Internal":true,"Count Failed Values":true},{"ID":180,"Name":"internal.metrics.executorRunTime","Value":4989,"Internal":true,"Count Failed Values":true},{"ID":181,"Name":"internal.metrics.executorCpuTime","Value":3904425626,"Internal":true,"Count Failed Values":true},{"ID":182,"Name":"internal.metrics.resultSize","Value":47702,"Internal":true,"Count Failed Values":true},{"ID":183,"Name":"internal.metrics.jvmGCTime","Value":235,"Internal":true,"Count Failed Values":true},{"ID":184,"Name":"internal.metrics.resultSerializationTime","Value":22,"Internal":true,"Count Failed Values":true},{"ID":187,"Name":"internal.metrics.peakExecutionMemory","Value":34603008,"Internal":true,"Count Failed Values":true},{"ID":189,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":60,"Internal":true,"Count Failed Values":true},{"ID":190,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":4,"Internal":true,"Count Failed Values":true},{"ID":191,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":602877,"Internal":true,"Count Failed Values":true},{"ID":192,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":193,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":39613,"Internal":true,"Count Failed Values":true},{"ID":194,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":195,"Name":"internal.metrics.shuffle.read.recordsRead","Value":23598,"Internal":true,"Count Failed Values":true},{"ID":201,"Name":"internal.metrics.output.bytesWritten","Value":990717,"Internal":true,"Count Failed Values":true},{"ID":202,"Name":"internal.metrics.output.recordsWritten","Value":23598,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} -{"Event":"SparkListenerJobEnd","Job ID":2,"Completion Time":1633789206006,"Job Result":{"Result":"JobSucceeded"}} -{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":11,"accumUpdates":[[47,8],[48,990717],[49,23598],[50,0]]} -{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":11,"time":1633789206627} -{"Event":"SparkListenerApplicationEnd","Timestamp":1633789206627} -{"Event":"SparkListenerLogStart","Spark Version":"3.1.2"} -{"Event":"SparkListenerResourceProfileAdded","Resource Profile Id":0,"Executor Resource Requests":{"cores":{"Resource Name":"cores","Amount":1,"Discovery Script":"","Vendor":""},"memory":{"Resource Name":"memory","Amount":4096,"Discovery Script":"","Vendor":""},"offHeap":{"Resource Name":"offHeap","Amount":0,"Discovery Script":"","Vendor":""}},"Task Resource Requests":{"cpus":{"Resource Name":"cpus","Amount":1.0}}} -{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"driver","Host":"hostb7.dsrg.utoronto.ca","Port":34077},"Maximum Memory":2101975449,"Timestamp":1633886612494,"Maximum Onheap Memory":2101975449,"Maximum Offheap Memory":0} -{"Event":"SparkListenerEnvironmentUpdate","JVM Information":{"Java Home":"/usr/lib/jvm/java-8-openjdk-amd64/jre","Java Version":"1.8.0_162 (Oracle Corporation)","Scala Version":"version 2.12.10"},"Spark Properties":{"spark.driver.host":"hostb7.dsrg.utoronto.ca","spark.eventLog.enabled":"true","spark.driver.port":"42853","spark.jars":"file:/home/usera112/projects/log-compressor/deployments/HiBench/sparkbench/assembly/target/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.app.name":"ScalaJoin","spark.scheduler.mode":"FIFO","spark.driver.memory":"4g","spark.submit.pyFiles":"","spark.default.parallelism":"8","spark.app.startTime":"1633886611309","spark.executor.id":"driver","spark.app.initial.jar.urls":"spark://hostb7.dsrg.utoronto.ca:42853/jars/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.submit.deployMode":"client","spark.master":"spark://hostb7:7077","spark.executor.memory":"4g","spark.eventLog.dir":"hdfs://hostb7:8020/directory","spark.app.id":"app-20211010132332-4636","spark.sql.shuffle.partitions":"8"},"Hadoop Properties":{"hadoop.service.shutdown.timeout":"30s","yarn.resourcemanager.amlauncher.thread-count":"50","yarn.sharedcache.enabled":"false","fs.s3a.connection.maximum":"15","yarn.nodemanager.numa-awareness.numactl.cmd":"/usr/bin/numactl","fs.s3a.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","yarn.app.mapreduce.am.scheduler.heartbeat.interval-ms":"1000","yarn.timeline-service.timeline-client.number-of-async-entities-to-merge":"10","hadoop.security.kms.client.timeout":"60","hadoop.http.authentication.kerberos.principal":"HTTP/_HOST@LOCALHOST","mapreduce.jobhistory.loadedjob.tasks.max":"-1","mapreduce.framework.name":"yarn","yarn.sharedcache.uploader.server.thread-count":"50","yarn.nodemanager.linux-container-executor.nonsecure-mode.user-pattern":"^[_.A-Za-z0-9][-@_.A-Za-z0-9]{0,255}?[$]?$","tfile.fs.output.buffer.size":"262144","yarn.app.mapreduce.am.job.task.listener.thread-count":"30","hadoop.security.groups.cache.background.reload.threads":"3","yarn.resourcemanager.webapp.cross-origin.enabled":"false","fs.AbstractFileSystem.ftp.impl":"org.apache.hadoop.fs.ftp.FtpFs","hadoop.registry.secure":"false","hadoop.shell.safely.delete.limit.num.files":"100","dfs.bytes-per-checksum":"512","mapreduce.job.acl-view-job":" ","fs.s3a.s3guard.ddb.background.sleep":"25ms","fs.s3a.retry.limit":"${fs.s3a.attempts.maximum}","mapreduce.jobhistory.loadedjobs.cache.size":"5","fs.s3a.s3guard.ddb.table.create":"false","yarn.nodemanager.amrmproxy.enabled":"false","yarn.timeline-service.entity-group-fs-store.with-user-dir":"false","mapreduce.input.fileinputformat.split.minsize":"0","yarn.resourcemanager.container.liveness-monitor.interval-ms":"600000","yarn.resourcemanager.client.thread-count":"50","io.seqfile.compress.blocksize":"1000000","yarn.sharedcache.checksum.algo.impl":"org.apache.hadoop.yarn.sharedcache.ChecksumSHA256Impl","yarn.nodemanager.amrmproxy.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.nodemanager.amrmproxy.DefaultRequestInterceptor","dfs.replication":"3","yarn.timeline-service.entity-group-fs-store.leveldb-cache-read-cache-size":"10485760","mapreduce.reduce.shuffle.fetch.retry.interval-ms":"1000","mapreduce.task.profile.maps":"0-2","yarn.scheduler.include-port-in-node-name":"false","yarn.nodemanager.admin-env":"MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX","yarn.resourcemanager.node-removal-untracked.timeout-ms":"60000","mapreduce.am.max-attempts":"2","hadoop.security.kms.client.failover.sleep.base.millis":"100","mapreduce.jobhistory.webapp.https.address":"0.0.0.0:19890","yarn.node-labels.fs-store.impl.class":"org.apache.hadoop.yarn.nodelabels.FileSystemNodeLabelsStore","yarn.nodemanager.collector-service.address":"${yarn.nodemanager.hostname}:8048","fs.trash.checkpoint.interval":"0","mapreduce.job.map.output.collector.class":"org.apache.hadoop.mapred.MapTask$MapOutputBuffer","yarn.resourcemanager.node-ip-cache.expiry-interval-secs":"-1","hadoop.http.authentication.signature.secret.file":"*********(redacted)","hadoop.jetty.logs.serve.aliases":"true","yarn.resourcemanager.placement-constraints.handler":"disabled","yarn.timeline-service.handler-thread-count":"10","yarn.resourcemanager.max-completed-applications":"1000","yarn.resourcemanager.system-metrics-publisher.enabled":"false","yarn.resourcemanager.placement-constraints.algorithm.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm.DefaultPlacementAlgorithm","yarn.sharedcache.webapp.address":"0.0.0.0:8788","yarn.resourcemanager.delegation.token.renew-interval":"*********(redacted)","yarn.sharedcache.nm.uploader.replication.factor":"10","hadoop.security.groups.negative-cache.secs":"30","yarn.app.mapreduce.task.container.log.backups":"0","mapreduce.reduce.skip.proc-count.auto-incr":"true","hadoop.security.group.mapping.ldap.posix.attr.gid.name":"gidNumber","rpc.engine.org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolPB":"org.apache.hadoop.ipc.ProtobufRpcEngine","ipc.client.fallback-to-simple-auth-allowed":"false","yarn.nodemanager.resource.memory.enforced":"true","yarn.client.failover-proxy-provider":"org.apache.hadoop.yarn.client.ConfiguredRMFailoverProxyProvider","yarn.timeline-service.http-authentication.simple.anonymous.allowed":"true","ha.health-monitor.check-interval.ms":"1000","yarn.acl.reservation-enable":"false","yarn.resourcemanager.store.class":"org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore","yarn.app.mapreduce.am.hard-kill-timeout-ms":"10000","fs.s3a.etag.checksum.enabled":"false","yarn.nodemanager.container-metrics.enable":"true","yarn.timeline-service.client.fd-clean-interval-secs":"60","yarn.resourcemanager.nodemanagers.heartbeat-interval-ms":"1000","hadoop.common.configuration.version":"3.0.0","fs.s3a.s3guard.ddb.table.capacity.read":"500","yarn.nodemanager.remote-app-log-dir-suffix":"logs","yarn.nodemanager.windows-container.cpu-limit.enabled":"false","yarn.nodemanager.runtime.linux.docker.privileged-containers.allowed":"false","file.blocksize":"67108864","hadoop.registry.zk.retry.ceiling.ms":"60000","yarn.scheduler.configuration.leveldb-store.path":"${hadoop.tmp.dir}/yarn/system/confstore","yarn.sharedcache.store.in-memory.initial-delay-mins":"10","mapreduce.jobhistory.principal":"jhs/_HOST@REALM.TLD","mapreduce.map.skip.proc-count.auto-incr":"true","fs.s3a.committer.name":"file","mapreduce.task.profile.reduces":"0-2","hadoop.zk.num-retries":"1000","yarn.webapp.xfs-filter.enabled":"true","seq.io.sort.mb":"100","yarn.scheduler.configuration.max.version":"100","yarn.timeline-service.webapp.https.address":"${yarn.timeline-service.hostname}:8190","yarn.resourcemanager.scheduler.address":"${yarn.resourcemanager.hostname}:8030","yarn.node-labels.enabled":"false","yarn.resourcemanager.webapp.ui-actions.enabled":"true","mapreduce.task.timeout":"600000","yarn.sharedcache.client-server.thread-count":"50","hadoop.security.groups.shell.command.timeout":"0s","hadoop.security.crypto.cipher.suite":"AES/CTR/NoPadding","yarn.nodemanager.elastic-memory-control.oom-handler":"org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.DefaultOOMHandler","yarn.resourcemanager.connect.max-wait.ms":"900000","fs.defaultFS":"hdfs://hostb7:8020","yarn.minicluster.use-rpc":"false","fs.har.impl.disable.cache":"true","yarn.webapp.ui2.enable":"false","io.compression.codec.bzip2.library":"system-native","yarn.nodemanager.distributed-scheduling.enabled":"false","mapreduce.shuffle.connection-keep-alive.timeout":"5","yarn.resourcemanager.webapp.https.address":"${yarn.resourcemanager.hostname}:8090","mapreduce.jobhistory.address":"0.0.0.0:10020","yarn.resourcemanager.nm-tokens.master-key-rolling-interval-secs":"*********(redacted)","yarn.is.minicluster":"false","yarn.nodemanager.address":"${yarn.nodemanager.hostname}:0","fs.abfss.impl":"org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","fs.AbstractFileSystem.s3a.impl":"org.apache.hadoop.fs.s3a.S3A","mapreduce.task.combine.progress.records":"10000","yarn.resourcemanager.epoch.range":"0","yarn.resourcemanager.am.max-attempts":"2","yarn.nodemanager.linux-container-executor.cgroups.hierarchy":"/hadoop-yarn","fs.AbstractFileSystem.wasbs.impl":"org.apache.hadoop.fs.azure.Wasbs","yarn.timeline-service.entity-group-fs-store.cache-store-class":"org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore","fs.ftp.transfer.mode":"BLOCK_TRANSFER_MODE","ipc.server.log.slow.rpc":"false","yarn.resourcemanager.node-labels.provider.fetch-interval-ms":"1800000","yarn.router.webapp.https.address":"0.0.0.0:8091","yarn.nodemanager.webapp.cross-origin.enabled":"false","fs.wasb.impl":"org.apache.hadoop.fs.azure.NativeAzureFileSystem","yarn.resourcemanager.auto-update.containers":"false","yarn.app.mapreduce.am.job.committer.cancel-timeout":"60000","yarn.scheduler.configuration.zk-store.parent-path":"/confstore","yarn.nodemanager.default-container-executor.log-dirs.permissions":"710","yarn.app.attempt.diagnostics.limit.kc":"64","ftp.bytes-per-checksum":"512","yarn.nodemanager.resource.memory-mb":"-1","fs.AbstractFileSystem.abfs.impl":"org.apache.hadoop.fs.azurebfs.Abfs","yarn.timeline-service.writer.flush-interval-seconds":"60","fs.s3a.fast.upload.active.blocks":"4","hadoop.security.credential.clear-text-fallback":"true","yarn.nodemanager.collector-service.thread-count":"5","fs.azure.secure.mode":"false","mapreduce.jobhistory.joblist.cache.size":"20000","fs.ftp.host":"0.0.0.0","yarn.resourcemanager.fs.state-store.num-retries":"0","yarn.resourcemanager.nodemanager-connect-retries":"10","yarn.nodemanager.log-aggregation.num-log-files-per-app":"30","hadoop.security.kms.client.encrypted.key.cache.low-watermark":"0.3f","fs.s3a.committer.magic.enabled":"false","yarn.timeline-service.client.max-retries":"30","dfs.ha.fencing.ssh.connect-timeout":"30000","yarn.log-aggregation-enable":"false","yarn.system-metrics-publisher.enabled":"false","mapreduce.reduce.markreset.buffer.percent":"0.0","fs.AbstractFileSystem.viewfs.impl":"org.apache.hadoop.fs.viewfs.ViewFs","mapreduce.task.io.sort.factor":"10","yarn.nodemanager.amrmproxy.client.thread-count":"25","ha.failover-controller.new-active.rpc-timeout.ms":"60000","yarn.nodemanager.container-localizer.java.opts":"-Xmx256m","mapreduce.jobhistory.datestring.cache.size":"200000","mapreduce.job.acl-modify-job":" ","yarn.nodemanager.windows-container.memory-limit.enabled":"false","yarn.timeline-service.webapp.address":"${yarn.timeline-service.hostname}:8188","yarn.app.mapreduce.am.job.committer.commit-window":"10000","yarn.nodemanager.container-manager.thread-count":"20","yarn.minicluster.fixed.ports":"false","hadoop.tags.system":"YARN,HDFS,NAMENODE,DATANODE,REQUIRED,SECURITY,KERBEROS,PERFORMANCE,CLIENT\n ,SERVER,DEBUG,DEPRECATED,COMMON,OPTIONAL","yarn.cluster.max-application-priority":"0","yarn.timeline-service.ttl-enable":"true","mapreduce.jobhistory.recovery.store.fs.uri":"${hadoop.tmp.dir}/mapred/history/recoverystore","hadoop.caller.context.signature.max.size":"40","yarn.client.load.resource-types.from-server":"false","ha.zookeeper.session-timeout.ms":"10000","tfile.io.chunk.size":"1048576","fs.s3a.s3guard.ddb.table.capacity.write":"100","mapreduce.job.speculative.slowtaskthreshold":"1.0","io.serializations":"org.apache.hadoop.io.serializer.WritableSerialization, org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization, org.apache.hadoop.io.serializer.avro.AvroReflectSerialization","hadoop.security.kms.client.failover.sleep.max.millis":"2000","hadoop.security.group.mapping.ldap.directory.search.timeout":"10000","yarn.scheduler.configuration.store.max-logs":"1000","yarn.nodemanager.node-attributes.provider.fetch-interval-ms":"600000","fs.swift.impl":"org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem","yarn.nodemanager.local-cache.max-files-per-directory":"8192","hadoop.http.cross-origin.enabled":"false","hadoop.zk.acl":"world:anyone:rwcda","mapreduce.map.sort.spill.percent":"0.80","yarn.timeline-service.entity-group-fs-store.scan-interval-seconds":"60","yarn.node-attribute.fs-store.impl.class":"org.apache.hadoop.yarn.server.resourcemanager.nodelabels.FileSystemNodeAttributeStore","fs.s3a.retry.interval":"500ms","yarn.timeline-service.client.best-effort":"false","yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled":"*********(redacted)","hadoop.security.group.mapping.ldap.posix.attr.uid.name":"uidNumber","fs.AbstractFileSystem.swebhdfs.impl":"org.apache.hadoop.fs.SWebHdfs","yarn.nodemanager.elastic-memory-control.timeout-sec":"5","mapreduce.ifile.readahead":"true","yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms":"300000","yarn.timeline-service.reader.webapp.address":"${yarn.timeline-service.webapp.address}","yarn.resourcemanager.placement-constraints.algorithm.pool-size":"1","yarn.timeline-service.hbase.coprocessor.jar.hdfs.location":"/hbase/coprocessor/hadoop-yarn-server-timelineservice.jar","hadoop.security.kms.client.encrypted.key.cache.num.refill.threads":"2","yarn.resourcemanager.scheduler.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler","yarn.app.mapreduce.am.command-opts":"-Xmx1024m","mapreduce.cluster.local.dir":"${hadoop.tmp.dir}/mapred/local","io.mapfile.bloom.error.rate":"0.005","fs.client.resolve.topology.enabled":"false","yarn.nodemanager.runtime.linux.allowed-runtimes":"default","yarn.sharedcache.store.class":"org.apache.hadoop.yarn.server.sharedcachemanager.store.InMemorySCMStore","ha.failover-controller.graceful-fence.rpc-timeout.ms":"5000","ftp.replication":"3","hadoop.security.uid.cache.secs":"14400","mapreduce.job.maxtaskfailures.per.tracker":"3","fs.s3a.metadatastore.impl":"org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore","io.skip.checksum.errors":"false","yarn.app.mapreduce.client-am.ipc.max-retries-on-timeouts":"3","yarn.timeline-service.webapp.xfs-filter.xframe-options":"SAMEORIGIN","fs.s3a.connection.timeout":"200000","mapreduce.job.max.split.locations":"15","yarn.resourcemanager.nm-container-queuing.max-queue-length":"15","hadoop.registry.zk.session.timeout.ms":"60000","yarn.federation.cache-ttl.secs":"300","mapreduce.jvm.system-properties-to-log":"os.name,os.version,java.home,java.runtime.version,java.vendor,java.version,java.vm.name,java.class.path,java.io.tmpdir,user.dir,user.name","yarn.resourcemanager.opportunistic-container-allocation.nodes-used":"10","yarn.timeline-service.entity-group-fs-store.active-dir":"/tmp/entity-file-history/active","mapreduce.shuffle.transfer.buffer.size":"131072","yarn.timeline-service.client.retry-interval-ms":"1000","yarn.http.policy":"HTTP_ONLY","fs.s3a.socket.send.buffer":"8192","fs.AbstractFileSystem.abfss.impl":"org.apache.hadoop.fs.azurebfs.Abfss","yarn.sharedcache.uploader.server.address":"0.0.0.0:8046","yarn.resourcemanager.delegation-token.max-conf-size-bytes":"*********(redacted)","hadoop.http.authentication.token.validity":"*********(redacted)","mapreduce.shuffle.max.connections":"0","yarn.minicluster.yarn.nodemanager.resource.memory-mb":"4096","mapreduce.job.emit-timeline-data":"false","yarn.nodemanager.resource.system-reserved-memory-mb":"-1","hadoop.kerberos.min.seconds.before.relogin":"60","mapreduce.jobhistory.move.thread-count":"3","yarn.resourcemanager.admin.client.thread-count":"1","yarn.dispatcher.drain-events.timeout":"300000","fs.s3a.buffer.dir":"${hadoop.tmp.dir}/s3a","hadoop.ssl.enabled.protocols":"TLSv1,SSLv2Hello,TLSv1.1,TLSv1.2","mapreduce.jobhistory.admin.address":"0.0.0.0:10033","yarn.log-aggregation-status.time-out.ms":"600000","fs.s3a.assumed.role.sts.endpoint.region":"us-west-1","mapreduce.shuffle.port":"13562","yarn.resourcemanager.max-log-aggregation-diagnostics-in-memory":"10","yarn.nodemanager.health-checker.interval-ms":"600000","yarn.router.clientrm.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.clientrm.DefaultClientRequestInterceptor","yarn.resourcemanager.zk-appid-node.split-index":"0","ftp.blocksize":"67108864","yarn.nodemanager.runtime.linux.sandbox-mode.local-dirs.permissions":"read","yarn.router.rmadmin.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.rmadmin.DefaultRMAdminRequestInterceptor","yarn.nodemanager.log-container-debug-info.enabled":"true","yarn.client.max-cached-nodemanagers-proxies":"0","yarn.nodemanager.linux-container-executor.cgroups.delete-delay-ms":"20","yarn.nodemanager.delete.debug-delay-sec":"0","yarn.nodemanager.pmem-check-enabled":"true","yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage":"90.0","mapreduce.app-submission.cross-platform":"false","yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms":"10000","yarn.nodemanager.container-retry-minimum-interval-ms":"1000","hadoop.security.groups.cache.secs":"300","yarn.federation.enabled":"false","fs.azure.local.sas.key.mode":"false","ipc.maximum.data.length":"67108864","mapreduce.shuffle.max.threads":"0","yarn.router.pipeline.cache-max-size":"25","yarn.resourcemanager.nm-container-queuing.load-comparator":"QUEUE_LENGTH","hadoop.security.authorization":"false","mapreduce.job.complete.cancel.delegation.tokens":"*********(redacted)","fs.s3a.paging.maximum":"5000","nfs.exports.allowed.hosts":"* rw","yarn.nodemanager.amrmproxy.ha.enable":"false","mapreduce.jobhistory.http.policy":"HTTP_ONLY","yarn.sharedcache.store.in-memory.check-period-mins":"720","hadoop.security.group.mapping.ldap.ssl":"false","yarn.client.application-client-protocol.poll-interval-ms":"200","yarn.scheduler.configuration.leveldb-store.compaction-interval-secs":"86400","yarn.timeline-service.writer.class":"org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl","ha.zookeeper.parent-znode":"/hadoop-ha","yarn.nodemanager.log-aggregation.policy.class":"org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.AllContainerLogAggregationPolicy","mapreduce.reduce.shuffle.merge.percent":"0.66","hadoop.security.group.mapping.ldap.search.filter.group":"(objectClass=group)","yarn.resourcemanager.placement-constraints.scheduler.pool-size":"1","yarn.nodemanager.resourcemanager.minimum.version":"NONE","mapreduce.job.speculative.speculative-cap-running-tasks":"0.1","yarn.admin.acl":"*","yarn.nodemanager.recovery.supervised":"false","yarn.sharedcache.admin.thread-count":"1","yarn.resourcemanager.ha.automatic-failover.enabled":"true","mapreduce.reduce.skip.maxgroups":"0","mapreduce.reduce.shuffle.connect.timeout":"180000","yarn.resourcemanager.address":"${yarn.resourcemanager.hostname}:8032","ipc.client.ping":"true","mapreduce.task.local-fs.write-limit.bytes":"-1","fs.adl.oauth2.access.token.provider.type":"*********(redacted)","mapreduce.shuffle.ssl.file.buffer.size":"65536","yarn.resourcemanager.ha.automatic-failover.embedded":"true","yarn.nodemanager.resource-plugins.gpu.docker-plugin":"nvidia-docker-v1","hadoop.ssl.enabled":"false","fs.s3a.multipart.purge":"false","yarn.scheduler.configuration.store.class":"file","yarn.resourcemanager.nm-container-queuing.queue-limit-stdev":"1.0f","mapreduce.job.end-notification.max.attempts":"5","mapreduce.output.fileoutputformat.compress.codec":"org.apache.hadoop.io.compress.DefaultCodec","yarn.nodemanager.container-monitor.procfs-tree.smaps-based-rss.enabled":"false","ipc.client.bind.wildcard.addr":"false","yarn.resourcemanager.webapp.rest-csrf.enabled":"false","ha.health-monitor.connect-retry-interval.ms":"1000","yarn.nodemanager.keytab":"/etc/krb5.keytab","mapreduce.jobhistory.keytab":"/etc/security/keytab/jhs.service.keytab","fs.s3a.threads.max":"10","mapreduce.reduce.shuffle.input.buffer.percent":"0.70","yarn.nodemanager.runtime.linux.docker.allowed-container-networks":"host,none,bridge","yarn.nodemanager.node-labels.resync-interval-ms":"120000","hadoop.tmp.dir":"/tmp/hadoop-${user.name}","mapreduce.job.maps":"2","mapreduce.jobhistory.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.job.end-notification.max.retry.interval":"5000","yarn.log-aggregation.retain-check-interval-seconds":"-1","yarn.resourcemanager.resource-tracker.client.thread-count":"50","yarn.rm.system-metrics-publisher.emit-container-events":"false","yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size":"10000","yarn.resourcemanager.ha.automatic-failover.zk-base-path":"/yarn-leader-election","io.seqfile.local.dir":"${hadoop.tmp.dir}/io/local","fs.s3a.s3guard.ddb.throttle.retry.interval":"100ms","fs.AbstractFileSystem.wasb.impl":"org.apache.hadoop.fs.azure.Wasb","mapreduce.client.submit.file.replication":"10","mapreduce.jobhistory.minicluster.fixed.ports":"false","fs.s3a.multipart.threshold":"2147483647","yarn.resourcemanager.webapp.xfs-filter.xframe-options":"SAMEORIGIN","mapreduce.jobhistory.done-dir":"${yarn.app.mapreduce.am.staging-dir}/history/done","ipc.client.idlethreshold":"4000","yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage":"false","mapreduce.reduce.input.buffer.percent":"0.0","yarn.nodemanager.runtime.linux.docker.userremapping-gid-threshold":"1","yarn.nodemanager.webapp.rest-csrf.enabled":"false","fs.ftp.host.port":"21","ipc.ping.interval":"60000","yarn.resourcemanager.history-writer.multi-threaded-dispatcher.pool-size":"10","yarn.resourcemanager.admin.address":"${yarn.resourcemanager.hostname}:8033","file.client-write-packet-size":"65536","ipc.client.kill.max":"10","mapreduce.reduce.speculative":"true","hadoop.security.key.default.bitlength":"128","mapreduce.job.reducer.unconditional-preempt.delay.sec":"300","yarn.nodemanager.disk-health-checker.interval-ms":"120000","yarn.nodemanager.log.deletion-threads-count":"4","yarn.webapp.filter-entity-list-by-user":"false","ipc.client.connection.maxidletime":"10000","mapreduce.task.io.sort.mb":"100","yarn.nodemanager.localizer.client.thread-count":"5","io.erasurecode.codec.rs.rawcoders":"rs_native,rs_java","io.erasurecode.codec.rs-legacy.rawcoders":"rs-legacy_java","yarn.sharedcache.admin.address":"0.0.0.0:8047","yarn.resourcemanager.placement-constraints.algorithm.iterator":"SERIAL","yarn.nodemanager.localizer.cache.cleanup.interval-ms":"600000","hadoop.security.crypto.codec.classes.aes.ctr.nopadding":"org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec, org.apache.hadoop.crypto.JceAesCtrCryptoCodec","mapreduce.job.cache.limit.max-resources-mb":"0","fs.s3a.connection.ssl.enabled":"true","yarn.nodemanager.process-kill-wait.ms":"5000","mapreduce.job.hdfs-servers":"${fs.defaultFS}","hadoop.workaround.non.threadsafe.getpwuid":"true","fs.df.interval":"60000","fs.s3a.multiobjectdelete.enable":"true","yarn.sharedcache.cleaner.resource-sleep-ms":"0","yarn.nodemanager.disk-health-checker.min-healthy-disks":"0.25","hadoop.shell.missing.defaultFs.warning":"false","io.file.buffer.size":"65536","hadoop.security.group.mapping.ldap.search.attr.member":"member","hadoop.security.random.device.file.path":"/dev/urandom","hadoop.security.sensitive-config-keys":"*********(redacted)","fs.s3a.s3guard.ddb.max.retries":"9","hadoop.rpc.socket.factory.class.default":"org.apache.hadoop.net.StandardSocketFactory","yarn.intermediate-data-encryption.enable":"false","yarn.resourcemanager.connect.retry-interval.ms":"30000","yarn.nodemanager.container.stderr.pattern":"{*stderr*,*STDERR*}","yarn.scheduler.minimum-allocation-mb":"1024","yarn.app.mapreduce.am.staging-dir":"/tmp/hadoop-yarn/staging","mapreduce.reduce.shuffle.read.timeout":"180000","hadoop.http.cross-origin.max-age":"1800","io.erasurecode.codec.xor.rawcoders":"xor_native,xor_java","fs.s3a.connection.establish.timeout":"5000","mapreduce.job.running.map.limit":"0","yarn.minicluster.control-resource-monitoring":"false","hadoop.ssl.require.client.cert":"false","hadoop.kerberos.kinit.command":"kinit","yarn.federation.state-store.class":"org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore","mapreduce.reduce.log.level":"INFO","hadoop.security.dns.log-slow-lookups.threshold.ms":"1000","mapreduce.job.ubertask.enable":"false","adl.http.timeout":"-1","yarn.resourcemanager.placement-constraints.retry-attempts":"3","hadoop.caller.context.enabled":"false","yarn.nodemanager.vmem-pmem-ratio":"2.1","hadoop.rpc.protection":"authentication","ha.health-monitor.rpc-timeout.ms":"45000","yarn.nodemanager.remote-app-log-dir":"/tmp/logs","hadoop.zk.timeout-ms":"10000","fs.s3a.s3guard.cli.prune.age":"86400000","yarn.nodemanager.resource.pcores-vcores-multiplier":"1.0","yarn.nodemanager.runtime.linux.sandbox-mode":"disabled","yarn.app.mapreduce.am.containerlauncher.threadpool-initial-size":"10","fs.s3a.committer.threads":"8","hadoop.zk.retry-interval-ms":"1000","hadoop.security.crypto.buffer.size":"8192","yarn.nodemanager.node-labels.provider.fetch-interval-ms":"600000","mapreduce.jobhistory.recovery.store.leveldb.path":"${hadoop.tmp.dir}/mapred/history/recoverystore","yarn.client.failover-retries-on-socket-timeouts":"0","yarn.nodemanager.resource.memory.enabled":"false","fs.azure.authorization.caching.enable":"true","hadoop.security.instrumentation.requires.admin":"false","yarn.nodemanager.delete.thread-count":"4","mapreduce.job.finish-when-all-reducers-done":"true","hadoop.registry.jaas.context":"Client","yarn.timeline-service.leveldb-timeline-store.path":"${hadoop.tmp.dir}/yarn/timeline","io.map.index.interval":"128","yarn.resourcemanager.nm-container-queuing.max-queue-wait-time-ms":"100","fs.abfs.impl":"org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","mapreduce.job.counters.max":"120","mapreduce.jobhistory.webapp.rest-csrf.enabled":"false","yarn.timeline-service.store-class":"org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore","mapreduce.jobhistory.move.interval-ms":"180000","yarn.nodemanager.localizer.fetch.thread-count":"4","yarn.resourcemanager.scheduler.client.thread-count":"50","hadoop.ssl.hostname.verifier":"DEFAULT","yarn.timeline-service.leveldb-state-store.path":"${hadoop.tmp.dir}/yarn/timeline","mapreduce.job.classloader":"false","mapreduce.task.profile.map.params":"${mapreduce.task.profile.params}","ipc.client.connect.timeout":"20000","hadoop.security.auth_to_local.mechanism":"hadoop","yarn.timeline-service.app-collector.linger-period.ms":"60000","yarn.nm.liveness-monitor.expiry-interval-ms":"600000","yarn.resourcemanager.reservation-system.planfollower.time-step":"1000","yarn.nodemanager.runtime.linux.docker.enable-userremapping.allowed":"true","yarn.webapp.api-service.enable":"false","yarn.nodemanager.recovery.enabled":"false","mapreduce.job.end-notification.retry.interval":"1000","fs.du.interval":"600000","fs.ftp.impl":"org.apache.hadoop.fs.ftp.FTPFileSystem","yarn.nodemanager.container.stderr.tail.bytes":"4096","hadoop.security.group.mapping.ldap.read.timeout.ms":"60000","hadoop.security.groups.cache.warn.after.ms":"5000","file.bytes-per-checksum":"512","mapreduce.outputcommitter.factory.scheme.s3a":"org.apache.hadoop.fs.s3a.commit.S3ACommitterFactory","hadoop.security.groups.cache.background.reload":"false","yarn.nodemanager.container-monitor.enabled":"true","yarn.nodemanager.elastic-memory-control.enabled":"false","net.topology.script.number.args":"100","mapreduce.task.merge.progress.records":"10000","yarn.nodemanager.localizer.address":"${yarn.nodemanager.hostname}:8040","yarn.timeline-service.keytab":"/etc/krb5.keytab","mapreduce.reduce.shuffle.fetch.retry.timeout-ms":"30000","yarn.resourcemanager.rm.container-allocation.expiry-interval-ms":"600000","mapreduce.fileoutputcommitter.algorithm.version":"1","yarn.resourcemanager.work-preserving-recovery.enabled":"true","mapreduce.map.skip.maxrecords":"0","yarn.sharedcache.root-dir":"/sharedcache","fs.s3a.retry.throttle.limit":"${fs.s3a.attempts.maximum}","hadoop.http.authentication.type":"simple","mapreduce.job.cache.limit.max-resources":"0","mapreduce.task.userlog.limit.kb":"0","yarn.resourcemanager.scheduler.monitor.enable":"false","ipc.client.connect.max.retries":"10","hadoop.registry.zk.retry.times":"5","yarn.nodemanager.resource-monitor.interval-ms":"3000","yarn.nodemanager.resource-plugins.gpu.allowed-gpu-devices":"auto","mapreduce.job.sharedcache.mode":"disabled","yarn.nodemanager.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.shuffle.listen.queue.size":"128","yarn.scheduler.configuration.mutation.acl-policy.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy","mapreduce.map.cpu.vcores":"1","yarn.log-aggregation.file-formats":"TFile","yarn.timeline-service.client.fd-retain-secs":"300","hadoop.user.group.static.mapping.overrides":"dr.who=;","fs.azure.sas.expiry.period":"90d","mapreduce.jobhistory.recovery.store.class":"org.apache.hadoop.mapreduce.v2.hs.HistoryServerFileSystemStateStoreService","yarn.resourcemanager.fail-fast":"${yarn.fail-fast}","yarn.resourcemanager.proxy-user-privileges.enabled":"false","yarn.router.webapp.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.webapp.DefaultRequestInterceptorREST","yarn.nodemanager.resource.memory.cgroups.soft-limit-percentage":"90.0","mapreduce.job.reducer.preempt.delay.sec":"0","hadoop.util.hash.type":"murmur","yarn.nodemanager.disk-validator":"basic","yarn.app.mapreduce.client.job.max-retries":"3","mapreduce.reduce.shuffle.retry-delay.max.ms":"60000","hadoop.security.group.mapping.ldap.connection.timeout.ms":"60000","mapreduce.task.profile.params":"-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,verbose=n,file=%s","yarn.app.mapreduce.shuffle.log.backups":"0","yarn.nodemanager.container-diagnostics-maximum-size":"10000","hadoop.registry.zk.retry.interval.ms":"1000","yarn.nodemanager.linux-container-executor.cgroups.delete-timeout-ms":"1000","fs.AbstractFileSystem.file.impl":"org.apache.hadoop.fs.local.LocalFs","yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds":"-1","mapreduce.jobhistory.cleaner.interval-ms":"86400000","hadoop.registry.zk.quorum":"localhost:2181","mapreduce.output.fileoutputformat.compress":"false","yarn.resourcemanager.am-rm-tokens.master-key-rolling-interval-secs":"*********(redacted)","fs.s3a.assumed.role.session.duration":"30m","hadoop.security.group.mapping.ldap.conversion.rule":"none","hadoop.ssl.server.conf":"ssl-server.xml","fs.s3a.retry.throttle.interval":"1000ms","seq.io.sort.factor":"100","yarn.sharedcache.cleaner.initial-delay-mins":"10","mapreduce.client.completion.pollinterval":"5000","hadoop.ssl.keystores.factory.class":"org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory","yarn.app.mapreduce.am.resource.cpu-vcores":"1","yarn.timeline-service.enabled":"false","yarn.nodemanager.runtime.linux.docker.capabilities":"CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE","yarn.acl.enable":"false","yarn.timeline-service.entity-group-fs-store.done-dir":"/tmp/entity-file-history/done/","mapreduce.task.profile":"false","yarn.resourcemanager.fs.state-store.uri":"${hadoop.tmp.dir}/yarn/system/rmstore","mapreduce.jobhistory.always-scan-user-dir":"false","yarn.nodemanager.opportunistic-containers-use-pause-for-preemption":"false","yarn.nodemanager.linux-container-executor.nonsecure-mode.local-user":"nobody","yarn.timeline-service.reader.class":"org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineReaderImpl","yarn.resourcemanager.configuration.provider-class":"org.apache.hadoop.yarn.LocalConfigurationProvider","yarn.nodemanager.runtime.linux.docker.userremapping-uid-threshold":"1","yarn.resourcemanager.configuration.file-system-based-store":"/yarn/conf","mapreduce.job.cache.limit.max-single-resource-mb":"0","yarn.nodemanager.runtime.linux.docker.stop.grace-period":"10","yarn.resourcemanager.resource-profiles.source-file":"resource-profiles.json","yarn.nodemanager.resource.percentage-physical-cpu-limit":"100","mapreduce.jobhistory.client.thread-count":"10","tfile.fs.input.buffer.size":"262144","mapreduce.client.progressmonitor.pollinterval":"1000","yarn.nodemanager.log-dirs":"${yarn.log.dir}/userlogs","fs.automatic.close":"true","yarn.nodemanager.hostname":"0.0.0.0","yarn.nodemanager.resource.memory.cgroups.swappiness":"0","ftp.stream-buffer-size":"4096","yarn.fail-fast":"false","yarn.timeline-service.app-aggregation-interval-secs":"15","hadoop.security.group.mapping.ldap.search.filter.user":"(&(objectClass=user)(sAMAccountName={0}))","yarn.nodemanager.container-localizer.log.level":"INFO","yarn.timeline-service.address":"${yarn.timeline-service.hostname}:10200","mapreduce.job.ubertask.maxmaps":"9","fs.s3a.threads.keepalivetime":"60","mapreduce.jobhistory.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","mapreduce.task.files.preserve.failedtasks":"false","yarn.app.mapreduce.client.job.retry-interval":"2000","ha.failover-controller.graceful-fence.connection.retries":"1","yarn.resourcemanager.delegation.token.max-lifetime":"*********(redacted)","yarn.timeline-service.client.drain-entities.timeout.ms":"2000","yarn.nodemanager.resource-plugins.fpga.vendor-plugin.class":"org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.IntelFpgaOpenclPlugin","yarn.timeline-service.entity-group-fs-store.summary-store":"org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore","mapreduce.reduce.cpu.vcores":"1","mapreduce.job.encrypted-intermediate-data.buffer.kb":"128","fs.client.resolve.remote.symlinks":"true","yarn.nodemanager.webapp.https.address":"0.0.0.0:8044","hadoop.http.cross-origin.allowed-origins":"*","mapreduce.job.encrypted-intermediate-data":"false","yarn.timeline-service.entity-group-fs-store.retain-seconds":"604800","yarn.resourcemanager.metrics.runtime.buckets":"60,300,1440","yarn.timeline-service.generic-application-history.max-applications":"10000","yarn.nodemanager.local-dirs":"${hadoop.tmp.dir}/nm-local-dir","mapreduce.shuffle.connection-keep-alive.enable":"false","yarn.node-labels.configuration-type":"centralized","fs.s3a.path.style.access":"false","yarn.nodemanager.aux-services.mapreduce_shuffle.class":"org.apache.hadoop.mapred.ShuffleHandler","yarn.sharedcache.store.in-memory.staleness-period-mins":"10080","fs.adl.impl":"org.apache.hadoop.fs.adl.AdlFileSystem","yarn.resourcemanager.nodemanager.minimum.version":"NONE","mapreduce.jobhistory.webapp.xfs-filter.xframe-options":"SAMEORIGIN","yarn.app.mapreduce.am.staging-dir.erasurecoding.enabled":"false","net.topology.impl":"org.apache.hadoop.net.NetworkTopology","io.map.index.skip":"0","yarn.timeline-service.reader.webapp.https.address":"${yarn.timeline-service.webapp.https.address}","fs.ftp.data.connection.mode":"ACTIVE_LOCAL_DATA_CONNECTION_MODE","mapreduce.job.local-fs.single-disk-limit.check.kill-limit-exceed":"true","yarn.scheduler.maximum-allocation-vcores":"4","hadoop.http.cross-origin.allowed-headers":"X-Requested-With,Content-Type,Accept,Origin","yarn.nodemanager.log-aggregation.compression-type":"none","yarn.timeline-service.version":"1.0f","yarn.ipc.rpc.class":"org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC","mapreduce.reduce.maxattempts":"4","hadoop.security.dns.log-slow-lookups.enabled":"false","mapreduce.job.committer.setup.cleanup.needed":"true","mapreduce.job.running.reduce.limit":"0","ipc.maximum.response.length":"134217728","yarn.resourcemanager.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","mapreduce.job.token.tracking.ids.enabled":"*********(redacted)","hadoop.caller.context.max.size":"128","yarn.nodemanager.runtime.linux.docker.host-pid-namespace.allowed":"false","yarn.nodemanager.runtime.linux.docker.delayed-removal.allowed":"false","hadoop.registry.system.acls":"sasl:yarn@, sasl:mapred@, sasl:hdfs@","yarn.nodemanager.recovery.dir":"${hadoop.tmp.dir}/yarn-nm-recovery","fs.s3a.fast.upload.buffer":"disk","mapreduce.jobhistory.intermediate-done-dir":"${yarn.app.mapreduce.am.staging-dir}/history/done_intermediate","yarn.app.mapreduce.shuffle.log.separate":"true","fs.s3a.max.total.tasks":"5","fs.s3a.readahead.range":"64K","hadoop.http.authentication.simple.anonymous.allowed":"true","fs.s3a.attempts.maximum":"20","hadoop.registry.zk.connection.timeout.ms":"15000","yarn.resourcemanager.delegation-token-renewer.thread-count":"*********(redacted)","yarn.nodemanager.health-checker.script.timeout-ms":"1200000","yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size":"10000","yarn.resourcemanager.resource-profiles.enabled":"false","yarn.timeline-service.hbase-schema.prefix":"prod.","fs.azure.authorization":"false","mapreduce.map.log.level":"INFO","yarn.resourcemanager.decommissioning-nodes-watcher.poll-interval-secs":"20","mapreduce.output.fileoutputformat.compress.type":"RECORD","yarn.resourcemanager.leveldb-state-store.path":"${hadoop.tmp.dir}/yarn/system/rmstore","yarn.timeline-service.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.ifile.readahead.bytes":"4194304","yarn.sharedcache.app-checker.class":"org.apache.hadoop.yarn.server.sharedcachemanager.RemoteAppChecker","yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users":"true","yarn.nodemanager.resource.detect-hardware-capabilities":"false","mapreduce.cluster.acls.enabled":"false","mapreduce.job.speculative.retry-after-no-speculate":"1000","hadoop.security.group.mapping.ldap.search.group.hierarchy.levels":"0","yarn.resourcemanager.fs.state-store.retry-interval-ms":"1000","file.stream-buffer-size":"4096","yarn.resourcemanager.application-timeouts.monitor.interval-ms":"3000","mapreduce.map.output.compress.codec":"org.apache.hadoop.io.compress.DefaultCodec","mapreduce.map.speculative":"true","mapreduce.job.speculative.retry-after-speculate":"15000","yarn.nodemanager.linux-container-executor.cgroups.mount":"false","yarn.app.mapreduce.am.container.log.backups":"0","yarn.app.mapreduce.am.log.level":"INFO","mapreduce.job.reduce.slowstart.completedmaps":"0.05","yarn.timeline-service.http-authentication.type":"simple","hadoop.security.group.mapping.ldap.search.attr.group.name":"cn","yarn.nodemanager.resource-plugins.fpga.allowed-fpga-devices":"0,1","yarn.timeline-service.client.internal-timers-ttl-secs":"420","hadoop.http.logs.enabled":"true","fs.s3a.block.size":"32M","yarn.sharedcache.client-server.address":"0.0.0.0:8045","yarn.nodemanager.logaggregation.threadpool-size-max":"100","yarn.resourcemanager.hostname":"hostb7","yarn.resourcemanager.delegation.key.update-interval":"86400000","mapreduce.reduce.shuffle.fetch.retry.enabled":"${yarn.nodemanager.recovery.enabled}","mapreduce.map.memory.mb":"-1","mapreduce.task.skip.start.attempts":"2","fs.AbstractFileSystem.hdfs.impl":"org.apache.hadoop.fs.Hdfs","yarn.nodemanager.disk-health-checker.enable":"true","ipc.client.tcpnodelay":"true","ipc.client.rpc-timeout.ms":"0","yarn.nodemanager.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","ipc.client.low-latency":"false","mapreduce.input.lineinputformat.linespermap":"1","yarn.router.interceptor.user.threadpool-size":"5","ipc.client.connect.max.retries.on.timeouts":"45","yarn.timeline-service.leveldb-timeline-store.read-cache-size":"104857600","fs.AbstractFileSystem.har.impl":"org.apache.hadoop.fs.HarFs","mapreduce.job.split.metainfo.maxsize":"10000000","yarn.am.liveness-monitor.expiry-interval-ms":"600000","yarn.resourcemanager.container-tokens.master-key-rolling-interval-secs":"*********(redacted)","yarn.timeline-service.entity-group-fs-store.app-cache-size":"10","fs.s3a.socket.recv.buffer":"8192","yarn.resourcemanager.resource-tracker.address":"${yarn.resourcemanager.hostname}:8031","yarn.nodemanager.node-labels.provider.fetch-timeout-ms":"1200000","mapreduce.job.heap.memory-mb.ratio":"0.8","yarn.resourcemanager.leveldb-state-store.compaction-interval-secs":"3600","yarn.resourcemanager.webapp.rest-csrf.custom-header":"X-XSRF-Header","yarn.scheduler.configuration.fs.path":"file://${hadoop.tmp.dir}/yarn/system/schedconf","mapreduce.client.output.filter":"FAILED","hadoop.http.filter.initializers":"org.apache.hadoop.http.lib.StaticUserWebFilter","mapreduce.reduce.memory.mb":"-1","yarn.timeline-service.hostname":"0.0.0.0","file.replication":"1","yarn.nodemanager.container-metrics.unregister-delay-ms":"10000","yarn.nodemanager.container-metrics.period-ms":"-1","mapreduce.fileoutputcommitter.task.cleanup.enabled":"false","yarn.nodemanager.log.retain-seconds":"10800","yarn.timeline-service.entity-group-fs-store.cleaner-interval-seconds":"3600","yarn.resourcemanager.keytab":"/etc/krb5.keytab","hadoop.security.group.mapping.providers.combined":"true","mapreduce.reduce.merge.inmem.threshold":"1000","yarn.timeline-service.recovery.enabled":"false","fs.azure.saskey.usecontainersaskeyforallaccess":"true","yarn.sharedcache.nm.uploader.thread-count":"20","yarn.resourcemanager.nodemanager-graceful-decommission-timeout-secs":"3600","mapreduce.shuffle.ssl.enabled":"false","yarn.timeline-service.hbase.coprocessor.app-final-value-retention-milliseconds":"259200000","fs.s3a.committer.staging.abort.pending.uploads":"true","yarn.nodemanager.opportunistic-containers-max-queue-length":"0","yarn.resourcemanager.state-store.max-completed-applications":"${yarn.resourcemanager.max-completed-applications}","mapreduce.job.speculative.minimum-allowed-tasks":"10","yarn.log-aggregation.retain-seconds":"-1","yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb":"0","mapreduce.jobhistory.max-age-ms":"604800000","hadoop.http.cross-origin.allowed-methods":"GET,POST,HEAD","yarn.resourcemanager.opportunistic-container-allocation.enabled":"false","mapreduce.jobhistory.webapp.address":"0.0.0.0:19888","hadoop.system.tags":"YARN,HDFS,NAMENODE,DATANODE,REQUIRED,SECURITY,KERBEROS,PERFORMANCE,CLIENT\n ,SERVER,DEBUG,DEPRECATED,COMMON,OPTIONAL","yarn.log-aggregation.file-controller.TFile.class":"org.apache.hadoop.yarn.logaggregation.filecontroller.tfile.LogAggregationTFileController","yarn.client.nodemanager-connect.max-wait-ms":"180000","yarn.resourcemanager.webapp.address":"${yarn.resourcemanager.hostname}:8088","mapreduce.jobhistory.recovery.enable":"false","mapreduce.reduce.shuffle.parallelcopies":"5","fs.AbstractFileSystem.webhdfs.impl":"org.apache.hadoop.fs.WebHdfs","fs.trash.interval":"0","yarn.app.mapreduce.client.max-retries":"3","hadoop.security.authentication":"simple","mapreduce.task.profile.reduce.params":"${mapreduce.task.profile.params}","yarn.app.mapreduce.am.resource.mb":"1536","mapreduce.input.fileinputformat.list-status.num-threads":"1","yarn.nodemanager.container-executor.class":"org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor","io.mapfile.bloom.size":"1048576","yarn.timeline-service.ttl-ms":"604800000","yarn.resourcemanager.nm-container-queuing.min-queue-length":"5","yarn.nodemanager.resource.cpu-vcores":"-1","mapreduce.job.reduces":"1","fs.s3a.multipart.size":"100M","yarn.scheduler.minimum-allocation-vcores":"1","mapreduce.job.speculative.speculative-cap-total-tasks":"0.01","hadoop.ssl.client.conf":"ssl-client.xml","mapreduce.job.queuename":"default","mapreduce.job.encrypted-intermediate-data-key-size-bits":"128","fs.s3a.metadatastore.authoritative":"false","yarn.nodemanager.webapp.xfs-filter.xframe-options":"SAMEORIGIN","ha.health-monitor.sleep-after-disconnect.ms":"1000","yarn.app.mapreduce.shuffle.log.limit.kb":"0","hadoop.security.group.mapping":"org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback","yarn.client.application-client-protocol.poll-timeout-ms":"-1","mapreduce.jobhistory.jhist.format":"binary","yarn.resourcemanager.ha.enabled":"false","hadoop.http.staticuser.user":"dr.who","mapreduce.task.exit.timeout.check-interval-ms":"20000","mapreduce.jobhistory.intermediate-user-done-dir.permissions":"770","mapreduce.task.exit.timeout":"60000","yarn.nodemanager.linux-container-executor.resources-handler.class":"org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler","mapreduce.reduce.shuffle.memory.limit.percent":"0.25","yarn.resourcemanager.reservation-system.enable":"false","mapreduce.map.output.compress":"false","ha.zookeeper.acl":"world:anyone:rwcda","ipc.server.max.connections":"0","yarn.nodemanager.aux-services":"mapreduce_shuffle","yarn.nodemanager.runtime.linux.docker.default-container-network":"host","yarn.router.webapp.address":"0.0.0.0:8089","yarn.scheduler.maximum-allocation-mb":"8192","yarn.resourcemanager.scheduler.monitor.policies":"org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy","yarn.sharedcache.cleaner.period-mins":"1440","yarn.nodemanager.resource-plugins.gpu.docker-plugin.nvidia-docker-v1.endpoint":"http://localhost:3476/v1.0/docker/cli","yarn.app.mapreduce.am.container.log.limit.kb":"0","ipc.client.connect.retry.interval":"1000","yarn.timeline-service.http-cross-origin.enabled":"false","fs.wasbs.impl":"org.apache.hadoop.fs.azure.NativeAzureFileSystem$Secure","yarn.federation.subcluster-resolver.class":"org.apache.hadoop.yarn.server.federation.resolver.DefaultSubClusterResolverImpl","yarn.resourcemanager.zk-state-store.parent-path":"/rmstore","mapreduce.jobhistory.cleaner.enable":"true","yarn.timeline-service.client.fd-flush-interval-secs":"10","hadoop.security.kms.client.encrypted.key.cache.expiry":"43200000","yarn.client.nodemanager-client-async.thread-pool-max-size":"500","mapreduce.map.maxattempts":"4","yarn.resourcemanager.nm-container-queuing.sorting-nodes-interval-ms":"1000","fs.s3a.committer.staging.tmp.path":"tmp/staging","yarn.nodemanager.sleep-delay-before-sigkill.ms":"250","yarn.resourcemanager.nm-container-queuing.min-queue-wait-time-ms":"10","mapreduce.job.end-notification.retry.attempts":"0","yarn.nodemanager.resource.count-logical-processors-as-cores":"false","hadoop.registry.zk.root":"/registry","adl.feature.ownerandgroup.enableupn":"false","yarn.resourcemanager.zk-max-znode-size.bytes":"1048576","mapreduce.job.reduce.shuffle.consumer.plugin.class":"org.apache.hadoop.mapreduce.task.reduce.Shuffle","yarn.resourcemanager.delayed.delegation-token.removal-interval-ms":"*********(redacted)","yarn.nodemanager.localizer.cache.target-size-mb":"10240","fs.s3a.committer.staging.conflict-mode":"fail","mapreduce.client.libjars.wildcard":"true","fs.s3a.committer.staging.unique-filenames":"true","yarn.nodemanager.node-attributes.provider.fetch-timeout-ms":"1200000","fs.s3a.list.version":"2","ftp.client-write-packet-size":"65536","fs.AbstractFileSystem.adl.impl":"org.apache.hadoop.fs.adl.Adl","hadoop.security.key.default.cipher":"AES/CTR/NoPadding","yarn.client.failover-retries":"0","fs.s3a.multipart.purge.age":"86400","mapreduce.job.local-fs.single-disk-limit.check.interval-ms":"5000","net.topology.node.switch.mapping.impl":"org.apache.hadoop.net.ScriptBasedMapping","yarn.nodemanager.amrmproxy.address":"0.0.0.0:8049","ipc.server.listen.queue.size":"128","map.sort.class":"org.apache.hadoop.util.QuickSort","fs.viewfs.rename.strategy":"SAME_MOUNTPOINT","hadoop.security.kms.client.authentication.retry-count":"1","fs.permissions.umask-mode":"022","fs.s3a.assumed.role.credentials.provider":"org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider","yarn.nodemanager.vmem-check-enabled":"true","yarn.nodemanager.numa-awareness.enabled":"false","yarn.nodemanager.recovery.compaction-interval-secs":"3600","yarn.app.mapreduce.client-am.ipc.max-retries":"3","yarn.federation.registry.base-dir":"yarnfederation/","mapreduce.job.max.map":"-1","mapreduce.job.local-fs.single-disk-limit.bytes":"-1","mapreduce.job.ubertask.maxreduces":"1","hadoop.security.kms.client.encrypted.key.cache.size":"500","hadoop.security.java.secure.random.algorithm":"SHA1PRNG","ha.failover-controller.cli-check.rpc-timeout.ms":"20000","mapreduce.jobhistory.jobname.limit":"50","mapreduce.application.classpath":"$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*:$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*","yarn.client.nodemanager-connect.retry-interval-ms":"10000","yarn.timeline-service.state-store-class":"org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore","yarn.nodemanager.env-whitelist":"JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,PATH,LANG,TZ,HADOOP_MAPRED_HOME","yarn.sharedcache.nested-level":"3","yarn.timeline-service.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","fs.azure.user.agent.prefix":"unknown","yarn.resourcemanager.zk-delegation-token-node.split-index":"*********(redacted)","yarn.nodemanager.numa-awareness.read-topology":"false","yarn.nodemanager.webapp.address":"${yarn.nodemanager.hostname}:8042","rpc.metrics.quantile.enable":"false","yarn.registry.class":"org.apache.hadoop.registry.client.impl.FSRegistryOperationsService","mapreduce.jobhistory.admin.acl":"*","yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size":"10","yarn.scheduler.queue-placement-rules":"user-group","hadoop.http.authentication.kerberos.keytab":"${user.home}/hadoop.keytab","yarn.resourcemanager.recovery.enabled":"false","yarn.timeline-service.webapp.rest-csrf.enabled":"false"},"System Properties":{"java.io.tmpdir":"/tmp","line.separator":"\n","path.separator":":","sun.management.compiler":"HotSpot 64-Bit Tiered Compilers","SPARK_SUBMIT":"true","sun.cpu.endian":"little","java.specification.version":"1.8","java.vm.specification.name":"Java Virtual Machine Specification","java.vendor":"Oracle Corporation","java.vm.specification.version":"1.8","user.home":"/home/usera112","file.encoding.pkg":"sun.io","sun.nio.ch.bugLevel":"","sun.arch.data.model":"64","sun.boot.library.path":"/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64","user.dir":"/home/usera112/projects/log-compressor/deployments/HiBench","java.library.path":"/usr/java/packages/lib/amd64:/usr/lib/x86_64-linux-gnu/jni:/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:/usr/lib/jni:/lib:/usr/lib","sun.cpu.isalist":"","os.arch":"amd64","java.vm.version":"25.162-b12","jetty.git.hash":"b881a572662e1943a14ae12e7e1207989f218b74","java.endorsed.dirs":"/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/endorsed","java.runtime.version":"1.8.0_162-8u162-b12-1-b12","java.vm.info":"mixed mode","java.ext.dirs":"/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/ext:/usr/java/packages/lib/ext","java.runtime.name":"OpenJDK Runtime Environment","file.separator":"/","java.class.version":"52.0","java.specification.name":"Java Platform API Specification","sun.boot.class.path":"/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/resources.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/rt.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/sunrsasign.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/jsse.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/jce.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/charsets.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/jfr.jar:/usr/lib/jvm/java-8-openjdk-amd64/jre/classes","file.encoding":"UTF-8","user.timezone":"America/Toronto","java.specification.vendor":"Oracle Corporation","sun.java.launcher":"SUN_STANDARD","os.version":"4.15.0-29-generic","sun.os.patch.level":"unknown","java.vm.specification.vendor":"Oracle Corporation","user.country":"US","sun.jnu.encoding":"UTF-8","user.language":"en","java.vendor.url":"http://java.oracle.com/","java.awt.printerjob":"sun.print.PSPrinterJob","java.awt.graphicsenv":"sun.awt.X11GraphicsEnvironment","awt.toolkit":"sun.awt.X11.XToolkit","os.name":"Linux","java.vm.vendor":"Oracle Corporation","java.vendor.url.bug":"http://bugreport.sun.com/bugreport/","user.name":"usera112","java.vm.name":"OpenJDK 64-Bit Server VM","sun.java.command":"org.apache.spark.deploy.SparkSubmit --master spark://hostb7:7077 --properties-file /home/usera112/projects/log-compressor/deployments/HiBench/report/join/spark/conf/sparkbench/spark.conf --class com.intel.hibench.sparkbench.sql.ScalaSparkSQLBench /home/usera112/projects/log-compressor/deployments/HiBench/sparkbench/assembly/target/sparkbench-assembly-8.0-SNAPSHOT-dist.jar ScalaJoin /home/usera112/projects/log-compressor/deployments/HiBench/report/join/spark/conf/../rankings_uservisits_join.hive","java.home":"/usr/lib/jvm/java-8-openjdk-amd64/jre","java.version":"1.8.0_162","sun.io.unicode.encoding":"UnicodeLittle"},"Classpath Entries":{"/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jackson-mapper-asl-1.9.13.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-yarn-api-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hk2-locator-2.6.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/nimbus-jose-jwt-4.41.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/htrace-core4-4.1.0-incubating.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/cats-kernel_2.12-2.0.0-M4.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/httpclient-4.5.6.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-collections-3.2.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-beeline-2.3.7.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/json-smart-2.3.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/curator-recipes-2.13.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/stream-2.9.6.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-graphx_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jline-2.14.6.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-admissionregistration-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/guava-14.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-sql_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/parquet-common-1.10.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/chill-java-0.9.5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-beanutils-1.9.4.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kryo-shaded-4.0.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hk2-api-2.6.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-io-2.5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/datanucleus-rdbms-4.1.19.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/metrics-graphite-4.1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/breeze_2.12-1.0.jar":"System Classpath","spark://hostb7.dsrg.utoronto.ca:42853/jars/sparkbench-assembly-8.0-SNAPSHOT-dist.jar":"Added By User","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jakarta.xml.bind-api-2.3.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jakarta.activation-api-1.2.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-lang3-3.10.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-mesos_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/avro-mapred-1.8.2-hadoop2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jsr305-3.0.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/HikariCP-2.5.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/ehcache-3.3.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-cli-1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-core-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/joda-time-2.10.5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/javassist-3.25.0-GA.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerby-config-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/token-provider-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/leveldbjni-all-1.8.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-sketch_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-mllib-local_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-apps-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/json4s-ast_2.12-3.7.0-M5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/json-1.8.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/datanucleus-core-4.1.17.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spire-macros_2.12-0.17.0-M1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/parquet-jackson-1.10.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-yarn-server-web-proxy-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-math3-3.4.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-vector-code-gen-2.3.7.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-network-shuffle_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/RoaringBitmap-0.9.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-mapreduce-client-jobclient-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/super-csv-2.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/pyrolite-4.30.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-yarn-common-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/slf4j-log4j12-1.7.30.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/macro-compat_2.12-1.1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-discovery-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-hive_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-metastore-2.3.7.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-streaming_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jackson-databind-2.10.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/snakeyaml-1.24.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-lang-2.6.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/libfb303-0.9.3.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/curator-framework-2.13.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerby-xdr-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/xz-1.5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-shims-common-2.3.7.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/mesos-1.4.0-shaded-protobuf.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerb-crypto-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-llap-common-2.3.7.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jersey-server-2.30.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/minlog-1.3.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jackson-module-scala_2.12-2.10.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/okhttp-2.7.5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/transaction-api-1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-exec-2.3.7-core.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-tags_2.12-3.1.2-tests.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/shims-0.9.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-jdbc-2.3.7.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/paranamer-2.8.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/woodstox-core-5.0.3.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/generex-1.0.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jackson-annotations-2.10.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spire-platform_2.12-0.17.0-M1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/chill_2.12-0.9.5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/JLargeArrays-1.5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-net-3.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-service-rpc-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/py4j-0.10.9.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/okhttp-3.12.12.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-crypto-1.1.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-core_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/zjsonpatch-0.3.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/guice-servlet-4.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/derby-10.12.1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-pool-1.5.4.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/parquet-encoding-1.10.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/scala-xml_2.12-1.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jersey-client-2.30.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerby-pkix-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerb-common-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-annotations-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-shims-2.3.7.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-compiler-3.0.16.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-kvstore_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-repl_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jcip-annotations-1.0-1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/avro-1.8.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/osgi-resource-locator-1.0.3.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/javolution-5.5.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/re2j-1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/orc-core-1.5.12.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/javax.jdo-3.2.0-m3.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/bonecp-0.8.0.RELEASE.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jakarta.inject-2.6.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/objenesis-2.6.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jersey-hk2-2.30.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spire-util_2.12-0.17.0-M1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/datanucleus-api-jdo-4.2.4.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-settings-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/aopalliance-1.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-kubernetes_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/httpcore-4.4.12.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/janino-3.0.16.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/scala-reflect-2.12.10.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/aopalliance-repackaged-2.6.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/libthrift-0.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/json4s-jackson_2.12-3.7.0-M5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-policy-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jackson-module-paranamer-2.10.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jaxb-runtime-2.3.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/orc-mapreduce-1.5.12.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-storageclass-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jpam-1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/gson-2.2.4.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/automaton-1.11-8.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/curator-client-2.13.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-codec-1.10.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-httpclient-3.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/flatbuffers-java-1.9.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-apiextensions-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/zookeeper-3.4.14.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/netty-all-4.1.51.Final.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-yarn-client-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jersey-container-servlet-2.30.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jackson-module-jaxb-annotations-2.10.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jakarta.ws.rs-api-2.1.6.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/xbean-asm7-shaded-4.15.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-auth-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-batch-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/arrow-vector-2.0.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jakarta.servlet-api-4.0.3.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-unsafe_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-client-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-cli-2.3.7.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-mllib_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/json4s-core_2.12-3.7.0-M5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/scala-parser-combinators_2.12-1.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/arrow-memory-core-2.0.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jersey-common-2.30.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerby-util-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jdo-api-3.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-metrics-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-storage-api-2.7.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jackson-jaxrs-base-2.9.5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/ivy-2.4.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-extensions-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-networking-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-events-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hk2-utils-2.6.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-common-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-rbac-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/protobuf-java-2.5.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/scala-compiler-2.12.10.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/lz4-java-1.7.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/threeten-extra-1.5.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/stax-api-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-client-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/parquet-hadoop-1.10.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/breeze-macros_2.12-1.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/ST4-4.0.4.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-serde-2.3.7.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/hadoop-3.3.1/etc/hadoop/":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/snappy-java-1.1.8.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/core-1.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerb-identity-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/orc-shims-1.5.12.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-configuration2-2.1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jaxb-api-2.2.11.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/arrow-format-2.0.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jsp-api-2.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/activation-1.1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jakarta.annotation-api-1.3.5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/oro-2.0.8.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/shapeless_2.12-2.3.3.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jackson-core-2.10.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-daemon-1.0.13.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-launcher_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/opencsv-2.3.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-tags_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-text-1.6.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/arrow-memory-netty-2.0.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerb-util-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerby-asn1-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jackson-core-asl-1.9.13.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-coordination-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jackson-dataformat-yaml-2.10.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerb-admin-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/avro-ipc-1.8.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-mapreduce-client-common-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/antlr4-runtime-4.8-1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jackson-jaxrs-json-provider-2.9.5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-common-2.3.7.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/arpack_combined_all-0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/geronimo-jcache_1.0_spec-1.0-alpha-1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/machinist_2.12-0.6.8.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/aircompressor-0.10.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/univocity-parsers-2.9.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerb-server-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-dbcp-1.4.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/stax2-api-3.1.4.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/istack-commons-runtime-3.0.8.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-yarn-server-common-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerb-simplekdc-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/antlr-runtime-3.5.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/JTransforms-3.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/guice-4.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-yarn_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/slf4j-api-1.7.30.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jackson-datatype-jsr310-2.11.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/parquet-format-2.4.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-yarn-registry-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/velocity-1.5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/javax.inject-1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-scheduling-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-autoscaling-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerb-client-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spire_2.12-0.17.0-M1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/log4j-1.2.17.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-network-common_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/logging-interceptor-3.12.12.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/dnsjava-2.1.7.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/audience-annotations-0.5.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/metrics-jmx-4.1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jta-1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-shims-scheduler-2.3.7.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/json4s-scalap_2.12-3.7.0-M5.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/conf/":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/metrics-json-4.1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jersey-container-servlet-core-2.30.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jodd-core-3.5.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/compress-lzf-1.0.3.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/metrics-jvm-4.1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/scala-collection-compat_2.12-2.1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/parquet-column-1.10.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/metrics-core-4.1.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/scala-library-2.12.10.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-common-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-hive-thriftserver_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/zstd-jni-1.4.8-1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jakarta.validation-api-2.0.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kerb-core-1.0.1.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/okio-1.14.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-logging-1.1.3.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jcl-over-slf4j-1.7.30.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jersey-media-jaxb-2.30.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/algebra_2.12-2.0.0-M2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/commons-compress-1.20.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-hdfs-client-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hive-shims-0.23-2.3.7.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/jul-to-slf4j-1.7.30.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/kubernetes-model-certificates-4.12.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/hadoop-mapreduce-client-core-3.2.0.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/spark-catalyst_2.12-3.1.2.jar":"System Classpath","/home/usera112/projects/log-compressor/deployments/spark-3.1.2-bin-hadoop3.2/jars/accessors-smart-1.2.jar":"System Classpath"}} -{"Event":"SparkListenerApplicationStart","App Name":"ScalaJoin","App ID":"app-20211010132332-4636","Timestamp":1633886611309,"User":"usera112"} -{"Event":"SparkListenerExecutorAdded","Timestamp":1633886614425,"Executor ID":"5","Executor Info":{"Host":"10.1.0.19","Total Cores":16,"Log Urls":{"stdout":"http://10.1.0.19:8081/logPage/?appId=app-20211010132332-4636&executorId=5&logType=stdout","stderr":"http://10.1.0.19:8081/logPage/?appId=app-20211010132332-4636&executorId=5&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0}} -{"Event":"SparkListenerExecutorAdded","Timestamp":1633886614427,"Executor ID":"10","Executor Info":{"Host":"10.1.0.16","Total Cores":16,"Log Urls":{"stdout":"http://10.1.0.16:8081/logPage/?appId=app-20211010132332-4636&executorId=10&logType=stdout","stderr":"http://10.1.0.16:8081/logPage/?appId=app-20211010132332-4636&executorId=10&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0}} -{"Event":"SparkListenerExecutorAdded","Timestamp":1633886614453,"Executor ID":"4","Executor Info":{"Host":"10.1.0.8","Total Cores":16,"Log Urls":{"stdout":"http://10.1.0.8:8081/logPage/?appId=app-20211010132332-4636&executorId=4&logType=stdout","stderr":"http://10.1.0.8:8081/logPage/?appId=app-20211010132332-4636&executorId=4&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0}} -{"Event":"SparkListenerExecutorAdded","Timestamp":1633886614472,"Executor ID":"11","Executor Info":{"Host":"10.1.0.13","Total Cores":16,"Log Urls":{"stdout":"http://10.1.0.13:8081/logPage/?appId=app-20211010132332-4636&executorId=11&logType=stdout","stderr":"http://10.1.0.13:8081/logPage/?appId=app-20211010132332-4636&executorId=11&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0}} -{"Event":"SparkListenerExecutorAdded","Timestamp":1633886614477,"Executor ID":"1","Executor Info":{"Host":"10.1.0.17","Total Cores":16,"Log Urls":{"stdout":"http://10.1.0.17:8081/logPage/?appId=app-20211010132332-4636&executorId=1&logType=stdout","stderr":"http://10.1.0.17:8081/logPage/?appId=app-20211010132332-4636&executorId=1&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0}} -{"Event":"SparkListenerExecutorAdded","Timestamp":1633886614486,"Executor ID":"0","Executor Info":{"Host":"10.1.0.12","Total Cores":16,"Log Urls":{"stdout":"http://10.1.0.12:8081/logPage/?appId=app-20211010132332-4636&executorId=0&logType=stdout","stderr":"http://10.1.0.12:8081/logPage/?appId=app-20211010132332-4636&executorId=0&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0}} -{"Event":"SparkListenerExecutorAdded","Timestamp":1633886614498,"Executor ID":"8","Executor Info":{"Host":"10.1.0.11","Total Cores":16,"Log Urls":{"stdout":"http://10.1.0.11:8081/logPage/?appId=app-20211010132332-4636&executorId=8&logType=stdout","stderr":"http://10.1.0.11:8081/logPage/?appId=app-20211010132332-4636&executorId=8&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0}} -{"Event":"SparkListenerExecutorAdded","Timestamp":1633886614514,"Executor ID":"9","Executor Info":{"Host":"10.1.0.14","Total Cores":16,"Log Urls":{"stdout":"http://10.1.0.14:8081/logPage/?appId=app-20211010132332-4636&executorId=9&logType=stdout","stderr":"http://10.1.0.14:8081/logPage/?appId=app-20211010132332-4636&executorId=9&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0}} -{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"5","Host":"10.1.0.19","Port":40959},"Maximum Memory":2101975449,"Timestamp":1633886614518,"Maximum Onheap Memory":2101975449,"Maximum Offheap Memory":0} -{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"10","Host":"10.1.0.16","Port":44733},"Maximum Memory":2101975449,"Timestamp":1633886614520,"Maximum Onheap Memory":2101975449,"Maximum Offheap Memory":0} -{"Event":"SparkListenerExecutorAdded","Timestamp":1633886614529,"Executor ID":"2","Executor Info":{"Host":"10.1.0.10","Total Cores":16,"Log Urls":{"stdout":"http://10.1.0.10:8081/logPage/?appId=app-20211010132332-4636&executorId=2&logType=stdout","stderr":"http://10.1.0.10:8081/logPage/?appId=app-20211010132332-4636&executorId=2&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0}} -{"Event":"SparkListenerExecutorAdded","Timestamp":1633886614532,"Executor ID":"6","Executor Info":{"Host":"10.1.0.18","Total Cores":16,"Log Urls":{"stdout":"http://10.1.0.18:8081/logPage/?appId=app-20211010132332-4636&executorId=6&logType=stdout","stderr":"http://10.1.0.18:8081/logPage/?appId=app-20211010132332-4636&executorId=6&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0}} -{"Event":"SparkListenerExecutorAdded","Timestamp":1633886614544,"Executor ID":"3","Executor Info":{"Host":"10.1.0.15","Total Cores":16,"Log Urls":{"stdout":"http://10.1.0.15:8081/logPage/?appId=app-20211010132332-4636&executorId=3&logType=stdout","stderr":"http://10.1.0.15:8081/logPage/?appId=app-20211010132332-4636&executorId=3&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0}} -{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"4","Host":"10.1.0.8","Port":39057},"Maximum Memory":2101975449,"Timestamp":1633886614548,"Maximum Onheap Memory":2101975449,"Maximum Offheap Memory":0} -{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"11","Host":"10.1.0.13","Port":41379},"Maximum Memory":2101975449,"Timestamp":1633886614572,"Maximum Onheap Memory":2101975449,"Maximum Offheap Memory":0} -{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"1","Host":"10.1.0.17","Port":36477},"Maximum Memory":2101975449,"Timestamp":1633886614575,"Maximum Onheap Memory":2101975449,"Maximum Offheap Memory":0} -{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"0","Host":"10.1.0.12","Port":46419},"Maximum Memory":2101975449,"Timestamp":1633886614585,"Maximum Onheap Memory":2101975449,"Maximum Offheap Memory":0} -{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"8","Host":"10.1.0.11","Port":40375},"Maximum Memory":2101975449,"Timestamp":1633886614593,"Maximum Onheap Memory":2101975449,"Maximum Offheap Memory":0} -{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"9","Host":"10.1.0.14","Port":34719},"Maximum Memory":2101975449,"Timestamp":1633886614613,"Maximum Onheap Memory":2101975449,"Maximum Offheap Memory":0} -{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"2","Host":"10.1.0.10","Port":45933},"Maximum Memory":2101975449,"Timestamp":1633886614628,"Maximum Onheap Memory":2101975449,"Maximum Offheap Memory":0} -{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"6","Host":"10.1.0.18","Port":33835},"Maximum Memory":2101975449,"Timestamp":1633886614633,"Maximum Onheap Memory":2101975449,"Maximum Offheap Memory":0} -{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"3","Host":"10.1.0.15","Port":36115},"Maximum Memory":2101975449,"Timestamp":1633886614646,"Maximum Onheap Memory":2101975449,"Maximum Offheap Memory":0} -{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":0,"description":"sql at ScalaSparkSQLBench.scala:43","details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nSetCatalogAndNamespace (1)\n\n\n(1) SetCatalogAndNamespace\nArguments: org.apache.spark.sql.connector.catalog.CatalogManager@2b1a901d, spark_catalog, ArrayBuffer(DEFAULT)\n\n","sparkPlanInfo":{"nodeName":"SetCatalogAndNamespace","simpleString":"SetCatalogAndNamespace org.apache.spark.sql.connector.catalog.CatalogManager@2b1a901d, spark_catalog, ArrayBuffer(DEFAULT)","children":[],"metadata":{},"metrics":[]},"time":1633886616285} -{"Event":"SparkListenerExecutorAdded","Timestamp":1633886619475,"Executor ID":"7","Executor Info":{"Host":"10.1.0.9","Total Cores":16,"Log Urls":{"stdout":"http://10.1.0.9:8081/logPage/?appId=app-20211010132332-4636&executorId=7&logType=stdout","stderr":"http://10.1.0.9:8081/logPage/?appId=app-20211010132332-4636&executorId=7&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0}} -{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"7","Host":"10.1.0.9","Port":35243},"Maximum Memory":2101975449,"Timestamp":1633886619587,"Maximum Onheap Memory":2101975449,"Maximum Offheap Memory":0} -{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":0,"time":1633886639860} -{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":1,"description":"sql at ScalaSparkSQLBench.scala:43","details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nExecute SetCommand (1)\n +- SetCommand (2)\n\n\n(1) Execute SetCommand\nOutput [2]: [key#8, value#9]\n\n(2) SetCommand\nArguments: (hive.input.format,Some(org.apache.hadoop.hive.ql.io.HiveInputFormat))\n\n","sparkPlanInfo":{"nodeName":"Execute SetCommand","simpleString":"Execute SetCommand","children":[],"metadata":{},"metrics":[]},"time":1633886639916} -{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":1,"time":1633886639921} -{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":2,"description":"sql at ScalaSparkSQLBench.scala:43","details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nExecute SetCommand (1)\n +- SetCommand (2)\n\n\n(1) Execute SetCommand\nOutput [2]: [key#18, value#19]\n\n(2) SetCommand\nArguments: (mapreduce.job.maps,Some(8))\n\n","sparkPlanInfo":{"nodeName":"Execute SetCommand","simpleString":"Execute SetCommand","children":[],"metadata":{},"metrics":[]},"time":1633886639938} -{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":2,"time":1633886639938} -{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":3,"description":"sql at ScalaSparkSQLBench.scala:43","details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nExecute SetCommand (1)\n +- SetCommand (2)\n\n\n(1) Execute SetCommand\nOutput [2]: [key#28, value#29]\n\n(2) SetCommand\nArguments: (mapreduce.job.reduces,Some(8))\n\n","sparkPlanInfo":{"nodeName":"Execute SetCommand","simpleString":"Execute SetCommand","children":[],"metadata":{},"metrics":[]},"time":1633886639954} -{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":3,"time":1633886639954} -{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":4,"description":"sql at ScalaSparkSQLBench.scala:43","details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nExecute SetCommand (1)\n +- SetCommand (2)\n\n\n(1) Execute SetCommand\nOutput [2]: [key#38, value#39]\n\n(2) SetCommand\nArguments: (hive.stats.autogather,Some(false))\n\n","sparkPlanInfo":{"nodeName":"Execute SetCommand","simpleString":"Execute SetCommand","children":[],"metadata":{},"metrics":[]},"time":1633886639966} -{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":4,"time":1633886639966} -{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":5,"description":"sql at ScalaSparkSQLBench.scala:43","details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nLocalTableScan (1)\n\n\n(1) LocalTableScan\nOutput: []\nArguments: \n\n","sparkPlanInfo":{"nodeName":"LocalTableScan","simpleString":"LocalTableScan ","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":0,"metricType":"sum"}]},"time":1633886640023} -{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":5,"time":1633886640023} -{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":6,"description":"sql at ScalaSparkSQLBench.scala:43","details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nExecute CreateTableCommand (1)\n +- CreateTableCommand (2)\n\n\n(1) Execute CreateTableCommand\nOutput: []\n\n(2) CreateTableCommand\nArguments: `default`.`rankings`, org.apache.hadoop.hive.serde2.OpenCSVSerde, false\n\n","sparkPlanInfo":{"nodeName":"Execute CreateTableCommand","simpleString":"Execute CreateTableCommand","children":[],"metadata":{},"metrics":[]},"time":1633886640087} -{"Event":"org.apache.spark.sql.catalyst.catalog.CreateTablePreEvent","database":"default","name":"rankings"} -{"Event":"org.apache.spark.sql.catalyst.catalog.CreateTableEvent","database":"default","name":"rankings"} -{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":6,"time":1633886640911} -{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":7,"description":"sql at ScalaSparkSQLBench.scala:43","details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nLocalTableScan (1)\n\n\n(1) LocalTableScan\nOutput: []\nArguments: \n\n","sparkPlanInfo":{"nodeName":"LocalTableScan","simpleString":"LocalTableScan ","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1,"metricType":"sum"}]},"time":1633886640934} -{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":7,"time":1633886640934} -{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":8,"description":"sql at ScalaSparkSQLBench.scala:43","details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nExecute CreateTableCommand (1)\n +- CreateTableCommand (2)\n\n\n(1) Execute CreateTableCommand\nOutput: []\n\n(2) CreateTableCommand\nArguments: `default`.`uservisits_copy`, org.apache.hadoop.hive.serde2.OpenCSVSerde, false\n\n","sparkPlanInfo":{"nodeName":"Execute CreateTableCommand","simpleString":"Execute CreateTableCommand","children":[],"metadata":{},"metrics":[]},"time":1633886640947} -{"Event":"org.apache.spark.sql.catalyst.catalog.CreateTablePreEvent","database":"default","name":"uservisits_copy"} -{"Event":"org.apache.spark.sql.catalyst.catalog.CreateTableEvent","database":"default","name":"uservisits_copy"} -{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":8,"time":1633886641021} -{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":9,"description":"sql at ScalaSparkSQLBench.scala:43","details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nLocalTableScan (1)\n\n\n(1) LocalTableScan\nOutput: []\nArguments: \n\n","sparkPlanInfo":{"nodeName":"LocalTableScan","simpleString":"LocalTableScan ","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2,"metricType":"sum"}]},"time":1633886641033} -{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":9,"time":1633886641034} -{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":10,"description":"sql at ScalaSparkSQLBench.scala:43","details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nExecute CreateTableCommand (1)\n +- CreateTableCommand (2)\n\n\n(1) Execute CreateTableCommand\nOutput: []\n\n(2) CreateTableCommand\nArguments: `default`.`rankings_uservisits_join`, org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, false\n\n","sparkPlanInfo":{"nodeName":"Execute CreateTableCommand","simpleString":"Execute CreateTableCommand","children":[],"metadata":{},"metrics":[]},"time":1633886641046} -{"Event":"org.apache.spark.sql.catalyst.catalog.CreateTablePreEvent","database":"default","name":"rankings_uservisits_join"} -{"Event":"org.apache.spark.sql.catalyst.catalog.CreateTableEvent","database":"default","name":"rankings_uservisits_join"} -{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":10,"time":1633886641212} -{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":11,"description":"sql at ScalaSparkSQLBench.scala:43","details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","physicalPlanDescription":"== Physical Plan ==\nExecute InsertIntoHiveTable (15)\n+- * Project (14)\n +- * Sort (13)\n +- Exchange (12)\n +- * HashAggregate (11)\n +- Exchange (10)\n +- * HashAggregate (9)\n +- * Project (8)\n +- * BroadcastHashJoin Inner BuildLeft (7)\n :- BroadcastExchange (3)\n : +- * Filter (2)\n : +- Scan hive default.rankings (1)\n +- * Project (6)\n +- * Filter (5)\n +- Scan hive default.uservisits_copy (4)\n\n\n(1) Scan hive default.rankings\nOutput [2]: [pageurl#41, pagerank#42]\nArguments: [pageurl#41, pagerank#42], HiveTableRelation [`default`.`rankings`, org.apache.hadoop.hive.serde2.OpenCSVSerde, Data Cols: [pageurl#41, pagerank#42, avgduration#43], Partition Cols: []]\n\n(2) Filter [codegen id : 1]\nInput [2]: [pageurl#41, pagerank#42]\nCondition : isnotnull(pageURL#41)\n\n(3) BroadcastExchange\nInput [2]: [pageurl#41, pagerank#42]\nArguments: HashedRelationBroadcastMode(List(input[0, string, false]),false), [id=#88]\n\n(4) Scan hive default.uservisits_copy\nOutput [4]: [adrevenue#47, desturl#45, sourceip#44, visitdate#46]\nArguments: [adRevenue#47, destURL#45, sourceIP#44, visitDate#46], HiveTableRelation [`default`.`uservisits_copy`, org.apache.hadoop.hive.serde2.OpenCSVSerde, Data Cols: [sourceip#44, desturl#45, visitdate#46, adrevenue#47, useragent#48, countrycode#49, languagecode#..., Partition Cols: []]\n\n(5) Filter\nInput [4]: [adrevenue#47, desturl#45, sourceip#44, visitdate#46]\nCondition : (((isnotnull(visitDate#46) AND (datediff(cast(visitDate#46 as date), 10592) >= 0)) AND (datediff(cast(visitDate#46 as date), 10957) <= 0)) AND isnotnull(destURL#45))\n\n(6) Project\nOutput [3]: [sourceIP#44, destURL#45, adRevenue#47]\nInput [4]: [adrevenue#47, desturl#45, sourceip#44, visitdate#46]\n\n(7) BroadcastHashJoin [codegen id : 2]\nLeft keys [1]: [pageURL#41]\nRight keys [1]: [destURL#45]\nJoin condition: None\n\n(8) Project [codegen id : 2]\nOutput [3]: [pagerank#42, sourceIP#44, adRevenue#47]\nInput [5]: [pageurl#41, pagerank#42, sourceIP#44, destURL#45, adRevenue#47]\n\n(9) HashAggregate [codegen id : 2]\nInput [3]: [pagerank#42, sourceIP#44, adRevenue#47]\nKeys [1]: [sourceIP#44]\nFunctions [2]: [partial_avg(cast(pageRank#42 as double)), partial_sum(cast(adRevenue#47 as double))]\nAggregate Attributes [3]: [sum#62, count#63L, sum#64]\nResults [4]: [sourceIP#44, sum#65, count#66L, sum#67]\n\n(10) Exchange\nInput [4]: [sourceIP#44, sum#65, count#66L, sum#67]\nArguments: hashpartitioning(sourceIP#44, 8), ENSURE_REQUIREMENTS, [id=#97]\n\n(11) HashAggregate [codegen id : 3]\nInput [4]: [sourceIP#44, sum#65, count#66L, sum#67]\nKeys [1]: [sourceIP#44]\nFunctions [2]: [avg(cast(pageRank#42 as double)), sum(cast(adRevenue#47 as double))]\nAggregate Attributes [2]: [avg(cast(pageRank#42 as double))#53, sum(cast(adRevenue#47 as double))#54]\nResults [3]: [sourceIP#44, avg(cast(pageRank#42 as double))#53 AS avg(CAST(pageRank AS DOUBLE))#55, sum(cast(adRevenue#47 as double))#54 AS totalRevenue#40]\n\n(12) Exchange\nInput [3]: [sourceIP#44, avg(CAST(pageRank AS DOUBLE))#55, totalRevenue#40]\nArguments: rangepartitioning(totalRevenue#40 DESC NULLS LAST, 8), ENSURE_REQUIREMENTS, [id=#101]\n\n(13) Sort [codegen id : 4]\nInput [3]: [sourceIP#44, avg(CAST(pageRank AS DOUBLE))#55, totalRevenue#40]\nArguments: [totalRevenue#40 DESC NULLS LAST], true, 0\n\n(14) Project [codegen id : 4]\nOutput [3]: [ansi_cast(sourceIP#44 as string) AS sourceIP#60, ansi_cast(avg(CAST(pageRank AS DOUBLE))#55 as double) AS avgPageRank#61, totalRevenue#40]\nInput [3]: [sourceIP#44, avg(CAST(pageRank AS DOUBLE))#55, totalRevenue#40]\n\n(15) Execute InsertIntoHiveTable\nInput [3]: [sourceIP#60, avgPageRank#61, totalRevenue#40]\nArguments: `default`.`rankings_uservisits_join`, org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, true, false, [sourceIP, avgPageRank, totalRevenue]\n\n","sparkPlanInfo":{"nodeName":"Execute InsertIntoHiveTable","simpleString":"Execute InsertIntoHiveTable `default`.`rankings_uservisits_join`, org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, true, false, [sourceIP, avgPageRank, totalRevenue]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"Project","simpleString":"Project [ansi_cast(sourceIP#44 as string) AS sourceIP#60, ansi_cast(avg(CAST(pageRank AS DOUBLE))#55 as double) AS avgPageRank#61, totalRevenue#40]","children":[{"nodeName":"Sort","simpleString":"Sort [totalRevenue#40 DESC NULLS LAST], true, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Exchange","simpleString":"Exchange rangepartitioning(totalRevenue#40 DESC NULLS LAST, 8), ENSURE_REQUIREMENTS, [id=#101]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[sourceIP#44], functions=[avg(cast(pageRank#42 as double)), sum(cast(adRevenue#47 as double))])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(sourceIP#44, 8), ENSURE_REQUIREMENTS, [id=#97]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[sourceIP#44], functions=[partial_avg(cast(pageRank#42 as double)), partial_sum(cast(adRevenue#47 as double))])","children":[{"nodeName":"Project","simpleString":"Project [pagerank#42, sourceIP#44, adRevenue#47]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [pageURL#41], [destURL#45], Inner, BuildLeft, false","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, false]),false), [id=#88]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Filter","simpleString":"Filter isnotnull(pageURL#41)","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan hive default.rankings","simpleString":"Scan hive default.rankings [pageurl#41, pagerank#42], HiveTableRelation [`default`.`rankings`, org.apache.hadoop.hive.serde2.OpenCSVSerde, Data Cols: [pageurl#41, pagerank#42, avgduration#43], Partition Cols: []]","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":75,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":74,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":73,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":72,"metricType":"timing"},{"name":"time to build","accumulatorId":71,"metricType":"timing"},{"name":"time to collect","accumulatorId":70,"metricType":"timing"},{"name":"number of output rows","accumulatorId":69,"metricType":"sum"},{"name":"data size","accumulatorId":68,"metricType":"size"}]}],"metadata":{},"metrics":[]},{"nodeName":"Project","simpleString":"Project [sourceIP#44, destURL#45, adRevenue#47]","children":[{"nodeName":"Filter","simpleString":"Filter (((isnotnull(visitDate#46) AND (datediff(cast(visitDate#46 as date), 10592) >= 0)) AND (datediff(cast(visitDate#46 as date), 10957) <= 0)) AND isnotnull(destURL#45))","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan hive default.uservisits_copy","simpleString":"Scan hive default.uservisits_copy [adRevenue#47, destURL#45, sourceIP#44, visitDate#46], HiveTableRelation [`default`.`uservisits_copy`, org.apache.hadoop.hive.serde2.OpenCSVSerde, Data Cols: [sourceip#44, desturl#45, visitdate#46, adrevenue#47, useragent#48, countrycode#49, languagecode#..., Partition Cols: []]","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":77,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":76,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":67,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":64,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":65,"metricType":"timing"},{"name":"peak memory","accumulatorId":63,"metricType":"size"},{"name":"number of output rows","accumulatorId":62,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":66,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":61,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":34,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":35,"metricType":"nsTiming"},{"name":"records read","accumulatorId":32,"metricType":"sum"},{"name":"local bytes read","accumulatorId":30,"metricType":"size"},{"name":"fetch wait time","accumulatorId":31,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":28,"metricType":"size"},{"name":"local blocks read","accumulatorId":27,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":26,"metricType":"sum"},{"name":"data size","accumulatorId":25,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":29,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":33,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":58,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":59,"metricType":"timing"},{"name":"peak memory","accumulatorId":57,"metricType":"size"},{"name":"number of output rows","accumulatorId":56,"metricType":"sum"},{"name":"avg hash probe bucket list iters","accumulatorId":60,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":55,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":45,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":46,"metricType":"nsTiming"},{"name":"records read","accumulatorId":43,"metricType":"sum"},{"name":"local bytes read","accumulatorId":41,"metricType":"size"},{"name":"fetch wait time","accumulatorId":42,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":39,"metricType":"size"},{"name":"local blocks read","accumulatorId":38,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":37,"metricType":"sum"},{"name":"data size","accumulatorId":36,"metricType":"size"},{"name":"remote bytes read to disk","accumulatorId":40,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":44,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":52,"metricType":"timing"},{"name":"peak memory","accumulatorId":53,"metricType":"size"},{"name":"spill size","accumulatorId":54,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":51,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"number of written files","accumulatorId":47,"metricType":"sum"},{"name":"written output","accumulatorId":48,"metricType":"size"},{"name":"number of output rows","accumulatorId":49,"metricType":"sum"},{"name":"number of dynamic part","accumulatorId":50,"metricType":"sum"}]},"time":1633886642667} -{"Event":"SparkListenerJobStart","Job ID":0,"Submission Time":1633886643553,"Stage Infos":[{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Number of Tasks":8,"RDD Info":[{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"2\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"Scan hive default.rankings\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"HadoopRDD","Scope":"{\"id\":\"7\",\"name\":\"Scan hive default.rankings\"}","Callsite":"","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"Scan hive default.rankings\"}","Callsite":"","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"Scan hive default.rankings\"}","Callsite":"","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:185)\njava.util.concurrent.FutureTask.run(FutureTask.java:266)\njava.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\njava.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\njava.lang.Thread.run(Thread.java:748)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[0],"Properties":{"spark.sql.warehouse.dir":"file:/home/usera112/projects/log-compressor/deployments/HiBench/spark-warehouse","spark.driver.host":"hostb7.dsrg.utoronto.ca","spark.eventLog.enabled":"true","spark.job.interruptOnCancel":"true","spark.driver.port":"42853","spark.jars":"file:/home/usera112/projects/log-compressor/deployments/HiBench/sparkbench/assembly/target/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.app.name":"ScalaJoin","spark.rdd.scope":"{\"id\":\"0\",\"name\":\"WholeStageCodegen (4)\"}","spark.driver.memory":"4g","spark.rdd.scope.noOverride":"true","spark.submit.pyFiles":"","spark.default.parallelism":"8","spark.job.description":"broadcast exchange (runId 8616dbde-088b-499b-bd85-94ba59933204)","spark.app.startTime":"1633886611309","spark.executor.id":"driver","spark.jobGroup.id":"8616dbde-088b-499b-bd85-94ba59933204","spark.app.initial.jar.urls":"spark://hostb7.dsrg.utoronto.ca:42853/jars/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.submit.deployMode":"client","spark.master":"spark://hostb7:7077","spark.executor.memory":"4g","spark.eventLog.dir":"hdfs://hostb7:8020/directory","spark.sql.execution.id":"11","spark.sql.catalogImplementation":"hive","spark.app.id":"app-20211010132332-4636","spark.sql.shuffle.partitions":"8"}} -{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Number of Tasks":8,"RDD Info":[{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"2\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"Scan hive default.rankings\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"HadoopRDD","Scope":"{\"id\":\"7\",\"name\":\"Scan hive default.rankings\"}","Callsite":"","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"Scan hive default.rankings\"}","Callsite":"","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"Scan hive default.rankings\"}","Callsite":"","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:185)\njava.util.concurrent.FutureTask.run(FutureTask.java:266)\njava.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\njava.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1633886643571,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.sql.warehouse.dir":"file:/home/usera112/projects/log-compressor/deployments/HiBench/spark-warehouse","spark.driver.host":"hostb7.dsrg.utoronto.ca","spark.eventLog.enabled":"true","spark.job.interruptOnCancel":"true","spark.driver.port":"42853","spark.jars":"file:/home/usera112/projects/log-compressor/deployments/HiBench/sparkbench/assembly/target/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.app.name":"ScalaJoin","spark.rdd.scope":"{\"id\":\"0\",\"name\":\"WholeStageCodegen (4)\"}","spark.driver.memory":"4g","spark.rdd.scope.noOverride":"true","spark.submit.pyFiles":"","spark.default.parallelism":"8","spark.job.description":"broadcast exchange (runId 8616dbde-088b-499b-bd85-94ba59933204)","spark.app.startTime":"1633886611309","spark.executor.id":"driver","spark.jobGroup.id":"8616dbde-088b-499b-bd85-94ba59933204","spark.app.initial.jar.urls":"spark://hostb7.dsrg.utoronto.ca:42853/jars/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.submit.deployMode":"client","spark.master":"spark://hostb7:7077","spark.executor.memory":"4g","spark.eventLog.dir":"hdfs://hostb7:8020/directory","spark.sql.execution.id":"11","spark.sql.catalogImplementation":"hive","spark.app.id":"app-20211010132332-4636","spark.sql.shuffle.partitions":"8"}} -{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Launch Time":1633886643618,"Executor ID":"6","Host":"10.1.0.18","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":1,"Index":1,"Attempt":0,"Launch Time":1633886643635,"Executor ID":"10","Host":"10.1.0.16","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":2,"Index":2,"Attempt":0,"Launch Time":1633886643635,"Executor ID":"0","Host":"10.1.0.12","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":3,"Index":3,"Attempt":0,"Launch Time":1633886643636,"Executor ID":"9","Host":"10.1.0.14","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":4,"Index":4,"Attempt":0,"Launch Time":1633886643636,"Executor ID":"8","Host":"10.1.0.11","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":5,"Index":5,"Attempt":0,"Launch Time":1633886643637,"Executor ID":"11","Host":"10.1.0.13","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":6,"Index":6,"Attempt":0,"Launch Time":1633886643637,"Executor ID":"5","Host":"10.1.0.19","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":7,"Index":7,"Attempt":0,"Launch Time":1633886643638,"Executor ID":"2","Host":"10.1.0.10","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskGettingResult","Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Launch Time":1633886643618,"Executor ID":"6","Host":"10.1.0.18","Locality":"ANY","Speculative":false,"Getting Result Time":1633886645916,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskGettingResult","Task Info":{"Task ID":6,"Index":6,"Attempt":0,"Launch Time":1633886643637,"Executor ID":"5","Host":"10.1.0.19","Locality":"ANY","Speculative":false,"Getting Result Time":1633886645946,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskGettingResult","Task Info":{"Task ID":4,"Index":4,"Attempt":0,"Launch Time":1633886643636,"Executor ID":"8","Host":"10.1.0.11","Locality":"ANY","Speculative":false,"Getting Result Time":1633886645949,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskGettingResult","Task Info":{"Task ID":1,"Index":1,"Attempt":0,"Launch Time":1633886643635,"Executor ID":"10","Host":"10.1.0.16","Locality":"ANY","Speculative":false,"Getting Result Time":1633886645952,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":6,"Index":6,"Attempt":0,"Launch Time":1633886643637,"Executor ID":"5","Host":"10.1.0.19","Locality":"ANY","Speculative":false,"Getting Result Time":1633886645946,"Finish Time":1633886645975,"Failed":false,"Killed":false,"Accumulables":[{"ID":73,"Name":"duration","Update":"206","Value":"206","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"number of output rows","Update":"15000","Value":"15000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":75,"Name":"number of output rows","Update":"15000","Value":"15000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":78,"Name":"internal.metrics.executorDeserializeTime","Update":684,"Value":684,"Internal":true,"Count Failed Values":true},{"ID":79,"Name":"internal.metrics.executorDeserializeCpuTime","Update":608558395,"Value":608558395,"Internal":true,"Count Failed Values":true},{"ID":80,"Name":"internal.metrics.executorRunTime","Update":1573,"Value":1573,"Internal":true,"Count Failed Values":true},{"ID":81,"Name":"internal.metrics.executorCpuTime","Update":1435649112,"Value":1435649112,"Internal":true,"Count Failed Values":true},{"ID":82,"Name":"internal.metrics.resultSize","Update":1067500,"Value":1067500,"Internal":true,"Count Failed Values":true},{"ID":83,"Name":"internal.metrics.jvmGCTime","Update":131,"Value":131,"Internal":true,"Count Failed Values":true},{"ID":84,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":99,"Name":"internal.metrics.input.bytesRead","Update":655811,"Value":655811,"Internal":true,"Count Failed Values":true},{"ID":100,"Name":"internal.metrics.input.recordsRead","Update":15000,"Value":15000,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":684,"Executor Deserialize CPU Time":608558395,"Executor Run Time":1573,"Executor CPU Time":1435649112,"Peak Execution Memory":0,"Result Size":1067500,"JVM GC Time":131,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":655811,"Records Read":15000},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":1,"Index":1,"Attempt":0,"Launch Time":1633886643635,"Executor ID":"10","Host":"10.1.0.16","Locality":"ANY","Speculative":false,"Getting Result Time":1633886645952,"Finish Time":1633886645980,"Failed":false,"Killed":false,"Accumulables":[{"ID":73,"Name":"duration","Update":"190","Value":"396","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"number of output rows","Update":"15000","Value":"30000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":75,"Name":"number of output rows","Update":"15000","Value":"30000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":78,"Name":"internal.metrics.executorDeserializeTime","Update":699,"Value":1383,"Internal":true,"Count Failed Values":true},{"ID":79,"Name":"internal.metrics.executorDeserializeCpuTime","Update":631462249,"Value":1240020644,"Internal":true,"Count Failed Values":true},{"ID":80,"Name":"internal.metrics.executorRunTime","Update":1552,"Value":3125,"Internal":true,"Count Failed Values":true},{"ID":81,"Name":"internal.metrics.executorCpuTime","Update":1399479944,"Value":2835129056,"Internal":true,"Count Failed Values":true},{"ID":82,"Name":"internal.metrics.resultSize","Update":1060240,"Value":2127740,"Internal":true,"Count Failed Values":true},{"ID":83,"Name":"internal.metrics.jvmGCTime","Update":131,"Value":262,"Internal":true,"Count Failed Values":true},{"ID":84,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":99,"Name":"internal.metrics.input.bytesRead","Update":651866,"Value":1307677,"Internal":true,"Count Failed Values":true},{"ID":100,"Name":"internal.metrics.input.recordsRead","Update":15000,"Value":30000,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":699,"Executor Deserialize CPU Time":631462249,"Executor Run Time":1552,"Executor CPU Time":1399479944,"Peak Execution Memory":0,"Result Size":1060240,"JVM GC Time":131,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":651866,"Records Read":15000},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":4,"Index":4,"Attempt":0,"Launch Time":1633886643636,"Executor ID":"8","Host":"10.1.0.11","Locality":"ANY","Speculative":false,"Getting Result Time":1633886645949,"Finish Time":1633886645980,"Failed":false,"Killed":false,"Accumulables":[{"ID":73,"Name":"duration","Update":"188","Value":"584","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"number of output rows","Update":"15000","Value":"45000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":75,"Name":"number of output rows","Update":"15000","Value":"45000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":78,"Name":"internal.metrics.executorDeserializeTime","Update":692,"Value":2075,"Internal":true,"Count Failed Values":true},{"ID":79,"Name":"internal.metrics.executorDeserializeCpuTime","Update":609287687,"Value":1849308331,"Internal":true,"Count Failed Values":true},{"ID":80,"Name":"internal.metrics.executorRunTime","Update":1566,"Value":4691,"Internal":true,"Count Failed Values":true},{"ID":81,"Name":"internal.metrics.executorCpuTime","Update":1391180520,"Value":4226309576,"Internal":true,"Count Failed Values":true},{"ID":82,"Name":"internal.metrics.resultSize","Update":1062242,"Value":3189982,"Internal":true,"Count Failed Values":true},{"ID":83,"Name":"internal.metrics.jvmGCTime","Update":153,"Value":415,"Internal":true,"Count Failed Values":true},{"ID":84,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":99,"Name":"internal.metrics.input.bytesRead","Update":653437,"Value":1961114,"Internal":true,"Count Failed Values":true},{"ID":100,"Name":"internal.metrics.input.recordsRead","Update":15000,"Value":45000,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":692,"Executor Deserialize CPU Time":609287687,"Executor Run Time":1566,"Executor CPU Time":1391180520,"Peak Execution Memory":0,"Result Size":1062242,"JVM GC Time":153,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":653437,"Records Read":15000},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Launch Time":1633886643618,"Executor ID":"6","Host":"10.1.0.18","Locality":"ANY","Speculative":false,"Getting Result Time":1633886645916,"Finish Time":1633886645980,"Failed":false,"Killed":false,"Accumulables":[{"ID":73,"Name":"duration","Update":"185","Value":"769","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"number of output rows","Update":"15000","Value":"60000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":75,"Name":"number of output rows","Update":"15000","Value":"60000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":78,"Name":"internal.metrics.executorDeserializeTime","Update":692,"Value":2767,"Internal":true,"Count Failed Values":true},{"ID":79,"Name":"internal.metrics.executorDeserializeCpuTime","Update":624637032,"Value":2473945363,"Internal":true,"Count Failed Values":true},{"ID":80,"Name":"internal.metrics.executorRunTime","Update":1528,"Value":6219,"Internal":true,"Count Failed Values":true},{"ID":81,"Name":"internal.metrics.executorCpuTime","Update":1361454940,"Value":5587764516,"Internal":true,"Count Failed Values":true},{"ID":82,"Name":"internal.metrics.resultSize","Update":1063975,"Value":4253957,"Internal":true,"Count Failed Values":true},{"ID":83,"Name":"internal.metrics.jvmGCTime","Update":141,"Value":556,"Internal":true,"Count Failed Values":true},{"ID":84,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":6,"Internal":true,"Count Failed Values":true},{"ID":99,"Name":"internal.metrics.input.bytesRead","Update":652408,"Value":2613522,"Internal":true,"Count Failed Values":true},{"ID":100,"Name":"internal.metrics.input.recordsRead","Update":15000,"Value":60000,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":692,"Executor Deserialize CPU Time":624637032,"Executor Run Time":1528,"Executor CPU Time":1361454940,"Peak Execution Memory":0,"Result Size":1063975,"JVM GC Time":141,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":652408,"Records Read":15000},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskGettingResult","Task Info":{"Task ID":3,"Index":3,"Attempt":0,"Launch Time":1633886643636,"Executor ID":"9","Host":"10.1.0.14","Locality":"ANY","Speculative":false,"Getting Result Time":1633886645980,"Finish Time":1633886645998,"Failed":false,"Killed":false,"Accumulables":[{"ID":73,"Name":"duration","Update":"197","Value":"966","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"number of output rows","Update":"15000","Value":"75000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":75,"Name":"number of output rows","Update":"15000","Value":"75000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":78,"Name":"internal.metrics.executorDeserializeTime","Update":671,"Value":3438,"Internal":true,"Count Failed Values":true},{"ID":79,"Name":"internal.metrics.executorDeserializeCpuTime","Update":607404581,"Value":3081349944,"Internal":true,"Count Failed Values":true},{"ID":80,"Name":"internal.metrics.executorRunTime","Update":1598,"Value":7817,"Internal":true,"Count Failed Values":true},{"ID":81,"Name":"internal.metrics.executorCpuTime","Update":1448214525,"Value":7035979041,"Internal":true,"Count Failed Values":true},{"ID":82,"Name":"internal.metrics.resultSize","Update":1057646,"Value":5311603,"Internal":true,"Count Failed Values":true},{"ID":83,"Name":"internal.metrics.jvmGCTime","Update":146,"Value":702,"Internal":true,"Count Failed Values":true},{"ID":84,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":99,"Name":"internal.metrics.input.bytesRead","Update":650282,"Value":3263804,"Internal":true,"Count Failed Values":true},{"ID":100,"Name":"internal.metrics.input.recordsRead","Update":15000,"Value":75000,"Internal":true,"Count Failed Values":true}]}} -{"Event":"SparkListenerTaskGettingResult","Task Info":{"Task ID":2,"Index":2,"Attempt":0,"Launch Time":1633886643635,"Executor ID":"0","Host":"10.1.0.12","Locality":"ANY","Speculative":false,"Getting Result Time":1633886645981,"Finish Time":1633886646005,"Failed":false,"Killed":false,"Accumulables":[{"ID":73,"Name":"duration","Update":"203","Value":"1169","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"number of output rows","Update":"15000","Value":"90000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":75,"Name":"number of output rows","Update":"15000","Value":"90000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":78,"Name":"internal.metrics.executorDeserializeTime","Update":723,"Value":4161,"Internal":true,"Count Failed Values":true},{"ID":79,"Name":"internal.metrics.executorDeserializeCpuTime","Update":634415441,"Value":3715765385,"Internal":true,"Count Failed Values":true},{"ID":80,"Name":"internal.metrics.executorRunTime","Update":1559,"Value":9376,"Internal":true,"Count Failed Values":true},{"ID":81,"Name":"internal.metrics.executorCpuTime","Update":1423102479,"Value":8459081520,"Internal":true,"Count Failed Values":true},{"ID":82,"Name":"internal.metrics.resultSize","Update":1064422,"Value":6376025,"Internal":true,"Count Failed Values":true},{"ID":83,"Name":"internal.metrics.jvmGCTime","Update":131,"Value":833,"Internal":true,"Count Failed Values":true},{"ID":84,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":9,"Internal":true,"Count Failed Values":true},{"ID":99,"Name":"internal.metrics.input.bytesRead","Update":654511,"Value":3918315,"Internal":true,"Count Failed Values":true},{"ID":100,"Name":"internal.metrics.input.recordsRead","Update":15000,"Value":90000,"Internal":true,"Count Failed Values":true}]}} -{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":3,"Index":3,"Attempt":0,"Launch Time":1633886643636,"Executor ID":"9","Host":"10.1.0.14","Locality":"ANY","Speculative":false,"Getting Result Time":1633886645980,"Finish Time":1633886645998,"Failed":false,"Killed":false,"Accumulables":[{"ID":73,"Name":"duration","Update":"197","Value":"966","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"number of output rows","Update":"15000","Value":"75000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":75,"Name":"number of output rows","Update":"15000","Value":"75000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":78,"Name":"internal.metrics.executorDeserializeTime","Update":671,"Value":3438,"Internal":true,"Count Failed Values":true},{"ID":79,"Name":"internal.metrics.executorDeserializeCpuTime","Update":607404581,"Value":3081349944,"Internal":true,"Count Failed Values":true},{"ID":80,"Name":"internal.metrics.executorRunTime","Update":1598,"Value":7817,"Internal":true,"Count Failed Values":true},{"ID":81,"Name":"internal.metrics.executorCpuTime","Update":1448214525,"Value":7035979041,"Internal":true,"Count Failed Values":true},{"ID":82,"Name":"internal.metrics.resultSize","Update":1057646,"Value":5311603,"Internal":true,"Count Failed Values":true},{"ID":83,"Name":"internal.metrics.jvmGCTime","Update":146,"Value":702,"Internal":true,"Count Failed Values":true},{"ID":84,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":99,"Name":"internal.metrics.input.bytesRead","Update":650282,"Value":3263804,"Internal":true,"Count Failed Values":true},{"ID":100,"Name":"internal.metrics.input.recordsRead","Update":15000,"Value":75000,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":671,"Executor Deserialize CPU Time":607404581,"Executor Run Time":1598,"Executor CPU Time":1448214525,"Peak Execution Memory":0,"Result Size":1057646,"JVM GC Time":146,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":650282,"Records Read":15000},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":2,"Index":2,"Attempt":0,"Launch Time":1633886643635,"Executor ID":"0","Host":"10.1.0.12","Locality":"ANY","Speculative":false,"Getting Result Time":1633886645981,"Finish Time":1633886646005,"Failed":false,"Killed":false,"Accumulables":[{"ID":73,"Name":"duration","Update":"203","Value":"1169","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"number of output rows","Update":"15000","Value":"90000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":75,"Name":"number of output rows","Update":"15000","Value":"90000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":78,"Name":"internal.metrics.executorDeserializeTime","Update":723,"Value":4161,"Internal":true,"Count Failed Values":true},{"ID":79,"Name":"internal.metrics.executorDeserializeCpuTime","Update":634415441,"Value":3715765385,"Internal":true,"Count Failed Values":true},{"ID":80,"Name":"internal.metrics.executorRunTime","Update":1559,"Value":9376,"Internal":true,"Count Failed Values":true},{"ID":81,"Name":"internal.metrics.executorCpuTime","Update":1423102479,"Value":8459081520,"Internal":true,"Count Failed Values":true},{"ID":82,"Name":"internal.metrics.resultSize","Update":1064422,"Value":6376025,"Internal":true,"Count Failed Values":true},{"ID":83,"Name":"internal.metrics.jvmGCTime","Update":131,"Value":833,"Internal":true,"Count Failed Values":true},{"ID":84,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":9,"Internal":true,"Count Failed Values":true},{"ID":99,"Name":"internal.metrics.input.bytesRead","Update":654511,"Value":3918315,"Internal":true,"Count Failed Values":true},{"ID":100,"Name":"internal.metrics.input.recordsRead","Update":15000,"Value":90000,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":723,"Executor Deserialize CPU Time":634415441,"Executor Run Time":1559,"Executor CPU Time":1423102479,"Peak Execution Memory":0,"Result Size":1064422,"JVM GC Time":131,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":654511,"Records Read":15000},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskGettingResult","Task Info":{"Task ID":5,"Index":5,"Attempt":0,"Launch Time":1633886643637,"Executor ID":"11","Host":"10.1.0.13","Locality":"ANY","Speculative":false,"Getting Result Time":1633886646029,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskGettingResult","Task Info":{"Task ID":7,"Index":7,"Attempt":0,"Launch Time":1633886643638,"Executor ID":"2","Host":"10.1.0.10","Locality":"ANY","Speculative":false,"Getting Result Time":1633886646035,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":5,"Index":5,"Attempt":0,"Launch Time":1633886643637,"Executor ID":"11","Host":"10.1.0.13","Locality":"ANY","Speculative":false,"Getting Result Time":1633886646029,"Finish Time":1633886646053,"Failed":false,"Killed":false,"Accumulables":[{"ID":73,"Name":"duration","Update":"182","Value":"1351","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"number of output rows","Update":"15000","Value":"105000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":75,"Name":"number of output rows","Update":"15000","Value":"105000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":78,"Name":"internal.metrics.executorDeserializeTime","Update":702,"Value":4863,"Internal":true,"Count Failed Values":true},{"ID":79,"Name":"internal.metrics.executorDeserializeCpuTime","Update":632593447,"Value":4348358832,"Internal":true,"Count Failed Values":true},{"ID":80,"Name":"internal.metrics.executorRunTime","Update":1633,"Value":11009,"Internal":true,"Count Failed Values":true},{"ID":81,"Name":"internal.metrics.executorCpuTime","Update":1479670691,"Value":9938752211,"Internal":true,"Count Failed Values":true},{"ID":82,"Name":"internal.metrics.resultSize","Update":1059139,"Value":7435164,"Internal":true,"Count Failed Values":true},{"ID":83,"Name":"internal.metrics.jvmGCTime","Update":148,"Value":981,"Internal":true,"Count Failed Values":true},{"ID":84,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":10,"Internal":true,"Count Failed Values":true},{"ID":99,"Name":"internal.metrics.input.bytesRead","Update":651025,"Value":4569340,"Internal":true,"Count Failed Values":true},{"ID":100,"Name":"internal.metrics.input.recordsRead","Update":15000,"Value":105000,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":702,"Executor Deserialize CPU Time":632593447,"Executor Run Time":1633,"Executor CPU Time":1479670691,"Peak Execution Memory":0,"Result Size":1059139,"JVM GC Time":148,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":651025,"Records Read":15000},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":7,"Index":7,"Attempt":0,"Launch Time":1633886643638,"Executor ID":"2","Host":"10.1.0.10","Locality":"ANY","Speculative":false,"Getting Result Time":1633886646035,"Finish Time":1633886646056,"Failed":false,"Killed":false,"Accumulables":[{"ID":73,"Name":"duration","Update":"181","Value":"1532","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"number of output rows","Update":"15000","Value":"120000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":75,"Name":"number of output rows","Update":"15000","Value":"120000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":78,"Name":"internal.metrics.executorDeserializeTime","Update":692,"Value":5555,"Internal":true,"Count Failed Values":true},{"ID":79,"Name":"internal.metrics.executorDeserializeCpuTime","Update":607426390,"Value":4955785222,"Internal":true,"Count Failed Values":true},{"ID":80,"Name":"internal.metrics.executorRunTime","Update":1654,"Value":12663,"Internal":true,"Count Failed Values":true},{"ID":81,"Name":"internal.metrics.executorCpuTime","Update":1490277458,"Value":11429029669,"Internal":true,"Count Failed Values":true},{"ID":82,"Name":"internal.metrics.resultSize","Update":1058522,"Value":8493686,"Internal":true,"Count Failed Values":true},{"ID":83,"Name":"internal.metrics.jvmGCTime","Update":147,"Value":1128,"Internal":true,"Count Failed Values":true},{"ID":84,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":11,"Internal":true,"Count Failed Values":true},{"ID":99,"Name":"internal.metrics.input.bytesRead","Update":650945,"Value":5220285,"Internal":true,"Count Failed Values":true},{"ID":100,"Name":"internal.metrics.input.recordsRead","Update":15000,"Value":120000,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":692,"Executor Deserialize CPU Time":607426390,"Executor Run Time":1654,"Executor CPU Time":1490277458,"Peak Execution Memory":0,"Result Size":1058522,"JVM GC Time":147,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":650945,"Records Read":15000},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Number of Tasks":8,"RDD Info":[{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"2\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"Scan hive default.rankings\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:266","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"HadoopRDD","Scope":"{\"id\":\"7\",\"name\":\"Scan hive default.rankings\"}","Callsite":"","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"Scan hive default.rankings\"}","Callsite":"","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"Scan hive default.rankings\"}","Callsite":"","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:185)\njava.util.concurrent.FutureTask.run(FutureTask.java:266)\njava.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\njava.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\njava.lang.Thread.run(Thread.java:748)","Submission Time":1633886643571,"Completion Time":1633886646058,"Accumulables":[{"ID":73,"Name":"duration","Value":"1532","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":74,"Name":"number of output rows","Value":"120000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":75,"Name":"number of output rows","Value":"120000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":78,"Name":"internal.metrics.executorDeserializeTime","Value":5555,"Internal":true,"Count Failed Values":true},{"ID":79,"Name":"internal.metrics.executorDeserializeCpuTime","Value":4955785222,"Internal":true,"Count Failed Values":true},{"ID":80,"Name":"internal.metrics.executorRunTime","Value":12663,"Internal":true,"Count Failed Values":true},{"ID":81,"Name":"internal.metrics.executorCpuTime","Value":11429029669,"Internal":true,"Count Failed Values":true},{"ID":82,"Name":"internal.metrics.resultSize","Value":8493686,"Internal":true,"Count Failed Values":true},{"ID":83,"Name":"internal.metrics.jvmGCTime","Value":1128,"Internal":true,"Count Failed Values":true},{"ID":84,"Name":"internal.metrics.resultSerializationTime","Value":11,"Internal":true,"Count Failed Values":true},{"ID":99,"Name":"internal.metrics.input.bytesRead","Value":5220285,"Internal":true,"Count Failed Values":true},{"ID":100,"Name":"internal.metrics.input.recordsRead","Value":120000,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} -{"Event":"SparkListenerJobEnd","Job ID":0,"Completion Time":1633886646065,"Job Result":{"Result":"JobSucceeded"}} -{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":11,"accumUpdates":[[72,111],[71,169],[70,3223],[69,120000],[68,29360128]]} -{"Event":"SparkListenerJobStart","Job ID":1,"Submission Time":1633886646652,"Stage Infos":[{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"sql at ScalaSparkSQLBench.scala:43","Number of Tasks":8,"RDD Info":[{"RDD ID":11,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"12\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[10],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":10,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"13\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[9],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":6,"Name":"HadoopRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":2,"Stage Attempt ID":0,"Stage Name":"sql at ScalaSparkSQLBench.scala:43","Number of Tasks":8,"RDD Info":[{"RDD ID":16,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[15],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":15,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[14],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":14,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":12,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"12\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[11],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[1],"Details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[1,2],"Properties":{"spark.sql.warehouse.dir":"file:/home/usera112/projects/log-compressor/deployments/HiBench/spark-warehouse","spark.driver.host":"hostb7.dsrg.utoronto.ca","spark.eventLog.enabled":"true","spark.driver.port":"42853","__fetch_continuous_blocks_in_batch_enabled":"true","spark.jars":"file:/home/usera112/projects/log-compressor/deployments/HiBench/sparkbench/assembly/target/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.app.name":"ScalaJoin","spark.rdd.scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","spark.driver.memory":"4g","spark.rdd.scope.noOverride":"true","spark.submit.pyFiles":"","spark.default.parallelism":"8","spark.app.startTime":"1633886611309","spark.executor.id":"driver","spark.app.initial.jar.urls":"spark://hostb7.dsrg.utoronto.ca:42853/jars/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.submit.deployMode":"client","spark.master":"spark://hostb7:7077","spark.executor.memory":"4g","spark.eventLog.dir":"hdfs://hostb7:8020/directory","spark.sql.execution.id":"11","spark.sql.catalogImplementation":"hive","spark.app.id":"app-20211010132332-4636","spark.sql.shuffle.partitions":"8"}} -{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"sql at ScalaSparkSQLBench.scala:43","Number of Tasks":8,"RDD Info":[{"RDD ID":11,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"12\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[10],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":10,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"13\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[9],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":6,"Name":"HadoopRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1633886646659,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.sql.warehouse.dir":"file:/home/usera112/projects/log-compressor/deployments/HiBench/spark-warehouse","spark.driver.host":"hostb7.dsrg.utoronto.ca","spark.eventLog.enabled":"true","spark.driver.port":"42853","__fetch_continuous_blocks_in_batch_enabled":"true","spark.jars":"file:/home/usera112/projects/log-compressor/deployments/HiBench/sparkbench/assembly/target/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.app.name":"ScalaJoin","spark.rdd.scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","spark.driver.memory":"4g","spark.rdd.scope.noOverride":"true","spark.submit.pyFiles":"","spark.default.parallelism":"8","spark.app.startTime":"1633886611309","spark.executor.id":"driver","spark.app.initial.jar.urls":"spark://hostb7.dsrg.utoronto.ca:42853/jars/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.submit.deployMode":"client","spark.master":"spark://hostb7:7077","spark.executor.memory":"4g","spark.eventLog.dir":"hdfs://hostb7:8020/directory","spark.sql.execution.id":"11","spark.sql.catalogImplementation":"hive","spark.app.id":"app-20211010132332-4636","spark.sql.shuffle.partitions":"8"}} -{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":8,"Index":0,"Attempt":0,"Launch Time":1633886646669,"Executor ID":"5","Host":"10.1.0.19","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":9,"Index":1,"Attempt":0,"Launch Time":1633886646670,"Executor ID":"2","Host":"10.1.0.10","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":10,"Index":2,"Attempt":0,"Launch Time":1633886646671,"Executor ID":"9","Host":"10.1.0.14","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":11,"Index":3,"Attempt":0,"Launch Time":1633886646671,"Executor ID":"3","Host":"10.1.0.15","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":12,"Index":4,"Attempt":0,"Launch Time":1633886646671,"Executor ID":"11","Host":"10.1.0.13","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":13,"Index":5,"Attempt":0,"Launch Time":1633886646672,"Executor ID":"6","Host":"10.1.0.18","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":14,"Index":6,"Attempt":0,"Launch Time":1633886646672,"Executor ID":"0","Host":"10.1.0.12","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":15,"Index":7,"Attempt":0,"Launch Time":1633886646672,"Executor ID":"10","Host":"10.1.0.16","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":12,"Index":4,"Attempt":0,"Launch Time":1633886646671,"Executor ID":"11","Host":"10.1.0.13","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":1633886648378,"Failed":false,"Killed":false,"Accumulables":[{"ID":25,"Name":"data size","Update":"161944","Value":"161944","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"shuffle bytes written","Update":"81819","Value":"81819","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"shuffle records written","Update":"2892","Value":"2892","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":35,"Name":"shuffle write time","Update":"7745450","Value":"7745450","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":61,"Name":"duration","Update":"1314","Value":"1314","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":62,"Name":"number of output rows","Update":"2892","Value":"2892","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":63,"Name":"peak memory","Update":"4456448","Value":"4456448","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":65,"Name":"time in aggregation build","Update":"999","Value":"999","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":66,"Name":"avg hash probe bucket list iters","Update":"10","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":67,"Name":"number of output rows","Update":"2892","Value":"2892","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":76,"Name":"number of output rows","Update":"2892","Value":"2892","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":77,"Name":"number of output rows","Update":"124603","Value":"124603","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":103,"Name":"internal.metrics.executorDeserializeTime","Update":194,"Value":194,"Internal":true,"Count Failed Values":true},{"ID":104,"Name":"internal.metrics.executorDeserializeCpuTime","Update":187312873,"Value":187312873,"Internal":true,"Count Failed Values":true},{"ID":105,"Name":"internal.metrics.executorRunTime","Update":1497,"Value":1497,"Internal":true,"Count Failed Values":true},{"ID":106,"Name":"internal.metrics.executorCpuTime","Update":1391505517,"Value":1391505517,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.resultSize","Update":2905,"Value":2905,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.jvmGCTime","Update":36,"Value":36,"Internal":true,"Count Failed Values":true},{"ID":112,"Name":"internal.metrics.peakExecutionMemory","Update":33816576,"Value":33816576,"Internal":true,"Count Failed Values":true},{"ID":121,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":81819,"Value":81819,"Internal":true,"Count Failed Values":true},{"ID":122,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":2892,"Value":2892,"Internal":true,"Count Failed Values":true},{"ID":123,"Name":"internal.metrics.shuffle.write.writeTime","Update":7745450,"Value":7745450,"Internal":true,"Count Failed Values":true},{"ID":124,"Name":"internal.metrics.input.bytesRead","Update":23064712,"Value":23064712,"Internal":true,"Count Failed Values":true},{"ID":125,"Name":"internal.metrics.input.recordsRead","Update":124603,"Value":124603,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":194,"Executor Deserialize CPU Time":187312873,"Executor Run Time":1497,"Executor CPU Time":1391505517,"Peak Execution Memory":33816576,"Result Size":2905,"JVM GC Time":36,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":81819,"Shuffle Write Time":7745450,"Shuffle Records Written":2892},"Input Metrics":{"Bytes Read":23064712,"Records Read":124603},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":15,"Index":7,"Attempt":0,"Launch Time":1633886646672,"Executor ID":"10","Host":"10.1.0.16","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":1633886648402,"Failed":false,"Killed":false,"Accumulables":[{"ID":25,"Name":"data size","Update":"167832","Value":"329776","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"shuffle bytes written","Update":"84637","Value":"166456","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"shuffle records written","Update":"2997","Value":"5889","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":35,"Name":"shuffle write time","Update":"7781822","Value":"15527272","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":61,"Name":"duration","Update":"1336","Value":"2650","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":62,"Name":"number of output rows","Update":"2997","Value":"5889","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":63,"Name":"peak memory","Update":"4456448","Value":"8912896","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":65,"Name":"time in aggregation build","Update":"1009","Value":"2008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":66,"Name":"avg hash probe bucket list iters","Update":"10","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":67,"Name":"number of output rows","Update":"2997","Value":"5889","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":76,"Name":"number of output rows","Update":"2997","Value":"5889","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":77,"Name":"number of output rows","Update":"125488","Value":"250091","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":103,"Name":"internal.metrics.executorDeserializeTime","Update":191,"Value":385,"Internal":true,"Count Failed Values":true},{"ID":104,"Name":"internal.metrics.executorDeserializeCpuTime","Update":184296885,"Value":371609758,"Internal":true,"Count Failed Values":true},{"ID":105,"Name":"internal.metrics.executorRunTime","Update":1526,"Value":3023,"Internal":true,"Count Failed Values":true},{"ID":106,"Name":"internal.metrics.executorCpuTime","Update":1410301401,"Value":2801806918,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.resultSize","Update":2905,"Value":5810,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.jvmGCTime","Update":38,"Value":74,"Internal":true,"Count Failed Values":true},{"ID":112,"Name":"internal.metrics.peakExecutionMemory","Update":33816576,"Value":67633152,"Internal":true,"Count Failed Values":true},{"ID":121,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":84637,"Value":166456,"Internal":true,"Count Failed Values":true},{"ID":122,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":2997,"Value":5889,"Internal":true,"Count Failed Values":true},{"ID":123,"Name":"internal.metrics.shuffle.write.writeTime","Update":7781822,"Value":15527272,"Internal":true,"Count Failed Values":true},{"ID":124,"Name":"internal.metrics.input.bytesRead","Update":23178313,"Value":46243025,"Internal":true,"Count Failed Values":true},{"ID":125,"Name":"internal.metrics.input.recordsRead","Update":125488,"Value":250091,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":191,"Executor Deserialize CPU Time":184296885,"Executor Run Time":1526,"Executor CPU Time":1410301401,"Peak Execution Memory":33816576,"Result Size":2905,"JVM GC Time":38,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":84637,"Shuffle Write Time":7781822,"Shuffle Records Written":2997},"Input Metrics":{"Bytes Read":23178313,"Records Read":125488},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":9,"Index":1,"Attempt":0,"Launch Time":1633886646670,"Executor ID":"2","Host":"10.1.0.10","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":1633886648410,"Failed":false,"Killed":false,"Accumulables":[{"ID":25,"Name":"data size","Update":"165920","Value":"495696","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"shuffle bytes written","Update":"83472","Value":"249928","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"shuffle records written","Update":"2963","Value":"8852","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":35,"Name":"shuffle write time","Update":"7910392","Value":"23437664","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":61,"Name":"duration","Update":"1353","Value":"4003","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":62,"Name":"number of output rows","Update":"2963","Value":"8852","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":63,"Name":"peak memory","Update":"4456448","Value":"13369344","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":65,"Name":"time in aggregation build","Update":"1030","Value":"3038","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":66,"Name":"avg hash probe bucket list iters","Update":"10","Value":"30","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":67,"Name":"number of output rows","Update":"2963","Value":"8852","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":76,"Name":"number of output rows","Update":"2963","Value":"8852","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":77,"Name":"number of output rows","Update":"125061","Value":"375152","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":103,"Name":"internal.metrics.executorDeserializeTime","Update":196,"Value":581,"Internal":true,"Count Failed Values":true},{"ID":104,"Name":"internal.metrics.executorDeserializeCpuTime","Update":189280000,"Value":560889758,"Internal":true,"Count Failed Values":true},{"ID":105,"Name":"internal.metrics.executorRunTime","Update":1531,"Value":4554,"Internal":true,"Count Failed Values":true},{"ID":106,"Name":"internal.metrics.executorCpuTime","Update":1417856615,"Value":4219663533,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.resultSize","Update":2904,"Value":8714,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.jvmGCTime","Update":36,"Value":110,"Internal":true,"Count Failed Values":true},{"ID":112,"Name":"internal.metrics.peakExecutionMemory","Update":33816576,"Value":101449728,"Internal":true,"Count Failed Values":true},{"ID":121,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":83472,"Value":249928,"Internal":true,"Count Failed Values":true},{"ID":122,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":2963,"Value":8852,"Internal":true,"Count Failed Values":true},{"ID":123,"Name":"internal.metrics.shuffle.write.writeTime","Update":7910392,"Value":23437664,"Internal":true,"Count Failed Values":true},{"ID":124,"Name":"internal.metrics.input.bytesRead","Update":23125676,"Value":69368701,"Internal":true,"Count Failed Values":true},{"ID":125,"Name":"internal.metrics.input.recordsRead","Update":125061,"Value":375152,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":196,"Executor Deserialize CPU Time":189280000,"Executor Run Time":1531,"Executor CPU Time":1417856615,"Peak Execution Memory":33816576,"Result Size":2904,"JVM GC Time":36,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":83472,"Shuffle Write Time":7910392,"Shuffle Records Written":2963},"Input Metrics":{"Bytes Read":23125676,"Records Read":125061},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":14,"Index":6,"Attempt":0,"Launch Time":1633886646672,"Executor ID":"0","Host":"10.1.0.12","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":1633886648439,"Failed":false,"Killed":false,"Accumulables":[{"ID":25,"Name":"data size","Update":"169456","Value":"665152","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"shuffle bytes written","Update":"85405","Value":"335333","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"shuffle records written","Update":"3026","Value":"11878","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":35,"Name":"shuffle write time","Update":"9110342","Value":"32548006","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":61,"Name":"duration","Update":"1344","Value":"5347","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":62,"Name":"number of output rows","Update":"3026","Value":"11878","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":63,"Name":"peak memory","Update":"4456448","Value":"17825792","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":65,"Name":"time in aggregation build","Update":"1018","Value":"4056","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":66,"Name":"avg hash probe bucket list iters","Update":"10","Value":"40","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":67,"Name":"number of output rows","Update":"3026","Value":"11878","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":76,"Name":"number of output rows","Update":"3026","Value":"11878","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":77,"Name":"number of output rows","Update":"125342","Value":"500494","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":103,"Name":"internal.metrics.executorDeserializeTime","Update":204,"Value":785,"Internal":true,"Count Failed Values":true},{"ID":104,"Name":"internal.metrics.executorDeserializeCpuTime","Update":197699444,"Value":758589202,"Internal":true,"Count Failed Values":true},{"ID":105,"Name":"internal.metrics.executorRunTime","Update":1549,"Value":6103,"Internal":true,"Count Failed Values":true},{"ID":106,"Name":"internal.metrics.executorCpuTime","Update":1437927392,"Value":5657590925,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.resultSize","Update":2904,"Value":11618,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.jvmGCTime","Update":37,"Value":147,"Internal":true,"Count Failed Values":true},{"ID":112,"Name":"internal.metrics.peakExecutionMemory","Update":33816576,"Value":135266304,"Internal":true,"Count Failed Values":true},{"ID":121,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":85405,"Value":335333,"Internal":true,"Count Failed Values":true},{"ID":122,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":3026,"Value":11878,"Internal":true,"Count Failed Values":true},{"ID":123,"Name":"internal.metrics.shuffle.write.writeTime","Update":9110342,"Value":32548006,"Internal":true,"Count Failed Values":true},{"ID":124,"Name":"internal.metrics.input.bytesRead","Update":23247031,"Value":92615732,"Internal":true,"Count Failed Values":true},{"ID":125,"Name":"internal.metrics.input.recordsRead","Update":125342,"Value":500494,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":204,"Executor Deserialize CPU Time":197699444,"Executor Run Time":1549,"Executor CPU Time":1437927392,"Peak Execution Memory":33816576,"Result Size":2904,"JVM GC Time":37,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":85405,"Shuffle Write Time":9110342,"Shuffle Records Written":3026},"Input Metrics":{"Bytes Read":23247031,"Records Read":125342},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":10,"Index":2,"Attempt":0,"Launch Time":1633886646671,"Executor ID":"9","Host":"10.1.0.14","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":1633886648463,"Failed":false,"Killed":false,"Accumulables":[{"ID":25,"Name":"data size","Update":"165872","Value":"831024","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"shuffle bytes written","Update":"83597","Value":"418930","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"shuffle records written","Update":"2962","Value":"14840","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":35,"Name":"shuffle write time","Update":"7636066","Value":"40184072","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":61,"Name":"duration","Update":"1391","Value":"6738","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":62,"Name":"number of output rows","Update":"2962","Value":"14840","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":63,"Name":"peak memory","Update":"4456448","Value":"22282240","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":65,"Name":"time in aggregation build","Update":"1061","Value":"5117","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":66,"Name":"avg hash probe bucket list iters","Update":"10","Value":"50","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":67,"Name":"number of output rows","Update":"2962","Value":"14840","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":76,"Name":"number of output rows","Update":"2962","Value":"14840","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":77,"Name":"number of output rows","Update":"125003","Value":"625497","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":103,"Name":"internal.metrics.executorDeserializeTime","Update":189,"Value":974,"Internal":true,"Count Failed Values":true},{"ID":104,"Name":"internal.metrics.executorDeserializeCpuTime","Update":184195217,"Value":942784419,"Internal":true,"Count Failed Values":true},{"ID":105,"Name":"internal.metrics.executorRunTime","Update":1590,"Value":7693,"Internal":true,"Count Failed Values":true},{"ID":106,"Name":"internal.metrics.executorCpuTime","Update":1481100516,"Value":7138691441,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.resultSize","Update":2904,"Value":14522,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.jvmGCTime","Update":37,"Value":184,"Internal":true,"Count Failed Values":true},{"ID":112,"Name":"internal.metrics.peakExecutionMemory","Update":33816576,"Value":169082880,"Internal":true,"Count Failed Values":true},{"ID":121,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":83597,"Value":418930,"Internal":true,"Count Failed Values":true},{"ID":122,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":2962,"Value":14840,"Internal":true,"Count Failed Values":true},{"ID":123,"Name":"internal.metrics.shuffle.write.writeTime","Update":7636066,"Value":40184072,"Internal":true,"Count Failed Values":true},{"ID":124,"Name":"internal.metrics.input.bytesRead","Update":23143498,"Value":115759230,"Internal":true,"Count Failed Values":true},{"ID":125,"Name":"internal.metrics.input.recordsRead","Update":125003,"Value":625497,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":189,"Executor Deserialize CPU Time":184195217,"Executor Run Time":1590,"Executor CPU Time":1481100516,"Peak Execution Memory":33816576,"Result Size":2904,"JVM GC Time":37,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":83597,"Shuffle Write Time":7636066,"Shuffle Records Written":2962},"Input Metrics":{"Bytes Read":23143498,"Records Read":125003},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":8,"Index":0,"Attempt":0,"Launch Time":1633886646669,"Executor ID":"5","Host":"10.1.0.19","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":1633886648477,"Failed":false,"Killed":false,"Accumulables":[{"ID":25,"Name":"data size","Update":"158088","Value":"989112","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"shuffle bytes written","Update":"79832","Value":"498762","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"shuffle records written","Update":"2823","Value":"17663","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":35,"Name":"shuffle write time","Update":"8489312","Value":"48673384","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":61,"Name":"duration","Update":"1391","Value":"8129","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":62,"Name":"number of output rows","Update":"2823","Value":"17663","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":63,"Name":"peak memory","Update":"262144","Value":"22544384","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":65,"Name":"time in aggregation build","Update":"1048","Value":"6165","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":67,"Name":"number of output rows","Update":"2823","Value":"17663","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":76,"Name":"number of output rows","Update":"2823","Value":"17663","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":77,"Name":"number of output rows","Update":"124858","Value":"750355","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":103,"Name":"internal.metrics.executorDeserializeTime","Update":198,"Value":1172,"Internal":true,"Count Failed Values":true},{"ID":104,"Name":"internal.metrics.executorDeserializeCpuTime","Update":190304790,"Value":1133089209,"Internal":true,"Count Failed Values":true},{"ID":105,"Name":"internal.metrics.executorRunTime","Update":1594,"Value":9287,"Internal":true,"Count Failed Values":true},{"ID":106,"Name":"internal.metrics.executorCpuTime","Update":1480262889,"Value":8618954330,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.resultSize","Update":2904,"Value":17426,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.jvmGCTime","Update":37,"Value":221,"Internal":true,"Count Failed Values":true},{"ID":112,"Name":"internal.metrics.peakExecutionMemory","Update":29622272,"Value":198705152,"Internal":true,"Count Failed Values":true},{"ID":121,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":79832,"Value":498762,"Internal":true,"Count Failed Values":true},{"ID":122,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":2823,"Value":17663,"Internal":true,"Count Failed Values":true},{"ID":123,"Name":"internal.metrics.shuffle.write.writeTime","Update":8489312,"Value":48673384,"Internal":true,"Count Failed Values":true},{"ID":124,"Name":"internal.metrics.input.bytesRead","Update":23122020,"Value":138881250,"Internal":true,"Count Failed Values":true},{"ID":125,"Name":"internal.metrics.input.recordsRead","Update":124858,"Value":750355,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":198,"Executor Deserialize CPU Time":190304790,"Executor Run Time":1594,"Executor CPU Time":1480262889,"Peak Execution Memory":29622272,"Result Size":2904,"JVM GC Time":37,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":79832,"Shuffle Write Time":8489312,"Shuffle Records Written":2823},"Input Metrics":{"Bytes Read":23122020,"Records Read":124858},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":13,"Index":5,"Attempt":0,"Launch Time":1633886646672,"Executor ID":"6","Host":"10.1.0.18","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":1633886648485,"Failed":false,"Killed":false,"Accumulables":[{"ID":25,"Name":"data size","Update":"170848","Value":"1159960","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"shuffle bytes written","Update":"85986","Value":"584748","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"shuffle records written","Update":"3051","Value":"20714","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":35,"Name":"shuffle write time","Update":"9141949","Value":"57815333","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":61,"Name":"duration","Update":"1393","Value":"9522","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":62,"Name":"number of output rows","Update":"3051","Value":"20714","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":63,"Name":"peak memory","Update":"4456448","Value":"27000832","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":65,"Name":"time in aggregation build","Update":"1049","Value":"7214","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":66,"Name":"avg hash probe bucket list iters","Update":"10","Value":"60","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":67,"Name":"number of output rows","Update":"3051","Value":"20714","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":76,"Name":"number of output rows","Update":"3051","Value":"20714","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":77,"Name":"number of output rows","Update":"124533","Value":"874888","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":103,"Name":"internal.metrics.executorDeserializeTime","Update":203,"Value":1375,"Internal":true,"Count Failed Values":true},{"ID":104,"Name":"internal.metrics.executorDeserializeCpuTime","Update":189570867,"Value":1322660076,"Internal":true,"Count Failed Values":true},{"ID":105,"Name":"internal.metrics.executorRunTime","Update":1597,"Value":10884,"Internal":true,"Count Failed Values":true},{"ID":106,"Name":"internal.metrics.executorCpuTime","Update":1475238135,"Value":10094192465,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.resultSize","Update":2904,"Value":20330,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.jvmGCTime","Update":38,"Value":259,"Internal":true,"Count Failed Values":true},{"ID":112,"Name":"internal.metrics.peakExecutionMemory","Update":33816576,"Value":232521728,"Internal":true,"Count Failed Values":true},{"ID":121,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":85986,"Value":584748,"Internal":true,"Count Failed Values":true},{"ID":122,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":3051,"Value":20714,"Internal":true,"Count Failed Values":true},{"ID":123,"Name":"internal.metrics.shuffle.write.writeTime","Update":9141949,"Value":57815333,"Internal":true,"Count Failed Values":true},{"ID":124,"Name":"internal.metrics.input.bytesRead","Update":23014586,"Value":161895836,"Internal":true,"Count Failed Values":true},{"ID":125,"Name":"internal.metrics.input.recordsRead","Update":124533,"Value":874888,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":119677952,"JVMOffHeapMemory":86570864,"OnHeapExecutionMemory":4456448,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":40864795,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":45321243,"OffHeapUnifiedMemory":0,"DirectPoolMemory":1171015,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":4,"MinorGCTime":42,"MajorGCCount":3,"MajorGCTime":145},"Task Metrics":{"Executor Deserialize Time":203,"Executor Deserialize CPU Time":189570867,"Executor Run Time":1597,"Executor CPU Time":1475238135,"Peak Execution Memory":33816576,"Result Size":2904,"JVM GC Time":38,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":85986,"Shuffle Write Time":9141949,"Shuffle Records Written":3051},"Input Metrics":{"Bytes Read":23014586,"Records Read":124533},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":11,"Index":3,"Attempt":0,"Launch Time":1633886646671,"Executor ID":"3","Host":"10.1.0.15","Locality":"ANY","Speculative":false,"Getting Result Time":0,"Finish Time":1633886650484,"Failed":false,"Killed":false,"Accumulables":[{"ID":25,"Name":"data size","Update":"161504","Value":"1321464","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"shuffle bytes written","Update":"81474","Value":"666222","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"shuffle records written","Update":"2884","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":35,"Name":"shuffle write time","Update":"9766392","Value":"67581725","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":61,"Name":"duration","Update":"1536","Value":"11058","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":62,"Name":"number of output rows","Update":"2884","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":63,"Name":"peak memory","Update":"4456448","Value":"31457280","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":65,"Name":"time in aggregation build","Update":"1168","Value":"8382","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":66,"Name":"avg hash probe bucket list iters","Update":"10","Value":"70","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":67,"Name":"number of output rows","Update":"2884","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":76,"Name":"number of output rows","Update":"2884","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":77,"Name":"number of output rows","Update":"125112","Value":"1000000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":103,"Name":"internal.metrics.executorDeserializeTime","Update":901,"Value":2276,"Internal":true,"Count Failed Values":true},{"ID":104,"Name":"internal.metrics.executorDeserializeCpuTime","Update":831523308,"Value":2154183384,"Internal":true,"Count Failed Values":true},{"ID":105,"Name":"internal.metrics.executorRunTime","Update":2873,"Value":13757,"Internal":true,"Count Failed Values":true},{"ID":106,"Name":"internal.metrics.executorCpuTime","Update":2636702759,"Value":12730895224,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.resultSize","Update":2904,"Value":23234,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.jvmGCTime","Update":168,"Value":427,"Internal":true,"Count Failed Values":true},{"ID":112,"Name":"internal.metrics.peakExecutionMemory","Update":33816576,"Value":266338304,"Internal":true,"Count Failed Values":true},{"ID":121,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":81474,"Value":666222,"Internal":true,"Count Failed Values":true},{"ID":122,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":2884,"Value":23598,"Internal":true,"Count Failed Values":true},{"ID":123,"Name":"internal.metrics.shuffle.write.writeTime","Update":9766392,"Value":67581725,"Internal":true,"Count Failed Values":true},{"ID":124,"Name":"internal.metrics.input.bytesRead","Update":23103793,"Value":184999629,"Internal":true,"Count Failed Values":true},{"ID":125,"Name":"internal.metrics.input.recordsRead","Update":125112,"Value":1000000,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":901,"Executor Deserialize CPU Time":831523308,"Executor Run Time":2873,"Executor CPU Time":2636702759,"Peak Execution Memory":33816576,"Result Size":2904,"JVM GC Time":168,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":81474,"Shuffle Write Time":9766392,"Shuffle Records Written":2884},"Input Metrics":{"Bytes Read":23103793,"Records Read":125112},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"sql at ScalaSparkSQLBench.scala:43","Number of Tasks":8,"RDD Info":[{"RDD ID":11,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"12\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[10],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":10,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"13\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[9],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":6,"Name":"HadoopRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1633886646659,"Completion Time":1633886650485,"Accumulables":[{"ID":25,"Name":"data size","Value":"1321464","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":33,"Name":"shuffle bytes written","Value":"666222","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":34,"Name":"shuffle records written","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":35,"Name":"shuffle write time","Value":"67581725","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":61,"Name":"duration","Value":"11058","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":62,"Name":"number of output rows","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":63,"Name":"peak memory","Value":"31457280","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":65,"Name":"time in aggregation build","Value":"8382","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":66,"Name":"avg hash probe bucket list iters","Value":"70","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":67,"Name":"number of output rows","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":76,"Name":"number of output rows","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":77,"Name":"number of output rows","Value":"1000000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":103,"Name":"internal.metrics.executorDeserializeTime","Value":2276,"Internal":true,"Count Failed Values":true},{"ID":104,"Name":"internal.metrics.executorDeserializeCpuTime","Value":2154183384,"Internal":true,"Count Failed Values":true},{"ID":105,"Name":"internal.metrics.executorRunTime","Value":13757,"Internal":true,"Count Failed Values":true},{"ID":106,"Name":"internal.metrics.executorCpuTime","Value":12730895224,"Internal":true,"Count Failed Values":true},{"ID":107,"Name":"internal.metrics.resultSize","Value":23234,"Internal":true,"Count Failed Values":true},{"ID":108,"Name":"internal.metrics.jvmGCTime","Value":427,"Internal":true,"Count Failed Values":true},{"ID":112,"Name":"internal.metrics.peakExecutionMemory","Value":266338304,"Internal":true,"Count Failed Values":true},{"ID":121,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":666222,"Internal":true,"Count Failed Values":true},{"ID":122,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":23598,"Internal":true,"Count Failed Values":true},{"ID":123,"Name":"internal.metrics.shuffle.write.writeTime","Value":67581725,"Internal":true,"Count Failed Values":true},{"ID":124,"Name":"internal.metrics.input.bytesRead","Value":184999629,"Internal":true,"Count Failed Values":true},{"ID":125,"Name":"internal.metrics.input.recordsRead","Value":1000000,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} -{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":2,"Stage Attempt ID":0,"Stage Name":"sql at ScalaSparkSQLBench.scala:43","Number of Tasks":8,"RDD Info":[{"RDD ID":16,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[15],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":15,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[14],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":14,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":12,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"12\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[11],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[1],"Details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1633886650501,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.sql.warehouse.dir":"file:/home/usera112/projects/log-compressor/deployments/HiBench/spark-warehouse","spark.driver.host":"hostb7.dsrg.utoronto.ca","spark.eventLog.enabled":"true","spark.driver.port":"42853","__fetch_continuous_blocks_in_batch_enabled":"true","spark.jars":"file:/home/usera112/projects/log-compressor/deployments/HiBench/sparkbench/assembly/target/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.app.name":"ScalaJoin","spark.rdd.scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","spark.driver.memory":"4g","spark.rdd.scope.noOverride":"true","spark.submit.pyFiles":"","spark.default.parallelism":"8","spark.app.startTime":"1633886611309","spark.executor.id":"driver","spark.app.initial.jar.urls":"spark://hostb7.dsrg.utoronto.ca:42853/jars/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.submit.deployMode":"client","spark.master":"spark://hostb7:7077","spark.executor.memory":"4g","spark.eventLog.dir":"hdfs://hostb7:8020/directory","spark.sql.execution.id":"11","spark.sql.catalogImplementation":"hive","spark.app.id":"app-20211010132332-4636","spark.sql.shuffle.partitions":"8"}} -{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":16,"Index":0,"Attempt":0,"Launch Time":1633886650509,"Executor ID":"6","Host":"10.1.0.18","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":17,"Index":1,"Attempt":0,"Launch Time":1633886650511,"Executor ID":"3","Host":"10.1.0.15","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":18,"Index":2,"Attempt":0,"Launch Time":1633886650511,"Executor ID":"0","Host":"10.1.0.12","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":19,"Index":3,"Attempt":0,"Launch Time":1633886650511,"Executor ID":"7","Host":"10.1.0.9","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":20,"Index":4,"Attempt":0,"Launch Time":1633886650512,"Executor ID":"10","Host":"10.1.0.16","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":21,"Index":5,"Attempt":0,"Launch Time":1633886650512,"Executor ID":"1","Host":"10.1.0.17","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":22,"Index":6,"Attempt":0,"Launch Time":1633886650513,"Executor ID":"4","Host":"10.1.0.8","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":23,"Index":7,"Attempt":0,"Launch Time":1633886650513,"Executor ID":"11","Host":"10.1.0.13","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":20,"Index":4,"Attempt":0,"Launch Time":1633886650512,"Executor ID":"10","Host":"10.1.0.16","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633886650735,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"7","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":27,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"72188","Value":"72188","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"local bytes read","Update":"10800","Value":"10800","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2937","Value":"2937","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"41","Value":"41","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2937","Value":"2937","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"4456448","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"16","Value":"16","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":128,"Name":"internal.metrics.executorDeserializeTime","Update":44,"Value":44,"Internal":true,"Count Failed Values":true},{"ID":129,"Name":"internal.metrics.executorDeserializeCpuTime","Update":39457154,"Value":39457154,"Internal":true,"Count Failed Values":true},{"ID":130,"Name":"internal.metrics.executorRunTime","Update":166,"Value":166,"Internal":true,"Count Failed Values":true},{"ID":131,"Name":"internal.metrics.executorCpuTime","Update":127369077,"Value":127369077,"Internal":true,"Count Failed Values":true},{"ID":132,"Name":"internal.metrics.resultSize","Update":14169,"Value":14169,"Internal":true,"Count Failed Values":true},{"ID":134,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":137,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":4456448,"Internal":true,"Count Failed Values":true},{"ID":139,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":140,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":141,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":72188,"Value":72188,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":10800,"Value":10800,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2937,"Value":2937,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":44,"Executor Deserialize CPU Time":39457154,"Executor Run Time":166,"Executor CPU Time":127369077,"Peak Execution Memory":4456448,"Result Size":14169,"JVM GC Time":0,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":72188,"Remote Bytes Read To Disk":0,"Local Bytes Read":10800,"Total Records Read":2937},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":16,"Index":0,"Attempt":0,"Launch Time":1633886650509,"Executor ID":"6","Host":"10.1.0.18","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633886650741,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"7","Value":"14","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":27,"Name":"local blocks read","Update":"1","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"71598","Value":"143786","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"local bytes read","Update":"10619","Value":"21419","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2905","Value":"5842","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"44","Value":"85","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2905","Value":"5842","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"8912896","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"18","Value":"34","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"22","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":128,"Name":"internal.metrics.executorDeserializeTime","Update":47,"Value":91,"Internal":true,"Count Failed Values":true},{"ID":129,"Name":"internal.metrics.executorDeserializeCpuTime","Update":40826189,"Value":80283343,"Internal":true,"Count Failed Values":true},{"ID":130,"Name":"internal.metrics.executorRunTime","Update":169,"Value":335,"Internal":true,"Count Failed Values":true},{"ID":131,"Name":"internal.metrics.executorCpuTime","Update":135097318,"Value":262466395,"Internal":true,"Count Failed Values":true},{"ID":132,"Name":"internal.metrics.resultSize","Update":14169,"Value":28338,"Internal":true,"Count Failed Values":true},{"ID":134,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":137,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":8912896,"Internal":true,"Count Failed Values":true},{"ID":139,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":14,"Internal":true,"Count Failed Values":true},{"ID":140,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":141,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":71598,"Value":143786,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":10619,"Value":21419,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2905,"Value":5842,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":47,"Executor Deserialize CPU Time":40826189,"Executor Run Time":169,"Executor CPU Time":135097318,"Peak Execution Memory":4456448,"Result Size":14169,"JVM GC Time":0,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":71598,"Remote Bytes Read To Disk":0,"Local Bytes Read":10619,"Total Records Read":2905},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":18,"Index":2,"Attempt":0,"Launch Time":1633886650511,"Executor ID":"0","Host":"10.1.0.12","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633886650741,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"7","Value":"21","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":27,"Name":"local blocks read","Update":"1","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"74808","Value":"218594","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"local bytes read","Update":"11096","Value":"32515","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"3049","Value":"8891","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"43","Value":"128","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"3049","Value":"8891","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"13369344","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"18","Value":"52","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"33","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":128,"Name":"internal.metrics.executorDeserializeTime","Update":46,"Value":137,"Internal":true,"Count Failed Values":true},{"ID":129,"Name":"internal.metrics.executorDeserializeCpuTime","Update":41224934,"Value":121508277,"Internal":true,"Count Failed Values":true},{"ID":130,"Name":"internal.metrics.executorRunTime","Update":167,"Value":502,"Internal":true,"Count Failed Values":true},{"ID":131,"Name":"internal.metrics.executorCpuTime","Update":132674598,"Value":395140993,"Internal":true,"Count Failed Values":true},{"ID":132,"Name":"internal.metrics.resultSize","Update":14169,"Value":42507,"Internal":true,"Count Failed Values":true},{"ID":134,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":137,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":13369344,"Internal":true,"Count Failed Values":true},{"ID":139,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":21,"Internal":true,"Count Failed Values":true},{"ID":140,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":141,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":74808,"Value":218594,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":11096,"Value":32515,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.shuffle.read.recordsRead","Update":3049,"Value":8891,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":46,"Executor Deserialize CPU Time":41224934,"Executor Run Time":167,"Executor CPU Time":132674598,"Peak Execution Memory":4456448,"Result Size":14169,"JVM GC Time":0,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":74808,"Remote Bytes Read To Disk":0,"Local Bytes Read":11096,"Total Records Read":3049},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":23,"Index":7,"Attempt":0,"Launch Time":1633886650513,"Executor ID":"11","Host":"10.1.0.13","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633886650747,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"7","Value":"28","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":27,"Name":"local blocks read","Update":"1","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"70635","Value":"289229","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"local bytes read","Update":"10061","Value":"42576","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2856","Value":"11747","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"43","Value":"171","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2856","Value":"11747","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"17825792","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"17","Value":"69","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"44","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":128,"Name":"internal.metrics.executorDeserializeTime","Update":47,"Value":184,"Internal":true,"Count Failed Values":true},{"ID":129,"Name":"internal.metrics.executorDeserializeCpuTime","Update":41600868,"Value":163109145,"Internal":true,"Count Failed Values":true},{"ID":130,"Name":"internal.metrics.executorRunTime","Update":174,"Value":676,"Internal":true,"Count Failed Values":true},{"ID":131,"Name":"internal.metrics.executorCpuTime","Update":136704415,"Value":531845408,"Internal":true,"Count Failed Values":true},{"ID":132,"Name":"internal.metrics.resultSize","Update":14169,"Value":56676,"Internal":true,"Count Failed Values":true},{"ID":134,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":137,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":17825792,"Internal":true,"Count Failed Values":true},{"ID":139,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":28,"Internal":true,"Count Failed Values":true},{"ID":140,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":141,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":70635,"Value":289229,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":10061,"Value":42576,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2856,"Value":11747,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":47,"Executor Deserialize CPU Time":41600868,"Executor Run Time":174,"Executor CPU Time":136704415,"Peak Execution Memory":4456448,"Result Size":14169,"JVM GC Time":0,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":70635,"Remote Bytes Read To Disk":0,"Local Bytes Read":10061,"Total Records Read":2856},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":17,"Index":1,"Attempt":0,"Launch Time":1633886650511,"Executor ID":"3","Host":"10.1.0.15","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633886650807,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"7","Value":"35","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":27,"Name":"local blocks read","Update":"1","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"74063","Value":"363292","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"local bytes read","Update":"10549","Value":"53125","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2999","Value":"14746","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"47","Value":"218","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2999","Value":"14746","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"22282240","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"19","Value":"88","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"55","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":128,"Name":"internal.metrics.executorDeserializeTime","Update":122,"Value":306,"Internal":true,"Count Failed Values":true},{"ID":129,"Name":"internal.metrics.executorDeserializeCpuTime","Update":116101291,"Value":279210436,"Internal":true,"Count Failed Values":true},{"ID":130,"Name":"internal.metrics.executorRunTime","Update":158,"Value":834,"Internal":true,"Count Failed Values":true},{"ID":131,"Name":"internal.metrics.executorCpuTime","Update":145272495,"Value":677117903,"Internal":true,"Count Failed Values":true},{"ID":132,"Name":"internal.metrics.resultSize","Update":14169,"Value":70845,"Internal":true,"Count Failed Values":true},{"ID":134,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":137,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":22282240,"Internal":true,"Count Failed Values":true},{"ID":139,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":35,"Internal":true,"Count Failed Values":true},{"ID":140,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":141,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":74063,"Value":363292,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":10549,"Value":53125,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2999,"Value":14746,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":122,"Executor Deserialize CPU Time":116101291,"Executor Run Time":158,"Executor CPU Time":145272495,"Peak Execution Memory":4456448,"Result Size":14169,"JVM GC Time":0,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":74063,"Remote Bytes Read To Disk":0,"Local Bytes Read":10549,"Total Records Read":2999},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":21,"Index":5,"Attempt":0,"Launch Time":1633886650512,"Executor ID":"1","Host":"10.1.0.17","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633886652103,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"8","Value":"43","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"83209","Value":"446501","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2948","Value":"17694","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"194","Value":"412","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2948","Value":"17694","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"26738688","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"44","Value":"132","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"66","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":128,"Name":"internal.metrics.executorDeserializeTime","Update":904,"Value":1210,"Internal":true,"Count Failed Values":true},{"ID":129,"Name":"internal.metrics.executorDeserializeCpuTime","Update":838498033,"Value":1117708469,"Internal":true,"Count Failed Values":true},{"ID":130,"Name":"internal.metrics.executorRunTime","Update":645,"Value":1479,"Internal":true,"Count Failed Values":true},{"ID":131,"Name":"internal.metrics.executorCpuTime","Update":616304248,"Value":1293422151,"Internal":true,"Count Failed Values":true},{"ID":132,"Name":"internal.metrics.resultSize","Update":14212,"Value":85057,"Internal":true,"Count Failed Values":true},{"ID":133,"Name":"internal.metrics.jvmGCTime","Update":33,"Value":33,"Internal":true,"Count Failed Values":true},{"ID":134,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":9,"Internal":true,"Count Failed Values":true},{"ID":137,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":26738688,"Internal":true,"Count Failed Values":true},{"ID":139,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":8,"Value":43,"Internal":true,"Count Failed Values":true},{"ID":140,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":141,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":83209,"Value":446501,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":53125,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2948,"Value":17694,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":904,"Executor Deserialize CPU Time":838498033,"Executor Run Time":645,"Executor CPU Time":616304248,"Peak Execution Memory":4456448,"Result Size":14212,"JVM GC Time":33,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":8,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":83209,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":2948},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":19,"Index":3,"Attempt":0,"Launch Time":1633886650511,"Executor ID":"7","Host":"10.1.0.9","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633886652176,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"8","Value":"51","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"84455","Value":"530956","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2994","Value":"20688","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"183","Value":"595","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2994","Value":"20688","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"31195136","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"39","Value":"171","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"77","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":128,"Name":"internal.metrics.executorDeserializeTime","Update":919,"Value":2129,"Internal":true,"Count Failed Values":true},{"ID":129,"Name":"internal.metrics.executorDeserializeCpuTime","Update":848576727,"Value":1966285196,"Internal":true,"Count Failed Values":true},{"ID":130,"Name":"internal.metrics.executorRunTime","Update":701,"Value":2180,"Internal":true,"Count Failed Values":true},{"ID":131,"Name":"internal.metrics.executorCpuTime","Update":649395251,"Value":1942817402,"Internal":true,"Count Failed Values":true},{"ID":132,"Name":"internal.metrics.resultSize","Update":14212,"Value":99269,"Internal":true,"Count Failed Values":true},{"ID":133,"Name":"internal.metrics.jvmGCTime","Update":33,"Value":66,"Internal":true,"Count Failed Values":true},{"ID":134,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":11,"Internal":true,"Count Failed Values":true},{"ID":137,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":31195136,"Internal":true,"Count Failed Values":true},{"ID":139,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":8,"Value":51,"Internal":true,"Count Failed Values":true},{"ID":140,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":141,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":84455,"Value":530956,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":53125,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2994,"Value":20688,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":919,"Executor Deserialize CPU Time":848576727,"Executor Run Time":701,"Executor CPU Time":649395251,"Peak Execution Memory":4456448,"Result Size":14212,"JVM GC Time":33,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":8,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":84455,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":2994},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":22,"Index":6,"Attempt":0,"Launch Time":1633886650513,"Executor ID":"4","Host":"10.1.0.8","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633886652199,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"8","Value":"59","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"82141","Value":"613097","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2910","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"196","Value":"791","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2910","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"35651584","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"40","Value":"211","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"88","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":128,"Name":"internal.metrics.executorDeserializeTime","Update":971,"Value":3100,"Internal":true,"Count Failed Values":true},{"ID":129,"Name":"internal.metrics.executorDeserializeCpuTime","Update":901930853,"Value":2868216049,"Internal":true,"Count Failed Values":true},{"ID":130,"Name":"internal.metrics.executorRunTime","Update":677,"Value":2857,"Internal":true,"Count Failed Values":true},{"ID":131,"Name":"internal.metrics.executorCpuTime","Update":642747927,"Value":2585565329,"Internal":true,"Count Failed Values":true},{"ID":132,"Name":"internal.metrics.resultSize","Update":14212,"Value":113481,"Internal":true,"Count Failed Values":true},{"ID":133,"Name":"internal.metrics.jvmGCTime","Update":32,"Value":98,"Internal":true,"Count Failed Values":true},{"ID":134,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":13,"Internal":true,"Count Failed Values":true},{"ID":137,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":35651584,"Internal":true,"Count Failed Values":true},{"ID":139,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":8,"Value":59,"Internal":true,"Count Failed Values":true},{"ID":140,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":141,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":82141,"Value":613097,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":53125,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2910,"Value":23598,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":137588888,"JVMOffHeapMemory":57420072,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":56143,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":56143,"OffHeapUnifiedMemory":0,"DirectPoolMemory":10053,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":2,"MinorGCTime":18,"MajorGCCount":2,"MajorGCTime":47},"Task Metrics":{"Executor Deserialize Time":971,"Executor Deserialize CPU Time":901930853,"Executor Run Time":677,"Executor CPU Time":642747927,"Peak Execution Memory":4456448,"Result Size":14212,"JVM GC Time":32,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":8,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":82141,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":2910},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":2,"Stage Attempt ID":0,"Stage Name":"sql at ScalaSparkSQLBench.scala:43","Number of Tasks":8,"RDD Info":[{"RDD ID":16,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[15],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":15,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[14],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":14,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":12,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"12\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[11],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[1],"Details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1633886650501,"Completion Time":1633886652200,"Accumulables":[{"ID":26,"Name":"remote blocks read","Value":"59","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":27,"Name":"local blocks read","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Value":"613097","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"local bytes read","Value":"53125","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Value":"791","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Value":"35651584","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Value":"211","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Value":"88","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":128,"Name":"internal.metrics.executorDeserializeTime","Value":3100,"Internal":true,"Count Failed Values":true},{"ID":129,"Name":"internal.metrics.executorDeserializeCpuTime","Value":2868216049,"Internal":true,"Count Failed Values":true},{"ID":130,"Name":"internal.metrics.executorRunTime","Value":2857,"Internal":true,"Count Failed Values":true},{"ID":131,"Name":"internal.metrics.executorCpuTime","Value":2585565329,"Internal":true,"Count Failed Values":true},{"ID":132,"Name":"internal.metrics.resultSize","Value":113481,"Internal":true,"Count Failed Values":true},{"ID":133,"Name":"internal.metrics.jvmGCTime","Value":98,"Internal":true,"Count Failed Values":true},{"ID":134,"Name":"internal.metrics.resultSerializationTime","Value":13,"Internal":true,"Count Failed Values":true},{"ID":137,"Name":"internal.metrics.peakExecutionMemory","Value":35651584,"Internal":true,"Count Failed Values":true},{"ID":139,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":59,"Internal":true,"Count Failed Values":true},{"ID":140,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":5,"Internal":true,"Count Failed Values":true},{"ID":141,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":613097,"Internal":true,"Count Failed Values":true},{"ID":142,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":143,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":53125,"Internal":true,"Count Failed Values":true},{"ID":144,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":145,"Name":"internal.metrics.shuffle.read.recordsRead","Value":23598,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} -{"Event":"SparkListenerJobEnd","Job ID":1,"Completion Time":1633886652201,"Job Result":{"Result":"JobSucceeded"}} -{"Event":"SparkListenerJobStart","Job ID":2,"Submission Time":1633886652274,"Stage Infos":[{"Stage ID":5,"Stage Attempt ID":0,"Stage Name":"sql at ScalaSparkSQLBench.scala:43","Number of Tasks":8,"RDD Info":[{"RDD ID":19,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[18],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":18,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[17],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[4],"Details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":3,"Stage Attempt ID":0,"Stage Name":"sql at ScalaSparkSQLBench.scala:43","Number of Tasks":8,"RDD Info":[{"RDD ID":11,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"12\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[10],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":10,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"13\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[9],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":6,"Name":"HadoopRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan hive default.uservisits_copy\"}","Callsite":"","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Accumulables":[],"Resource Profile Id":0},{"Stage ID":4,"Stage Attempt ID":0,"Stage Name":"sql at ScalaSparkSQLBench.scala:43","Number of Tasks":8,"RDD Info":[{"RDD ID":17,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":12,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"12\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[11],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[3],"Details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Accumulables":[],"Resource Profile Id":0}],"Stage IDs":[5,3,4],"Properties":{"spark.sql.warehouse.dir":"file:/home/usera112/projects/log-compressor/deployments/HiBench/spark-warehouse","spark.driver.host":"hostb7.dsrg.utoronto.ca","spark.eventLog.enabled":"true","spark.driver.port":"42853","__fetch_continuous_blocks_in_batch_enabled":"true","spark.jars":"file:/home/usera112/projects/log-compressor/deployments/HiBench/sparkbench/assembly/target/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.app.name":"ScalaJoin","spark.driver.memory":"4g","spark.submit.pyFiles":"","spark.default.parallelism":"8","spark.app.startTime":"1633886611309","spark.executor.id":"driver","spark.app.initial.jar.urls":"spark://hostb7.dsrg.utoronto.ca:42853/jars/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.submit.deployMode":"client","spark.master":"spark://hostb7:7077","spark.executor.memory":"4g","spark.eventLog.dir":"hdfs://hostb7:8020/directory","spark.sql.execution.id":"11","spark.sql.catalogImplementation":"hive","spark.app.id":"app-20211010132332-4636","spark.sql.shuffle.partitions":"8"}} -{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":4,"Stage Attempt ID":0,"Stage Name":"sql at ScalaSparkSQLBench.scala:43","Number of Tasks":8,"RDD Info":[{"RDD ID":17,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":12,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"12\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[11],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[3],"Details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1633886652279,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.sql.warehouse.dir":"file:/home/usera112/projects/log-compressor/deployments/HiBench/spark-warehouse","spark.driver.host":"hostb7.dsrg.utoronto.ca","spark.eventLog.enabled":"true","spark.driver.port":"42853","__fetch_continuous_blocks_in_batch_enabled":"true","spark.jars":"file:/home/usera112/projects/log-compressor/deployments/HiBench/sparkbench/assembly/target/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.app.name":"ScalaJoin","spark.driver.memory":"4g","spark.submit.pyFiles":"","spark.default.parallelism":"8","spark.app.startTime":"1633886611309","spark.executor.id":"driver","spark.app.initial.jar.urls":"spark://hostb7.dsrg.utoronto.ca:42853/jars/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.submit.deployMode":"client","spark.master":"spark://hostb7:7077","spark.executor.memory":"4g","spark.eventLog.dir":"hdfs://hostb7:8020/directory","spark.sql.execution.id":"11","spark.sql.catalogImplementation":"hive","spark.app.id":"app-20211010132332-4636","spark.sql.shuffle.partitions":"8"}} -{"Event":"SparkListenerTaskStart","Stage ID":4,"Stage Attempt ID":0,"Task Info":{"Task ID":24,"Index":0,"Attempt":0,"Launch Time":1633886652293,"Executor ID":"7","Host":"10.1.0.9","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":4,"Stage Attempt ID":0,"Task Info":{"Task ID":25,"Index":1,"Attempt":0,"Launch Time":1633886652293,"Executor ID":"10","Host":"10.1.0.16","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":4,"Stage Attempt ID":0,"Task Info":{"Task ID":26,"Index":2,"Attempt":0,"Launch Time":1633886652293,"Executor ID":"3","Host":"10.1.0.15","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":4,"Stage Attempt ID":0,"Task Info":{"Task ID":27,"Index":3,"Attempt":0,"Launch Time":1633886652293,"Executor ID":"11","Host":"10.1.0.13","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":4,"Stage Attempt ID":0,"Task Info":{"Task ID":28,"Index":4,"Attempt":0,"Launch Time":1633886652294,"Executor ID":"4","Host":"10.1.0.8","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":4,"Stage Attempt ID":0,"Task Info":{"Task ID":29,"Index":5,"Attempt":0,"Launch Time":1633886652294,"Executor ID":"9","Host":"10.1.0.14","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":4,"Stage Attempt ID":0,"Task Info":{"Task ID":30,"Index":6,"Attempt":0,"Launch Time":1633886652294,"Executor ID":"1","Host":"10.1.0.17","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":4,"Stage Attempt ID":0,"Task Info":{"Task ID":31,"Index":7,"Attempt":0,"Launch Time":1633886652294,"Executor ID":"8","Host":"10.1.0.11","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":4,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":25,"Index":1,"Attempt":0,"Launch Time":1633886652293,"Executor ID":"10","Host":"10.1.0.16","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633886652392,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"7","Value":"66","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":27,"Name":"local blocks read","Update":"1","Value":"6","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"73963","Value":"687060","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"local bytes read","Update":"10649","Value":"63774","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2999","Value":"26597","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":36,"Name":"data size","Update":"143952","Value":"143952","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":44,"Name":"shuffle bytes written","Update":"81470","Value":"81470","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":45,"Name":"shuffle records written","Update":"2999","Value":"2999","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":46,"Name":"shuffle write time","Update":"2979545","Value":"2979545","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"27","Value":"818","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2999","Value":"26597","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"40108032","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"6","Value":"217","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"99","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":153,"Name":"internal.metrics.executorDeserializeTime","Update":50,"Value":50,"Internal":true,"Count Failed Values":true},{"ID":154,"Name":"internal.metrics.executorDeserializeCpuTime","Update":42880086,"Value":42880086,"Internal":true,"Count Failed Values":true},{"ID":155,"Name":"internal.metrics.executorRunTime","Update":39,"Value":39,"Internal":true,"Count Failed Values":true},{"ID":156,"Name":"internal.metrics.executorCpuTime","Update":36997640,"Value":36997640,"Internal":true,"Count Failed Values":true},{"ID":157,"Name":"internal.metrics.resultSize","Update":4237,"Value":4237,"Internal":true,"Count Failed Values":true},{"ID":162,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":4456448,"Internal":true,"Count Failed Values":true},{"ID":164,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":165,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":166,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":73963,"Value":73963,"Internal":true,"Count Failed Values":true},{"ID":167,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":168,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":10649,"Value":10649,"Internal":true,"Count Failed Values":true},{"ID":169,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":170,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2999,"Value":2999,"Internal":true,"Count Failed Values":true},{"ID":171,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":81470,"Value":81470,"Internal":true,"Count Failed Values":true},{"ID":172,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":2999,"Value":2999,"Internal":true,"Count Failed Values":true},{"ID":173,"Name":"internal.metrics.shuffle.write.writeTime","Update":2979545,"Value":2979545,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":50,"Executor Deserialize CPU Time":42880086,"Executor Run Time":39,"Executor CPU Time":36997640,"Peak Execution Memory":4456448,"Result Size":4237,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":73963,"Remote Bytes Read To Disk":0,"Local Bytes Read":10649,"Total Records Read":2999},"Shuffle Write Metrics":{"Shuffle Bytes Written":81470,"Shuffle Write Time":2979545,"Shuffle Records Written":2999},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":4,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":26,"Index":2,"Attempt":0,"Launch Time":1633886652293,"Executor ID":"3","Host":"10.1.0.15","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633886652403,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"7","Value":"73","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":27,"Name":"local blocks read","Update":"1","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"75292","Value":"762352","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"local bytes read","Update":"10612","Value":"74386","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"3049","Value":"29646","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":36,"Name":"data size","Update":"146336","Value":"290288","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":44,"Name":"shuffle bytes written","Update":"82863","Value":"164333","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":45,"Name":"shuffle records written","Update":"3049","Value":"6048","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":46,"Name":"shuffle write time","Update":"4012945","Value":"6992490","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"34","Value":"852","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"3049","Value":"29646","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"44564480","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"7","Value":"224","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"110","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":153,"Name":"internal.metrics.executorDeserializeTime","Update":54,"Value":104,"Internal":true,"Count Failed Values":true},{"ID":154,"Name":"internal.metrics.executorDeserializeCpuTime","Update":45521713,"Value":88401799,"Internal":true,"Count Failed Values":true},{"ID":155,"Name":"internal.metrics.executorRunTime","Update":45,"Value":84,"Internal":true,"Count Failed Values":true},{"ID":156,"Name":"internal.metrics.executorCpuTime","Update":42313212,"Value":79310852,"Internal":true,"Count Failed Values":true},{"ID":157,"Name":"internal.metrics.resultSize","Update":4236,"Value":8473,"Internal":true,"Count Failed Values":true},{"ID":162,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":8912896,"Internal":true,"Count Failed Values":true},{"ID":164,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":14,"Internal":true,"Count Failed Values":true},{"ID":165,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":166,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":75292,"Value":149255,"Internal":true,"Count Failed Values":true},{"ID":167,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":168,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":10612,"Value":21261,"Internal":true,"Count Failed Values":true},{"ID":169,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":170,"Name":"internal.metrics.shuffle.read.recordsRead","Update":3049,"Value":6048,"Internal":true,"Count Failed Values":true},{"ID":171,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":82863,"Value":164333,"Internal":true,"Count Failed Values":true},{"ID":172,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":3049,"Value":6048,"Internal":true,"Count Failed Values":true},{"ID":173,"Name":"internal.metrics.shuffle.write.writeTime","Update":4012945,"Value":6992490,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":54,"Executor Deserialize CPU Time":45521713,"Executor Run Time":45,"Executor CPU Time":42313212,"Peak Execution Memory":4456448,"Result Size":4236,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":75292,"Remote Bytes Read To Disk":0,"Local Bytes Read":10612,"Total Records Read":3049},"Shuffle Write Metrics":{"Shuffle Bytes Written":82863,"Shuffle Write Time":4012945,"Shuffle Records Written":3049},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":4,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":27,"Index":3,"Attempt":0,"Launch Time":1633886652293,"Executor ID":"11","Host":"10.1.0.13","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633886652450,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"7","Value":"80","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":27,"Name":"local blocks read","Update":"1","Value":"8","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"73685","Value":"836037","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"local bytes read","Update":"10770","Value":"85156","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2994","Value":"32640","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":36,"Name":"data size","Update":"143712","Value":"434000","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":44,"Name":"shuffle bytes written","Update":"81340","Value":"245673","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":45,"Name":"shuffle records written","Update":"2994","Value":"9042","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":46,"Name":"shuffle write time","Update":"3068160","Value":"10060650","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"30","Value":"882","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2994","Value":"32640","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"49020928","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"5","Value":"229","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"121","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":153,"Name":"internal.metrics.executorDeserializeTime","Update":104,"Value":208,"Internal":true,"Count Failed Values":true},{"ID":154,"Name":"internal.metrics.executorDeserializeCpuTime","Update":47825831,"Value":136227630,"Internal":true,"Count Failed Values":true},{"ID":155,"Name":"internal.metrics.executorRunTime","Update":42,"Value":126,"Internal":true,"Count Failed Values":true},{"ID":156,"Name":"internal.metrics.executorCpuTime","Update":38610610,"Value":117921462,"Internal":true,"Count Failed Values":true},{"ID":157,"Name":"internal.metrics.resultSize","Update":4237,"Value":12710,"Internal":true,"Count Failed Values":true},{"ID":162,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":13369344,"Internal":true,"Count Failed Values":true},{"ID":164,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":21,"Internal":true,"Count Failed Values":true},{"ID":165,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":166,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":73685,"Value":222940,"Internal":true,"Count Failed Values":true},{"ID":167,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":168,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":10770,"Value":32031,"Internal":true,"Count Failed Values":true},{"ID":169,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":170,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2994,"Value":9042,"Internal":true,"Count Failed Values":true},{"ID":171,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":81340,"Value":245673,"Internal":true,"Count Failed Values":true},{"ID":172,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":2994,"Value":9042,"Internal":true,"Count Failed Values":true},{"ID":173,"Name":"internal.metrics.shuffle.write.writeTime","Update":3068160,"Value":10060650,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":104,"Executor Deserialize CPU Time":47825831,"Executor Run Time":42,"Executor CPU Time":38610610,"Peak Execution Memory":4456448,"Result Size":4237,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":73685,"Remote Bytes Read To Disk":0,"Local Bytes Read":10770,"Total Records Read":2994},"Shuffle Write Metrics":{"Shuffle Bytes Written":81340,"Shuffle Write Time":3068160,"Shuffle Records Written":2994},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":4,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":30,"Index":6,"Attempt":0,"Launch Time":1633886652294,"Executor ID":"1","Host":"10.1.0.17","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633886652452,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"8","Value":"88","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"82141","Value":"918178","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2910","Value":"35550","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":36,"Name":"data size","Update":"139680","Value":"573680","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":44,"Name":"shuffle bytes written","Update":"79297","Value":"324970","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":45,"Name":"shuffle records written","Update":"2910","Value":"11952","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":46,"Name":"shuffle write time","Update":"12386955","Value":"22447605","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"42","Value":"924","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2910","Value":"35550","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"53477376","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"7","Value":"236","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"132","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":153,"Name":"internal.metrics.executorDeserializeTime","Update":61,"Value":269,"Internal":true,"Count Failed Values":true},{"ID":154,"Name":"internal.metrics.executorDeserializeCpuTime","Update":54460754,"Value":190688384,"Internal":true,"Count Failed Values":true},{"ID":155,"Name":"internal.metrics.executorRunTime","Update":86,"Value":212,"Internal":true,"Count Failed Values":true},{"ID":156,"Name":"internal.metrics.executorCpuTime","Update":82291839,"Value":200213301,"Internal":true,"Count Failed Values":true},{"ID":157,"Name":"internal.metrics.resultSize","Update":4236,"Value":16946,"Internal":true,"Count Failed Values":true},{"ID":162,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":17825792,"Internal":true,"Count Failed Values":true},{"ID":164,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":8,"Value":29,"Internal":true,"Count Failed Values":true},{"ID":165,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":166,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":82141,"Value":305081,"Internal":true,"Count Failed Values":true},{"ID":167,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":168,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":32031,"Internal":true,"Count Failed Values":true},{"ID":169,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":170,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2910,"Value":11952,"Internal":true,"Count Failed Values":true},{"ID":171,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":79297,"Value":324970,"Internal":true,"Count Failed Values":true},{"ID":172,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":2910,"Value":11952,"Internal":true,"Count Failed Values":true},{"ID":173,"Name":"internal.metrics.shuffle.write.writeTime","Update":12386955,"Value":22447605,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":61,"Executor Deserialize CPU Time":54460754,"Executor Run Time":86,"Executor CPU Time":82291839,"Peak Execution Memory":4456448,"Result Size":4236,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":8,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":82141,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":2910},"Shuffle Write Metrics":{"Shuffle Bytes Written":79297,"Shuffle Write Time":12386955,"Shuffle Records Written":2910},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":4,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":24,"Index":0,"Attempt":0,"Launch Time":1633886652293,"Executor ID":"7","Host":"10.1.0.9","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633886652456,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"8","Value":"96","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"82217","Value":"1000395","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2905","Value":"38455","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":36,"Name":"data size","Update":"139440","Value":"713120","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":44,"Name":"shuffle bytes written","Update":"79034","Value":"404004","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":45,"Name":"shuffle records written","Update":"2905","Value":"14857","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":46,"Name":"shuffle write time","Update":"11902912","Value":"34350517","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"42","Value":"966","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2905","Value":"38455","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"57933824","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"6","Value":"242","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"143","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":153,"Name":"internal.metrics.executorDeserializeTime","Update":69,"Value":338,"Internal":true,"Count Failed Values":true},{"ID":154,"Name":"internal.metrics.executorDeserializeCpuTime","Update":60065310,"Value":250753694,"Internal":true,"Count Failed Values":true},{"ID":155,"Name":"internal.metrics.executorRunTime","Update":83,"Value":295,"Internal":true,"Count Failed Values":true},{"ID":156,"Name":"internal.metrics.executorCpuTime","Update":80941330,"Value":281154631,"Internal":true,"Count Failed Values":true},{"ID":157,"Name":"internal.metrics.resultSize","Update":4235,"Value":21181,"Internal":true,"Count Failed Values":true},{"ID":162,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":22282240,"Internal":true,"Count Failed Values":true},{"ID":164,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":8,"Value":37,"Internal":true,"Count Failed Values":true},{"ID":165,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":166,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":82217,"Value":387298,"Internal":true,"Count Failed Values":true},{"ID":167,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":168,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":32031,"Internal":true,"Count Failed Values":true},{"ID":169,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":170,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2905,"Value":14857,"Internal":true,"Count Failed Values":true},{"ID":171,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":79034,"Value":404004,"Internal":true,"Count Failed Values":true},{"ID":172,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":2905,"Value":14857,"Internal":true,"Count Failed Values":true},{"ID":173,"Name":"internal.metrics.shuffle.write.writeTime","Update":11902912,"Value":34350517,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":69,"Executor Deserialize CPU Time":60065310,"Executor Run Time":83,"Executor CPU Time":80941330,"Peak Execution Memory":4456448,"Result Size":4235,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":8,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":82217,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":2905},"Shuffle Write Metrics":{"Shuffle Bytes Written":79034,"Shuffle Write Time":11902912,"Shuffle Records Written":2905},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":4,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":28,"Index":4,"Attempt":0,"Launch Time":1633886652294,"Executor ID":"4","Host":"10.1.0.8","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633886652456,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"8","Value":"104","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"82988","Value":"1083383","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2937","Value":"41392","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":36,"Name":"data size","Update":"140976","Value":"854096","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":44,"Name":"shuffle bytes written","Update":"79743","Value":"483747","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":45,"Name":"shuffle records written","Update":"2937","Value":"17794","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":46,"Name":"shuffle write time","Update":"11348691","Value":"45699208","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"43","Value":"1009","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2937","Value":"41392","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"62390272","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"7","Value":"249","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"154","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":153,"Name":"internal.metrics.executorDeserializeTime","Update":69,"Value":407,"Internal":true,"Count Failed Values":true},{"ID":154,"Name":"internal.metrics.executorDeserializeCpuTime","Update":60955899,"Value":311709593,"Internal":true,"Count Failed Values":true},{"ID":155,"Name":"internal.metrics.executorRunTime","Update":83,"Value":378,"Internal":true,"Count Failed Values":true},{"ID":156,"Name":"internal.metrics.executorCpuTime","Update":79088856,"Value":360243487,"Internal":true,"Count Failed Values":true},{"ID":157,"Name":"internal.metrics.resultSize","Update":4235,"Value":25416,"Internal":true,"Count Failed Values":true},{"ID":162,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":26738688,"Internal":true,"Count Failed Values":true},{"ID":164,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":8,"Value":45,"Internal":true,"Count Failed Values":true},{"ID":165,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":166,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":82988,"Value":470286,"Internal":true,"Count Failed Values":true},{"ID":167,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":168,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":32031,"Internal":true,"Count Failed Values":true},{"ID":169,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":170,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2937,"Value":17794,"Internal":true,"Count Failed Values":true},{"ID":171,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":79743,"Value":483747,"Internal":true,"Count Failed Values":true},{"ID":172,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":2937,"Value":17794,"Internal":true,"Count Failed Values":true},{"ID":173,"Name":"internal.metrics.shuffle.write.writeTime","Update":11348691,"Value":45699208,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":69,"Executor Deserialize CPU Time":60955899,"Executor Run Time":83,"Executor CPU Time":79088856,"Peak Execution Memory":4456448,"Result Size":4235,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":8,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":82988,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":2937},"Shuffle Write Metrics":{"Shuffle Bytes Written":79743,"Shuffle Write Time":11348691,"Shuffle Records Written":2937},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":4,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":29,"Index":5,"Attempt":0,"Launch Time":1633886652294,"Executor ID":"9","Host":"10.1.0.14","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633886652527,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"7","Value":"111","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":27,"Name":"local blocks read","Update":"1","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"72944","Value":"1156327","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"local bytes read","Update":"10265","Value":"95421","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2948","Value":"44340","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":36,"Name":"data size","Update":"141504","Value":"995600","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":44,"Name":"shuffle bytes written","Update":"80386","Value":"564133","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":45,"Name":"shuffle records written","Update":"2948","Value":"20742","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":46,"Name":"shuffle write time","Update":"3682339","Value":"49381547","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"47","Value":"1056","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2948","Value":"44340","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"66846720","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"17","Value":"266","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"165","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":153,"Name":"internal.metrics.executorDeserializeTime","Update":70,"Value":477,"Internal":true,"Count Failed Values":true},{"ID":154,"Name":"internal.metrics.executorDeserializeCpuTime","Update":63682790,"Value":375392383,"Internal":true,"Count Failed Values":true},{"ID":155,"Name":"internal.metrics.executorRunTime","Update":153,"Value":531,"Internal":true,"Count Failed Values":true},{"ID":156,"Name":"internal.metrics.executorCpuTime","Update":141854217,"Value":502097704,"Internal":true,"Count Failed Values":true},{"ID":157,"Name":"internal.metrics.resultSize","Update":4236,"Value":29652,"Internal":true,"Count Failed Values":true},{"ID":162,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":31195136,"Internal":true,"Count Failed Values":true},{"ID":164,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":52,"Internal":true,"Count Failed Values":true},{"ID":165,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":166,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":72944,"Value":543230,"Internal":true,"Count Failed Values":true},{"ID":167,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":168,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":10265,"Value":42296,"Internal":true,"Count Failed Values":true},{"ID":169,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":170,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2948,"Value":20742,"Internal":true,"Count Failed Values":true},{"ID":171,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":80386,"Value":564133,"Internal":true,"Count Failed Values":true},{"ID":172,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":2948,"Value":20742,"Internal":true,"Count Failed Values":true},{"ID":173,"Name":"internal.metrics.shuffle.write.writeTime","Update":3682339,"Value":49381547,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":70,"Executor Deserialize CPU Time":63682790,"Executor Run Time":153,"Executor CPU Time":141854217,"Peak Execution Memory":4456448,"Result Size":4236,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":72944,"Remote Bytes Read To Disk":0,"Local Bytes Read":10265,"Total Records Read":2948},"Shuffle Write Metrics":{"Shuffle Bytes Written":80386,"Shuffle Write Time":3682339,"Shuffle Records Written":2948},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":4,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":31,"Index":7,"Attempt":0,"Launch Time":1633886652294,"Executor ID":"8","Host":"10.1.0.11","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633886652866,"Failed":false,"Killed":false,"Accumulables":[{"ID":26,"Name":"remote blocks read","Update":"8","Value":"119","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Update":"80696","Value":"1237023","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Update":"2856","Value":"47196","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":36,"Name":"data size","Update":"137080","Value":"1132680","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":44,"Name":"shuffle bytes written","Update":"77822","Value":"641955","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":45,"Name":"shuffle records written","Update":"2856","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":46,"Name":"shuffle write time","Update":"10030040","Value":"59411587","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Update":"143","Value":"1199","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Update":"2856","Value":"47196","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Update":"4456448","Value":"71303168","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Update":"37","Value":"303","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Update":"11","Value":"176","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":153,"Name":"internal.metrics.executorDeserializeTime","Update":258,"Value":735,"Internal":true,"Count Failed Values":true},{"ID":154,"Name":"internal.metrics.executorDeserializeCpuTime","Update":235875863,"Value":611268246,"Internal":true,"Count Failed Values":true},{"ID":155,"Name":"internal.metrics.executorRunTime","Update":300,"Value":831,"Internal":true,"Count Failed Values":true},{"ID":156,"Name":"internal.metrics.executorCpuTime","Update":281219809,"Value":783317513,"Internal":true,"Count Failed Values":true},{"ID":157,"Name":"internal.metrics.resultSize","Update":4236,"Value":33888,"Internal":true,"Count Failed Values":true},{"ID":162,"Name":"internal.metrics.peakExecutionMemory","Update":4456448,"Value":35651584,"Internal":true,"Count Failed Values":true},{"ID":164,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":8,"Value":60,"Internal":true,"Count Failed Values":true},{"ID":165,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":166,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":80696,"Value":623926,"Internal":true,"Count Failed Values":true},{"ID":167,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":168,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":42296,"Internal":true,"Count Failed Values":true},{"ID":169,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":170,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2856,"Value":23598,"Internal":true,"Count Failed Values":true},{"ID":171,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":77822,"Value":641955,"Internal":true,"Count Failed Values":true},{"ID":172,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":2856,"Value":23598,"Internal":true,"Count Failed Values":true},{"ID":173,"Name":"internal.metrics.shuffle.write.writeTime","Update":10030040,"Value":59411587,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":258,"Executor Deserialize CPU Time":235875863,"Executor Run Time":300,"Executor CPU Time":281219809,"Peak Execution Memory":4456448,"Result Size":4236,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":8,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":80696,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":2856},"Shuffle Write Metrics":{"Shuffle Bytes Written":77822,"Shuffle Write Time":10030040,"Shuffle Records Written":2856},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}} -{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":4,"Stage Attempt ID":0,"Stage Name":"sql at ScalaSparkSQLBench.scala:43","Number of Tasks":8,"RDD Info":[{"RDD ID":17,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":12,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"12\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[11],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[3],"Details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1633886652279,"Completion Time":1633886652866,"Accumulables":[{"ID":26,"Name":"remote blocks read","Value":"119","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":27,"Name":"local blocks read","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":28,"Name":"remote bytes read","Value":"1237023","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":30,"Name":"local bytes read","Value":"95421","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":31,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":32,"Name":"records read","Value":"47196","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":36,"Name":"data size","Value":"1132680","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":44,"Name":"shuffle bytes written","Value":"641955","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":45,"Name":"shuffle records written","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":46,"Name":"shuffle write time","Value":"59411587","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":55,"Name":"duration","Value":"1199","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":56,"Name":"number of output rows","Value":"47196","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":57,"Name":"peak memory","Value":"71303168","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":59,"Name":"time in aggregation build","Value":"303","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":60,"Name":"avg hash probe bucket list iters","Value":"176","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":153,"Name":"internal.metrics.executorDeserializeTime","Value":735,"Internal":true,"Count Failed Values":true},{"ID":154,"Name":"internal.metrics.executorDeserializeCpuTime","Value":611268246,"Internal":true,"Count Failed Values":true},{"ID":155,"Name":"internal.metrics.executorRunTime","Value":831,"Internal":true,"Count Failed Values":true},{"ID":156,"Name":"internal.metrics.executorCpuTime","Value":783317513,"Internal":true,"Count Failed Values":true},{"ID":157,"Name":"internal.metrics.resultSize","Value":33888,"Internal":true,"Count Failed Values":true},{"ID":162,"Name":"internal.metrics.peakExecutionMemory","Value":35651584,"Internal":true,"Count Failed Values":true},{"ID":164,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":60,"Internal":true,"Count Failed Values":true},{"ID":165,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":4,"Internal":true,"Count Failed Values":true},{"ID":166,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":623926,"Internal":true,"Count Failed Values":true},{"ID":167,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":168,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":42296,"Internal":true,"Count Failed Values":true},{"ID":169,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":170,"Name":"internal.metrics.shuffle.read.recordsRead","Value":23598,"Internal":true,"Count Failed Values":true},{"ID":171,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":641955,"Internal":true,"Count Failed Values":true},{"ID":172,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":23598,"Internal":true,"Count Failed Values":true},{"ID":173,"Name":"internal.metrics.shuffle.write.writeTime","Value":59411587,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} -{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":5,"Stage Attempt ID":0,"Stage Name":"sql at ScalaSparkSQLBench.scala:43","Number of Tasks":8,"RDD Info":[{"RDD ID":19,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[18],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":18,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[17],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[4],"Details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1633886652870,"Accumulables":[],"Resource Profile Id":0},"Properties":{"spark.sql.warehouse.dir":"file:/home/usera112/projects/log-compressor/deployments/HiBench/spark-warehouse","spark.driver.host":"hostb7.dsrg.utoronto.ca","spark.eventLog.enabled":"true","spark.driver.port":"42853","__fetch_continuous_blocks_in_batch_enabled":"true","spark.jars":"file:/home/usera112/projects/log-compressor/deployments/HiBench/sparkbench/assembly/target/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.app.name":"ScalaJoin","spark.driver.memory":"4g","spark.submit.pyFiles":"","spark.default.parallelism":"8","spark.app.startTime":"1633886611309","spark.executor.id":"driver","spark.app.initial.jar.urls":"spark://hostb7.dsrg.utoronto.ca:42853/jars/sparkbench-assembly-8.0-SNAPSHOT-dist.jar","spark.submit.deployMode":"client","spark.master":"spark://hostb7:7077","spark.executor.memory":"4g","spark.eventLog.dir":"hdfs://hostb7:8020/directory","spark.sql.execution.id":"11","spark.sql.catalogImplementation":"hive","spark.app.id":"app-20211010132332-4636","spark.sql.shuffle.partitions":"8"}} -{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":32,"Index":0,"Attempt":0,"Launch Time":1633886652903,"Executor ID":"2","Host":"10.1.0.10","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":33,"Index":1,"Attempt":0,"Launch Time":1633886652904,"Executor ID":"11","Host":"10.1.0.13","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":34,"Index":2,"Attempt":0,"Launch Time":1633886652904,"Executor ID":"4","Host":"10.1.0.8","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":35,"Index":3,"Attempt":0,"Launch Time":1633886652904,"Executor ID":"7","Host":"10.1.0.9","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":36,"Index":4,"Attempt":0,"Launch Time":1633886652905,"Executor ID":"0","Host":"10.1.0.12","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":37,"Index":5,"Attempt":0,"Launch Time":1633886652905,"Executor ID":"5","Host":"10.1.0.19","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":38,"Index":6,"Attempt":0,"Launch Time":1633886652905,"Executor ID":"3","Host":"10.1.0.15","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":39,"Index":7,"Attempt":0,"Launch Time":1633886652905,"Executor ID":"9","Host":"10.1.0.14","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":33,"Index":1,"Attempt":0,"Launch Time":1633886652904,"Executor ID":"11","Host":"10.1.0.13","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633886653449,"Failed":false,"Killed":false,"Accumulables":[{"ID":37,"Name":"remote blocks read","Update":"7","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":38,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":39,"Name":"remote bytes read","Update":"72246","Value":"72246","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":41,"Name":"local bytes read","Update":"10594","Value":"10594","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":42,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":43,"Name":"records read","Update":"3059","Value":"3059","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":51,"Name":"duration","Update":"310","Value":"310","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":52,"Name":"sort time","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":53,"Name":"peak memory","Update":"4325376","Value":"4325376","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":54,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":178,"Name":"internal.metrics.executorDeserializeTime","Update":75,"Value":75,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.executorDeserializeCpuTime","Update":68491859,"Value":68491859,"Internal":true,"Count Failed Values":true},{"ID":180,"Name":"internal.metrics.executorRunTime","Update":453,"Value":453,"Internal":true,"Count Failed Values":true},{"ID":181,"Name":"internal.metrics.executorCpuTime","Update":330296732,"Value":330296732,"Internal":true,"Count Failed Values":true},{"ID":182,"Name":"internal.metrics.resultSize","Update":5952,"Value":5952,"Internal":true,"Count Failed Values":true},{"ID":184,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":187,"Name":"internal.metrics.peakExecutionMemory","Update":4325376,"Value":4325376,"Internal":true,"Count Failed Values":true},{"ID":189,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":190,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":191,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":72246,"Value":72246,"Internal":true,"Count Failed Values":true},{"ID":192,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":193,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":10594,"Value":10594,"Internal":true,"Count Failed Values":true},{"ID":194,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":195,"Name":"internal.metrics.shuffle.read.recordsRead","Update":3059,"Value":3059,"Internal":true,"Count Failed Values":true},{"ID":201,"Name":"internal.metrics.output.bytesWritten","Update":127517,"Value":127517,"Internal":true,"Count Failed Values":true},{"ID":202,"Name":"internal.metrics.output.recordsWritten","Update":3059,"Value":3059,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":75,"Executor Deserialize CPU Time":68491859,"Executor Run Time":453,"Executor CPU Time":330296732,"Peak Execution Memory":4325376,"Result Size":5952,"JVM GC Time":0,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":72246,"Remote Bytes Read To Disk":0,"Local Bytes Read":10594,"Total Records Read":3059},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":127517,"Records Written":3059},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":36,"Index":4,"Attempt":0,"Launch Time":1633886652905,"Executor ID":"0","Host":"10.1.0.12","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633886653449,"Failed":false,"Killed":false,"Accumulables":[{"ID":37,"Name":"remote blocks read","Update":"8","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":39,"Name":"remote bytes read","Update":"77456","Value":"149702","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":42,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":43,"Name":"records read","Update":"2839","Value":"5898","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":51,"Name":"duration","Update":"312","Value":"622","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":52,"Name":"sort time","Update":"6","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":53,"Name":"peak memory","Update":"4325376","Value":"8650752","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":54,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":178,"Name":"internal.metrics.executorDeserializeTime","Update":80,"Value":155,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.executorDeserializeCpuTime","Update":74216019,"Value":142707878,"Internal":true,"Count Failed Values":true},{"ID":180,"Name":"internal.metrics.executorRunTime","Update":449,"Value":902,"Internal":true,"Count Failed Values":true},{"ID":181,"Name":"internal.metrics.executorCpuTime","Update":338833998,"Value":669130730,"Internal":true,"Count Failed Values":true},{"ID":182,"Name":"internal.metrics.resultSize","Update":5952,"Value":11904,"Internal":true,"Count Failed Values":true},{"ID":184,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":187,"Name":"internal.metrics.peakExecutionMemory","Update":4325376,"Value":8650752,"Internal":true,"Count Failed Values":true},{"ID":189,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":8,"Value":15,"Internal":true,"Count Failed Values":true},{"ID":190,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":191,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":77456,"Value":149702,"Internal":true,"Count Failed Values":true},{"ID":192,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":193,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":10594,"Internal":true,"Count Failed Values":true},{"ID":194,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":195,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2839,"Value":5898,"Internal":true,"Count Failed Values":true},{"ID":201,"Name":"internal.metrics.output.bytesWritten","Update":119261,"Value":246778,"Internal":true,"Count Failed Values":true},{"ID":202,"Name":"internal.metrics.output.recordsWritten","Update":2839,"Value":5898,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":80,"Executor Deserialize CPU Time":74216019,"Executor Run Time":449,"Executor CPU Time":338833998,"Peak Execution Memory":4325376,"Result Size":5952,"JVM GC Time":0,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":8,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":77456,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":2839},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":119261,"Records Written":2839},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":39,"Index":7,"Attempt":0,"Launch Time":1633886652905,"Executor ID":"9","Host":"10.1.0.14","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633886653449,"Failed":false,"Killed":false,"Accumulables":[{"ID":37,"Name":"remote blocks read","Update":"7","Value":"22","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":38,"Name":"local blocks read","Update":"1","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":39,"Name":"remote bytes read","Update":"68649","Value":"218351","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":41,"Name":"local bytes read","Update":"10663","Value":"21257","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":42,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":43,"Name":"records read","Update":"2902","Value":"8800","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":51,"Name":"duration","Update":"308","Value":"930","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":52,"Name":"sort time","Update":"6","Value":"17","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":53,"Name":"peak memory","Update":"4325376","Value":"12976128","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":54,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":178,"Name":"internal.metrics.executorDeserializeTime","Update":80,"Value":235,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.executorDeserializeCpuTime","Update":74914209,"Value":217622087,"Internal":true,"Count Failed Values":true},{"ID":180,"Name":"internal.metrics.executorRunTime","Update":448,"Value":1350,"Internal":true,"Count Failed Values":true},{"ID":181,"Name":"internal.metrics.executorCpuTime","Update":323882943,"Value":993013673,"Internal":true,"Count Failed Values":true},{"ID":182,"Name":"internal.metrics.resultSize","Update":5952,"Value":17856,"Internal":true,"Count Failed Values":true},{"ID":184,"Name":"internal.metrics.resultSerializationTime","Update":3,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":187,"Name":"internal.metrics.peakExecutionMemory","Update":4325376,"Value":12976128,"Internal":true,"Count Failed Values":true},{"ID":189,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":22,"Internal":true,"Count Failed Values":true},{"ID":190,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":191,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":68649,"Value":218351,"Internal":true,"Count Failed Values":true},{"ID":192,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":193,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":10663,"Value":21257,"Internal":true,"Count Failed Values":true},{"ID":194,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":195,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2902,"Value":8800,"Internal":true,"Count Failed Values":true},{"ID":201,"Name":"internal.metrics.output.bytesWritten","Update":124359,"Value":371137,"Internal":true,"Count Failed Values":true},{"ID":202,"Name":"internal.metrics.output.recordsWritten","Update":2902,"Value":8800,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":80,"Executor Deserialize CPU Time":74914209,"Executor Run Time":448,"Executor CPU Time":323882943,"Peak Execution Memory":4325376,"Result Size":5952,"JVM GC Time":0,"Result Serialization Time":3,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":68649,"Remote Bytes Read To Disk":0,"Local Bytes Read":10663,"Total Records Read":2902},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":124359,"Records Written":2902},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":38,"Index":6,"Attempt":0,"Launch Time":1633886652905,"Executor ID":"3","Host":"10.1.0.15","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633886653462,"Failed":false,"Killed":false,"Accumulables":[{"ID":37,"Name":"remote blocks read","Update":"7","Value":"29","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":38,"Name":"local blocks read","Update":"1","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":39,"Name":"remote bytes read","Update":"69262","Value":"287613","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":41,"Name":"local bytes read","Update":"9703","Value":"30960","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":42,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":43,"Name":"records read","Update":"2891","Value":"11691","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":51,"Name":"duration","Update":"303","Value":"1233","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":52,"Name":"sort time","Update":"6","Value":"23","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":53,"Name":"peak memory","Update":"4325376","Value":"17301504","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":54,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":178,"Name":"internal.metrics.executorDeserializeTime","Update":100,"Value":335,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.executorDeserializeCpuTime","Update":93582698,"Value":311204785,"Internal":true,"Count Failed Values":true},{"ID":180,"Name":"internal.metrics.executorRunTime","Update":446,"Value":1796,"Internal":true,"Count Failed Values":true},{"ID":181,"Name":"internal.metrics.executorCpuTime","Update":335659027,"Value":1328672700,"Internal":true,"Count Failed Values":true},{"ID":182,"Name":"internal.metrics.resultSize","Update":5952,"Value":23808,"Internal":true,"Count Failed Values":true},{"ID":184,"Name":"internal.metrics.resultSerializationTime","Update":3,"Value":10,"Internal":true,"Count Failed Values":true},{"ID":187,"Name":"internal.metrics.peakExecutionMemory","Update":4325376,"Value":17301504,"Internal":true,"Count Failed Values":true},{"ID":189,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":29,"Internal":true,"Count Failed Values":true},{"ID":190,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":191,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":69262,"Value":287613,"Internal":true,"Count Failed Values":true},{"ID":192,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":193,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":9703,"Value":30960,"Internal":true,"Count Failed Values":true},{"ID":194,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":195,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2891,"Value":11691,"Internal":true,"Count Failed Values":true},{"ID":201,"Name":"internal.metrics.output.bytesWritten","Update":121918,"Value":493055,"Internal":true,"Count Failed Values":true},{"ID":202,"Name":"internal.metrics.output.recordsWritten","Update":2891,"Value":11691,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":100,"Executor Deserialize CPU Time":93582698,"Executor Run Time":446,"Executor CPU Time":335659027,"Peak Execution Memory":4325376,"Result Size":5952,"JVM GC Time":0,"Result Serialization Time":3,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":69262,"Remote Bytes Read To Disk":0,"Local Bytes Read":9703,"Total Records Read":2891},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":121918,"Records Written":2891},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":37,"Index":5,"Attempt":0,"Launch Time":1633886652905,"Executor ID":"5","Host":"10.1.0.19","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633886653487,"Failed":false,"Killed":false,"Accumulables":[{"ID":37,"Name":"remote blocks read","Update":"8","Value":"37","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":39,"Name":"remote bytes read","Update":"85304","Value":"372917","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":42,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":43,"Name":"records read","Update":"3132","Value":"14823","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":51,"Name":"duration","Update":"294","Value":"1527","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":52,"Name":"sort time","Update":"5","Value":"28","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":53,"Name":"peak memory","Update":"4325376","Value":"21626880","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":54,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":178,"Name":"internal.metrics.executorDeserializeTime","Update":87,"Value":422,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.executorDeserializeCpuTime","Update":81387229,"Value":392592014,"Internal":true,"Count Failed Values":true},{"ID":180,"Name":"internal.metrics.executorRunTime","Update":484,"Value":2280,"Internal":true,"Count Failed Values":true},{"ID":181,"Name":"internal.metrics.executorCpuTime","Update":381920658,"Value":1710593358,"Internal":true,"Count Failed Values":true},{"ID":182,"Name":"internal.metrics.resultSize","Update":5952,"Value":29760,"Internal":true,"Count Failed Values":true},{"ID":184,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":12,"Internal":true,"Count Failed Values":true},{"ID":187,"Name":"internal.metrics.peakExecutionMemory","Update":4325376,"Value":21626880,"Internal":true,"Count Failed Values":true},{"ID":189,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":8,"Value":37,"Internal":true,"Count Failed Values":true},{"ID":190,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":191,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":85304,"Value":372917,"Internal":true,"Count Failed Values":true},{"ID":192,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":193,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":30960,"Internal":true,"Count Failed Values":true},{"ID":194,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":195,"Name":"internal.metrics.shuffle.read.recordsRead","Update":3132,"Value":14823,"Internal":true,"Count Failed Values":true},{"ID":201,"Name":"internal.metrics.output.bytesWritten","Update":131756,"Value":624811,"Internal":true,"Count Failed Values":true},{"ID":202,"Name":"internal.metrics.output.recordsWritten","Update":3132,"Value":14823,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":87,"Executor Deserialize CPU Time":81387229,"Executor Run Time":484,"Executor CPU Time":381920658,"Peak Execution Memory":4325376,"Result Size":5952,"JVM GC Time":0,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":8,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":85304,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":3132},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":131756,"Records Written":3132},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":32,"Index":0,"Attempt":0,"Launch Time":1633886652903,"Executor ID":"2","Host":"10.1.0.10","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633886653497,"Failed":false,"Killed":false,"Accumulables":[{"ID":37,"Name":"remote blocks read","Update":"8","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":39,"Name":"remote bytes read","Update":"84078","Value":"456995","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":42,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":43,"Name":"records read","Update":"3101","Value":"17924","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":51,"Name":"duration","Update":"307","Value":"1834","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":52,"Name":"sort time","Update":"5","Value":"33","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":53,"Name":"peak memory","Update":"4325376","Value":"25952256","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":54,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":178,"Name":"internal.metrics.executorDeserializeTime","Update":90,"Value":512,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.executorDeserializeCpuTime","Update":85994483,"Value":478586497,"Internal":true,"Count Failed Values":true},{"ID":180,"Name":"internal.metrics.executorRunTime","Update":489,"Value":2769,"Internal":true,"Count Failed Values":true},{"ID":181,"Name":"internal.metrics.executorCpuTime","Update":393474482,"Value":2104067840,"Internal":true,"Count Failed Values":true},{"ID":182,"Name":"internal.metrics.resultSize","Update":5952,"Value":35712,"Internal":true,"Count Failed Values":true},{"ID":184,"Name":"internal.metrics.resultSerializationTime","Update":4,"Value":16,"Internal":true,"Count Failed Values":true},{"ID":187,"Name":"internal.metrics.peakExecutionMemory","Update":4325376,"Value":25952256,"Internal":true,"Count Failed Values":true},{"ID":189,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":8,"Value":45,"Internal":true,"Count Failed Values":true},{"ID":190,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":0,"Value":3,"Internal":true,"Count Failed Values":true},{"ID":191,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":84078,"Value":456995,"Internal":true,"Count Failed Values":true},{"ID":192,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":193,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":0,"Value":30960,"Internal":true,"Count Failed Values":true},{"ID":194,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":195,"Name":"internal.metrics.shuffle.read.recordsRead","Update":3101,"Value":17924,"Internal":true,"Count Failed Values":true},{"ID":201,"Name":"internal.metrics.output.bytesWritten","Update":129343,"Value":754154,"Internal":true,"Count Failed Values":true},{"ID":202,"Name":"internal.metrics.output.recordsWritten","Update":3101,"Value":17924,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":90,"Executor Deserialize CPU Time":85994483,"Executor Run Time":489,"Executor CPU Time":393474482,"Peak Execution Memory":4325376,"Result Size":5952,"JVM GC Time":0,"Result Serialization Time":4,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":8,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":84078,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":3101},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":129343,"Records Written":3101},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":35,"Index":3,"Attempt":0,"Launch Time":1633886652904,"Executor ID":"7","Host":"10.1.0.9","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633886654337,"Failed":false,"Killed":false,"Accumulables":[{"ID":37,"Name":"remote blocks read","Update":"7","Value":"52","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":38,"Name":"local blocks read","Update":"1","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":39,"Name":"remote bytes read","Update":"76289","Value":"533284","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":41,"Name":"local bytes read","Update":"10900","Value":"41860","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":42,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":43,"Name":"records read","Update":"3223","Value":"21147","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":51,"Name":"duration","Update":"1157","Value":"2991","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":52,"Name":"sort time","Update":"6","Value":"39","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":53,"Name":"peak memory","Update":"4325376","Value":"30277632","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":54,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":178,"Name":"internal.metrics.executorDeserializeTime","Update":109,"Value":621,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.executorDeserializeCpuTime","Update":101726190,"Value":580312687,"Internal":true,"Count Failed Values":true},{"ID":180,"Name":"internal.metrics.executorRunTime","Update":1312,"Value":4081,"Internal":true,"Count Failed Values":true},{"ID":181,"Name":"internal.metrics.executorCpuTime","Update":1055555672,"Value":3159623512,"Internal":true,"Count Failed Values":true},{"ID":182,"Name":"internal.metrics.resultSize","Update":5995,"Value":41707,"Internal":true,"Count Failed Values":true},{"ID":183,"Name":"internal.metrics.jvmGCTime","Update":119,"Value":119,"Internal":true,"Count Failed Values":true},{"ID":184,"Name":"internal.metrics.resultSerializationTime","Update":3,"Value":19,"Internal":true,"Count Failed Values":true},{"ID":187,"Name":"internal.metrics.peakExecutionMemory","Update":4325376,"Value":30277632,"Internal":true,"Count Failed Values":true},{"ID":189,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":52,"Internal":true,"Count Failed Values":true},{"ID":190,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":191,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":76289,"Value":533284,"Internal":true,"Count Failed Values":true},{"ID":192,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":193,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":10900,"Value":41860,"Internal":true,"Count Failed Values":true},{"ID":194,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":195,"Name":"internal.metrics.shuffle.read.recordsRead","Update":3223,"Value":21147,"Internal":true,"Count Failed Values":true},{"ID":201,"Name":"internal.metrics.output.bytesWritten","Update":134365,"Value":888519,"Internal":true,"Count Failed Values":true},{"ID":202,"Name":"internal.metrics.output.recordsWritten","Update":3223,"Value":21147,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":109,"Executor Deserialize CPU Time":101726190,"Executor Run Time":1312,"Executor CPU Time":1055555672,"Peak Execution Memory":4325376,"Result Size":5995,"JVM GC Time":119,"Result Serialization Time":3,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":76289,"Remote Bytes Read To Disk":0,"Local Bytes Read":10900,"Total Records Read":3223},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":134365,"Records Written":3223},"Updated Blocks":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":34,"Index":2,"Attempt":0,"Launch Time":1633886652904,"Executor ID":"4","Host":"10.1.0.8","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1633886654355,"Failed":false,"Killed":false,"Accumulables":[{"ID":37,"Name":"remote blocks read","Update":"7","Value":"59","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":38,"Name":"local blocks read","Update":"1","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":39,"Name":"remote bytes read","Update":"58219","Value":"591503","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":41,"Name":"local bytes read","Update":"8592","Value":"50452","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":42,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":43,"Name":"records read","Update":"2451","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":51,"Name":"duration","Update":"1139","Value":"4130","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":52,"Name":"sort time","Update":"5","Value":"44","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":53,"Name":"peak memory","Update":"4325376","Value":"34603008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":54,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":178,"Name":"internal.metrics.executorDeserializeTime","Update":118,"Value":739,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.executorDeserializeCpuTime","Update":108815452,"Value":689128139,"Internal":true,"Count Failed Values":true},{"ID":180,"Name":"internal.metrics.executorRunTime","Update":1319,"Value":5400,"Internal":true,"Count Failed Values":true},{"ID":181,"Name":"internal.metrics.executorCpuTime","Update":1051976887,"Value":4211600399,"Internal":true,"Count Failed Values":true},{"ID":182,"Name":"internal.metrics.resultSize","Update":5995,"Value":47702,"Internal":true,"Count Failed Values":true},{"ID":183,"Name":"internal.metrics.jvmGCTime","Update":109,"Value":228,"Internal":true,"Count Failed Values":true},{"ID":184,"Name":"internal.metrics.resultSerializationTime","Update":4,"Value":23,"Internal":true,"Count Failed Values":true},{"ID":187,"Name":"internal.metrics.peakExecutionMemory","Update":4325376,"Value":34603008,"Internal":true,"Count Failed Values":true},{"ID":189,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":7,"Value":59,"Internal":true,"Count Failed Values":true},{"ID":190,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":191,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":58219,"Value":591503,"Internal":true,"Count Failed Values":true},{"ID":192,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":193,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":8592,"Value":50452,"Internal":true,"Count Failed Values":true},{"ID":194,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":195,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2451,"Value":23598,"Internal":true,"Count Failed Values":true},{"ID":201,"Name":"internal.metrics.output.bytesWritten","Update":102178,"Value":990697,"Internal":true,"Count Failed Values":true},{"ID":202,"Name":"internal.metrics.output.recordsWritten","Update":2451,"Value":23598,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0},"Task Metrics":{"Executor Deserialize Time":118,"Executor Deserialize CPU Time":108815452,"Executor Run Time":1319,"Executor CPU Time":1051976887,"Peak Execution Memory":4325376,"Result Size":5995,"JVM GC Time":109,"Result Serialization Time":4,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":7,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":58219,"Remote Bytes Read To Disk":0,"Local Bytes Read":8592,"Total Records Read":2451},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":102178,"Records Written":2451},"Updated Blocks":[]}} -{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":5,"Stage Attempt ID":0,"Stage Name":"sql at ScalaSparkSQLBench.scala:43","Number of Tasks":8,"RDD Info":[{"RDD ID":19,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[18],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":18,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"8\",\"name\":\"Exchange\"}","Callsite":"sql at ScalaSparkSQLBench.scala:43","Parent IDs":[17],"Storage Level":{"Use Disk":false,"Use Memory":false,"Deserialized":false,"Replication":1},"Barrier":false,"Number of Partitions":8,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[4],"Details":"org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.$anonfun$main$1(ScalaSparkSQLBench.scala:43)\nscala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)\nscala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)\nscala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench$.main(ScalaSparkSQLBench.scala:41)\ncom.intel.hibench.sparkbench.sql.ScalaSparkSQLBench.main(ScalaSparkSQLBench.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:951)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1039)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1048)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1633886652870,"Completion Time":1633886654356,"Accumulables":[{"ID":37,"Name":"remote blocks read","Value":"59","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":38,"Name":"local blocks read","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":39,"Name":"remote bytes read","Value":"591503","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":41,"Name":"local bytes read","Value":"50452","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":42,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":43,"Name":"records read","Value":"23598","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":51,"Name":"duration","Value":"4130","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":52,"Name":"sort time","Value":"44","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":53,"Name":"peak memory","Value":"34603008","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":54,"Name":"spill size","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":178,"Name":"internal.metrics.executorDeserializeTime","Value":739,"Internal":true,"Count Failed Values":true},{"ID":179,"Name":"internal.metrics.executorDeserializeCpuTime","Value":689128139,"Internal":true,"Count Failed Values":true},{"ID":180,"Name":"internal.metrics.executorRunTime","Value":5400,"Internal":true,"Count Failed Values":true},{"ID":181,"Name":"internal.metrics.executorCpuTime","Value":4211600399,"Internal":true,"Count Failed Values":true},{"ID":182,"Name":"internal.metrics.resultSize","Value":47702,"Internal":true,"Count Failed Values":true},{"ID":183,"Name":"internal.metrics.jvmGCTime","Value":228,"Internal":true,"Count Failed Values":true},{"ID":184,"Name":"internal.metrics.resultSerializationTime","Value":23,"Internal":true,"Count Failed Values":true},{"ID":187,"Name":"internal.metrics.peakExecutionMemory","Value":34603008,"Internal":true,"Count Failed Values":true},{"ID":189,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":59,"Internal":true,"Count Failed Values":true},{"ID":190,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":5,"Internal":true,"Count Failed Values":true},{"ID":191,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":591503,"Internal":true,"Count Failed Values":true},{"ID":192,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":193,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":50452,"Internal":true,"Count Failed Values":true},{"ID":194,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":195,"Name":"internal.metrics.shuffle.read.recordsRead","Value":23598,"Internal":true,"Count Failed Values":true},{"ID":201,"Name":"internal.metrics.output.bytesWritten","Value":990697,"Internal":true,"Count Failed Values":true},{"ID":202,"Name":"internal.metrics.output.recordsWritten","Value":23598,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0}} -{"Event":"SparkListenerJobEnd","Job ID":2,"Completion Time":1633886654356,"Job Result":{"Result":"JobSucceeded"}} -{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":11,"accumUpdates":[[47,8],[48,990697],[49,23598],[50,0]]} -{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":11,"time":1633886655028} -{"Event":"SparkListenerApplicationEnd","Timestamp":1633886655029} -{} -{"empty":{},"null":null} From 8a7241009d4c36fc45039300c8d3add410eeaef6 Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Mon, 18 Nov 2024 20:16:21 -0500 Subject: [PATCH 46/50] Merge guard changes --- src/clp_ffi_py/PyObjectUtils.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/clp_ffi_py/PyObjectUtils.hpp b/src/clp_ffi_py/PyObjectUtils.hpp index 0b671af0..c8168792 100644 --- a/src/clp_ffi_py/PyObjectUtils.hpp +++ b/src/clp_ffi_py/PyObjectUtils.hpp @@ -5,6 +5,8 @@ #include +#include + namespace clp_ffi_py { /** * A specialized deleter for PyObjectPtr which decrements the pointed PyObject reference count when @@ -58,10 +60,10 @@ using PyObjectStaticPtr = std::unique_ptr Date: Thu, 21 Nov 2024 18:58:28 -0500 Subject: [PATCH 47/50] Apply suggestions from code review Co-authored-by: haiqi96 <14502009+haiqi96@users.noreply.github.com> --- src/clp_ffi_py/ir/native/PySerializer.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/clp_ffi_py/ir/native/PySerializer.cpp b/src/clp_ffi_py/ir/native/PySerializer.cpp index 4699ae20..a09da5de 100644 --- a/src/clp_ffi_py/ir/native/PySerializer.cpp +++ b/src/clp_ffi_py/ir/native/PySerializer.cpp @@ -28,7 +28,7 @@ namespace { PyDoc_STRVAR( cPySerializerDoc, "Serializer for serializing CLP key-value pair IR streams.\n" - "This class serializes log events using the CLP key-value pair IR format and writes the" + "This class serializes log events into the CLP key-value pair IR format and writes the" " serialized data to a specified byte stream object.\n\n" "__init__(self, output_stream, buffer_size_limit=65536)\n\n" "Initializes a :class:`Serializer` instance with the given output stream. Note that each" @@ -37,7 +37,7 @@ PyDoc_STRVAR( ":param output_stream: A writable byte output stream to which the serializer will write the" " serialized IR byte sequences.\n" ":type output_stream: IO[bytes]\n" - ":param buffer_size_limit: Threshold of how much serialized data to buffer before flushing" + ":param buffer_size_limit: The maximum amount of serialized data to buffer before flushing" " it to `output_stream`. Defaults to 64 KiB.\n" ":type buffer_size_limit: int\n" ); @@ -379,9 +379,9 @@ CLP_FFI_PY_METHOD auto PySerializer_dealloc(PySerializer* self) -> void { if (0 != PyErr_WarnEx( PyExc_ResourceWarning, - "`Serializer.close()` is not called before object destruction. Forget to call " - "`close` will leave the stream incomplete, and potentially resulting in data " - "loss due to data buffering", + "`Serializer.close()` is not called before object destruction, which will leave" + " the stream incomplete, and potentially resulting in data" + " loss due to data buffering", 1 )) { From 836c1dd2bd0be5fb2e92fc2079bdacf4380f86fa Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Thu, 21 Nov 2024 19:14:40 -0500 Subject: [PATCH 48/50] Apply code review comments --- src/clp_ffi_py/ir/native/PySerializer.hpp | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/clp_ffi_py/ir/native/PySerializer.hpp b/src/clp_ffi_py/ir/native/PySerializer.hpp index 37bfcb94..c5cecb40 100644 --- a/src/clp_ffi_py/ir/native/PySerializer.hpp +++ b/src/clp_ffi_py/ir/native/PySerializer.hpp @@ -77,15 +77,8 @@ class PySerializer { [[nodiscard]] auto is_closed() const -> bool { return nullptr == m_serializer; } - /** - * Asserts the serializer has not been closed. - * @return true on success, false if it's already been closed with `IOError` set. - */ - [[nodiscard]] auto assert_is_not_closed() const -> bool; - /** * Serializes the log event from the given msgpack map into IR format. - * NOTE: the serializer must not be closed to call this method. * @param msgpack_byte_sequence * @return the number of bytes serialized on success. * @return std::nullptr on failure with the relevant Python exception and error set. @@ -100,7 +93,6 @@ class PySerializer { /** * Flushes the underlying IR buffer and `m_output_stream`. - * NOTE: the serializer must not be closed to call this method. * @return true on success. * @return false on failure with the relevant Python exception and error set. */ @@ -109,7 +101,6 @@ class PySerializer { /** * Closes the serializer by writing the buffered results into the output stream with * end-of-stream IR Unit appended in the end. - * NOTE: the serializer must not be closed to call this method. * @return true on success. * @return false on failure with the relevant Python exception and error set. */ @@ -132,6 +123,12 @@ class PySerializer { [[nodiscard]] static auto module_level_init(PyObject* py_module) -> bool; private: + /** + * Asserts the serializer has not been closed. + * @return true on success, false if it's already been closed with `IOError` set. + */ + [[nodiscard]] auto assert_is_not_closed() const -> bool; + [[nodiscard]] auto get_ir_buf_size() const -> Py_ssize_t { return static_cast(m_serializer->get_ir_buf_view().size()); } From c11c97574e26af64bfb06ea7d9b6f6e086317b57 Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Thu, 21 Nov 2024 19:25:02 -0500 Subject: [PATCH 49/50] Apply code review comments --- src/clp_ffi_py/ir/native/PySerializer.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/clp_ffi_py/ir/native/PySerializer.cpp b/src/clp_ffi_py/ir/native/PySerializer.cpp index a09da5de..79704d60 100644 --- a/src/clp_ffi_py/ir/native/PySerializer.cpp +++ b/src/clp_ffi_py/ir/native/PySerializer.cpp @@ -236,12 +236,11 @@ PySerializer_init(PySerializer* self, PyObject* args, PyObject* keywords) -> int // Ensure the `output_stream` has `write`, `flush`, and `close` methods auto output_stream_has_method = [&](char const* method_name) -> bool { - PyObjectPtr const write_method{PyObject_GetAttrString(output_stream, method_name) - }; - if (nullptr == write_method) { + PyObjectPtr const method{PyObject_GetAttrString(output_stream, method_name)}; + if (nullptr == method) { return false; } - if (false == static_cast(PyCallable_Check(write_method.get()))) { + if (false == static_cast(PyCallable_Check(method.get()))) { PyErr_SetString(PyExc_TypeError, ""); PyErr_Format( PyExc_TypeError, From 1bb026bfaa560dacd44da7e4d03ae281c9c262c4 Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Thu, 21 Nov 2024 19:33:23 -0500 Subject: [PATCH 50/50] Use n to parse Py_ssize_t --- src/clp_ffi_py/ir/native/PySerializer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/clp_ffi_py/ir/native/PySerializer.cpp b/src/clp_ffi_py/ir/native/PySerializer.cpp index 79704d60..7b473451 100644 --- a/src/clp_ffi_py/ir/native/PySerializer.cpp +++ b/src/clp_ffi_py/ir/native/PySerializer.cpp @@ -225,7 +225,7 @@ PySerializer_init(PySerializer* self, PyObject* args, PyObject* keywords) -> int == static_cast(PyArg_ParseTupleAndKeywords( args, keywords, - "O|L", + "O|n", static_cast(keyword_table), &output_stream, &buffer_size_limit