diff --git a/be/src/runtime/memory/mem_tracker_limiter.cpp b/be/src/runtime/memory/mem_tracker_limiter.cpp index c2f27a74a093801..ec89e1ea52c0a05 100644 --- a/be/src/runtime/memory/mem_tracker_limiter.cpp +++ b/be/src/runtime/memory/mem_tracker_limiter.cpp @@ -132,7 +132,7 @@ MemTrackerLimiter::~MemTrackerLimiter() { fmt::format("mem tracker label: {}, consumption: {}, peak consumption: {}, {}.", label(), _consumption->current_value(), _consumption->peak_value(), mem_tracker_inaccurate_msg); - LOG(FATAL) << err_msg << print_address_sanitizers(); + LOG(INFO) << err_msg << print_address_sanitizers(); } #endif if (ExecEnv::tracking_memory()) { @@ -141,10 +141,10 @@ MemTrackerLimiter::~MemTrackerLimiter() { _consumption->set(0); #ifndef NDEBUG } else if (!_address_sanitizers.empty()) { - LOG(INFO) << "[Address Sanitizer] consumption is 0, but address sanitizers not empty. " - << ", mem tracker label: " << _label - << ", peak consumption: " << _consumption->peak_value() - << print_address_sanitizers(); + LOG(FATAL) << "[Address Sanitizer] consumption is 0, but address sanitizers not empty. " + << ", mem tracker label: " << _label + << ", peak consumption: " << _consumption->peak_value() + << print_address_sanitizers(); #endif } memory_memtrackerlimiter_cnt << -1; @@ -156,13 +156,13 @@ void MemTrackerLimiter::add_address_sanitizers(void* buf, size_t size) { std::lock_guard l(_address_sanitizers_mtx); auto it = _address_sanitizers.find(buf); if (it != _address_sanitizers.end()) { - LOG(INFO) << "[Address Sanitizer] memory buf repeat add, mem tracker label: " << _label - << ", consumption: " << _consumption->current_value() - << ", peak consumption: " << _consumption->peak_value() << ", buf: " << buf - << ", size: " << size << ", old buf: " << it->first - << ", old size: " << it->second.size - << ", new stack_trace: " << get_stack_trace(1, "DISABLED") - << ", old stack_trace: " << it->second.stack_trace; + _error_address_sanitizers.emplace_back( + fmt::format("[Address Sanitizer] memory buf repeat add, mem tracker label: {}, " + "consumption: {}, peak consumption: {}, buf: {}, size: {}, old " + "buf: {}, old size: {}, new stack_trace: {}, old stack_trace: {}.", + _label, _consumption->current_value(), _consumption->peak_value(), + buf, size, it->first, it->second.size, + get_stack_trace(1, "FULL_WITH_INLINE"), it->second.stack_trace);); } // if alignment not equal to 0, maybe usable_size > size. @@ -179,21 +179,21 @@ void MemTrackerLimiter::remove_address_sanitizers(void* buf, size_t size) { auto it = _address_sanitizers.find(buf); if (it != _address_sanitizers.end()) { if (it->second.size != size) { - LOG(INFO) << "[Address Sanitizer] free memory buf size inaccurate, mem tracker " - "label: " - << _label << ", consumption: " << _consumption->current_value() - << ", peak consumption: " << _consumption->peak_value() - << ", buf: " << buf << ", size: " << size << ", old buf: " << it->first - << ", old size: " << it->second.size - << ", new stack_trace: " << get_stack_trace(1, "DISABLED") - << ", old stack_trace: " << it->second.stack_trace; + _error_address_sanitizers.emplace_back(fmt::format( + "[Address Sanitizer] free memory buf size inaccurate, mem tracker label: " + "{}, consumption: {}, peak consumption: {}, buf: {}, size: {}, old buf: " + "{}, old size: {}, new stack_trace: {}, old stack_trace: {}.", + _label, _consumption->current_value(), _consumption->peak_value(), buf, + size, it->first, it->second.size, get_stack_trace(1, "FULL_WITH_INLINE"), + it->second.stack_trace)); } _address_sanitizers.erase(buf); } else { - LOG(INFO) << "[Address Sanitizer] memory buf not exist, mem tracker label: " << _label - << ", consumption: " << _consumption->current_value() - << ", peak consumption: " << _consumption->peak_value() << ", buf: " << buf - << ", size: " << size << ", stack_trace: " << get_stack_trace(1, "DISABLED"); + _error_address_sanitizers.emplace_back(fmt::format( + "[Address Sanitizer] memory buf not exist, mem tracker label: {}, consumption: " + "{}, peak consumption: {}, buf: {}, size: {}, stack_trace: {}.", + _label, _consumption->current_value(), _consumption->peak_value(), buf, size, + get_stack_trace(1, "FULL_WITH_INLINE"))); } } } @@ -201,10 +201,15 @@ void MemTrackerLimiter::remove_address_sanitizers(void* buf, size_t size) { std::string MemTrackerLimiter::print_address_sanitizers() { std::lock_guard l(_address_sanitizers_mtx); std::string detail = "[Address Sanitizer]:"; + detail += "\n memory not be freed:"; for (const auto& it : _address_sanitizers) { detail += fmt::format("\n {}, size {}, strack trace: {}", it.first, it.second.size, it.second.stack_trace); } + detail += "\n incorrect memory alloc and free:"; + for (const auto& err_msg : _error_address_sanitizers) { + detail += fmt::format("\n {}", err_msg); + } return detail; } #endif diff --git a/be/src/runtime/memory/mem_tracker_limiter.h b/be/src/runtime/memory/mem_tracker_limiter.h index e5c5cb1bc0369c6..1118c1a5247f54f 100644 --- a/be/src/runtime/memory/mem_tracker_limiter.h +++ b/be/src/runtime/memory/mem_tracker_limiter.h @@ -260,6 +260,7 @@ class MemTrackerLimiter final : public MemTracker { std::mutex _address_sanitizers_mtx; std::unordered_map _address_sanitizers; + std::vector _error_address_sanitizers; #endif }; diff --git a/be/src/util/faststring.h b/be/src/util/faststring.h index 3ec0acbda01d79e..f82bcfdb0d9585e 100644 --- a/be/src/util/faststring.h +++ b/be/src/util/faststring.h @@ -85,7 +85,8 @@ class faststring : private Allocator(Allocator::alloc(len_)); + ret = reinterpret_cast(Allocator::alloc(capacity_)); + DCHECK(len_ <= capacity_); memcpy(ret, data_, len_); } OwnedSlice result(ret, len_, capacity_); diff --git a/be/src/util/slice.h b/be/src/util/slice.h index 1c8579ffed45d59..b8ff313e76d4771 100644 --- a/be/src/util/slice.h +++ b/be/src/util/slice.h @@ -362,7 +362,14 @@ class OwnedSlice : private Allocator