Skip to content

Commit

Permalink
[feature](decimal) support decimal256: compile ok
Browse files Browse the repository at this point in the history
  • Loading branch information
jacktengg committed Sep 28, 2023
1 parent 8679095 commit c96091c
Show file tree
Hide file tree
Showing 49 changed files with 3,071 additions and 294 deletions.
1 change: 1 addition & 0 deletions be/src/common/consts.h
Original file line number Diff line number Diff line change
Expand Up @@ -33,5 +33,6 @@ const std::string DYNAMIC_COLUMN_NAME = "__DORIS_DYNAMIC_COL__";
constexpr int MAX_DECIMAL32_PRECISION = 9;
constexpr int MAX_DECIMAL64_PRECISION = 18;
constexpr int MAX_DECIMAL128_PRECISION = 38;
constexpr int MAX_DECIMAL256_PRECISION = 76;
} // namespace BeConsts
} // namespace doris
2 changes: 2 additions & 0 deletions be/src/exec/olap_common.h
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,8 @@ std::string cast_to_string(T value, int scale) {
return ((vectorized::Decimal<int64_t>)value).to_string(scale);
} else if constexpr (primitive_type == TYPE_DECIMAL128I) {
return ((vectorized::Decimal<int128_t>)value).to_string(scale);
} else if constexpr (primitive_type == TYPE_DECIMAL256) {
return ((vectorized::Decimal<Int256>)value).to_string(scale);
} else if constexpr (primitive_type == TYPE_TINYINT) {
return std::to_string(static_cast<int>(value));
} else if constexpr (primitive_type == TYPE_LARGEINT) {
Expand Down
6 changes: 6 additions & 0 deletions be/src/exec/schema_scanner.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -287,6 +287,12 @@ Status SchemaScanner::fill_dest_column_for_range(vectorized::Block* block, size_
reinterpret_cast<const char*>(&num), 0);
break;
}
// case TYPE_DECIMAL256: {
// const vectorized::Int256 num = (reinterpret_cast<PackedInt256*>(data))->value;
// reinterpret_cast<vectorized::ColumnDecimal256*>(col_ptr)->insert_data(
// reinterpret_cast<const char*>(&num), 0);
// break;
// }

case TYPE_DECIMAL32: {
const int32_t num = *reinterpret_cast<int32_t*>(data);
Expand Down
3 changes: 2 additions & 1 deletion be/src/olap/olap_common.h
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,8 @@ enum class FieldType {
OLAP_FIELD_TYPE_DECIMAL128I = 33,
OLAP_FIELD_TYPE_JSONB = 34,
OLAP_FIELD_TYPE_VARIANT = 35,
OLAP_FIELD_TYPE_AGG_STATE = 36
OLAP_FIELD_TYPE_AGG_STATE = 36,
OLAP_FIELD_TYPE_DECIMAL256 = 37,
};

// Define all aggregation methods supported by Field
Expand Down
1 change: 1 addition & 0 deletions be/src/runtime/define_primitive_type.h
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ enum PrimitiveType : PrimitiveNative {
TYPE_VARIANT, /* 32 */
TYPE_LAMBDA_FUNCTION, /* 33 */
TYPE_AGG_STATE, /* 34 */
TYPE_DECIMAL256, /* 35 */
};

constexpr PrimitiveNative BEGIN_OF_PRIMITIVE_TYPE = INVALID_TYPE;
Expand Down
6 changes: 6 additions & 0 deletions be/src/runtime/primitive_type.h
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ constexpr bool is_enumeration_type(PrimitiveType type) {
case TYPE_DECIMAL32:
case TYPE_DECIMAL64:
case TYPE_DECIMAL128I:
case TYPE_DECIMAL256:
case TYPE_BOOLEAN:
case TYPE_ARRAY:
case TYPE_STRUCT:
Expand Down Expand Up @@ -205,6 +206,11 @@ struct PrimitiveTypeTraits<TYPE_DECIMAL128I> {
using ColumnType = vectorized::ColumnDecimal<vectorized::Decimal128I>;
};
template <>
struct PrimitiveTypeTraits<TYPE_DECIMAL256> {
using CppType = vectorized::Decimal256;
using ColumnType = vectorized::ColumnDecimal<vectorized::Decimal256>;
};
template <>
struct PrimitiveTypeTraits<TYPE_LARGEINT> {
using CppType = __int128_t;
using ColumnType = vectorized::ColumnInt128;
Expand Down
4 changes: 4 additions & 0 deletions be/src/runtime/runtime_predicate.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,10 @@ Status RuntimePredicate::init(const PrimitiveType type, const bool nulls_first)
_get_value_fn = get_decimal128_value;
break;
}
case PrimitiveType::TYPE_DECIMAL256: {
_get_value_fn = get_decimal256_value;
break;
}
default:
return Status::InvalidArgument("unsupported runtime predicate type {}", type);
}
Expand Down
6 changes: 6 additions & 0 deletions be/src/runtime/runtime_predicate.h
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,12 @@ class RuntimePredicate {
auto v = field.get<DecimalField<Decimal128I>>();
return cast_to_string<TYPE_DECIMAL128I, ValueType>(v.get_value(), v.get_scale());
}

static std::string get_decimal256_value(const Field& field) {
using ValueType = typename PrimitiveTypeTraits<TYPE_DECIMAL256>::CppType;
auto v = field.get<DecimalField<Decimal256>>();
return cast_to_string<TYPE_DECIMAL256, ValueType>(v.get_value(), v.get_scale());
}
};

} // namespace vectorized
Expand Down
8 changes: 8 additions & 0 deletions be/src/runtime/type_limit.h
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
#include "runtime/datetime_value.h"
#include "runtime/decimalv2_value.h"
#include "vec/common/string_ref.h"
#include "vec/core/wide_integer.h"

namespace doris {

Expand Down Expand Up @@ -70,6 +71,13 @@ struct type_limit<vectorized::Decimal128> {
}
static vectorized::Decimal128 min() { return -max(); }
};
static Int256 MAX_DECIMAL256_INT({18446744073709551615ul, 8607968719199866879ul,
532749306367912313ul, 1593091911132452277ul});
template <>
struct type_limit<vectorized::Decimal256> {
static vectorized::Decimal256 max() { return vectorized::Decimal256(MAX_DECIMAL256_INT); }
static vectorized::Decimal256 min() { return vectorized::Decimal256(-MAX_DECIMAL256_INT); }
};

template <>
struct type_limit<vectorized::VecDateTimeValue> {
Expand Down
6 changes: 4 additions & 2 deletions be/src/util/binary_cast.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
#include "runtime/datetime_value.h"
#include "runtime/decimalv2_value.h"
#include "util/types.h"
#include "vec/core/wide_integer.h"
#include "vec/runtime/vdatetime_value.h"
namespace doris {
union TypeConverter {
Expand Down Expand Up @@ -79,6 +80,7 @@ To binary_cast(From from) {
match_v<From, doris::vectorized::VecDateTimeValue, To, __int64_t>;
constexpr bool from_i128_to_decv2 = match_v<From, __int128_t, To, DecimalV2Value>;
constexpr bool from_decv2_to_i128 = match_v<From, DecimalV2Value, To, __int128_t>;
constexpr bool from_decv2_to_i256 = match_v<From, DecimalV2Value, To, wide::Int256>;

constexpr bool from_ui32_to_date_v2 =
match_v<From, uint32_t, To,
Expand All @@ -98,8 +100,8 @@ To binary_cast(From from) {

static_assert(from_u64_to_db || from_i64_to_db || from_db_to_i64 || from_db_to_u64 ||
from_i64_to_vec_dt || from_vec_dt_to_i64 || from_i128_to_decv2 ||
from_decv2_to_i128 || from_ui32_to_date_v2 || from_date_v2_to_ui32 ||
from_ui64_to_datetime_v2 || from_datetime_v2_to_ui64);
from_decv2_to_i128 || from_decv2_to_i256 || from_ui32_to_date_v2 ||
from_date_v2_to_ui32 || from_ui64_to_datetime_v2 || from_datetime_v2_to_ui64);

if constexpr (from_u64_to_db) {
TypeConverter conv;
Expand Down
5 changes: 4 additions & 1 deletion be/src/util/string_parser.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
#include "runtime/large_int_value.h"
#include "runtime/primitive_type.h"
#include "vec/common/int_exp.h"
#include "vec/core/wide_integer.h"
#include "vec/data_types/data_type_decimal.h"

namespace doris {
Expand Down Expand Up @@ -91,14 +92,16 @@ class StringParser {
template <typename T>
static T get_scale_multiplier(int scale) {
static_assert(std::is_same_v<T, int32_t> || std::is_same_v<T, int64_t> ||
std::is_same_v<T, __int128>,
std::is_same_v<T, __int128> || std::is_same_v<T, Int256>,
"You can only instantiate as int32_t, int64_t, __int128.");
if constexpr (std::is_same_v<T, int32_t>) {
return common::exp10_i32(scale);
} else if constexpr (std::is_same_v<T, int64_t>) {
return common::exp10_i64(scale);
} else if constexpr (std::is_same_v<T, __int128>) {
return common::exp10_i128(scale);
} else if constexpr (std::is_same_v<T, Int256>) {
return common::exp10_i256(scale);
}
}

Expand Down
12 changes: 11 additions & 1 deletion be/src/vec/columns/column_decimal.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -122,8 +122,10 @@ template <typename T>
UInt64 ColumnDecimal<T>::get64(size_t n) const {
if constexpr (sizeof(T) > sizeof(UInt64)) {
LOG(FATAL) << "Method get64 is not supported for " << get_family_name();
return 0;
} else {
return static_cast<typename T::NativeType>(data[n]);
}
return static_cast<typename T::NativeType>(data[n]);
}

template <typename T>
Expand Down Expand Up @@ -527,6 +529,13 @@ Decimal128I ColumnDecimal<Decimal128I>::get_scale_multiplier() const {
return common::exp10_i128(scale);
}

// duplicate with
// Decimal256 DataTypeDecimal<Decimal256>::get_scale_multiplier(UInt32 scale) {
template <>
Decimal256 ColumnDecimal<Decimal256>::get_scale_multiplier() const {
return Decimal256(common::exp10_i256(scale));
}

template <typename T>
ColumnPtr ColumnDecimal<T>::index(const IColumn& indexes, size_t limit) const {
return select_index_impl(*this, indexes, limit);
Expand All @@ -536,4 +545,5 @@ template class ColumnDecimal<Decimal32>;
template class ColumnDecimal<Decimal64>;
template class ColumnDecimal<Decimal128>;
template class ColumnDecimal<Decimal128I>;
template class ColumnDecimal<Decimal256>;
} // namespace doris::vectorized
4 changes: 2 additions & 2 deletions be/src/vec/columns/column_decimal.h
Original file line number Diff line number Diff line change
Expand Up @@ -201,8 +201,8 @@ class ColumnDecimal final : public COWHelper<ColumnVectorHelper, ColumnDecimal<T
return StringRef(reinterpret_cast<const char*>(&data[n]), sizeof(data[n]));
}
void get(size_t n, Field& res) const override { res = (*this)[n]; }
bool get_bool(size_t n) const override { return bool(data[n]); }
Int64 get_int(size_t n) const override { return Int64(data[n] * scale); }
bool get_bool(size_t n) const override { return bool(data[n].value); }
Int64 get_int(size_t n) const override { return Int64(data[n].value * scale); }
UInt64 get64(size_t n) const override;
bool is_default_at(size_t n) const override { return data[n].value == 0; }

Expand Down
1 change: 1 addition & 0 deletions be/src/vec/columns/columns_number.h
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ using ColumnDecimal32 = ColumnDecimal<Decimal32>;
using ColumnDecimal64 = ColumnDecimal<Decimal64>;
using ColumnDecimal128 = ColumnDecimal<Decimal128>;
using ColumnDecimal128I = ColumnDecimal<Decimal128I>;
using ColumnDecimal256 = ColumnDecimal<Decimal256>;

template <typename T>
struct IsFixLenColumnType {
Expand Down
13 changes: 13 additions & 0 deletions be/src/vec/common/arithmetic_overflow.h
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

#pragma once

#include "vec/core/wide_integer.h"
namespace common {
template <typename T>
inline bool add_overflow(T x, T y, T& res) {
Expand Down Expand Up @@ -79,6 +80,12 @@ inline bool sub_overflow(__int128 x, __int128 y, __int128& res) {
return (y < 0 && x > max_int128 + y) || (y > 0 && x < min_int128 + y);
}

/// Multiply and ignore overflow.
template <typename T1, typename T2>
inline auto mul_ignore_overflow(T1 x, T2 y) {
return x * y;
}

template <typename T>
inline bool mul_overflow(T x, T y, T& res) {
return __builtin_mul_overflow(x, y, &res);
Expand Down Expand Up @@ -109,4 +116,10 @@ inline bool mul_overflow(__int128 x, __int128 y, __int128& res) {
unsigned __int128 b = (y > 0) ? y : -y;
return (a * b) / b != a;
}

template <>
inline bool mul_overflow(wide::Int256 x, wide::Int256 y, wide::Int256& res) {
res = mul_ignore_overflow(x, y);
return false;
}
} // namespace common
2 changes: 2 additions & 0 deletions be/src/vec/common/field_visitors.h
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,8 @@ typename std::decay_t<Visitor>::ResultType apply_visitor(Visitor&& visitor, F&&
return visitor(field.template get<DecimalField<Decimal128>>());
case Field::Types::Decimal128I:
return visitor(field.template get<DecimalField<Decimal128I>>());
case Field::Types::Decimal256:
return visitor(field.template get<DecimalField<Decimal256>>());
default:
LOG(FATAL) << "Bad type of Field";
return {};
Expand Down
92 changes: 92 additions & 0 deletions be/src/vec/common/int_exp.h
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@
#include <limits>
#include <utility>

#include "vec/core/wide_integer.h"

namespace exp_details {

// compile-time exp(v, n) by linear recursion
Expand Down Expand Up @@ -78,4 +80,94 @@ inline constexpr __int128 exp10_i128(int x) {
return exp_details::get_exp<__int128, 10, 39>(x);
}

using wide::Int256;
inline Int256 exp10_i256(int x) {
if (x < 0) return 0;
if (x > 76) return std::numeric_limits<Int256>::max();

using Int256 = Int256;
static constexpr Int256 i10e18 {1000000000000000000ll};
static const Int256 values[] = {
static_cast<Int256>(1ll),
static_cast<Int256>(10ll),
static_cast<Int256>(100ll),
static_cast<Int256>(1000ll),
static_cast<Int256>(10000ll),
static_cast<Int256>(100000ll),
static_cast<Int256>(1000000ll),
static_cast<Int256>(10000000ll),
static_cast<Int256>(100000000ll),
static_cast<Int256>(1000000000ll),
static_cast<Int256>(10000000000ll),
static_cast<Int256>(100000000000ll),
static_cast<Int256>(1000000000000ll),
static_cast<Int256>(10000000000000ll),
static_cast<Int256>(100000000000000ll),
static_cast<Int256>(1000000000000000ll),
static_cast<Int256>(10000000000000000ll),
static_cast<Int256>(100000000000000000ll),
i10e18,
i10e18 * 10ll,
i10e18 * 100ll,
i10e18 * 1000ll,
i10e18 * 10000ll,
i10e18 * 100000ll,
i10e18 * 1000000ll,
i10e18 * 10000000ll,
i10e18 * 100000000ll,
i10e18 * 1000000000ll,
i10e18 * 10000000000ll,
i10e18 * 100000000000ll,
i10e18 * 1000000000000ll,
i10e18 * 10000000000000ll,
i10e18 * 100000000000000ll,
i10e18 * 1000000000000000ll,
i10e18 * 10000000000000000ll,
i10e18 * 100000000000000000ll,
i10e18 * 100000000000000000ll * 10ll,
i10e18 * 100000000000000000ll * 100ll,
i10e18 * 100000000000000000ll * 1000ll,
i10e18 * 100000000000000000ll * 10000ll,
i10e18 * 100000000000000000ll * 100000ll,
i10e18 * 100000000000000000ll * 1000000ll,
i10e18 * 100000000000000000ll * 10000000ll,
i10e18 * 100000000000000000ll * 100000000ll,
i10e18 * 100000000000000000ll * 1000000000ll,
i10e18 * 100000000000000000ll * 10000000000ll,
i10e18 * 100000000000000000ll * 100000000000ll,
i10e18 * 100000000000000000ll * 1000000000000ll,
i10e18 * 100000000000000000ll * 10000000000000ll,
i10e18 * 100000000000000000ll * 100000000000000ll,
i10e18 * 100000000000000000ll * 1000000000000000ll,
i10e18 * 100000000000000000ll * 10000000000000000ll,
i10e18 * 100000000000000000ll * 100000000000000000ll,
i10e18 * 100000000000000000ll * 100000000000000000ll * 10ll,
i10e18 * 100000000000000000ll * 100000000000000000ll * 100ll,
i10e18 * 100000000000000000ll * 100000000000000000ll * 1000ll,
i10e18 * 100000000000000000ll * 100000000000000000ll * 10000ll,
i10e18 * 100000000000000000ll * 100000000000000000ll * 100000ll,
i10e18 * 100000000000000000ll * 100000000000000000ll * 1000000ll,
i10e18 * 100000000000000000ll * 100000000000000000ll * 10000000ll,
i10e18 * 100000000000000000ll * 100000000000000000ll * 100000000ll,
i10e18 * 100000000000000000ll * 100000000000000000ll * 1000000000ll,
i10e18 * 100000000000000000ll * 100000000000000000ll * 10000000000ll,
i10e18 * 100000000000000000ll * 100000000000000000ll * 100000000000ll,
i10e18 * 100000000000000000ll * 100000000000000000ll * 1000000000000ll,
i10e18 * 100000000000000000ll * 100000000000000000ll * 10000000000000ll,
i10e18 * 100000000000000000ll * 100000000000000000ll * 100000000000000ll,
i10e18 * 100000000000000000ll * 100000000000000000ll * 1000000000000000ll,
i10e18 * 100000000000000000ll * 100000000000000000ll * 10000000000000000ll,
i10e18 * 100000000000000000ll * 100000000000000000ll * 100000000000000000ll,
i10e18 * 100000000000000000ll * 100000000000000000ll * 100000000000000000ll * 10ll,
i10e18 * 100000000000000000ll * 100000000000000000ll * 100000000000000000ll * 100ll,
i10e18 * 100000000000000000ll * 100000000000000000ll * 100000000000000000ll * 1000ll,
i10e18 * 100000000000000000ll * 100000000000000000ll * 100000000000000000ll * 10000ll,
i10e18 * 100000000000000000ll * 100000000000000000ll * 100000000000000000ll * 100000ll,
i10e18 * 100000000000000000ll * 100000000000000000ll * 100000000000000000ll * 1000000ll,
i10e18 * 100000000000000000ll * 100000000000000000ll * 100000000000000000ll *
10000000ll,
};
return values[x];
}

} // namespace common
Loading

0 comments on commit c96091c

Please sign in to comment.