Skip to content

Commit

Permalink
add json/ipv4
Browse files Browse the repository at this point in the history
  • Loading branch information
@zhangstar333
zhangstar333 committed Apr 3, 2024
1 parent b003858 commit bd2784d
Show file tree
Hide file tree
Showing 6 changed files with 81 additions and 9 deletions.
28 changes: 28 additions & 0 deletions be/src/vec/data_types/serde/data_type_ipv4_serde.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,5 +88,33 @@ Status DataTypeIPv4SerDe::deserialize_one_cell_from_json(IColumn& column, Slice&
return Status::OK();
}

Status DataTypeIPv4SerDe::write_column_to_pb(const IColumn& column, PValues& result, int start,
int end) const {
const auto& column_data = assert_cast<const ColumnIPv4&>(column);
result.mutable_string_value()->Reserve(end - start);
auto* ptype = result.mutable_type();
ptype->set_id(PGenericType::IPV4);
for (int i = start; i < end; ++i) {
IPv4Value ipv4_value(column_data.get_data()[i]);
result.add_string_value(ipv4_value.to_string());
}
return Status::OK();
}

Status DataTypeIPv4SerDe::read_column_from_pb(IColumn& column, const PValues& arg) const {
auto& col_data = static_cast<ColumnIPv4&>(column).get_data();
col_data.reserve(arg.string_value_size());
for (int i = 0; i < arg.string_value_size(); ++i) {
IPv4 ipv4_val;
if (!IPv4Value::from_string(ipv4_val, arg.string_value(i).c_str(),
arg.string_value(i).size())) {
throw doris::Exception(
ErrorCode::INVALID_ARGUMENT, "parse number fail in ipv4, string: '{}'",
std::string(arg.string_value(i).c_str(), arg.string_value(i).size()).c_str());
}
col_data.emplace_back(ipv4_val);
}
return Status::OK();
}
} // namespace vectorized
} // namespace doris
3 changes: 3 additions & 0 deletions be/src/vec/data_types/serde/data_type_ipv4_serde.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,9 @@ class DataTypeIPv4SerDe : public DataTypeNumberSerDe<IPv4> {
FormatOptions& options) const override;
Status deserialize_one_cell_from_json(IColumn& column, Slice& slice,
const FormatOptions& options) const override;
Status write_column_to_pb(const IColumn& column, PValues& result, int start,
int end) const override;
Status read_column_from_pb(IColumn& column, const PValues& arg) const override;

private:
template <bool is_binary_format>
Expand Down
6 changes: 3 additions & 3 deletions be/src/vec/data_types/serde/data_type_ipv6_serde.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ Status DataTypeIPv6SerDe::write_column_to_pb(const IColumn& column, PValues& res
result.add_string_value(ipv6_value.to_string());
}
return Status::OK();
};
}

Status DataTypeIPv6SerDe::read_column_from_pb(IColumn& column, const PValues& arg) const {
auto& col_data = static_cast<ColumnIPv6&>(column).get_data();
Expand All @@ -109,13 +109,13 @@ Status DataTypeIPv6SerDe::read_column_from_pb(IColumn& column, const PValues& ar
if (!IPv6Value::from_string(ipv6_val, arg.string_value(i).c_str(),
arg.string_value(i).size())) {
throw doris::Exception(
ErrorCode::INVALID_ARGUMENT, "parse number fail, string: '{}'",
ErrorCode::INVALID_ARGUMENT, "parse number fail in ipv6, string: '{}'",
std::string(arg.string_value(i).c_str(), arg.string_value(i).size()).c_str());
}
col_data.emplace_back(ipv6_val);
}
return Status::OK();
};
}

} // namespace vectorized
} // namespace doris
27 changes: 27 additions & 0 deletions be/src/vec/data_types/serde/data_type_jsonb_serde.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -240,5 +240,32 @@ Status DataTypeJsonbSerDe::read_one_cell_from_json(IColumn& column,
return Status::OK();
}

Status DataTypeJsonbSerDe::write_column_to_pb(const IColumn& column, PValues& result, int start,
int end) const {
const auto& string_column = assert_cast<const ColumnString&>(column);
result.mutable_string_value()->Reserve(end - start);
auto* ptype = result.mutable_type();
ptype->set_id(PGenericType::JSONB);
for (size_t row_num = start; row_num < end; ++row_num) {
std::string_view string_ref = string_column.get_data_at(row_num).to_string_view();
std::string json_string =
JsonbToJson::jsonb_to_json_string(string_ref.data(), string_ref.size());
result.add_string_value(json_string);
}
return Status::OK();
}

Status DataTypeJsonbSerDe::read_column_from_pb(IColumn& column, const PValues& arg) const {
auto& column_string = assert_cast<ColumnString&>(column);
column_string.reserve(arg.string_value_size());
for (int i = 0; i < arg.string_value_size(); ++i) {
JsonBinaryValue value;
RETURN_IF_ERROR(
value.from_json_string(arg.string_value(i).c_str(), arg.string_value(i).size()));
column_string.insert_data(value.value(), value.size());
}
return Status::OK();
}

} // namespace vectorized
} // namespace doris
3 changes: 3 additions & 0 deletions be/src/vec/data_types/serde/data_type_jsonb_serde.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,9 @@ class DataTypeJsonbSerDe : public DataTypeStringSerDe {
rapidjson::Document::AllocatorType& allocator,
int row_num) const override;
Status read_one_cell_from_json(IColumn& column, const rapidjson::Value& result) const override;
Status write_column_to_pb(const IColumn& column, PValues& result, int start,
int end) const override;
Status read_column_from_pb(IColumn& column, const PValues& arg) const override;

private:
template <bool is_binary_format>
Expand Down
23 changes: 17 additions & 6 deletions ...e/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,10 @@
import org.apache.doris.nereids.trees.expressions.literal.DecimalV3Literal;
import org.apache.doris.nereids.trees.expressions.literal.DoubleLiteral;
import org.apache.doris.nereids.trees.expressions.literal.FloatLiteral;
import org.apache.doris.nereids.trees.expressions.literal.IPv4Literal;
import org.apache.doris.nereids.trees.expressions.literal.IPv6Literal;
import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral;
import org.apache.doris.nereids.trees.expressions.literal.JsonLiteral;
import org.apache.doris.nereids.trees.expressions.literal.LargeIntLiteral;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.expressions.literal.MapLiteral;
Expand Down Expand Up @@ -183,7 +185,9 @@ private static void collectConst(Expression expr, Map<String, Expression> constM
return;
}
// eg: avg_state(1) return is agg function serialize data
if (expr.getDataType().isAggStateType() || expr.getDataType().isObjectType()) {
// and some type can't find a literal to represent
if (expr.getDataType().isAggStateType() || expr.getDataType().isObjectType()
|| expr.getDataType().isVariantType() || expr.getDataType().isTimeV2Type()) {
return;
}
if (skipSleepFunction(expr)) {
Expand Down Expand Up @@ -388,12 +392,12 @@ public static List<Literal> getResultExpression(DataType type, PValues resultCon
localDate.getDayOfMonth());
res.add(dateV2Literal);
}
} else if (type.isTimeV2Type()) {
int num = resultContent.getDoubleValueCount();
} else if (type.isIPv4Type()) {
int num = resultContent.getStringValueCount();
for (int i = 0; i < num; ++i) {
double doubleValue = resultContent.getDoubleValue(i);
DoubleLiteral doubleLiteral = new DoubleLiteral(doubleValue);
res.add(doubleLiteral);
String stringValue = resultContent.getStringValue(i);
IPv4Literal iPv4Literal = new IPv4Literal(stringValue);
res.add(iPv4Literal);
}
} else if (type.isIPv6Type()) {
int num = resultContent.getStringValueCount();
Expand All @@ -402,6 +406,13 @@ public static List<Literal> getResultExpression(DataType type, PValues resultCon
IPv6Literal iPv6Literal = new IPv6Literal(stringValue);
res.add(iPv6Literal);
}
} else if (type.isJsonType()) {
int num = resultContent.getStringValueCount();
for (int i = 0; i < num; ++i) {
String stringValue = resultContent.getStringValue(i);
JsonLiteral jsonLiteral = new JsonLiteral(stringValue);
res.add(jsonLiteral);
}
} else if (type.isStringLikeType()) {
int num = resultContent.getStringValueCount();
for (int i = 0; i < num; ++i) {
Expand Down

0 comments on commit bd2784d

Please sign in to comment.