Skip to content

Commit

Permalink
[FIX](complextype) fix complextype has empty literal may make be core a…
Browse files Browse the repository at this point in the history 
…pache#28233 (apache#28820)
  • Loading branch information
@amorynan
amorynan authored Dec 27, 2023
1 parent ce0fcb5 commit 3befd30
Show file tree
Hide file tree
Showing 12 changed files with 237 additions and 71 deletions.
8 changes: 3 additions & 5 deletions fe/fe-common/src/main/java/org/apache/doris/catalog/ArrayType.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,13 +88,11 @@ public boolean matchesType(Type t) {
return false;
}

// Array(Null) is a virtual Array type, can match any Array(...) type
if (itemType.isNull() || ((ArrayType) t).getItemType().isNull()) {
return true;
if (((ArrayType) t).getContainsNull() != getContainsNull()) {
return false;
}

return itemType.matchesType(((ArrayType) t).itemType)
&& (((ArrayType) t).containsNull || !containsNull);
return itemType.matchesType(((ArrayType) t).itemType);
}

@Override
Expand Down
8 changes: 5 additions & 3 deletions fe/fe-common/src/main/java/org/apache/doris/catalog/MapType.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,9 +126,11 @@ public boolean matchesType(Type t) {
return false;
}

if ((keyType.isNull() || ((MapType) t).getKeyType().isNull())
&& (valueType.isNull() || ((MapType) t).getKeyType().isNull())) {
return true;
if (((MapType) t).getIsKeyContainsNull() != getIsKeyContainsNull()) {
return false;
}
if (((MapType) t).getIsValueContainsNull() != getIsValueContainsNull()) {
return false;
}

return keyType.matchesType(((MapType) t).keyType)
Expand Down
3 changes: 2 additions & 1 deletion fe/fe-common/src/main/java/org/apache/doris/catalog/TemplateType.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,7 @@ public Type specializeTemplateType(Type specificType, Map<String, Type> speciali
}

if (specializedType != null
&& !specializedType.isNull()
&& !specificType.equals(specializedType)
&& !specificType.matchesType(specializedType)
&& !Type.isImplicitlyCastable(specificType, specializedType, true)
Expand All @@ -104,7 +105,7 @@ public Type specializeTemplateType(Type specificType, Map<String, Type> speciali
name, specificType, specializedType));
}

if (specializedType == null) {
if (specializedType == null || specializedType.isNull()) {
specializedTypeMap.put(name, specificType);
}
return specializedTypeMap.get(name);
Expand Down
17 changes: 1 addition & 16 deletions fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1943,6 +1943,7 @@ public static boolean matchExactType(Type type1, Type type2) {
}

public static boolean matchExactType(Type type1, Type type2, boolean ignorePrecision) {
// we should make type decide to match other for itself to impl matchesType instead of switch case types
if (type1.matchesType(type2)) {
if (PrimitiveType.typeWithPrecision.contains(type2.getPrimitiveType())) {
// For types which has precision and scale, we also need to check quality between precisions and scales
Expand All @@ -1955,22 +1956,6 @@ public static boolean matchExactType(Type type1, Type type2, boolean ignorePreci
return isSameDecimalTypeWithDifferentPrecision(((ScalarType) type2).decimalPrecision(),
((ScalarType) type1).decimalPrecision());
}
} else if (type2.isArrayType()) {
// For types array, we also need to check contains null for case like
// cast(array<not_null(int)> as array<int>)
if (((ArrayType) type2).getContainsNull() != ((ArrayType) type1).getContainsNull()) {
return false;
}
return matchExactType(((ArrayType) type2).getItemType(), ((ArrayType) type1).getItemType());
} else if (type2.isMapType()) {
if (((MapType) type2).getIsKeyContainsNull() != ((MapType) type1).getIsKeyContainsNull()) {
return false;
}
if (((MapType) type2).getIsValueContainsNull() != ((MapType) type1).getIsValueContainsNull()) {
return false;
}
return matchExactType(((MapType) type2).getKeyType(), ((MapType) type1).getKeyType())
&& matchExactType(((MapType) type2).getValueType(), ((MapType) type1).getValueType());
} else {
return true;
}
Expand Down
33 changes: 30 additions & 3 deletions fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1614,8 +1614,25 @@ && collectChildReturnTypes()[0].isDecimalV3()) {
// now first find function in built-in functions
if (Strings.isNullOrEmpty(fnName.getDb())) {
Type[] childTypes = collectChildReturnTypes();
fn = getBuiltinFunction(fnName.getFunction(), childTypes,
Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF);
// when we call count<Array<T>> with nested type is not null type which is defined in FunctionSet
// so here aim to make function signature to match builtln func we defined in fe code
if (fnName.getFunction().equalsIgnoreCase("count") && childTypes.length > 0
&& childTypes[0].isComplexType()) {
// get origin type to match builtln func
Type[] matchFuncChildTypes = new Type[1];
if (childTypes[0].isArrayType()) {
matchFuncChildTypes[0] = Type.ARRAY;
} else if (childTypes[0].isMapType()) {
matchFuncChildTypes[0] = Type.MAP;
} else if (childTypes[0].isStructType()) {
matchFuncChildTypes[0] = Type.GENERIC_STRUCT;
}
fn = getBuiltinFunction(fnName.getFunction(), matchFuncChildTypes,
Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF);
} else {
fn = getBuiltinFunction(fnName.getFunction(), childTypes,
Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF);
}
}

// find user defined functions
Expand Down Expand Up @@ -1822,10 +1839,20 @@ && collectChildReturnTypes()[0].isDecimalV3()) {
ix = i % 2 == 0 ? 0 : 1;
}

// array_zip varargs special case array_zip(array1, array2, ...)
// we only specialize array_zip with first array type, next type we same with custom type
if (i >= args.length && (fnName.getFunction().equalsIgnoreCase("array_zip"))) {
if (argTypes[i].isNull()) {
uncheckedCastChild(args[i - 1], i);
}
continue;
}
if (i == 0 && (fnName.getFunction().equalsIgnoreCase("char"))) {
continue;
}

if (fnName.getFunction().equalsIgnoreCase("count") && args[i].isComplexType()) {
continue;
}
if ((fnName.getFunction().equalsIgnoreCase("money_format") || fnName.getFunction()
.equalsIgnoreCase("histogram")
|| fnName.getFunction().equalsIgnoreCase("hist"))
Expand Down
17 changes: 7 additions & 10 deletions fe/fe-core/src/main/java/org/apache/doris/analysis/IsNullPredicate.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,17 +47,14 @@ public static void initBuiltins(FunctionSet functionSet) {

functionSet.addBuiltinBothScalaAndVectorized(ScalarFunction.createBuiltinOperator(IS_NOT_NULL,
null, Lists.newArrayList(t), Type.BOOLEAN, NullableMode.ALWAYS_NOT_NULLABLE));
}
// for array type
for (Type complexType : Lists.newArrayList(Type.ARRAY, Type.MAP, Type.GENERIC_STRUCT)) {
functionSet.addBuiltinBothScalaAndVectorized(ScalarFunction.createBuiltinOperator(IS_NULL, null,
Lists.newArrayList(complexType), Type.BOOLEAN, NullableMode.ALWAYS_NOT_NULLABLE));

// for array type
for (Type complexType : Lists.newArrayList(Type.ARRAY, Type.MAP, Type.GENERIC_STRUCT)) {
functionSet.addBuiltinBothScalaAndVectorized(ScalarFunction.createBuiltinOperator(IS_NULL, null,
Lists.newArrayList(complexType), Type.BOOLEAN, NullableMode.ALWAYS_NOT_NULLABLE));

functionSet.addBuiltinBothScalaAndVectorized(ScalarFunction.createBuiltinOperator(IS_NOT_NULL,
null, Lists.newArrayList(complexType), Type.BOOLEAN,
NullableMode.ALWAYS_NOT_NULLABLE));
}

functionSet.addBuiltinBothScalaAndVectorized(ScalarFunction.createBuiltinOperator(IS_NOT_NULL, null,
Lists.newArrayList(complexType), Type.BOOLEAN, NullableMode.ALWAYS_NOT_NULLABLE));
}
}

Expand Down
7 changes: 7 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/analysis/StructLiteral.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@
import org.apache.doris.common.AnalysisException;
import org.apache.doris.thrift.TExprNode;
import org.apache.doris.thrift.TExprNodeType;
import org.apache.doris.thrift.TTypeDesc;
import org.apache.doris.thrift.TTypeNode;

import com.google.common.base.Preconditions;
import org.apache.commons.lang3.StringUtils;
Expand Down Expand Up @@ -86,6 +88,11 @@ public String getStringValueForArray() {
@Override
protected void toThrift(TExprNode msg) {
msg.node_type = TExprNodeType.STRUCT_LITERAL;
((StructType) type).getFields().forEach(v -> msg.setChildType(v.getType().getPrimitiveType().toThrift()));
TTypeDesc container = new TTypeDesc();
container.setTypes(new ArrayList<TTypeNode>());
type.toThrift(container);
msg.setType(container);
}

@Override
Expand Down
51 changes: 45 additions & 6 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -385,14 +385,40 @@ public Function specializeTemplateFunction(Function templateFunction, Function r
throw new TypeException(templateFunction
+ " is not support for template since it's not a ScalarFunction");
}
Type[] args = specializedFunction.getArgs();
ArrayList<Type> args = new ArrayList<>();
Collections.addAll(args, specializedFunction.getArgs());
Map<String, Type> specializedTypeMap = Maps.newHashMap();
for (int i = 0; i < args.length; i++) {
if (args[i].hasTemplateType()) {
int i = 0;
for (; i < args.size(); i++) {
if (args.get(i).hasTemplateType()) {
hasTemplateType = true;
args[i] = args[i].specializeTemplateType(requestFunction.getArgs()[i], specializedTypeMap, false);
// if args[i] is template type, and requestFunction.getArgs()[i] NULL_TYPE, we need call function
// deduce to get the specific type
Type deduceType = requestFunction.getArgs()[i];
if (requestFunction.getArgs()[i].isNull()
|| (requestFunction.getArgs()[i] instanceof ArrayType
&& ((ArrayType) requestFunction.getArgs()[i]).getItemType().isNull())
&& FunctionTypeDeducers.DEDUCERS.containsKey(specializedFunction.functionName())) {
deduceType = FunctionTypeDeducers.deduce(specializedFunction.functionName(), i, requestFunction.getArgs());
args.set(i, args.get(i).specializeTemplateType(deduceType == null ? requestFunction.getArgs()[i]
: deduceType, specializedTypeMap, false));
} else {
args.set(i, args.get(i).specializeTemplateType(requestFunction.getArgs()[i],
specializedTypeMap, false));
}
}
}
// here need to support varArgs template according to request data
if (specializedFunction.hasVarArgs() && i < requestFunction.getNumArgs()) {
for (; i < requestFunction.getNumArgs(); i++) {
if (requestFunction.getArgs()[i].isNull()) {
args.add(args.get(i - 1));
} else {
args.add(requestFunction.getArgs()[i]);
}
}
}
specializedFunction.setArgs(args);
if (specializedFunction.getReturnType().hasTemplateType()) {
hasTemplateType = true;
specializedFunction.setReturnType(
Expand Down Expand Up @@ -423,7 +449,7 @@ public Function resolveInferenceFunction(Function inferenceFunction, Function re
newTypes[i] = inputType;
}
}
Type newRetType = FunctionTypeDeducers.deduce(inferenceFunction.functionName(), newTypes);
Type newRetType = FunctionTypeDeducers.deduce(inferenceFunction.functionName(), 0, newTypes);
if (newRetType != null && inferenceFunction instanceof ScalarFunction) {
ScalarFunction f = (ScalarFunction) inferenceFunction;
return new ScalarFunction(f.getFunctionName(), Lists.newArrayList(newTypes), newRetType, f.hasVarArgs(),
Expand All @@ -445,7 +471,20 @@ public static boolean isCastMatchAllowed(Function desc, Function candicate) {
final Type[] candicateArgTypes = candicate.getArgs();
if (!(descArgTypes[0] instanceof ScalarType)
|| !(candicateArgTypes[0] instanceof ScalarType)) {
if (candicateArgTypes[0] instanceof ArrayType || candicateArgTypes[0] instanceof MapType) {
if (candicateArgTypes[0] instanceof ArrayType) {
// match is exactly type. but for null type , with in array|map elem can not return true, because for
// be will make null_type to uint8
// so here meet null_type just make true as allowed, descArgTypes[0]).getItemType().isNull() is for
// empty literal like: []|{}
if (descArgTypes[0] instanceof ArrayType && ((ArrayType) descArgTypes[0]).getItemType().isNull()) {
return true;
}
return descArgTypes[0].matchesType(candicateArgTypes[0]);
} else if (candicateArgTypes[0] instanceof MapType) {
if (descArgTypes[0] instanceof MapType && ((MapType) descArgTypes[0]).getKeyType().isNull()
&& ((MapType) descArgTypes[0]).getValueType().isNull()) {
return true;
}
return descArgTypes[0].matchesType(candicateArgTypes[0]);
}
return false;
Expand Down
35 changes: 29 additions & 6 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionTypeDeducers.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,26 +23,49 @@
import java.util.List;

public class FunctionTypeDeducers {

public interface TypeDeducer {
public Type deduce(Type[] args);
public Type deduce(int argIdx, Type[] args);
}

public static final ImmutableMap<String, TypeDeducer> DEDUCERS = ImmutableMap.<String, TypeDeducer>builder()
.put("named_struct", new NamedStructDeducer())
.put("struct_element", new StructElementDeducer())
.put("array_contains", new ArrayElemFuncDeducer())
.put("array_pushback", new ArrayElemFuncDeducer())
.put("element_at", new ArrayElemFuncDeducer())
.build();

public static Type deduce(String fnName, Type[] args) {
public static Type deduce(String fnName, int argIdx, Type[] args) {
if (DEDUCERS.containsKey(fnName)) {
return DEDUCERS.get(fnName).deduce(args);
return DEDUCERS.get(fnName).deduce(argIdx, args);
}
return null;
}

public static class ArrayElemFuncDeducer implements TypeDeducer {
@Override
public Type deduce(int argIdx, Type[] args) {
if (args.length >= 2) {
if (argIdx == 0) {
// first args should only to be array or null
return args[0] instanceof ArrayType || args[0].isNull() ? new ArrayType(args[1]) : args[0];
} else if (args[0].isNull()) {
// first arg is null, later element is not contains
return args[argIdx];
} else if (args[0] instanceof ArrayType
&& Type.isImplicitlyCastable(args[argIdx], ((ArrayType) args[0]).getItemType(), false)) {
return args[argIdx];
} else {
return null;
}
}
return null;
}
}

public static class NamedStructDeducer implements TypeDeducer {
@Override
public Type deduce(Type[] args) {
public Type deduce(int argIdx, Type[] args) {
List<Type> evenArgs = Lists.newArrayList();
for (int i = 0; i < args.length; i++) {
if ((i & 1) == 1) {
Expand All @@ -55,7 +78,7 @@ public Type deduce(Type[] args) {

public static class StructElementDeducer implements TypeDeducer {
@Override
public Type deduce(Type[] args) {
public Type deduce(int argIdx, Type[] args) {
if (args[0] instanceof StructType) {
return Type.ANY_ELEMENT_TYPE;
}
Expand Down
Loading

0 comments on commit 3befd30

Please sign in to comment.