From 2636930600971763d1d9e5c9a923e5d8847c68f2 Mon Sep 17 00:00:00 2001 From: morrySnow <101034200+morrySnow@users.noreply.github.com> Date: Thu, 14 Mar 2024 14:57:32 +0800 Subject: [PATCH] [feature](Nereids) support agg state type in create table (#32171) this PR introduce a behavior change, syntax of create table with agg_state type is changed. --- .../sql-reference/Data-Types/AGG_STATE.md | 4 +- .../sql-reference/Data-Types/AGG_STATE.md | 4 +- .../apache/doris/catalog/AggStateType.java | 4 +- .../org/apache/doris/nereids/DorisLexer.g4 | 1 + .../org/apache/doris/nereids/DorisParser.g4 | 15 ++++- fe/fe-core/src/main/cup/sql_parser.cup | 33 ++++------ .../org/apache/doris/analysis/ColumnDef.java | 4 +- .../analysis/CreateMaterializedViewStmt.java | 2 +- .../apache/doris/catalog/AggregateType.java | 2 +- .../java/org/apache/doris/catalog/Column.java | 35 ++-------- .../nereids/parser/LogicalPlanBuilder.java | 34 +++++++++- .../SelectMaterializedIndexWithAggregate.java | 2 +- .../plans/commands/info/ColumnDefinition.java | 64 ++++++++++--------- .../doris/nereids/types/AggStateType.java | 7 -- fe/fe-core/src/main/jflex/sql_scanner.flex | 1 + .../test_vertical_compaction_agg_state.groovy | 2 +- .../agg_state/avg/test_agg_state_avg.groovy | 2 +- .../test_agg_state_group_concat.groovy | 2 +- .../agg_state/max/test_agg_state_max.groovy | 4 +- .../nereids/test_agg_state_nereids.groovy | 2 +- .../test_agg_state_quantile_union.groovy | 2 +- .../agg_state/test_agg_state.groovy | 2 +- .../suites/mv_p0/dis_26495/dis_26495.groovy | 2 +- .../statistics/test_agg_complex_type.groovy | 2 +- 24 files changed, 121 insertions(+), 111 deletions(-) diff --git a/docs/en/docs/sql-manual/sql-reference/Data-Types/AGG_STATE.md b/docs/en/docs/sql-manual/sql-reference/Data-Types/AGG_STATE.md index 41535d923b7d8b..9dea25a52a5e04 100644 --- a/docs/en/docs/sql-manual/sql-reference/Data-Types/AGG_STATE.md +++ b/docs/en/docs/sql-manual/sql-reference/Data-Types/AGG_STATE.md @@ -41,8 +41,8 @@ Create table example: ```sql create table a_table( k1 int null, - k2 agg_state max_by(int not null,int), - k3 agg_state group_concat(string) + k2 agg_state generic, + k3 agg_state generic ) aggregate key (k1) distributed BY hash(k1) buckets 3 diff --git a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Types/AGG_STATE.md b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Types/AGG_STATE.md index 18dc95c324915e..aa459dd7483761 100644 --- a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Types/AGG_STATE.md +++ b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Types/AGG_STATE.md @@ -41,8 +41,8 @@ under the License. ```sql create table a_table( k1 int null, - k2 agg_state max_by(int not null,int), - k3 agg_state group_concat(string) + k2 agg_state generic, + k3 agg_state generic ) aggregate key (k1) distributed BY hash(k1) buckets 3 diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/AggStateType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/AggStateType.java index 35301fa704e5e6..72f93a2f696805 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/AggStateType.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/AggStateType.java @@ -72,7 +72,7 @@ public boolean getResultIsNullable() { @Override public String toSql(int depth) { StringBuilder stringBuilder = new StringBuilder(); - stringBuilder.append("AGG_STATE("); + stringBuilder.append("AGG_STATE<").append(functionName).append("("); for (int i = 0; i < subTypes.size(); i++) { if (i > 0) { stringBuilder.append(", "); @@ -82,7 +82,7 @@ public String toSql(int depth) { stringBuilder.append(" NULL"); } } - stringBuilder.append(")"); + stringBuilder.append(")>"); return stringBuilder.toString(); } diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 index 02304ee30c6ac1..256888c2fc1409 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 @@ -271,6 +271,7 @@ FRONTENDS: 'FRONTENDS'; FULL: 'FULL'; FUNCTION: 'FUNCTION'; FUNCTIONS: 'FUNCTIONS'; +GENERIC: 'GENERIC'; GLOBAL: 'GLOBAL'; GRANT: 'GRANT'; GRANTS: 'GRANTS'; diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index 133eea3ed7efad..e0a306196713e2 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -530,7 +530,10 @@ columnDefs columnDef : colName=identifier type=dataType - KEY? (aggType=aggTypeDef)? ((NOT NULL) | NULL)? (AUTO_INCREMENT (LEFT_PAREN autoIncInitValue=number RIGHT_PAREN)?)? + KEY? + (aggType=aggTypeDef)? + ((NOT)? NULL)? + (AUTO_INCREMENT (LEFT_PAREN autoIncInitValue=number RIGHT_PAREN)?)? (DEFAULT (nullValue=NULL | INTEGER_VALUE | stringValue=STRING_LITERAL | CURRENT_TIMESTAMP (LEFT_PAREN defaultValuePrecision=number RIGHT_PAREN)?))? (ON UPDATE CURRENT_TIMESTAMP (LEFT_PAREN onUpdateValuePrecision=number RIGHT_PAREN)?)? @@ -587,7 +590,7 @@ rollupDef ; aggTypeDef - : MAX | MIN | SUM | REPLACE | REPLACE_IF_NOT_NULL | HLL_UNION | BITMAP_UNION | QUANTILE_UNION + : MAX | MIN | SUM | REPLACE | REPLACE_IF_NOT_NULL | HLL_UNION | BITMAP_UNION | QUANTILE_UNION | GENERIC ; tabletList @@ -846,10 +849,17 @@ unitIdentifier : YEAR | MONTH | WEEK | DAY | HOUR | MINUTE | SECOND ; +dataTypeWithNullable + : dataType ((NOT)? NULL)? + ; + dataType : complex=ARRAY LT dataType GT #complexDataType | complex=MAP LT dataType COMMA dataType GT #complexDataType | complex=STRUCT LT complexColTypeList GT #complexDataType + | AGG_STATE LT functionNameIdentifier + LEFT_PAREN dataTypes+=dataTypeWithNullable + (COMMA dataTypes+=dataTypeWithNullable)* RIGHT_PAREN GT #aggStateDataType | primitiveColType (LEFT_PAREN (INTEGER_VALUE | ASTERISK) (COMMA INTEGER_VALUE)* RIGHT_PAREN)? #primitiveDataType ; @@ -1061,6 +1071,7 @@ nonReserved | FREE | FRONTENDS | FUNCTION + | GENERIC | GLOBAL | GRAPH | GROUPING diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup index 23efc77cc05fa7..0770ac73da7675 100644 --- a/fe/fe-core/src/main/cup/sql_parser.cup +++ b/fe/fe-core/src/main/cup/sql_parser.cup @@ -402,6 +402,7 @@ terminal String KW_FULL, KW_FUNCTION, KW_FUNCTIONS, + KW_GENERIC, KW_GLOBAL, KW_GRANT, KW_GRANTS, @@ -3157,6 +3158,10 @@ opt_agg_type ::= {: RESULT = AggregateType.QUANTILE_UNION; :} + | KW_GENERIC + {: + RESULT = AggregateType.GENERIC; + :} ; opt_partition ::= @@ -3756,31 +3761,11 @@ column_definition ::= ColumnDef columnDef = new ColumnDef(columnName, typeDef, isKey, null, isAllowNull, autoIncInitValue, defaultValue, comment); RESULT = columnDef; :} - | ident:columnName KW_AGG_STATE IDENT:fnName LPAREN type_def_nullable_list:list RPAREN opt_auto_inc_init_value:autoIncInitValue opt_default_value:defaultValue opt_comment:comment - {: - for (TypeDef def : list) { - def.analyze(null); - } - ColumnDef columnDef = new ColumnDef(columnName, new TypeDef(Expr.createAggStateType(fnName, - list.stream().map(TypeDef::getType).collect(Collectors.toList()), - list.stream().map(TypeDef::getNullable).collect(Collectors.toList()))), false, AggregateType.GENERIC_AGGREGATION, false, defaultValue, comment); - RESULT = columnDef; - :} | ident:columnName type_def:typeDef opt_is_key:isKey opt_agg_type:aggType opt_is_allow_null:isAllowNull opt_auto_inc_init_value:autoIncInitValue opt_default_value:defaultValue opt_comment:comment {: ColumnDef columnDef = new ColumnDef(columnName, typeDef, isKey, aggType, isAllowNull, autoIncInitValue, defaultValue, comment); RESULT = columnDef; :} - | ident:columnName KW_AGG_STATE opt_is_key:isKey opt_agg_type:aggType LPAREN type_def_nullable_list:list RPAREN opt_default_value:defaultValue opt_comment:comment - {: - for (TypeDef def : list) { - def.analyze(null); - } - ColumnDef columnDef = new ColumnDef(columnName, new TypeDef(Expr.createAggStateType(aggType.name().toLowerCase(), - list.stream().map(TypeDef::getType).collect(Collectors.toList()), - list.stream().map(TypeDef::getNullable).collect(Collectors.toList()))), isKey, AggregateType.GENERIC_AGGREGATION, false, defaultValue, comment); - RESULT = columnDef; - :} ; index_definition ::= @@ -6599,6 +6584,12 @@ type ::= {: ScalarType type = ScalarType.createHllType(); RESULT = type; :} + | KW_AGG_STATE LESSTHAN IDENT:fnName LPAREN type_def_nullable_list:list RPAREN GREATERTHAN + {: + RESULT = Expr.createAggStateType(fnName, + list.stream().map(TypeDef::getType).collect(Collectors.toList()), + list.stream().map(TypeDef::getNullable).collect(Collectors.toList())); + :} | KW_ALL {: RESULT = Type.ALL; :} ; @@ -7828,6 +7819,8 @@ keyword ::= {: RESULT = id; :} | KW_GLOBAL:id {: RESULT = id; :} + | KW_GENERIC:id + {: RESULT = id; :} | KW_GRAPH:id {: RESULT = id; :} | KW_HASH:id diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java index 59e5be4725b5fb..d202467dfb37ba 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java @@ -332,12 +332,12 @@ public void analyze(boolean isOlap) throws AnalysisException { } // check if aggregate type is valid - if (aggregateType != AggregateType.GENERIC_AGGREGATION + if (aggregateType != AggregateType.GENERIC && !aggregateType.checkCompatibility(type.getPrimitiveType())) { throw new AnalysisException(String.format("Aggregate type %s is not compatible with primitive type %s", toString(), type.toSql())); } - if (aggregateType == AggregateType.GENERIC_AGGREGATION) { + if (aggregateType == AggregateType.GENERIC) { if (!SessionVariable.enableAggState()) { throw new AnalysisException("agg state not enable, need set enable_agg_state=true"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java index 5686877a751d1e..f95b83ae404695 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java @@ -549,7 +549,7 @@ private MVColumnItem buildMVColumnItem(Analyzer analyzer, FunctionCallExpr funct type = Type.BIGINT; break; default: - mvAggregateType = AggregateType.GENERIC_AGGREGATION; + mvAggregateType = AggregateType.GENERIC; if (functionCallExpr.getParams().isDistinct() || functionCallExpr.getParams().isStar()) { throw new AnalysisException( "The Materialized-View's generic aggregation not support star or distinct"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/AggregateType.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/AggregateType.java index 47ee91c0b5ca02..94a788f0b9afe0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/AggregateType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/AggregateType.java @@ -37,7 +37,7 @@ public enum AggregateType { NONE("NONE"), BITMAP_UNION("BITMAP_UNION"), QUANTILE_UNION("QUANTILE_UNION"), - GENERIC_AGGREGATION("GENERIC_AGGREGATION"); + GENERIC("GENERIC"); private static EnumMap> compatibilityMap; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java index 748f6f7b1beefa..82bac846d7d9d3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java @@ -133,9 +133,6 @@ public class Column implements Writable, GsonPostProcessable { @SerializedName(value = "uniqueId") private int uniqueId; - @SerializedName(value = "genericAggregationName") - private String genericAggregationName; - @SerializedName(value = "clusterKeyId") private int clusterKeyId = -1; @@ -249,8 +246,8 @@ public Column(String name, Type type, boolean isKey, AggregateType aggregateType c.setIsAllowNull(aggState.getSubTypeNullables().get(i)); addChildrenColumn(c); } - this.genericAggregationName = aggState.getFunctionName(); - this.aggregationType = AggregateType.GENERIC_AGGREGATION; + this.isAllowNull = false; + this.aggregationType = AggregateType.GENERIC; } } @@ -454,11 +451,7 @@ public AggregateType getAggregationType() { } public String getAggregationString() { - if (getAggregationType() == AggregateType.GENERIC_AGGREGATION) { - return getGenericAggregationString(); - } else { - return getAggregationType().name(); - } + return getAggregationType().name(); } public boolean isAggregated() { @@ -915,22 +908,6 @@ public String toSql(boolean isUniqueTable) { return toSql(isUniqueTable, false); } - public String getGenericAggregationString() { - StringBuilder sb = new StringBuilder(); - sb.append(genericAggregationName).append("("); - for (int i = 0; i < children.size(); i++) { - if (i != 0) { - sb.append(", "); - } - sb.append(children.get(i).getType().toSql()); - if (children.get(i).isAllowNull()) { - sb.append(" NULL"); - } - } - sb.append(")"); - return sb.toString(); - } - public String toSql(boolean isUniqueTable, boolean isCompatible) { StringBuilder sb = new StringBuilder(); sb.append("`").append(name).append("` "); @@ -942,11 +919,9 @@ public String toSql(boolean isUniqueTable, boolean isCompatible) { } else { sb.append(typeStr); } - if (aggregationType == AggregateType.GENERIC_AGGREGATION) { - sb.append(" ").append(getGenericAggregationString()); - } else if (aggregationType != null && aggregationType != AggregateType.NONE && !isUniqueTable + if (aggregationType != null && aggregationType != AggregateType.NONE && !isUniqueTable && !isAggregationTypeImplicit) { - sb.append(" ").append(aggregationType.name()); + sb.append(" ").append(aggregationType.toSql()); } if (isAllowNull) { sb.append(" NULL"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 90a9b63f4f533f..1e2f8d2f23089b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -23,6 +23,7 @@ import org.apache.doris.analysis.TableName; import org.apache.doris.analysis.UserIdentity; import org.apache.doris.catalog.AggregateType; +import org.apache.doris.catalog.BuiltinAggregateFunctions; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.ScalarType; @@ -42,6 +43,7 @@ import org.apache.doris.nereids.DorisParser; import org.apache.doris.nereids.DorisParser.AddConstraintContext; import org.apache.doris.nereids.DorisParser.AggClauseContext; +import org.apache.doris.nereids.DorisParser.AggStateDataTypeContext; import org.apache.doris.nereids.DorisParser.AliasQueryContext; import org.apache.doris.nereids.DorisParser.AliasedQueryContext; import org.apache.doris.nereids.DorisParser.AlterMTMVContext; @@ -75,6 +77,7 @@ import org.apache.doris.nereids.DorisParser.CreateRowPolicyContext; import org.apache.doris.nereids.DorisParser.CreateTableContext; import org.apache.doris.nereids.DorisParser.CteContext; +import org.apache.doris.nereids.DorisParser.DataTypeWithNullableContext; import org.apache.doris.nereids.DorisParser.DateCeilContext; import org.apache.doris.nereids.DorisParser.DateFloorContext; import org.apache.doris.nereids.DorisParser.Date_addContext; @@ -422,6 +425,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias; import org.apache.doris.nereids.trees.plans.logical.LogicalUnion; import org.apache.doris.nereids.trees.plans.logical.UsingJoin; +import org.apache.doris.nereids.types.AggStateType; import org.apache.doris.nereids.types.ArrayType; import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.MapType; @@ -2523,7 +2527,9 @@ public ColumnDefinition visitColumnDef(ColumnDefContext ctx) { String colName = ctx.colName.getText(); DataType colType = ctx.type instanceof PrimitiveDataTypeContext ? visitPrimitiveDataType(((PrimitiveDataTypeContext) ctx.type)) - : visitComplexDataType(((ComplexDataTypeContext) ctx.type)); + : ctx.type instanceof ComplexDataTypeContext + ? visitComplexDataType((ComplexDataTypeContext) ctx.type) + : visitAggStateDataType((AggStateDataTypeContext) ctx.type); colType = colType.conversion(); boolean isKey = ctx.KEY() != null; boolean isNotNull = ctx.NOT() != null; @@ -3256,6 +3262,32 @@ private ExplainLevel parseExplainPlanType(PlanTypeContext planTypeContext) { return ExplainLevel.ALL_PLAN; } + @Override + public Pair visitDataTypeWithNullable(DataTypeWithNullableContext ctx) { + return ParserUtils.withOrigin(ctx, () -> Pair.of(typedVisit(ctx.dataType()), ctx.NOT() == null)); + } + + @Override + public DataType visitAggStateDataType(AggStateDataTypeContext ctx) { + return ParserUtils.withOrigin(ctx, () -> { + List> dataTypeWithNullables = ctx.dataTypes.stream() + .map(this::visitDataTypeWithNullable) + .collect(Collectors.toList()); + List dataTypes = dataTypeWithNullables.stream() + .map(dt -> dt.first) + .collect(ImmutableList.toImmutableList()); + List nullables = dataTypeWithNullables.stream() + .map(dt -> dt.second) + .collect(ImmutableList.toImmutableList()); + String functionName = ctx.functionNameIdentifier().getText(); + if (!BuiltinAggregateFunctions.INSTANCE.aggFuncNames.contains(functionName)) { + // TODO use function binder to check function exists + throw new ParseException("Can not found function '" + functionName + "'", ctx); + } + return new AggStateType(functionName, dataTypes, nullables); + }); + } + @Override public DataType visitPrimitiveDataType(PrimitiveDataTypeContext ctx) { return ParserUtils.withOrigin(ctx, () -> { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java index 3857a04ba1c7a9..ea09b25ba6f493 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java @@ -1487,7 +1487,7 @@ public Expression visitSum(Sum sum, RewriteContext context) { @Override public Expression visitAggregateFunction(AggregateFunction aggregateFunction, RewriteContext context) { String aggStateName = normalizeName(CreateMaterializedViewStmt.mvColumnBuilder( - AggregateType.GENERIC_AGGREGATION, StateCombinator.create(aggregateFunction).toSql())); + AggregateType.GENERIC, StateCombinator.create(aggregateFunction).toSql())); Column mvColumn = context.checkContext.getColumn(aggStateName); if (mvColumn != null && context.checkContext.valueNameToColumn.containsValue(mvColumn)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java index eac17f05d370f6..9360f2d608a482 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java @@ -246,6 +246,20 @@ public void validate(boolean isOlap, Set keysSet, boolean isEnableMergeO } } + if (aggType != null) { + // check if aggregate type is valid + if (aggType != AggregateType.GENERIC + && !aggType.checkCompatibility(type.toCatalogDataType().getPrimitiveType())) { + throw new AnalysisException(String.format("Aggregate type %s is not compatible with primitive type %s", + aggType, type.toSql())); + } + if (aggType == AggregateType.GENERIC) { + if (!SessionVariable.enableAggState()) { + throw new AnalysisException("agg state not enable, need set enable_agg_state=true"); + } + } + } + if (isOlap) { if (!isKey && keysType.equals(KeysType.UNIQUE_KEYS)) { aggTypeImplicit = true; @@ -334,14 +348,28 @@ public void validate(boolean isOlap, Set keysSet, boolean isEnableMergeO // from old planner CreateTableStmt's analyze method, after call columnDef.analyze(engineName.equals("olap")); if (isOlap && type.isComplexType()) { - if (aggType != null && aggType != AggregateType.NONE - && aggType != AggregateType.REPLACE) { - throw new AnalysisException(type.toCatalogDataType().getPrimitiveType() - + " column can't support aggregation " + aggType); - } if (isKey) { throw new AnalysisException(type.toCatalogDataType().getPrimitiveType() - + " can only be used in the non-key column of the duplicate table at present."); + + " can only be used in the non-key column at present."); + } + if (type.isAggStateType()) { + if (aggType == null) { + throw new AnalysisException(type.toCatalogDataType().getPrimitiveType() + + " column must have aggregation type"); + } else { + if (aggType != AggregateType.GENERIC + && aggType != AggregateType.NONE + && aggType != AggregateType.REPLACE) { + throw new AnalysisException(type.toCatalogDataType().getPrimitiveType() + + " column can't support aggregation " + aggType); + } + } + isNullable = false; + } else { + if (aggType != null && aggType != AggregateType.NONE && aggType != AggregateType.REPLACE) { + throw new AnalysisException(type.toCatalogDataType().getPrimitiveType() + + " column can't support aggregation " + aggType); + } } } @@ -350,30 +378,6 @@ public void validate(boolean isOlap, Set keysSet, boolean isEnableMergeO } } - /** - * check if is nested complex type. - */ - private boolean isNestedComplexType(DataType dataType) { - if (!dataType.isComplexType()) { - return false; - } - if (dataType instanceof ArrayType) { - if (((ArrayType) dataType).getItemType() instanceof ArrayType) { - return isNestedComplexType(((ArrayType) dataType).getItemType()); - } else { - return ((ArrayType) dataType).getItemType().isComplexType(); - } - } - if (dataType instanceof MapType) { - return ((MapType) dataType).getKeyType().isComplexType() - || ((MapType) dataType).getValueType().isComplexType(); - } - if (dataType instanceof StructType) { - return ((StructType) dataType).getFields().stream().anyMatch(f -> f.getDataType().isComplexType()); - } - return false; - } - // from TypeDef.java analyze() private void validateDataType(Type catalogType) { if (catalogType.exceedsMaxNestingDepth()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/AggStateType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/AggStateType.java index 6680a6ccee0827..e7d8dbdfe3dad4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/AggStateType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/AggStateType.java @@ -35,8 +35,6 @@ */ public class AggStateType extends DataType { - public static final AggStateType SYSTEM_DEFAULT = new AggStateType(null, ImmutableList.of(), ImmutableList.of()); - public static final int WIDTH = 16; private final List subTypes; @@ -94,11 +92,6 @@ public String simpleString() { return "agg_state"; } - @Override - public DataType defaultConcreteType() { - return SYSTEM_DEFAULT; - } - @Override public boolean equals(Object o) { if (!(o instanceof AggStateType)) { diff --git a/fe/fe-core/src/main/jflex/sql_scanner.flex b/fe/fe-core/src/main/jflex/sql_scanner.flex index 175744532c033f..43dd60a4ce1fb9 100644 --- a/fe/fe-core/src/main/jflex/sql_scanner.flex +++ b/fe/fe-core/src/main/jflex/sql_scanner.flex @@ -251,6 +251,7 @@ import org.apache.doris.qe.SqlModeHelper; keywordMap.put("function", new Integer(SqlParserSymbols.KW_FUNCTION)); keywordMap.put("functions", new Integer(SqlParserSymbols.KW_FUNCTIONS)); keywordMap.put("type_cast", new Integer(SqlParserSymbols.KW_TYPECAST)); + keywordMap.put("generic", new Integer(SqlParserSymbols.KW_GENERIC)); keywordMap.put("global", new Integer(SqlParserSymbols.KW_GLOBAL)); keywordMap.put("grant", new Integer(SqlParserSymbols.KW_GRANT)); keywordMap.put("grants", new Integer(SqlParserSymbols.KW_GRANTS)); diff --git a/regression-test/suites/compaction/test_vertical_compaction_agg_state.groovy b/regression-test/suites/compaction/test_vertical_compaction_agg_state.groovy index 9e0f99dd0c35b5..209f816785ca0a 100644 --- a/regression-test/suites/compaction/test_vertical_compaction_agg_state.groovy +++ b/regression-test/suites/compaction/test_vertical_compaction_agg_state.groovy @@ -46,7 +46,7 @@ suite("test_vertical_compaction_agg_state") { sql """ CREATE TABLE IF NOT EXISTS ${tableName} ( user_id VARCHAR, - agg_user_id agg_state collect_set(string) + agg_user_id agg_state generic )ENGINE=OLAP AGGREGATE KEY(`user_id`) COMMENT 'OLAP' diff --git a/regression-test/suites/datatype_p0/agg_state/avg/test_agg_state_avg.groovy b/regression-test/suites/datatype_p0/agg_state/avg/test_agg_state_avg.groovy index 977f855f52da55..fb86b4428f0c7d 100644 --- a/regression-test/suites/datatype_p0/agg_state/avg/test_agg_state_avg.groovy +++ b/regression-test/suites/datatype_p0/agg_state/avg/test_agg_state_avg.groovy @@ -21,7 +21,7 @@ suite("test_agg_state_avg") { sql """ create table a_table( k1 int not null, - k2 agg_state avg(int not null) + k2 agg_state generic ) aggregate key (k1) distributed BY hash(k1) diff --git a/regression-test/suites/datatype_p0/agg_state/group_concat/test_agg_state_group_concat.groovy b/regression-test/suites/datatype_p0/agg_state/group_concat/test_agg_state_group_concat.groovy index fed7d075af5fcf..f17e44a1596114 100644 --- a/regression-test/suites/datatype_p0/agg_state/group_concat/test_agg_state_group_concat.groovy +++ b/regression-test/suites/datatype_p0/agg_state/group_concat/test_agg_state_group_concat.groovy @@ -21,7 +21,7 @@ suite("test_agg_state_group_concat") { sql """ create table a_table( k1 int null, - k2 agg_state group_concat(string) + k2 agg_state generic ) aggregate key (k1) distributed BY hash(k1) buckets 3 diff --git a/regression-test/suites/datatype_p0/agg_state/max/test_agg_state_max.groovy b/regression-test/suites/datatype_p0/agg_state/max/test_agg_state_max.groovy index 45b6e5463e9192..4bcc98a8b22baf 100644 --- a/regression-test/suites/datatype_p0/agg_state/max/test_agg_state_max.groovy +++ b/regression-test/suites/datatype_p0/agg_state/max/test_agg_state_max.groovy @@ -21,7 +21,7 @@ suite("test_agg_state_max") { sql """ create table a_table( k1 int not null, - k2 agg_state max(int not null) + k2 agg_state generic ) aggregate key (k1) distributed BY hash(k1) @@ -60,7 +60,7 @@ suite("test_agg_state_max") { sql """ create table a_table2( k1 int not null, - k2 agg_state max(int null) + k2 agg_state generic ) aggregate key (k1) distributed BY hash(k1) diff --git a/regression-test/suites/datatype_p0/agg_state/nereids/test_agg_state_nereids.groovy b/regression-test/suites/datatype_p0/agg_state/nereids/test_agg_state_nereids.groovy index 3adbfe9e43e968..0d9115664a53c1 100644 --- a/regression-test/suites/datatype_p0/agg_state/nereids/test_agg_state_nereids.groovy +++ b/regression-test/suites/datatype_p0/agg_state/nereids/test_agg_state_nereids.groovy @@ -48,7 +48,7 @@ suite("test_agg_state_nereids") { sql """ create table a_table( k1 int null, - k2 agg_state max_by(int not null,int) + k2 agg_state generic ) aggregate key (k1) distributed BY hash(k1) buckets 3 diff --git a/regression-test/suites/datatype_p0/agg_state/quantile_union/test_agg_state_quantile_union.groovy b/regression-test/suites/datatype_p0/agg_state/quantile_union/test_agg_state_quantile_union.groovy index 92f9f80a38dbd5..41d40ce8f49b03 100644 --- a/regression-test/suites/datatype_p0/agg_state/quantile_union/test_agg_state_quantile_union.groovy +++ b/regression-test/suites/datatype_p0/agg_state/quantile_union/test_agg_state_quantile_union.groovy @@ -21,7 +21,7 @@ suite("test_agg_state_quantile_union") { sql """ create table a_table( k1 int not null, - k2 agg_state quantile_union(quantile_state not null) + k2 agg_state generic ) aggregate key (k1) distributed BY hash(k1) diff --git a/regression-test/suites/datatype_p0/agg_state/test_agg_state.groovy b/regression-test/suites/datatype_p0/agg_state/test_agg_state.groovy index 19443191588b1e..78edeaaa340909 100644 --- a/regression-test/suites/datatype_p0/agg_state/test_agg_state.groovy +++ b/regression-test/suites/datatype_p0/agg_state/test_agg_state.groovy @@ -45,7 +45,7 @@ suite("test_agg_state") { sql """ create table a_table( k1 int null, - k2 agg_state max_by(int not null,int) + k2 agg_state generic ) aggregate key (k1) distributed BY hash(k1) buckets 3 diff --git a/regression-test/suites/mv_p0/dis_26495/dis_26495.groovy b/regression-test/suites/mv_p0/dis_26495/dis_26495.groovy index 4cc3842b9914f4..e30af70b0041b3 100644 --- a/regression-test/suites/mv_p0/dis_26495/dis_26495.groovy +++ b/regression-test/suites/mv_p0/dis_26495/dis_26495.groovy @@ -22,7 +22,7 @@ suite ("dis_26495") { sql """ DROP TABLE IF EXISTS doris_test; """ sql """ - create table doris_test (a int,b int, agg_st_1 agg_state max_by(int ,int)) + create table doris_test (a int,b int, agg_st_1 agg_state generic) DISTRIBUTED BY HASH(a) BUCKETS 1 properties("replication_num" = "1"); """ diff --git a/regression-test/suites/statistics/test_agg_complex_type.groovy b/regression-test/suites/statistics/test_agg_complex_type.groovy index 0f10234ae6ac0a..43cdeff4776533 100644 --- a/regression-test/suites/statistics/test_agg_complex_type.groovy +++ b/regression-test/suites/statistics/test_agg_complex_type.groovy @@ -24,7 +24,7 @@ suite("test_analyze_with_agg_complex_type") { device_id bitmap BITMAP_UNION NULL, hll_test hll hll_union, qs QUANTILE_STATE QUANTILE_UNION, - agg_st_1 agg_state max_by(int ,int) + agg_st_1 agg_state GENERIC ) aggregate key (datekey) distributed by hash(datekey) buckets 1