From 95ce8c05e75d6dd16f65042ba08ff57e5a395d96 Mon Sep 17 00:00:00 2001 From: minghong Date: Tue, 17 Dec 2024 19:10:21 +0800 Subject: [PATCH] tmp --- .../apache/doris/nereids/NereidsPlanner.java | 16 ++- .../nereids/commonCTE/ConsumerRewriter.java | 37 +++++++ .../commonCTE/ConsumerRewriterContext.java | 22 +++++ .../doris/nereids/commonCTE/CteExtractor.java | 97 ++++++++++++++++--- .../nereids/commonCTE/SignatureVisitor.java | 15 +-- .../rules/analysis/CheckAfterRewrite.java | 10 ++ .../plans/logical/LogicalCTEConsumer.java | 3 +- .../trees/plans/logical/LogicalOlapScan.java | 2 +- .../org/apache/doris/qe/SessionVariable.java | 7 ++ .../data/query_p0/common_cte/common_cte.out | 17 ++++ .../query_p0/common_cte/common_cte.groovy | 25 +++++ .../query_p0/common_cte/ddl/lineitem.sql | 18 ++++ .../suites/query_p0/common_cte/ddl/nation.sql | 12 +++ .../suites/query_p0/common_cte/ddl/part.sql | 11 +++ .../query_p0/common_cte/ddl/partsupp.sql | 7 ++ .../suites/query_p0/common_cte/ddl/region.sql | 10 ++ .../query_p0/common_cte/ddl/supplier.sql | 9 ++ .../query_p0/common_cte/ddl/workers.sql | 10 ++ .../suites/query_p0/common_cte/load.groovy | 63 ++++++++++++ 19 files changed, 364 insertions(+), 27 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/commonCTE/ConsumerRewriter.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/commonCTE/ConsumerRewriterContext.java create mode 100644 regression-test/data/query_p0/common_cte/common_cte.out create mode 100644 regression-test/suites/query_p0/common_cte/common_cte.groovy create mode 100644 regression-test/suites/query_p0/common_cte/ddl/lineitem.sql create mode 100644 regression-test/suites/query_p0/common_cte/ddl/nation.sql create mode 100644 regression-test/suites/query_p0/common_cte/ddl/part.sql create mode 100644 regression-test/suites/query_p0/common_cte/ddl/partsupp.sql create mode 100644 regression-test/suites/query_p0/common_cte/ddl/region.sql create mode 100644 regression-test/suites/query_p0/common_cte/ddl/supplier.sql create mode 100644 regression-test/suites/query_p0/common_cte/ddl/workers.sql create mode 100644 regression-test/suites/query_p0/common_cte/load.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java index c0ab8aa6d53d7a..fc8758125eb766 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java @@ -117,6 +117,8 @@ public class NereidsPlanner extends Planner { private double cost = 0; private LogicalPlanAdapter logicalPlanAdapter; + private CteExtractor cteExtractor; + public NereidsPlanner(StatementContext statementContext) { this.statementContext = statementContext; } @@ -264,6 +266,9 @@ protected Plan planWithoutLock( statementContext.getConnectContext().getExecutor().getSummaryProfile().setNereidsRewriteTime(); } + if (ConnectContext.get() != null && ConnectContext.get().getSessionVariable().enableCommonCte) { + extractCommonCTE(); + } if (explainLevel == ExplainLevel.REWRITTEN_PLAN || explainLevel == ExplainLevel.ALL_PLAN) { rewrittenPlan = cascadesContext.getRewritePlan(); if (explainLevel == ExplainLevel.REWRITTEN_PLAN) { @@ -286,8 +291,6 @@ protected Plan planWithoutLock( setRuntimeFilterWaitTimeByTableRowCountAndType(); - extractCommonCTE(); - optimize(); if (statementContext.getConnectContext().getExecutor() != null) { statementContext.getConnectContext().getExecutor().getSummaryProfile().setNereidsOptimizeTime(); @@ -325,8 +328,12 @@ protected LogicalPlan preprocess(LogicalPlan logicalPlan) { } private void extractCommonCTE() { - CteExtractor commonCTE = new CteExtractor((AbstractLogicalPlan) cascadesContext.getRewritePlan()); - commonCTE.execute(); + CteExtractor commonCTE = new CteExtractor( + (AbstractLogicalPlan) cascadesContext.getRewritePlan(), + cascadesContext); + List plans = commonCTE.execute(); + cascadesContext.setRewritePlan(plans.get(0)); + cteExtractor = commonCTE; } /** @@ -666,6 +673,7 @@ public String getExplainString(ExplainOptions explainOptions) { break; case REWRITTEN_PLAN: plan = rewrittenPlan.treeString(); + plan += "\n common cte info \n" + cteExtractor; break; case OPTIMIZED_PLAN: plan = "cost = " + cost + "\n" + optimizedPlan.treeString() + mvSummary; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/commonCTE/ConsumerRewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/commonCTE/ConsumerRewriter.java new file mode 100644 index 00000000000000..a06c46e3c5bbe3 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/commonCTE/ConsumerRewriter.java @@ -0,0 +1,37 @@ +package org.apache.doris.nereids.commonCTE; + +import org.apache.doris.nereids.trees.expressions.CTEId; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.AbstractLogicalPlan; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMultimap; + +import java.util.ArrayList; +import java.util.List; + +public class ConsumerRewriter extends DefaultPlanRewriter { + public static final ConsumerRewriter INSTANCE = new ConsumerRewriter(); + @Override + public Plan visit(Plan plan, ConsumerRewriterContext context) { + if (context.isTarget(plan)) { + context.producer.getOutput(); + plan.getOutput(); + ImmutableMap.Builder cToPBuilder = ImmutableMap.builder(); + ImmutableMultimap.Builder pToCBuilder = ImmutableMultimap.builder(); + for (int i = 0; i< context.producer.getOutput().size(); i++) { + cToPBuilder.put(plan.getOutput().get(i), context.producer.getOutput().get(i)); + pToCBuilder.put(context.producer.getOutput().get(i), plan.getOutput().get(i)); + } + return new LogicalCTEConsumer(StatementScopeIdGenerator.newRelationId(), context.cteId, + "comm_cte_" + context.cteId.asInt(), cToPBuilder.build(), pToCBuilder.build()); + } + return visitChildren(this, plan, context); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/commonCTE/ConsumerRewriterContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/commonCTE/ConsumerRewriterContext.java new file mode 100644 index 00000000000000..be66e7d30f6e3c --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/commonCTE/ConsumerRewriterContext.java @@ -0,0 +1,22 @@ +package org.apache.doris.nereids.commonCTE; + +import org.apache.doris.nereids.trees.expressions.CTEId; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.AbstractLogicalPlan; + +import java.util.List; + +public class ConsumerRewriterContext { + private final List targets; + public Plan producer; + public final CTEId cteId; + + public ConsumerRewriterContext(List targets, Plan producer, CTEId cteId) { + this.targets = targets; + this.producer = producer; + this.cteId = cteId; + } + public boolean isTarget(Plan plan) { + return targets.contains(plan); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/commonCTE/CteExtractor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/commonCTE/CteExtractor.java index ab907b45626784..480c5038f3e12f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/commonCTE/CteExtractor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/commonCTE/CteExtractor.java @@ -1,43 +1,112 @@ package org.apache.doris.nereids.commonCTE; +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.rules.analysis.CheckAfterRewrite; +import org.apache.doris.nereids.rules.exploration.mv.ComparisonResult; +import org.apache.doris.nereids.rules.exploration.mv.LogicalCompatibilityContext; +import org.apache.doris.nereids.rules.exploration.mv.MaterializationContext; +import org.apache.doris.nereids.rules.exploration.mv.StructInfo; +import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping; +import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; +import org.apache.doris.nereids.trees.expressions.CTEId; +import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; +import org.apache.doris.nereids.trees.plans.AbstractPlan; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.AbstractLogicalPlan; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer; + +import com.google.common.collect.Lists; import java.util.ArrayList; +import java.util.BitSet; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; public class CteExtractor { - private AbstractLogicalPlan plan; - private Map anchorToSignature = new HashMap<>(); - private Map> signatureToAnchorList = new HashMap<>(); + private final CascadesContext cascadesContext; + private final AbstractLogicalPlan plan; + private final Map planToSignature = new HashMap<>(); + private final Map> signatureToPlanList = new HashMap<>(); - public CteExtractor(AbstractLogicalPlan plan) { + public CteExtractor(AbstractLogicalPlan plan, CascadesContext cascadesContext) { this.plan = plan; + this.cascadesContext = cascadesContext; } - public AbstractLogicalPlan execute() { + public List execute() { sign(); - return plan; + return extract(); } private void sign() { SignatureVisitor visitor = new SignatureVisitor(); - visitor.visit(plan, anchorToSignature); - extract(); + visitor.visit(plan, planToSignature); } - private void extract() { - List a = anchorToSignature.values().stream().collect(Collectors.toList()); + public List extract() { + for (AbstractLogicalPlan plan : planToSignature.keySet()) { + TableSignature signature = planToSignature.get(plan); + List plans = signatureToPlanList.computeIfAbsent(signature, key -> new ArrayList<>()); + plans.add(plan); + } + TableSignature targetSignature = null; + for (TableSignature signature : signatureToPlanList.keySet()) { + if (signature.isContainsAggregation()) { + targetSignature = signature; + } + } + if (targetSignature != null) { + List targetPlans = signatureToPlanList.get(targetSignature); + Plan target0 = targetPlans.get(0); + Optional optStruct0 = MaterializationContext.constructStructInfo(target0, target0, cascadesContext, new BitSet()); + StructInfo viewStructInfo = optStruct0.get(); + Plan target1 = targetPlans.get(1); + Optional optStruct1 = MaterializationContext.constructStructInfo(target1, target1, cascadesContext, new BitSet()); + StructInfo queryStructInfo = optStruct1.get(); + List queryToViewTableMappings = RelationMapping.generate(optStruct0.get().getRelations(), + optStruct1.get().getRelations()); + RelationMapping queryToViewTableMapping = queryToViewTableMappings.get(0); + SlotMapping queryToViewSlotMapping = SlotMapping.generate(queryToViewTableMapping); + SlotMapping viewToQuerySlotMapping = queryToViewSlotMapping.inverse(); + LogicalCompatibilityContext compatibilityContext = LogicalCompatibilityContext.from( + queryToViewTableMapping, viewToQuerySlotMapping, optStruct0.get(), optStruct1.get()); + ComparisonResult comparisonResult = StructInfo.isGraphLogicalEquals(queryStructInfo, viewStructInfo, + compatibilityContext); + if (!comparisonResult.isInvalid()) { + CTEId cteId = StatementScopeIdGenerator.newCTEId(); + LogicalCTEProducer producer = new LogicalCTEProducer(cteId, target0); + ConsumerRewriterContext context = new ConsumerRewriterContext(Lists.newArrayList(target0, target1), producer, cteId); + Plan rewritten = plan.accept(ConsumerRewriter.INSTANCE, context); + AbstractLogicalPlan root = new LogicalCTEAnchor(cteId, producer, rewritten); + CheckAfterRewrite checker = new CheckAfterRewrite(); + checker.checkValidation(root); + return Lists.newArrayList(root); + } + System.out.println(queryToViewTableMappings); + } + return Lists.newArrayList(); + } + private void checkPlanValidation(Plan plan) { - for (Plan plan : anchorToSignature.keySet()) { - TableSignature signature = anchorToSignature.get(plan); - List plans = signatureToAnchorList.computeIfAbsent(signature, key -> new ArrayList<>()); - plans.add(plan); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + for (TableSignature signature : signatureToPlanList.keySet()) { + sb.append(String.format("%s\n", signature.toString())); + sb.append(" "); + for (AbstractLogicalPlan plan : signatureToPlanList.get(signature)) { + sb.append(plan.getId()).append(" "); + } + sb.append("\n"); } + return sb.toString(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/commonCTE/SignatureVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/commonCTE/SignatureVisitor.java index 38ec46b7822658..89b8ea0670ee7b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/commonCTE/SignatureVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/commonCTE/SignatureVisitor.java @@ -1,6 +1,7 @@ package org.apache.doris.nereids.commonCTE; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.AbstractLogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; @@ -12,11 +13,11 @@ import java.util.Map; -public class SignatureVisitor extends DefaultPlanVisitor >{ +public class SignatureVisitor extends DefaultPlanVisitor >{ @Override public TableSignature visitLogicalCatalogRelation(LogicalCatalogRelation relation, - Map signatureMap) { + Map signatureMap) { TableSignature signature = new TableSignature(true, false, ImmutableSet.of(relation.getTable().getId())); signatureMap.put(relation, signature); @@ -25,7 +26,7 @@ public TableSignature visitLogicalCatalogRelation(LogicalCatalogRelation relatio @Override public TableSignature visitLogicalFilter(LogicalFilter filter, - Map signatureMap) { + Map signatureMap) { TableSignature childSignature = filter.child().accept(this, signatureMap); if (filter.child() instanceof LogicalAggregate) { return TableSignature.EMPTY; @@ -36,7 +37,7 @@ public TableSignature visitLogicalFilter(LogicalFilter filter, @Override public TableSignature visitLogicalJoin(LogicalJoin join, - Map signatureMap) { + Map signatureMap) { TableSignature signature = TableSignature.EMPTY; TableSignature leftSignature = join.left().accept(this, signatureMap); @@ -56,7 +57,7 @@ public TableSignature visitLogicalJoin(LogicalJoin aggregate, - Map signatureMap) { + Map signatureMap) { TableSignature signature = TableSignature.EMPTY; TableSignature childSignature = aggregate.child().accept(this, signatureMap); if (childSignature != TableSignature.EMPTY) { @@ -68,7 +69,7 @@ public TableSignature visitLogicalAggregate(LogicalAggregate agg @Override public TableSignature visitLogicalProject(LogicalProject project, - Map signatureMap) { + Map signatureMap) { TableSignature childSignature = project.child().accept(this, signatureMap); if (childSignature != TableSignature.EMPTY) { signatureMap.put(project, childSignature); @@ -77,7 +78,7 @@ public TableSignature visitLogicalProject(LogicalProject project } @Override - public TableSignature visit(Plan plan, Map signatureMap) { + public TableSignature visit(Plan plan, Map signatureMap) { for (Plan child : plan.children()) { child.accept(this, signatureMap); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java index 47cffe28c55c31..4c6c7a24579b81 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java @@ -208,4 +208,14 @@ private void checkMatchIsUsedCorrectly(Plan plan) { } } } + + public void checkValidation(Plan plan) { + checkAllSlotReferenceFromChildren(plan); + checkUnexpectedExpression(plan); + checkMetricTypeIsUsedCorrectly(plan); + checkMatchIsUsedCorrectly(plan); + for (Plan child : plan.children()) { + checkValidation(child); + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEConsumer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEConsumer.java index 415fdddf80b449..eacfb3889bf32f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEConsumer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEConsumer.java @@ -196,6 +196,7 @@ public String toString() { return Utils.toSqlString("LogicalCteConsumer[" + id.asInt() + "]", "cteId", cteId, "relationId", relationId, - "name", name); + "name", name, + "map", consumerToProducerOutputMap); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java index 2216e58c4fa3b8..ec1094bc13ba5a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java @@ -228,7 +228,7 @@ public OlapTable getTable() { @Override public String toString() { - return Utils.toSqlString("LogicalOlapScan", + return Utils.toSqlString("LogicalOlapScan[" + id.asInt() + "]", "qualified", qualifiedName(), "indexName", getSelectedMaterializedIndexName().orElse(""), "selectedIndexId", selectedIndexId, diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 380c758e5751c8..96b0157b6ab9d2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -2214,6 +2214,13 @@ public void setIgnoreShapePlanNodes(String ignoreShapePlanNodes) { ) public boolean enableCompressMaterialize = false; + @VariableMgr.VarAttr( + name = "enable_common_cte", + needForward = true, fuzzy = false, + varType = VariableAnnotation.EXPERIMENTAL + ) + public boolean enableCommonCte = true; + @VariableMgr.VarAttr( name = ENABLE_AGG_SPILL, description = {"控制是否启用聚合算子落盘。默认为 false。", diff --git a/regression-test/data/query_p0/common_cte/common_cte.out b/regression-test/data/query_p0/common_cte/common_cte.out new file mode 100644 index 00000000000000..2207a229095611 --- /dev/null +++ b/regression-test/data/query_p0/common_cte/common_cte.out @@ -0,0 +1,17 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !1 -- +LogicalResultSink[168] ( outputExprs=[a#34] ) ++--LogicalAggregate[166] ( groupByExpr=[a#34], outputExpr=[a#34], hasRepeat=false ) + +--LogicalUnion ( qualifier=ALL, outputs=[a#34], regularChildrenOutputs=[[a#32], [b#33]], constantExprsList=[], hasPushedFilter=false ) + |--LogicalAggregate[157] ( groupByExpr=[], outputExpr=[count(*) AS `a`#32], hasRepeat=false ) + | +--LogicalProject[155] ( distinct=false, projects=[1 AS `1`#41] ) + | +--LogicalOlapScan[129] ( qualified=internal.regression_test_query_p0_common_cte.lineitem, indexName=, selectedIndexId=1596500, preAgg=ON ) + +--LogicalAggregate[160] ( groupByExpr=[], outputExpr=[count(*) AS `b`#33], hasRepeat=false ) + +--LogicalProject[158] ( distinct=false, projects=[1 AS `1`#42] ) + +--LogicalOlapScan[132] ( qualified=internal.regression_test_query_p0_common_cte.lineitem, indexName=, selectedIndexId=1596500, preAgg=ON ) + common cte info +SPJG [1596499] + 155 129 132 158 +SPJG AGG [1596499] + 157 160 + diff --git a/regression-test/suites/query_p0/common_cte/common_cte.groovy b/regression-test/suites/query_p0/common_cte/common_cte.groovy new file mode 100644 index 00000000000000..154a0d176bdddd --- /dev/null +++ b/regression-test/suites/query_p0/common_cte/common_cte.groovy @@ -0,0 +1,25 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("common_cte") { + qt_1 " explain rewritten plan select * from (select count() a from lineitem union all select count() b from lineitem) T" +} + +set enable_common_cte=false; +explain physical plan +with x as (select count() a from lineitem) +select * from (select * from x union all select * from x) T; \ No newline at end of file diff --git a/regression-test/suites/query_p0/common_cte/ddl/lineitem.sql b/regression-test/suites/query_p0/common_cte/ddl/lineitem.sql new file mode 100644 index 00000000000000..b4dc8f24551f94 --- /dev/null +++ b/regression-test/suites/query_p0/common_cte/ddl/lineitem.sql @@ -0,0 +1,18 @@ +CREATE TABLE IF NOT EXISTS lineitem ( + orderkey bigint, + partkey bigint, + suppkey bigint, + linenumber integer, + quantity double, + extendedprice double, + discount double, + tax double, + returnflag varchar(1), + linestatus varchar(1), + shipdate date, + commitdate date, + receiptdate date, + shipinstruct varchar(25), + shipmode varchar(10), + comment varchar(44) +) DUPLICATE KEY(orderkey) DISTRIBUTED BY HASH(orderkey) BUCKETS 3 PROPERTIES ("replication_num" = "1") \ No newline at end of file diff --git a/regression-test/suites/query_p0/common_cte/ddl/nation.sql b/regression-test/suites/query_p0/common_cte/ddl/nation.sql new file mode 100644 index 00000000000000..42446a423e76c7 --- /dev/null +++ b/regression-test/suites/query_p0/common_cte/ddl/nation.sql @@ -0,0 +1,12 @@ + +CREATE TABLE IF NOT EXISTS nation ( + nationkey BIGINT NOT NULL, + name VARCHAR(25) NOT NULL, + regionkey BIGINT NOT NULL, + comment VARCHAR(152) +) +DUPLICATE KEY(nationkey) +DISTRIBUTED BY HASH(nationkey) BUCKETS 3 +PROPERTIES ( + "replication_num" = "1" +) \ No newline at end of file diff --git a/regression-test/suites/query_p0/common_cte/ddl/part.sql b/regression-test/suites/query_p0/common_cte/ddl/part.sql new file mode 100644 index 00000000000000..09338f5770740b --- /dev/null +++ b/regression-test/suites/query_p0/common_cte/ddl/part.sql @@ -0,0 +1,11 @@ +CREATE TABLE IF NOT EXISTS part ( + p_partkey INTEGER NOT NULL, + p_name VARCHAR(55) NOT NULL, + p_mfgr CHAR(25) NOT NULL, + p_brand CHAR(10) NOT NULL, + p_type VARCHAR(25) NOT NULL, + p_size INTEGER NOT NULL, + p_container CHAR(10) NOT NULL, + p_retailprice DECIMAL(15, 2) NOT NULL, + p_comment VARCHAR(23) NOT NULL +) DUPLICATE KEY(p_partkey) DISTRIBUTED BY HASH(p_partkey) BUCKETS 3 PROPERTIES ("replication_num" = "1") \ No newline at end of file diff --git a/regression-test/suites/query_p0/common_cte/ddl/partsupp.sql b/regression-test/suites/query_p0/common_cte/ddl/partsupp.sql new file mode 100644 index 00000000000000..c0fd9378698f0f --- /dev/null +++ b/regression-test/suites/query_p0/common_cte/ddl/partsupp.sql @@ -0,0 +1,7 @@ +CREATE TABLE IF NOT EXISTS partsupp ( + partkey bigint, + suppkey bigint, + availqty integer, + supplycost double, + comment varchar(199) +) DUPLICATE KEY(partkey, suppkey) DISTRIBUTED BY HASH(partkey) BUCKETS 3 PROPERTIES ("replication_num" = "1") diff --git a/regression-test/suites/query_p0/common_cte/ddl/region.sql b/regression-test/suites/query_p0/common_cte/ddl/region.sql new file mode 100644 index 00000000000000..80eae844eda251 --- /dev/null +++ b/regression-test/suites/query_p0/common_cte/ddl/region.sql @@ -0,0 +1,10 @@ +CREATE TABLE IF NOT EXISTS region ( + r_regionkey INTEGER NOT NULL, + r_name CHAR(25) NOT NULL, + r_comment VARCHAR(152) +) +DUPLICATE KEY(r_regionkey) +DISTRIBUTED BY HASH(r_regionkey) BUCKETS 3 +PROPERTIES ( + "replication_num" = "1" +) \ No newline at end of file diff --git a/regression-test/suites/query_p0/common_cte/ddl/supplier.sql b/regression-test/suites/query_p0/common_cte/ddl/supplier.sql new file mode 100644 index 00000000000000..42cec55af2537d --- /dev/null +++ b/regression-test/suites/query_p0/common_cte/ddl/supplier.sql @@ -0,0 +1,9 @@ +CREATE TABLE IF NOT EXISTS supplier ( + s_suppkey INTEGER NOT NULL, + s_name CHAR(25) NOT NULL, + s_address VARCHAR(40) NOT NULL, + s_nationkey INTEGER NOT NULL, + s_phone CHAR(15) NOT NULL, + s_acctbal DECIMAL(15, 2) NOT NULL, + s_comment VARCHAR(101) NOT NULL +) DUPLICATE KEY(s_suppkey) DISTRIBUTED BY HASH(s_suppkey) BUCKETS 3 PROPERTIES ("replication_num" = "1") \ No newline at end of file diff --git a/regression-test/suites/query_p0/common_cte/ddl/workers.sql b/regression-test/suites/query_p0/common_cte/ddl/workers.sql new file mode 100644 index 00000000000000..c3d2e6c6e59363 --- /dev/null +++ b/regression-test/suites/query_p0/common_cte/ddl/workers.sql @@ -0,0 +1,10 @@ +CREATE TABLE IF NOT EXISTS workers ( + id_employee INT, + first_name VARCHAR(32), + last_name VARCHAR(32), + date_of_employment DATE, + department TINYINT(1), + id_department INT, + name VARCHAR(32), + salary INT +) DUPLICATE KEY(id_employee) DISTRIBUTED BY HASH(first_name) BUCKETS 3 PROPERTIES ("replication_num" = "1") diff --git a/regression-test/suites/query_p0/common_cte/load.groovy b/regression-test/suites/query_p0/common_cte/load.groovy new file mode 100644 index 00000000000000..756d503d1e9d73 --- /dev/null +++ b/regression-test/suites/query_p0/common_cte/load.groovy @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// The cases is copied from https://github.com/trinodb/trino/tree/master +// /testing/trino-product-tests/src/main/resources/sql-tests/testcases/window_functions +// and modified by Doris. + +suite("load") { + def tables = ['lineitem', 'region', 'nation', 'part', 'supplier', 'partsupp', 'workers'] + + for (String table in tables) { + sql """ DROP TABLE IF EXISTS ${table} """ + } + + for (String table in tables) { + sql new File("""${context.file.parent}/ddl/${table}.sql""").text + } + + for (String tableName in tables) { + streamLoad { + // you can skip declare db, because a default db already specify in ${DORIS_HOME}/conf/regression-conf.groovy + // db 'regression_test' + table "${tableName}" + + // default column_separator is specify in doris fe config, usually is '\t'. + // this line change to ',' + set 'column_separator', '|' + set 'compress_type', 'GZ' + // relate to ${DORIS_HOME}/regression-test/data/demo/streamload_input.csv. + // also, you can stream load a http stream, e.g. http://xxx/some.csv + file """${getS3Url() + '/regression/tpch/sf0.01/'+ tableName}.csv.gz""" + + time 10000 // limit inflight 10s + + // if declared a check callback, the default check condition will ignore. + // So you must check all condition + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + assertEquals('success', json.Status.toLowerCase()) + assertEquals(json.NumberTotalRows, json.NumberLoadedRows) + assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0) + } + } + } +} \ No newline at end of file