diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java index 8f695fa56216..a37943ff8338 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java +++ b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java @@ -2363,7 +2363,6 @@ private static RexNode isDistinctFromInternal( RexNode y, boolean neg) { - // see https://olapio.atlassian.net/browse/KE-42039 if (neg) { // x is not distinct from y // x=y IS TRUE or ((x is null) and (y is null)), @@ -2894,127 +2893,6 @@ public static boolean classifyFilters( return !filtersToRemove.isEmpty(); } - /** - * see https://olapio.atlassian.net/browse/KE-42039 - * Calcite 1.30 copy from following method and add new parameter - shiftedMapping - * @see RelOptUtil#classifyFilters(RelNode, List, boolean, boolean, boolean, List, List, List) - * - * Classifies filters according to where they should be processed. They - * either stay where they are, are pushed to the join (if they originated - * from above the join), or are pushed to one of the children. Filters that - * are pushed are added to list passed in as input parameters. - * - * @param joinRel join node - * @param filters filters to be classified - * @param pushInto whether filters can be pushed into the join - * @param pushLeft true if filters can be pushed to the left - * @param pushRight true if filters can be pushed to the right - * @param joinFilters list of filters to push to the join - * @param leftFilters list of filters to push to the left child - * @param rightFilters list of filters to push to the right child - * @param shiftedMapping list of leftFilters or rightFilters to push to the child - * @return whether at least one filter was pushed - */ - public static boolean classifyFilters( - RelNode joinRel, - List filters, - boolean pushInto, - boolean pushLeft, - boolean pushRight, - List joinFilters, - List leftFilters, - List rightFilters, - Map shiftedMapping) { - RexBuilder rexBuilder = joinRel.getCluster().getRexBuilder(); - List joinFields = joinRel.getRowType().getFieldList(); - final int nSysFields = 0; // joinRel.getSystemFieldList().size(); - final List leftFields = - joinRel.getInputs().get(0).getRowType().getFieldList(); - final int nFieldsLeft = leftFields.size(); - final List rightFields = - joinRel.getInputs().get(1).getRowType().getFieldList(); - final int nFieldsRight = rightFields.size(); - final int nTotalFields = nFieldsLeft + nFieldsRight; - - // set the reference bitmaps for the left and right children - ImmutableBitSet leftBitmap = - ImmutableBitSet.range(nSysFields, nSysFields + nFieldsLeft); - ImmutableBitSet rightBitmap = - ImmutableBitSet.range(nSysFields + nFieldsLeft, nTotalFields); - - final List filtersToRemove = new ArrayList<>(); - for (RexNode filter : filters) { - final InputFinder inputFinder = InputFinder.analyze(filter); - final ImmutableBitSet inputBits = inputFinder.build(); - - // REVIEW - are there any expressions that need special handling - // and therefore cannot be pushed? - - if (pushLeft && leftBitmap.contains(inputBits)) { - // ignore filters that always evaluate to true - if (!filter.isAlwaysTrue()) { - // adjust the field references in the filter to reflect - // that fields in the left now shift over by the number - // of system fields - final RexNode shiftedFilter = - shiftFilter( - nSysFields, - nSysFields + nFieldsLeft, - -nSysFields, - rexBuilder, - joinFields, - nTotalFields, - leftFields, - filter); - - leftFilters.add(shiftedFilter); - shiftedMapping.put(shiftedFilter, filter); - } - filtersToRemove.add(filter); - } else if (pushRight && rightBitmap.contains(inputBits)) { - if (!filter.isAlwaysTrue()) { - // adjust the field references in the filter to reflect - // that fields in the right now shift over to the left - final RexNode shiftedFilter = - shiftFilter( - nSysFields + nFieldsLeft, - nTotalFields, - -(nSysFields + nFieldsLeft), - rexBuilder, - joinFields, - nTotalFields, - rightFields, - filter); - rightFilters.add(shiftedFilter); - shiftedMapping.put(shiftedFilter, filter); - } - filtersToRemove.add(filter); - - } else { - // If the filter can't be pushed to either child, we may push them into the join - if (pushInto) { - // Calcite 1.30 do not do the acutal push into - // remove above filters only if the filter is within the filter condition - /*if (!joinFilters.contains(filter)) { - joinFilters.add(filter); - } - filtersToRemove.add(filter);*/ - if (joinFilters.stream().anyMatch(filter::equals)) { - filtersToRemove.add(filter); - } - } - } - } - - // Remove filters after the loop, to prevent concurrent modification. - if (!filtersToRemove.isEmpty()) { - filters.removeAll(filtersToRemove); - } - - // Did anything change? - return !filtersToRemove.isEmpty(); - } - /** * Classifies filters according to where they should be processed. They * either stay where they are, are pushed to the join (if they originated diff --git a/core/src/main/java/org/apache/calcite/rel/rules/ReduceExpressionsRule.java b/core/src/main/java/org/apache/calcite/rel/rules/ReduceExpressionsRule.java index 4546db40af26..dbaa5972496f 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/ReduceExpressionsRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/ReduceExpressionsRule.java @@ -29,7 +29,6 @@ import org.apache.calcite.rel.core.Filter; import org.apache.calcite.rel.core.Join; import org.apache.calcite.rel.core.Project; -import org.apache.calcite.rel.core.Values; import org.apache.calcite.rel.core.Window; import org.apache.calcite.rel.logical.LogicalCalc; import org.apache.calcite.rel.logical.LogicalFilter; @@ -180,8 +179,7 @@ public FilterReduceExpressionsRule(Class filterClass, filter.getInput()); } else if (newConditionExp instanceof RexLiteral || RexUtil.isNullLiteral(newConditionExp, true)) { - Values values = (Values) createEmptyRelOrEquivalent(call, filter); - call.transformTo(values.copy(call.rel(0).getTraitSet(), values.getInputs())); + call.transformTo(createEmptyRelOrEquivalent(call, filter)); } else if (reduced) { call.transformTo(call.builder() .push(filter.getInput()) diff --git a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java index 8a8a71015fdf..353662e8be2e 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java @@ -211,17 +211,6 @@ import static java.util.Objects.requireNonNull; -/* - * The code has synced with calcite. Hope one day, we could remove the hardcode override point. - * OVERRIDE POINT: - * - DEFAULT_IN_SUB_QUERY_THRESHOLD, was `20`, now `Integer.MAX_VALUE` - * - isTrimUnusedFields(), override to false - * - AggConverter.translateAgg(...), skip column reading - * for COUNT(COL), for https://jirap.corp.ebay.com/browse/KYLIN-104 - * - convertQuery(), call hackSelectStar() at the end - * - createJoin() check cast operation - */ - /** * Converts a SQL parse tree (consisting of * {@link org.apache.calcite.sql.SqlNode} objects) into a relational algebra @@ -1232,266 +1221,6 @@ private void replaceSubQueries( } } - /** - * OVERRIDE POINT - * see https://olapio.atlassian.net/browse/KE-42057 - * Calcite 1.30 Keeps the same changes with AL-5295 as the previous Calcite version - * FYI: https://github.com/Kyligence/KAP/issues/13872 - */ - /*private void substituteSubQuery(Blackboard bb, SubQuery subQuery) { - final RexNode expr = subQuery.expr; - if (expr != null) { - // Already done. - return; - } - - final SqlBasicCall call; - final RelNode rel; - final SqlNode query; - final RelOptUtil.Exists converted; - switch (subQuery.node.getKind()) { - case CURSOR: - convertCursor(bb, subQuery); - return; - - case ARRAY_QUERY_CONSTRUCTOR: - case MAP_QUERY_CONSTRUCTOR: - case MULTISET_QUERY_CONSTRUCTOR: - if (!config.isExpand()) { - return; - } - // fall through - case MULTISET_VALUE_CONSTRUCTOR: - rel = convertMultisets(ImmutableList.of(subQuery.node), bb); - subQuery.expr = bb.register(rel, JoinRelType.INNER); - return; - - case IN: - case NOT_IN: - case SOME: - case ALL: - call = (SqlBasicCall) subQuery.node; - query = call.operand(1); - if (!config.isExpand() && !(query instanceof SqlNodeList)) { - return; - } - final SqlNode leftKeyNode = call.operand(0); - - final List leftKeys; - switch (leftKeyNode.getKind()) { - case ROW: - leftKeys = new ArrayList<>(); - for (SqlNode sqlExpr : ((SqlBasicCall) leftKeyNode).getOperandList()) { - leftKeys.add(bb.convertExpression(sqlExpr)); - } - break; - default: - leftKeys = ImmutableList.of(bb.convertExpression(leftKeyNode)); - } - - if (query instanceof SqlNodeList) { - SqlNodeList valueList = (SqlNodeList) query; - - // keep in clause. - if (!valueList.accept(new SqlIdentifierFinder()) - && Boolean.parseBoolean(System.getProperty("calcite.keep-in-clause", "false")) - && (leftKeys.size() <= 1 - || !Boolean.parseBoolean( - System.getProperty("calcite.convert-multiple-columns-in-to-or", - "false")))) { - RexNode subQueryExpr = constructIn(bb, leftKeys, valueList, call.getOperator().kind); - if (subQueryExpr != null) { - subQuery.expr = subQueryExpr; - return; - } - } - // When the list size under the threshold or the list references columns, we convert to OR. - if (valueList.size() < config.getInSubQueryThreshold() - || valueList.accept(new SqlIdentifierFinder())) { - subQuery.expr = - convertInToOr( - bb, - leftKeys, - valueList, - (SqlInOperator) call.getOperator()); - return; - } - - // Otherwise, let convertExists translate - // values list into an inline table for the - // reference to Q below. - } - - // Project out the search columns from the left side - - // Q1: - // "select from emp where emp.deptno in (select col1 from T)" - // - // is converted to - // - // "select from - // emp inner join (select distinct col1 from T)) q - // on emp.deptno = q.col1 - // - // Q2: - // "select from emp where emp.deptno not in (Q)" - // - // is converted to - // - // "select from - // emp left outer join (select distinct col1, TRUE from T) q - // on emp.deptno = q.col1 - // where emp.deptno <> null - // and q.indicator <> TRUE" - // - // Note: Sub-query can be used as SqlUpdate#condition like below: - // - // UPDATE emp - // SET empno = 1 WHERE emp.empno IN ( - // SELECT emp.empno FROM emp WHERE emp.empno = 2) - // - // In such case, when converting SqlUpdate#condition, bb.root is null - // and it makes no sense to do the sub-query substitution. - if (bb.root == null) { - return; - } - final RelDataType targetRowType = - SqlTypeUtil.promoteToRowType(typeFactory, - validator().getValidatedNodeType(leftKeyNode), null); - final boolean notIn = call.getOperator().kind == SqlKind.NOT_IN; - converted = - convertExists(query, RelOptUtil.SubQueryType.IN, subQuery.logic, - notIn, targetRowType); - if (converted.indicator) { - // Generate - // emp CROSS JOIN (SELECT COUNT(*) AS c, - // COUNT(deptno) AS ck FROM dept) - final RelDataType longType = - typeFactory.createSqlType(SqlTypeName.BIGINT); - final RelNode seek = converted.r.getInput(0); // fragile - final int keyCount = leftKeys.size(); - final List args = ImmutableIntList.range(0, keyCount); - LogicalAggregate aggregate = - LogicalAggregate.create(seek, - ImmutableList.of(), - ImmutableBitSet.of(), - null, - ImmutableList.of( - AggregateCall.create(SqlStdOperatorTable.COUNT, false, - false, false, ImmutableList.of(), -1, null, - RelCollations.EMPTY, longType, null), - AggregateCall.create(SqlStdOperatorTable.COUNT, false, - false, false, args, -1, null, - RelCollations.EMPTY, longType, null))); - LogicalJoin join = - LogicalJoin.create(bb.root(), aggregate, ImmutableList.of(), - rexBuilder.makeLiteral(true), ImmutableSet.of(), JoinRelType.INNER); - bb.setRoot(join, false); - } - final RexNode rex = - bb.register(converted.r, - converted.outerJoin ? JoinRelType.LEFT : JoinRelType.INNER, - leftKeys); - - RelOptUtil.Logic logic = subQuery.logic; - switch (logic) { - case TRUE_FALSE_UNKNOWN: - case UNKNOWN_AS_TRUE: - if (!converted.indicator) { - logic = RelOptUtil.Logic.TRUE_FALSE; - } - break; - default: - break; - } - subQuery.expr = translateIn(logic, bb.root, rex); - if (notIn) { - subQuery.expr = - rexBuilder.makeCall(SqlStdOperatorTable.NOT, subQuery.expr); - } - return; - - case EXISTS: - // "select from emp where exists (select a from T)" - // - // is converted to the following if the sub-query is correlated: - // - // "select from emp left outer join (select AGG_TRUE() as indicator - // from T group by corr_var) q where q.indicator is true" - // - // If there is no correlation, the expression is replaced with a - // boolean indicating whether the sub-query returned 0 or >= 1 row. - if (!config.isExpand()) { - return; - } - call = (SqlBasicCall) subQuery.node; - query = call.operand(0); - final SqlValidatorScope seekScope = - (query instanceof SqlSelect) - ? validator().getSelectScope((SqlSelect) query) - : null; - final Blackboard seekBb = createBlackboard(seekScope, null, false); - final RelNode seekRel = convertQueryOrInList(seekBb, query, null); - requireNonNull(seekRel, () -> "seekRel is null for query " + query); - // An EXIST sub-query whose inner child has at least 1 tuple - // (e.g. an Aggregate with no grouping columns or non-empty Values - // node) should be simplified to a Boolean constant expression. - final RelMetadataQuery mq = seekRel.getCluster().getMetadataQuery(); - final Double minRowCount = mq.getMinRowCount(seekRel); - if (minRowCount != null && minRowCount >= 1D) { - subQuery.expr = rexBuilder.makeLiteral(true); - return; - } - converted = RelOptUtil.createExistsPlan(seekRel, - RelOptUtil.SubQueryType.EXISTS, subQuery.logic, true, relBuilder); - assert !converted.indicator; - if (convertNonCorrelatedSubQuery(subQuery, bb, converted.r, true)) { - return; - } - subQuery.expr = bb.register(converted.r, JoinRelType.LEFT); - return; - case UNIQUE: - return; - case SCALAR_QUERY: - // Convert the sub-query. If it's non-correlated, convert it - // to a constant expression. - if (!config.isExpand()) { - return; - } - call = (SqlBasicCall) subQuery.node; - query = call.operand(0); - converted = convertExists(query, RelOptUtil.SubQueryType.SCALAR, - subQuery.logic, true, null); - assert !converted.indicator; - if (convertNonCorrelatedSubQuery(subQuery, bb, converted.r, false)) { - return; - } - rel = convertToSingleValueSubq(query, converted.r); - subQuery.expr = bb.register(rel, JoinRelType.LEFT); - return; - - case SELECT: - // This is used when converting multiset queries: - // - // select * from unnest(select multiset[deptno] from emps); - // - converted = convertExists(subQuery.node, RelOptUtil.SubQueryType.SCALAR, - subQuery.logic, true, null); - assert !converted.indicator; - subQuery.expr = bb.register(converted.r, JoinRelType.LEFT); - - // This is used when converting window table functions: - // - // select * from table(tumble(table emps, descriptor(deptno), interval '3' DAY)) - // - bb.cursors.add(converted.r); - return; - - default: - throw new AssertionError("unexpected kind of sub-query: " - + subQuery.node); - } - }*/ private void substituteSubQuery(Blackboard bb, SubQuery subQuery) { final RexNode expr = subQuery.expr; if (expr != null) { @@ -1961,57 +1690,6 @@ public RelNode convertToSingleValueSubq( * @param op The operator (IN, NOT IN, > SOME, ...) * @return converted expression */ - /*private @Nullable RexNode convertInToOr( - final Blackboard bb, - final List leftKeys, - SqlNodeList valuesList, - SqlInOperator op) { - final List comparisons = new ArrayList<>(); - for (SqlNode rightVals : valuesList) { - RexNode rexComparison; - final SqlOperator comparisonOp; - if (op instanceof SqlQuantifyOperator) { - comparisonOp = RelOptUtil.op(((SqlQuantifyOperator) op).comparisonKind, - SqlStdOperatorTable.EQUALS); - } else { - comparisonOp = SqlStdOperatorTable.EQUALS; - } - if (leftKeys.size() == 1) { - rexComparison = - rexBuilder.makeCall(comparisonOp, - leftKeys.get(0), - ensureSqlType(leftKeys.get(0).getType(), - bb.convertExpression(rightVals))); - } else { - assert rightVals instanceof SqlCall; - final SqlBasicCall call = (SqlBasicCall) rightVals; - assert (call.getOperator() instanceof SqlRowOperator) - && call.operandCount() == leftKeys.size(); - rexComparison = - RexUtil.composeConjunction(rexBuilder, - Util.transform( - Pair.zip(leftKeys, call.getOperandList()), - pair -> rexBuilder.makeCall(comparisonOp, pair.left, - // TODO: remove requireNonNull when checkerframework issue resolved - ensureSqlType(requireNonNull(pair.left, "pair.left").getType(), - bb.convertExpression(pair.right))))); - } - comparisons.add(rexComparison); - } - - switch (op.kind) { - case ALL: - return RexUtil.composeConjunction(rexBuilder, comparisons, true); - case NOT_IN: - return rexBuilder.makeCall(SqlStdOperatorTable.NOT, - RexUtil.composeDisjunction(rexBuilder, comparisons)); - case IN: - case SOME: - return RexUtil.composeDisjunction(rexBuilder, comparisons, true); - default: - throw new AssertionError(); - } - }*/ private @Nullable RexNode convertInToOr( final Blackboard bb, SqlNode leftKeyNode,