From c793609b1944e33362c10a7f6b3756bcc643e242 Mon Sep 17 00:00:00 2001 From: Frank McSherry Date: Fri, 13 Aug 2021 01:35:39 -0400 Subject: [PATCH] Relation CSE (#7715) * rebase * More tests updated for viewing * reorganize logic into Bindings struct * typos * additional tests for Common Subexpression Elimination (CSE) Co-authored-by: Philip Stoev --- src/transform/src/cse/mod.rs | 1 + src/transform/src/cse/relation_cse.rs | 138 ++ src/transform/src/lib.rs | 10 + .../tests/testdata/join-implementation | 11 +- src/transform/tests/testdata/keys | 38 +- src/transform/tests/testdata/lifting | 14 +- src/transform/tests/testdata/steps | 52 +- test/sqllogictest/chbench.slt | 215 +-- test/sqllogictest/column_knowledge.slt | 195 +-- test/sqllogictest/joins.slt | 55 +- test/sqllogictest/relation-cse.slt | 1514 +++++++++++++++++ test/sqllogictest/tpch.slt | 283 ++- test/sqllogictest/transform/union.slt | 14 +- 13 files changed, 2068 insertions(+), 472 deletions(-) create mode 100644 src/transform/src/cse/relation_cse.rs create mode 100644 test/sqllogictest/relation-cse.slt diff --git a/src/transform/src/cse/mod.rs b/src/transform/src/cse/mod.rs index 60e7af4b28e56..2dd9db7320cba 100644 --- a/src/transform/src/cse/mod.rs +++ b/src/transform/src/cse/mod.rs @@ -10,3 +10,4 @@ //! Common subexpression elimination. pub mod map; +pub mod relation_cse; diff --git a/src/transform/src/cse/relation_cse.rs b/src/transform/src/cse/relation_cse.rs new file mode 100644 index 0000000000000..86c8657177781 --- /dev/null +++ b/src/transform/src/cse/relation_cse.rs @@ -0,0 +1,138 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +//! Identifies common relation subexpressions and places them behind `Let` bindings. +//! +//! All structurally equivalent expressions, defined recursively as having structurally +//! equivalent inputs, and identical parameters, will be placed behind `Let` bindings. +//! The resulting expressions likely have an excess of `Let` expressions, and should be +//! subjected to the `InlineLet` transformation to remove those that are not necessary. + +use std::collections::HashMap; + +use expr::{Id, LocalId, MirRelationExpr}; + +use crate::TransformArgs; + +/// Identifies common relation subexpressions and places them behind `Let` bindings. +#[derive(Debug)] +pub struct RelationCSE; + +impl crate::Transform for RelationCSE { + fn transform( + &self, + relation: &mut MirRelationExpr, + _: TransformArgs, + ) -> Result<(), crate::TransformError> { + let mut bindings = Bindings::default(); + bindings.intern_expression(relation); + bindings.populate_expression(relation); + Ok(()) + } +} + +/// Maintains `Let` bindings in a compact, explicit representation. +/// +/// The `bindings` map contains neither `Let` bindings nor two structurally +/// equivalent expressions. +/// +/// The bindings can be interpreted as an ordered sequence of let bindings, +/// ordered by their identifier, that should be applied in order before the +/// use of the expression from which they have been extracted. +#[derive(Debug, Default)] +pub struct Bindings { + /// A list of let-bound expressions and their order / identifier. + bindings: HashMap, + /// Mapping from conventional local `Get` identifiers to new ones. + rebindings: HashMap, +} + +impl Bindings { + /// Replace `relation` with an equivalent `Get` expression referencing a location in `bindings`. + /// + /// The algorithm performs a post-order traversal of the expression tree, binding each distinct + /// expression to a new local identifier. It maintains the invariant that `bindings` contains no + /// `Let` expressions, nor any two structurally equivalent expressions. + /// + /// Once each sub-expression is replaced by a canonical `Get` expression, each expression is also + /// in a canonical representation, which is used to check for prior instances and drives re-use. + fn intern_expression(&mut self, relation: &mut MirRelationExpr) { + match relation { + MirRelationExpr::Let { id, value, body } => { + self.intern_expression(value); + let new_id = if let MirRelationExpr::Get { + id: Id::Local(x), .. + } = **value + { + x + } else { + panic!("Invariant violated") + }; + self.rebindings.insert(*id, new_id); + self.intern_expression(body); + let body = body.take_dangerous(); + self.rebindings.remove(id); + *relation = body; + } + MirRelationExpr::Get { id, .. } => { + if let Id::Local(id) = id { + *id = self.rebindings[id]; + } + } + + _ => { + // All other expressions just need to apply the logic recursively. + relation.visit1_mut(&mut |expr| { + self.intern_expression(expr); + }) + } + }; + + // This should be fast, as it depends directly on only `Get` expressions. + let typ = relation.typ(); + // We want to maintain the invariant that `relation` ends up as a local `Get`. + if let MirRelationExpr::Get { + id: Id::Local(_), .. + } = relation + { + // Do nothing, as the expression is already a local `Get` expression. + } else { + // Either find an instance of `relation` or insert this one. + let bindings_len = self.bindings.len() as u64; + let id = self + .bindings + .entry(relation.take_dangerous()) + .or_insert(bindings_len); + *relation = MirRelationExpr::Get { + id: Id::Local(LocalId::new(*id)), + typ, + } + } + } + + /// Populates `expression` with necessary `Let` bindings. + /// + /// This population may result in substantially more `Let` bindings that one + /// might expect. It is very appropriate to run the `InlineLet` transformation + /// afterwards to remove `Let` bindings that it deems unhelpful. + fn populate_expression(self, expression: &mut MirRelationExpr) { + // Convert the bindings in to a sequence, by the local identifier. + let mut bindings = self.bindings.into_iter().collect::>(); + bindings.sort_by_key(|(_, i)| *i); + + for (value, index) in bindings.into_iter().rev() { + let new_expression = MirRelationExpr::Let { + id: LocalId::new(index), + value: Box::new(value), + body: Box::new(expression.take_dangerous()), + }; + *expression = new_expression; + } + } +} diff --git a/src/transform/src/lib.rs b/src/transform/src/lib.rs index 4ea28aaaa0ef6..937f287f9b787 100644 --- a/src/transform/src/lib.rs +++ b/src/transform/src/lib.rs @@ -270,6 +270,11 @@ impl Optimizer { // `Map {Cross Join (Input, Constant()), Literal}`. // Join fusion will clean this up to `Map{Input, Literal}` Box::new(crate::map_lifting::LiteralLifting), + // Identifies common relation subexpressions. + // Must be followed by let inlining, to keep under control. + Box::new(crate::cse::relation_cse::RelationCSE), + Box::new(crate::inline_let::InlineLet), + Box::new(crate::update_let::UpdateLet), Box::new(crate::FuseAndCollapse::default()), ], }), @@ -306,6 +311,11 @@ impl Optimizer { Box::new(crate::projection_lifting::ProjectionLifting), Box::new(crate::join_implementation::JoinImplementation), Box::new(crate::fusion::project::Project), + // Identifies common relation subexpressions. + // Must be followed by let inlining, to keep under control. + Box::new(crate::cse::relation_cse::RelationCSE), + Box::new(crate::inline_let::InlineLet), + Box::new(crate::update_let::UpdateLet), Box::new(crate::reduction::FoldConstants { limit: Some(10000) }), ]; let mut optimizer = Self::for_view(); diff --git a/src/transform/tests/testdata/join-implementation b/src/transform/tests/testdata/join-implementation index f775135943d55..cd58077adbe5b 100644 --- a/src/transform/tests/testdata/join-implementation +++ b/src/transform/tests/testdata/join-implementation @@ -66,18 +66,17 @@ opt (join [(get x) (get x)] [[5 #0 5 #3]]) ---- ---- -%0 = +%0 = Let l0 = | Get x (u0) | Filter (#0 = 5) -| ArrangeBy () %1 = -| Get x (u0) -| Filter (#0 = 5) +| Get %0 (l0) +| ArrangeBy () %2 = -| Join %0 %1 -| | implementation = Differential %1 %0.() +| Join %1 %0 +| | implementation = Differential %0 %1.() | | demand = (#0..#5) ---- ---- diff --git a/src/transform/tests/testdata/keys b/src/transform/tests/testdata/keys index b3ae337f9c1f4..d5d0ad30d9144 100644 --- a/src/transform/tests/testdata/keys +++ b/src/transform/tests/testdata/keys @@ -75,7 +75,43 @@ Applied Fixpoint { transforms: [FuseAndCollapse { transforms: [ProjectionExtract | | keys = ((#0), (#1)) ==== -No change: Fixpoint { transforms: [PredicatePushdown, NonNullable, ColumnKnowledge, Demand, FuseAndCollapse { transforms: [ProjectionExtraction, ProjectionLifting, Map, Filter, Project, Join, InlineLet, Reduce, Union, UnionBranchCancellation, UpdateLet, RedundantJoin, FoldConstants { limit: Some(10000) }] }], limit: 100 }, Fixpoint { transforms: [ReductionPushdown, ReduceElision, LiteralLifting, FuseAndCollapse { transforms: [ProjectionExtraction, ProjectionLifting, Map, Filter, Project, Join, InlineLet, Reduce, Union, UnionBranchCancellation, UpdateLet, RedundantJoin, FoldConstants { limit: Some(10000) }] }], limit: 100 }, Fixpoint { transforms: [ProjectionLifting, JoinImplementation, ColumnKnowledge, FoldConstants { limit: Some(10000) }, Filter, Demand, LiteralLifting, Map], limit: 100 }, ReductionPushdown, Map, ProjectionLifting, JoinImplementation, Project, FoldConstants { limit: Some(10000) } +No change: Fixpoint { transforms: [PredicatePushdown, NonNullable, ColumnKnowledge, Demand, FuseAndCollapse { transforms: [ProjectionExtraction, ProjectionLifting, Map, Filter, Project, Join, InlineLet, Reduce, Union, UnionBranchCancellation, UpdateLet, RedundantJoin, FoldConstants { limit: Some(10000) }] }], limit: 100 }, Fixpoint { transforms: [ReductionPushdown, ReduceElision, LiteralLifting, RelationCSE, InlineLet, UpdateLet, FuseAndCollapse { transforms: [ProjectionExtraction, ProjectionLifting, Map, Filter, Project, Join, InlineLet, Reduce, Union, UnionBranchCancellation, UpdateLet, RedundantJoin, FoldConstants { limit: Some(10000) }] }], limit: 100 }, Fixpoint { transforms: [ProjectionLifting, JoinImplementation, ColumnKnowledge, FoldConstants { limit: Some(10000) }, Filter, Demand, LiteralLifting, Map], limit: 100 }, ReductionPushdown, Map, ProjectionLifting, JoinImplementation, Project +==== +Applied RelationCSE: +%0 = Let l0 = +| Get x (u0) +| | types = (Int32?, Int64?, Int32?) +| | keys = ((#0), (#1)) + +%1 = Let l1 = +| Get %0 (l0) +| | types = (Int32?, Int64?, Int32?) +| | keys = ((#0), (#1)) +| Project (#0..#2, #0..#2) +| | types = (Int32?, Int64?, Int32?, Int32?, Int64?, Int32?) +| | keys = ((#0), (#1)) + +%2 = +| Get %1 (l1) +| | types = (Int32?, Int64?, Int32?, Int32?, Int64?, Int32?) +| | keys = ((#0), (#1)) +| | types = (Int32?, Int64?, Int32?, Int32?, Int64?, Int32?) +| | keys = ((#0), (#1)) +| | types = (Int32?, Int64?, Int32?, Int32?, Int64?, Int32?) +| | keys = ((#0), (#1)) + +==== +Applied InlineLet: +%0 = +| Get x (u0) +| | types = (Int32?, Int64?, Int32?) +| | keys = ((#0), (#1)) +| Project (#0..#2, #0..#2) +| | types = (Int32?, Int64?, Int32?, Int32?, Int64?, Int32?) +| | keys = ((#0), (#1)) + +==== +No change: UpdateLet, FoldConstants { limit: Some(10000) } ==== Final: %0 = diff --git a/src/transform/tests/testdata/lifting b/src/transform/tests/testdata/lifting index 12ed73f4bb8c9..6694bf3b4a573 100644 --- a/src/transform/tests/testdata/lifting +++ b/src/transform/tests/testdata/lifting @@ -352,17 +352,19 @@ opt [#2 #1])]) ---- ---- -%0 = +%0 = Let l0 = | Get y (u1) | Map 1 -| Project (#2, #0) %1 = -| Get y (u1) -| Map 1 -| Project (#2, #1) +| Get %0 (l0) +| Project (#2, #0) %2 = -| Union %0 %1 +| Get %0 (l0) +| Project (#2, #1) + +%3 = +| Union %1 %2 ---- ---- diff --git a/src/transform/tests/testdata/steps b/src/transform/tests/testdata/steps index 1fe56a09d1265..6e88cc12e426a 100644 --- a/src/transform/tests/testdata/steps +++ b/src/transform/tests/testdata/steps @@ -30,19 +30,61 @@ steps | Union %0 %1 ==== -No change: TopKElision, NonNullRequirements, Fixpoint { transforms: [FuseAndCollapse { transforms: [ProjectionExtraction, ProjectionLifting, Map, Filter, Project, Join, InlineLet, Reduce, Union, UnionBranchCancellation, UpdateLet, RedundantJoin, FoldConstants { limit: Some(10000) }] }], limit: 100 }, Fixpoint { transforms: [PredicatePushdown, NonNullable, ColumnKnowledge, Demand, FuseAndCollapse { transforms: [ProjectionExtraction, ProjectionLifting, Map, Filter, Project, Join, InlineLet, Reduce, Union, UnionBranchCancellation, UpdateLet, RedundantJoin, FoldConstants { limit: Some(10000) }] }], limit: 100 }, Fixpoint { transforms: [ReductionPushdown, ReduceElision, LiteralLifting, FuseAndCollapse { transforms: [ProjectionExtraction, ProjectionLifting, Map, Filter, Project, Join, InlineLet, Reduce, Union, UnionBranchCancellation, UpdateLet, RedundantJoin, FoldConstants { limit: Some(10000) }] }], limit: 100 }, Fixpoint { transforms: [ProjectionLifting, JoinImplementation, ColumnKnowledge, FoldConstants { limit: Some(10000) }, Filter, Demand, LiteralLifting, Map], limit: 100 }, ReductionPushdown, Map, ProjectionLifting, JoinImplementation, Project, FoldConstants { limit: Some(10000) } +No change: TopKElision, NonNullRequirements, Fixpoint { transforms: [FuseAndCollapse { transforms: [ProjectionExtraction, ProjectionLifting, Map, Filter, Project, Join, InlineLet, Reduce, Union, UnionBranchCancellation, UpdateLet, RedundantJoin, FoldConstants { limit: Some(10000) }] }], limit: 100 }, Fixpoint { transforms: [PredicatePushdown, NonNullable, ColumnKnowledge, Demand, FuseAndCollapse { transforms: [ProjectionExtraction, ProjectionLifting, Map, Filter, Project, Join, InlineLet, Reduce, Union, UnionBranchCancellation, UpdateLet, RedundantJoin, FoldConstants { limit: Some(10000) }] }], limit: 100 } ==== -Final: -%0 = +Applied Fixpoint { transforms: [ReductionPushdown, ReduceElision, LiteralLifting, RelationCSE, InlineLet, UpdateLet, FuseAndCollapse { transforms: [ProjectionExtraction, ProjectionLifting, Map, Filter, Project, Join, InlineLet, Reduce, Union, UnionBranchCancellation, UpdateLet, RedundantJoin, FoldConstants { limit: Some(10000) }] }], limit: 100 }: +%0 = Let l0 = | Get x (u0) | Filter #0 %1 = +| Union %0 %0 + +==== +No change: Fixpoint { transforms: [ProjectionLifting, JoinImplementation, ColumnKnowledge, FoldConstants { limit: Some(10000) }, Filter, Demand, LiteralLifting, Map], limit: 100 }, ReductionPushdown, Map, ProjectionLifting, JoinImplementation, Project +==== +Applied RelationCSE: +%0 = Let l0 = | Get x (u0) + +%1 = Let l1 = +| Get %0 (l0) | Filter #0 -%2 = -| Union %0 %1 +%2 = Let l2 = +| Union %1 %1 + +%3 = +| Get %2 (l2) + +==== +Applied InlineLet: +%0 = Let l1 = +| Get x (u0) +| Filter #0 + +%1 = +| Union %0 %0 + +==== +Applied UpdateLet: +%0 = Let l0 = +| Get x (u0) +| Filter #0 + +%1 = +| Union %0 %0 + +==== +No change: FoldConstants { limit: Some(10000) } +==== +Final: +%0 = Let l0 = +| Get x (u0) +| Filter #0 + +%1 = +| Union %0 %0 ==== ---- diff --git a/test/sqllogictest/chbench.slt b/test/sqllogictest/chbench.slt index 03a91f81a1d3b..d2f439feee556 100644 --- a/test/sqllogictest/chbench.slt +++ b/test/sqllogictest/chbench.slt @@ -300,24 +300,25 @@ AND i_id = m_i_id AND s_quantity = m_s_quantity ORDER BY n_name, su_name, i_id ---- -%0 = -| Get materialize.public.item (u24) +%0 = Let l0 = +| Get materialize.public.region (u37) +| Filter "^EUROP.*$" ~(padchar(#1)) | ArrangeBy (#0) %1 = -| Get materialize.public.supplier (u34) +| Get materialize.public.item (u24) | ArrangeBy (#0) %2 = -| Get materialize.public.stock (u26) -| Filter !(isnull(#2)) +| Get materialize.public.supplier (u34) +| ArrangeBy (#0) %3 = -| Get materialize.public.nation (u31) -| ArrangeBy (#0) +| Get materialize.public.stock (u26) +| Filter !(isnull(#2)) %4 = -| Get materialize.public.region (u37) +| Get materialize.public.nation (u31) | ArrangeBy (#0) %5 = @@ -333,28 +334,23 @@ ORDER BY n_name, su_name, i_id | ArrangeBy (#0) (#2) %8 = -| Get materialize.public.region (u37) -| ArrangeBy (#0) - -%9 = -| Join %5 %6 %7 %8 (= #17 #18) (= #21 #25) (= #27 #29) +| Join %5 %6 %7 %0 (= #17 #18) (= #21 #25) (= #27 #29) | | implementation = DeltaQuery -| | delta %5 %6.(#0) %7.(#0) %8.(#0) -| | delta %6 %7.(#0) %8.(#0) %5.(#17) -| | delta %7 %8.(#0) %6.(#3) %5.(#17) -| | delta %8 %7.(#2) %6.(#3) %5.(#17) -| | demand = (#0, #2, #30) -| Filter "^EUROP.*$" ~(padchar(#30)) +| | delta %5 %6.(#0) %7.(#0) %0.(#0) +| | delta %6 %7.(#0) %0.(#0) %5.(#17) +| | delta %7 %0.(#0) %6.(#3) %5.(#17) +| | delta %0 %7.(#2) %6.(#3) %5.(#17) +| | demand = (#0, #2) | Reduce group=(#0) | | agg min(#2) | Filter !(isnull(#1)) | ArrangeBy (#0, #1) -%10 = -| Join %0 %1 %2 %3 %4 %9 (= #0 #12 #37) (= #5 #29) (= #8 #30) (= #14 #38) (= #32 #34) -| | implementation = Differential %2 %9.(#0, #1) %0.(#0) %1.(#0) %3.(#0) %4.(#0) -| | demand = (#0, #2, #4..#7, #9, #11, #31, #35) -| Filter "^.*b$" ~(padchar(#4)), "^EUROP.*$" ~(padchar(#35)) +%9 = +| Join %1 %2 %3 %4 %0 %8 (= #0 #12 #37) (= #5 #29) (= #8 #30) (= #14 #38) (= #32 #34) +| | implementation = Differential %3 %8.(#0, #1) %1.(#0) %2.(#0) %4.(#0) %0.(#0) +| | demand = (#0, #2, #4..#7, #9, #11, #31) +| Filter "^.*b$" ~(padchar(#4)) | Project (#5, #6, #31, #0, #2, #7, #9, #11) Finish order_by=(#2 asc, #1 asc, #3 asc) limit=none offset=0 project=(#0..#7) @@ -589,44 +585,40 @@ AND ol_delivery_d BETWEEN TIMESTAMP '2007-01-02 00:00:00.000000' AND TIMESTAMP ' GROUP BY su_nationkey, substr(c_state, 1, 1), EXTRACT(year FROM o_entry_d) ORDER BY su_nationkey, cust_nation, l_year ---- -%0 = +%0 = Let l0 = +| Get materialize.public.nation (u31) +| ArrangeBy (#0) + +%1 = | Get materialize.public.supplier (u34) | ArrangeBy (#0) (#3) -%1 = +%2 = | Get materialize.public.stock (u26) | ArrangeBy (#0, #1) (#17) -%2 = +%3 = | Get materialize.public.orderline (u19) | ArrangeBy (#2, #1, #0) (#5, #4) -%3 = +%4 = | Get materialize.public.order (u16) | ArrangeBy (#0, #1, #2) (#2, #1, #3) -%4 = +%5 = | Get materialize.public.customer (u6) | ArrangeBy (#0, #1, #2) (#21) -%5 = -| Get materialize.public.nation (u31) -| ArrangeBy (#0) - %6 = -| Get materialize.public.nation (u31) -| ArrangeBy (#0) - -%7 = -| Join %0 %1 %2 %3 %4 %5 %6 (= #0 #24) (= #3 #65) (= #7 #29) (= #8 #30) (= #25 #35) (= #26 #36 #44) (= #27 #37 #45) (= #38 #43) (= #64 #69) +| Join %1 %2 %3 %4 %5 %0 %0 (= #0 #24) (= #3 #65) (= #7 #29) (= #8 #30) (= #25 #35) (= #26 #36 #44) (= #27 #37 #45) (= #38 #43) (= #64 #69) | | implementation = DeltaQuery -| | delta %0 %5.(#0) %1.(#17) %2.(#5, #4) %3.(#0, #1, #2) %4.(#0, #1, #2) %6.(#0) -| | delta %1 %0.(#0) %5.(#0) %2.(#5, #4) %3.(#0, #1, #2) %4.(#0, #1, #2) %6.(#0) -| | delta %2 %3.(#0, #1, #2) %4.(#0, #1, #2) %1.(#0, #1) %0.(#0) %5.(#0) %6.(#0) -| | delta %3 %4.(#0, #1, #2) %6.(#0) %2.(#2, #1, #0) %1.(#0, #1) %0.(#0) %5.(#0) -| | delta %4 %6.(#0) %3.(#2, #1, #3) %2.(#2, #1, #0) %1.(#0, #1) %0.(#0) %5.(#0) -| | delta %5 %0.(#3) %1.(#17) %2.(#5, #4) %3.(#0, #1, #2) %4.(#0, #1, #2) %6.(#0) -| | delta %6 %4.(#21) %3.(#2, #1, #3) %2.(#2, #1, #0) %1.(#0, #1) %0.(#0) %5.(#0) +| | delta %1 %0.(#0) %2.(#17) %3.(#5, #4) %4.(#0, #1, #2) %5.(#0, #1, #2) %0.(#0) +| | delta %2 %1.(#0) %0.(#0) %3.(#5, #4) %4.(#0, #1, #2) %5.(#0, #1, #2) %0.(#0) +| | delta %3 %4.(#0, #1, #2) %5.(#0, #1, #2) %2.(#0, #1) %1.(#0) %0.(#0) %0.(#0) +| | delta %4 %5.(#0, #1, #2) %0.(#0) %3.(#2, #1, #0) %2.(#0, #1) %1.(#0) %0.(#0) +| | delta %5 %0.(#0) %4.(#2, #1, #3) %3.(#2, #1, #0) %2.(#0, #1) %1.(#0) %0.(#0) +| | delta %0 %1.(#3) %2.(#17) %3.(#5, #4) %4.(#0, #1, #2) %5.(#0, #1, #2) %0.(#0) +| | delta %0 %5.(#21) %4.(#2, #1, #3) %3.(#2, #1, #0) %2.(#0, #1) %1.(#0) %0.(#0) | | demand = (#3, #31, #33, #39, #52, #66, #70) | Filter (((#66 = "GERMANY") && (#70 = "CAMBODIA")) || ((#66 = "CAMBODIA") && (#70 = "GERMANY"))), (datetots(#31) <= 2012-01-02 00:00:00), (datetots(#31) >= 2007-01-02 00:00:00) | Reduce group=(#3, substr(chartostr(#52), 1, 1), date_part_year_tstz(datetotstz(#39))) @@ -866,7 +858,7 @@ ORDER BY ordercount DESC | Get materialize.public.nation (u31) | ArrangeBy (#0) -%3 = +%3 = Let l0 = | Join %0 %1 %2 (= #17 #18) (= #21 #25) | | implementation = DeltaQuery | | delta %0 %1.(#0) %2.(#0) @@ -874,37 +866,22 @@ ORDER BY ordercount DESC | | delta %2 %1.(#3) %0.(#17) | | demand = (#0, #14, #26) | Filter (#26 = "GERMANY") -| Reduce group=(#0) -| | agg sum(#14) %4 = -| Get materialize.public.stock (u26) -| ArrangeBy (#17) +| Get %3 (l0) +| Reduce group=(#0) +| | agg sum(#14) %5 = -| Get materialize.public.supplier (u34) -| ArrangeBy (#0) (#3) - -%6 = -| Get materialize.public.nation (u31) -| ArrangeBy (#0) - -%7 = -| Join %4 %5 %6 (= #17 #18) (= #21 #25) -| | implementation = DeltaQuery -| | delta %4 %5.(#0) %6.(#0) -| | delta %5 %6.(#0) %4.(#17) -| | delta %6 %5.(#3) %4.(#17) -| | demand = (#14, #26) -| Filter (#26 = "GERMANY") +| Get %3 (l0) | Reduce group=() | | agg sum(#14) | Map (i64tonumeric(#0) * 0.005) | ArrangeBy () -%8 = -| Join %3 %7 -| | implementation = Differential %3 %7.() +%6 = +| Join %4 %5 +| | implementation = Differential %4 %5.() | | demand = (#0, #1, #3) | Filter (i64tonumeric(#1) > #3) | Project (#0, #1) @@ -1089,53 +1066,41 @@ AND total_revenue = ( ORDER BY su_suppkey ---- %0 = -| Get materialize.public.supplier (u34) -| ArrangeBy (#0) - -%1 = | Get materialize.public.orderline (u19) | ArrangeBy (#5, #4) -%2 = +%1 = | Get materialize.public.stock (u26) | ArrangeBy (#0, #1) -%3 = -| Join %1 %2 (= #4 #10) (= #5 #11) +%2 = Let l0 = +| Join %0 %1 (= #4 #10) (= #5 #11) | | implementation = DeltaQuery -| | delta %1 %2.(#0, #1) -| | delta %2 %1.(#5, #4) +| | delta %0 %1.(#0, #1) +| | delta %1 %0.(#5, #4) | | demand = (#6, #8, #27) | Filter (datetots(#6) >= 2007-01-02 00:00:00) | Reduce group=(#27) | | agg sum(#8) + +%3 = +| Get materialize.public.supplier (u34) | ArrangeBy (#0) %4 = -| Get materialize.public.orderline (u19) -| ArrangeBy (#5, #4) +| Get %2 (l0) +| ArrangeBy (#0) %5 = -| Get materialize.public.stock (u26) -| ArrangeBy (#0, #1) - -%6 = -| Join %4 %5 (= #4 #10) (= #5 #11) -| | implementation = DeltaQuery -| | delta %4 %5.(#0, #1) -| | delta %5 %4.(#5, #4) -| | demand = (#6, #8, #27) -| Filter (datetots(#6) >= 2007-01-02 00:00:00) -| Reduce group=(#27) -| | agg sum(#8) +| Get %2 (l0) | Reduce group=() | | agg max(#1) | Filter !(isnull(#0)) | ArrangeBy (#0) -%7 = -| Join %0 %3 %6 (= #0 #7) (= #8 #9) -| | implementation = Differential %0.(#0) %3.(#0) %6.(#0) +%6 = +| Join %3 %4 %5 (= #0 #7) (= #8 #9) +| | implementation = Differential %3.(#0) %4.(#0) %5.(#0) | | demand = (#0..#2, #4, #8) | Project (#0..#2, #4, #8) @@ -1232,23 +1197,17 @@ FROM WHERE ol_i_id = t.i_id AND ol_quantity < t.a ---- -%0 = +%0 = Let l0 = | Get materialize.public.orderline (u19) -| ArrangeBy (#4) +| Filter !(isnull(#4)) %1 = | Get materialize.public.item (u24) | ArrangeBy (#0) %2 = -| Get materialize.public.orderline (u19) -| ArrangeBy (#4) - -%3 = -| Join %1 %2 (= #0 #9) -| | implementation = DeltaQuery -| | delta %1 %2.(#4) -| | delta %2 %1.(#0) +| Join %1 %0 (= #0 #9) +| | implementation = Differential %0 %1.(#0) | | demand = (#0, #4, #12) | Filter "^.*b$" ~(padchar(#4)) | Reduce group=(#0) @@ -1257,30 +1216,28 @@ AND ol_quantity < t.a | Map (i64tof64(#1) / i64tof64(if (#2 = 0) then {null} else {#2})) | ArrangeBy (#0) -%4 = Let l0 = -| Join %0 %3 (= #4 #10) -| | implementation = DeltaQuery -| | delta %0 %3.(#0) -| | delta %3 %0.(#4) +%3 = Let l1 = +| Join %0 %2 (= #4 #10) +| | implementation = Differential %0 %2.(#0) | | demand = (#7, #8, #13) | Filter (i32tof64(#7) < #13) | Reduce group=() | | agg sum(#8) -%5 = -| Get %4 (l0) +%4 = +| Get %3 (l1) | Negate | Project () -%6 = +%5 = | Constant () -%7 = -| Union %5 %6 +%6 = +| Union %4 %5 | Map null -%8 = -| Union %4 %7 +%7 = +| Union %3 %6 | Map (#0 / 2) | Project (#1) @@ -1595,37 +1552,33 @@ ORDER BY substr(c_state, 1, 1) | | demand = (#0..#2, #9, #16, #24) | Filter (numerictof64(#16) > #24) -%3 = +%3 = Let l1 = | Get %2 (l0) | ArrangeBy (#0, #1, #2) %4 = -| Get %2 (l0) -| ArrangeBy (#0, #1, #2) - -%5 = | Get materialize.public.order (u16) | ArrangeBy (#2, #1, #3) -%6 = -| Join %4 %5 (= #0 #28) (= #1 #26) (= #2 #27) +%5 = +| Join %3 %4 (= #0 #28) (= #1 #26) (= #2 #27) | | implementation = DeltaQuery -| | delta %4 %5.(#2, #1, #3) -| | delta %5 %4.(#0, #1, #2) +| | delta %3 %4.(#2, #1, #3) +| | delta %4 %3.(#0, #1, #2) | | demand = (#0..#2) | Distinct group=(#0, #1, #2) | Negate -%7 = +%6 = | Get %2 (l0) | Project (#0..#2) -%8 = -| Union %6 %7 +%7 = +| Union %5 %6 -%9 = -| Join %3 %8 (= #0 #25) (= #1 #26) (= #2 #27) -| | implementation = Differential %8 %3.(#0, #1, #2) +%8 = +| Join %3 %7 (= #0 #25) (= #1 #26) (= #2 #27) +| | implementation = Differential %7 %3.(#0, #1, #2) | | demand = (#9, #16) | Reduce group=(substr(chartostr(#9), 1, 1)) | | agg count(true) diff --git a/test/sqllogictest/column_knowledge.slt b/test/sqllogictest/column_knowledge.slt index a87fabb695fb0..6a2df82f6b402 100644 --- a/test/sqllogictest/column_knowledge.slt +++ b/test/sqllogictest/column_knowledge.slt @@ -61,80 +61,74 @@ EOF query T multiline EXPLAIN SELECT * FROM t1 LEFT JOIN t2 ON (t1.f1 = t2.f1) WHERE t1.f1 = 123; ---- -%0 = +%0 = Let l0 = | Get materialize.public.t1 (u1) | Filter (#0 = 123) -| ArrangeBy () %1 = +| Get %0 (l0) +| ArrangeBy () + +%2 = | Get materialize.public.t2 (u3) | Filter (#0 = 123) -%2 = Let l0 = -| Join %0 %1 -| | implementation = Differential %1 %0.() +%3 = Let l1 = +| Join %1 %2 +| | implementation = Differential %2 %1.() | | demand = (#0..#3) -%3 = -| Get materialize.public.t1 (u1) -| Filter (#0 = 123) -| ArrangeBy () - %4 = -| Join %3 %2 -| | implementation = Differential %2 %3.() -| | demand = (#0, #1) +| Get %3 (l1) | Negate | Project (#0, #1) %5 = -| Get materialize.public.t1 (u1) -| Filter (#0 = 123) - -%6 = -| Union %4 %5 +| Union %4 %0 | Map null, null -%7 = -| Union %6 %2 +%6 = +| Union %5 %3 EOF query T multiline EXPLAIN SELECT * FROM t1 LEFT JOIN t2 USING (f1) WHERE t1.f1 = 123; ---- -%0 = +%0 = Let l0 = | Get materialize.public.t1 (u1) | Filter (#0 = 123) -| ArrangeBy () %1 = +| Get %0 (l0) +| ArrangeBy () + +%2 = | Get materialize.public.t2 (u3) | Filter (#0 = 123) -%2 = Let l0 = -| Join %0 %1 -| | implementation = Differential %1 %0.() +%3 = Let l1 = +| Join %1 %2 +| | implementation = Differential %2 %1.() | | demand = (#0, #1, #3) -%3 = -| Get %2 (l0) +%4 = +| Get %3 (l1) | Negate | Map 123 | Project (#4, #1) -%4 = -| Get materialize.public.t1 (u1) -| Filter (#0 = 123) +%5 = +| Get %0 (l0) | Map 123 | Project (#2, #1) -%5 = -| Union %3 %4 +%6 = +| Union %4 %5 | Map null, null -%6 = -| Union %2 %5 +%7 = +| Union %3 %6 | Project (#0, #1, #3) EOF @@ -218,7 +212,7 @@ EXPLAIN SELECT (SELECT t1.f1 FROM t1 WHERE t1.f1 = t2.f1) FROM t2 WHERE t2.f1 = | Get materialize.public.t2 (u3) | Filter (#0 = 123) -%1 = +%1 = Let l1 = | Get %0 (l0) | ArrangeBy () @@ -226,41 +220,37 @@ EXPLAIN SELECT (SELECT t1.f1 FROM t1 WHERE t1.f1 = t2.f1) FROM t2 WHERE t2.f1 = | Get materialize.public.t1 (u1) | Filter (#0 = 123) -%3 = Let l1 = +%3 = Let l2 = | Join %1 %2 | | implementation = Differential %2 %1.() | | demand = (#2) %4 = -| Get %0 (l0) -| ArrangeBy () - -%5 = -| Get %3 (l1) +| Get %3 (l2) | Map 123 | Project (#4, #2) -%6 = -| Get %3 (l1) +%5 = +| Get %3 (l2) | Negate | Map 123 | Project (#4) -%7 = +%6 = | Get %0 (l0) | Map 123 | Project (#2) -%8 = -| Union %6 %7 +%7 = +| Union %5 %6 | Map null -%9 = -| Union %5 %8 +%8 = +| Union %4 %7 -%10 = -| Join %4 %9 -| | implementation = Differential %9 %4.() +%9 = +| Join %1 %8 +| | implementation = Differential %8 %1.() | | demand = (#3) | Project (#3) @@ -456,54 +446,38 @@ EXPLAIN SELECT * FROM t4 AS a1, t4 AS a2 WHERE a1.f1 = 123 AND a1.f2 = 234 AND a %0 = | Get materialize.public.t4 (u7) | Filter (#0 = 123), (#1 = 234) -| ArrangeBy () - -%1 = -| Get materialize.public.t4 (u7) -| Filter (#0 = 123), (#1 = 234) - -%2 = -| Join %0 %1 -| | implementation = Differential %1 %0.() -| | demand = (#0..#3) +| Project (#0, #1, #0, #1) EOF query T multiline EXPLAIN SELECT * FROM t4 AS a1 LEFT JOIN t4 AS a2 USING (f1, f2) WHERE a1.f1 = 123 AND a1.f2 = 234; ---- -%0 = +%0 = Let l0 = | Get materialize.public.t4 (u7) | Filter (#0 = 123), (#1 = 234) -| ArrangeBy () %1 = -| Get materialize.public.t4 (u7) -| Filter (#0 = 123), (#1 = 234) - -%2 = Let l0 = -| Join %0 %1 -| | implementation = Differential %1 %0.() -| | demand = (#0, #1) +| Get %0 (l0) +| Project (#0, #1, #0, #1) -%3 = -| Get %2 (l0) +%2 = +| Get %0 (l0) | Negate | Map 123, 234 -| Project (#4, #5) +| Project (#2, #3) -%4 = -| Get materialize.public.t4 (u7) -| Filter (#0 = 123), (#1 = 234) +%3 = +| Get %0 (l0) | Map 123, 234 | Project (#2, #3) -%5 = -| Union %3 %4 +%4 = +| Union %2 %3 | Map null, null -%6 = -| Union %2 %5 +%5 = +| Union %1 %4 | Project (#0, #1) EOF @@ -515,38 +489,31 @@ EOF query T multiline EXPLAIN SELECT * FROM t4 AS a1 LEFT JOIN t4 AS a2 USING (f1, f2) WHERE a1.f1 = 123 AND a2.f2 = 234; ---- -%0 = +%0 = Let l0 = | Get materialize.public.t4 (u7) | Filter (#0 = 123), (#1 = 234) -| ArrangeBy () %1 = -| Get materialize.public.t4 (u7) -| Filter (#0 = 123), (#1 = 234) - -%2 = Let l0 = -| Join %0 %1 -| | implementation = Differential %1 %0.() -| | demand = (#0, #1) +| Get %0 (l0) +| Project (#0, #1, #0, #1) -%3 = -| Get %2 (l0) +%2 = +| Get %0 (l0) | Negate | Map 123, 234 -| Project (#4, #5) +| Project (#2, #3) -%4 = -| Get materialize.public.t4 (u7) -| Filter (#0 = 123), (#1 = 234) +%3 = +| Get %0 (l0) | Map 123, 234 | Project (#2, #3) -%5 = -| Union %3 %4 +%4 = +| Union %2 %3 | Map null, null -%6 = -| Union %2 %5 +%5 = +| Union %1 %4 | Project (#0, #1) EOF @@ -558,38 +525,40 @@ EOF query T multiline EXPLAIN SELECT * FROM t1 LEFT JOIN t2 USING (f1) WHERE t1.f1 = 123 AND t2.f1 = 234; ---- -%0 = +%0 = Let l0 = | Get materialize.public.t1 (u1) | Filter (#0 = 123), (#0 = 234) -| ArrangeBy () %1 = +| Get %0 (l0) +| ArrangeBy () + +%2 = | Get materialize.public.t2 (u3) | Filter (#0 = 123), (#0 = 234) -%2 = Let l0 = -| Join %0 %1 -| | implementation = Differential %1 %0.() +%3 = Let l1 = +| Join %1 %2 +| | implementation = Differential %2 %1.() | | demand = (#0, #1, #3) -%3 = -| Get %2 (l0) +%4 = +| Get %3 (l1) | Negate | Map 123 | Project (#4, #1) -%4 = -| Get materialize.public.t1 (u1) -| Filter (#0 = 123), (#0 = 234) +%5 = +| Get %0 (l0) | Map 123 | Project (#2, #1) -%5 = -| Union %3 %4 +%6 = +| Union %4 %5 | Map null, null -%6 = -| Union %2 %5 +%7 = +| Union %3 %6 | Project (#0, #1, #3) EOF diff --git a/test/sqllogictest/joins.slt b/test/sqllogictest/joins.slt index 6b2efa269fd1d..e866627888400 100644 --- a/test/sqllogictest/joins.slt +++ b/test/sqllogictest/joins.slt @@ -110,23 +110,17 @@ SELECT l1.la, l2.lb, l3.lb FROM l as l1, l as l2, l as l3 WHERE l1.la + 1 = l2.la AND l3.la = l1.la + l2.la ---- -%0 = +%0 = Let l0 = | Get materialize.public.l (u1) | Filter !(isnull(#0)) -%1 = -| Get materialize.public.l (u1) -| Filter !(isnull(#0)) +%1 = Let l1 = +| Get %0 (l0) | ArrangeBy (#0) %2 = -| Get materialize.public.l (u1) -| Filter !(isnull(#0)) -| ArrangeBy (#0) - -%3 = -| Join %0 %1 %2 (= #2 (#0 + 1)) (= #4 (#0 + #2)) -| | implementation = Differential %0 %1.(#0) %2.(#0) +| Join %0 %1 %1 (= #2 (#0 + 1)) (= #4 (#0 + #2)) +| | implementation = Differential %0 %1.(#0) %1.(#0) | | demand = (#0, #3, #5) | Project (#0, #3, #5) @@ -429,56 +423,49 @@ SELECT * FROM l FULL JOIN r ON l.la = r.ra %3 = Let l1 = | Get %2 (l0) | Distinct group=(#0) +| ArrangeBy (#0) %4 = | Get materialize.public.r (u3) %5 = -| Get %3 (l1) -| ArrangeBy (#0) - -%6 = -| Join %4 %5 (= #0 #2) -| | implementation = Differential %4 %5.(#0) +| Join %4 %3 (= #0 #2) +| | implementation = Differential %4 %3.(#0) | | demand = (#0, #1) | Negate | Project (#0, #1) -%7 = +%6 = | Get materialize.public.r (u3) -%8 = -| Union %6 %7 +%7 = +| Union %5 %6 | Map null, null | Project (#2, #3, #0, #1) -%9 = +%8 = | Get materialize.public.l (u1) -%10 = -| Get %3 (l1) -| ArrangeBy (#0) - -%11 = -| Join %9 %10 (= #0 #2) -| | implementation = Differential %9 %10.(#0) +%9 = +| Join %8 %3 (= #0 #2) +| | implementation = Differential %8 %3.(#0) | | demand = (#0, #1) | Negate | Project (#0, #1) -%12 = +%10 = | Get materialize.public.l (u1) -%13 = -| Union %11 %12 +%11 = +| Union %9 %10 | Map null, null -%14 = +%12 = | Get %2 (l0) | Project (#0, #1, #0, #3) -%15 = -| Union %8 %13 %14 +%13 = +| Union %7 %11 %12 EOF diff --git a/test/sqllogictest/relation-cse.slt b/test/sqllogictest/relation-cse.slt new file mode 100644 index 0000000000000..f6fc55ac9a7e9 --- /dev/null +++ b/test/sqllogictest/relation-cse.slt @@ -0,0 +1,1514 @@ +# Copyright Materialize, Inc. and contributors. All rights reserved. +# +# Use of this software is governed by the Business Source License +# included in the LICENSE file at the root of this repository. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0. + +# +# Test Common subexpression elimination for Relations. +# PR https://github.com/MaterializeInc/materialize/pull/7715 +# + +statement ok +CREATE TABLE t1 (f1 INTEGER, f2 INTEGER); + +statement ok +CREATE INDEX i1 ON t1 (f1); + +statement ok +CREATE TABLE t2 (f1 INTEGER, f2 INTEGER); + +## +## Tests around the placement of CSEs in the top-level query itself +## + +# no CSE detection here, no predicates in query +query T multiline +EXPLAIN SELECT * FROM t1 AS a1 , t1 AS a2; +---- +%0 = +| Get materialize.public.t1 (u1) +| ArrangeBy () + +%1 = +| Get materialize.public.t1 (u1) + +%2 = +| Join %0 %1 +| | implementation = Differential %1 %0.() +| | demand = (#0..#3) + +EOF + +# +# Inner joins +# + +query T multiline +EXPLAIN SELECT * FROM t1 AS a1 , t1 AS a2, t1 AS a3; +---- +%0 = Let l0 = +| Get materialize.public.t1 (u1) +| ArrangeBy () + +%1 = +| Get materialize.public.t1 (u1) + +%2 = +| Join %0 %0 %1 +| | implementation = Differential %1 %0.() %0.() +| | demand = (#0..#5) + +EOF + +query T multiline +EXPLAIN SELECT * FROM t1 AS a1 , t1 AS a2 WHERE a1.f1 = 1 AND a2.f1 = 1; +---- +%0 = Let l0 = +| Get materialize.public.t1 (u1) +| Filter (#0 = 1) + +%1 = +| Get %0 (l0) +| ArrangeBy () + +%2 = +| Join %1 %0 +| | implementation = Differential %0 %1.() +| | demand = (#0..#3) + +EOF + +query T multiline +EXPLAIN SELECT * FROM t1 AS a1 , t1 AS a2, t1 AS a3 WHERE a1.f1 = 1 AND a2.f1 = 1 AND a3.f1 = 1; +---- +%0 = Let l0 = +| Get materialize.public.t1 (u1) +| Filter (#0 = 1) + +%1 = Let l1 = +| Get %0 (l0) +| ArrangeBy () + +%2 = +| Join %1 %1 %0 +| | implementation = Differential %0 %1.() %1.() +| | demand = (#0..#5) + +EOF + +# +# Outer join +# + +query T multiline +EXPLAIN SELECT * FROM t1 AS a1 LEFT JOIN t1 AS a2 USING (f1) WHERE a1.f1 = 1 AND a2.f1 = 1; +---- +%0 = Let l0 = +| Get materialize.public.t1 (u1) +| Filter !(isnull(#0)) + +%1 = +| Get %0 (l0) +| ArrangeBy (#0) + +%2 = Let l1 = +| Join %1 %0 (= #0 #2) +| | implementation = Differential %0 %1.(#0) +| | demand = (#0, #1, #3) + +%3 = +| Get %2 (l1) +| Project (#0, #1, #0, #3) + +%4 = +| Get %2 (l1) +| Distinct group=(#0, #1) +| Negate + +%5 = +| Get materialize.public.t1 (u1) +| Distinct group=(#0, #1) + +%6 = +| Union %4 %5 + +%7 = +| Get materialize.public.t1 (u1) +| ArrangeBy (#0, #1) + +%8 = +| Join %6 %7 (= #0 #2) (= #1 #3) +| | implementation = Differential %6 %7.(#0, #1) +| | demand = (#0, #1) +| Map null, null +| Project (#0, #1, #4, #5) + +%9 = +| Union %3 %8 +| Map coalesce(#0, #2) +| Filter (#4 = 1) +| Project (#4, #1, #3) + +EOF + +# +# Subqueries +# + +query T multiline +EXPLAIN SELECT * FROM t1 WHERE f1 = (SELECT f1 FROM t1) AND f2 = (SELECT f1 FROM t1); +---- +%0 = +| Get materialize.public.t1 (u1) +| Filter !(isnull(#0)) +| Project (#0) + +%1 = +| Get materialize.public.t1 (u1) +| Reduce group=() +| | agg count(true) +| Filter (err: more than one record produced in subquery), (#0 > 1) +| Map (err: more than one record produced in subquery) +| Project (#1) + +%2 = Let l0 = +| Union %0 %1 + +%3 = +| Get materialize.public.t1 (u1) +| Filter !(isnull(#0)), !(isnull(#1)) +| ArrangeBy (#1) + +%4 = +| Get %2 (l0) +| ArrangeBy (#0) + +%5 = +| Join %3 %4 %2 (= #0 #2) (= #1 #3) +| | implementation = Differential %2 %3.(#1) %4.(#0) +| | demand = (#0, #1) +| Project (#0, #1) + +EOF + +query T multiline +EXPLAIN SELECT * FROM t1 WHERE f1 = (SELECT f1 FROM t1 WHERE f1 = 1) AND f2 = (SELECT f1 FROM t1 WHERE f1 = 1); +---- +%0 = Let l0 = +| Get materialize.public.t1 (u1) +| Filter (#0 = 1) + +%1 = +| Get %0 (l0) +| Project (#0) + +%2 = +| Get %0 (l0) +| Reduce group=() +| | agg count(true) +| Filter (err: more than one record produced in subquery), (#0 > 1) +| Map (err: more than one record produced in subquery) +| Project (#1) + +%3 = Let l1 = +| Union %1 %2 + +%4 = +| Get materialize.public.t1 (u1) +| Filter !(isnull(#0)), !(isnull(#1)) +| ArrangeBy (#1) + +%5 = +| Get %3 (l1) +| ArrangeBy (#0) + +%6 = +| Join %4 %5 %3 (= #0 #2) (= #1 #3) +| | implementation = Differential %3 %4.(#1) %5.(#0) +| | demand = (#0, #1) +| Project (#0, #1) + +EOF + +query T multiline +EXPLAIN SELECT * FROM t1 WHERE f1 = (SELECT f1 FROM t1) AND EXISTS (SELECT f1 FROM t1); +---- +%0 = Let l0 = +| Get materialize.public.t1 (u1) +| Filter !(isnull(#0)) + +%1 = +| Get %0 (l0) +| ArrangeBy (#0) + +%2 = +| Get %0 (l0) +| Project (#0) + +%3 = +| Get materialize.public.t1 (u1) +| Reduce group=() +| | agg count(true) +| Filter (err: more than one record produced in subquery), (#0 > 1) +| Map (err: more than one record produced in subquery) +| Project (#1) + +%4 = +| Union %2 %3 + +%5 = +| Get materialize.public.t1 (u1) +| Distinct group=() +| ArrangeBy () + +%6 = +| Join %1 %4 %5 (= #0 #2) +| | implementation = Differential %4 %5.() %1.(#0) +| | demand = (#0, #1) +| Project (#0, #1) + +EOF + +query T multiline +EXPLAIN SELECT * FROM t1 WHERE f1 = (SELECT f1 FROM t1) OR f2 = (SELECT f1 FROM t1); +---- +%0 = +| Get materialize.public.t1 (u1) +| Project (#0) + +%1 = +| Get materialize.public.t1 (u1) +| Reduce group=() +| | agg count(true) +| Filter (#0 > 1) +| Map (err: more than one record produced in subquery) +| Project (#1) + +%2 = Let l0 = +| Union %0 %1 + +%3 = +| Get %2 (l0) +| Distinct group=() +| Negate + +%4 = +| Constant () + +%5 = +| Union %3 %4 +| Map null + +%6 = Let l1 = +| Union %2 %5 + +%7 = +| Get materialize.public.t1 (u1) +| ArrangeBy () + +%8 = +| Get %6 (l1) +| ArrangeBy () + +%9 = +| Join %7 %8 %6 +| | implementation = Differential %6 %7.() %8.() +| | demand = (#0..#3) +| Filter ((#0 = #2) || (#1 = #3)) +| Project (#0, #1) + +EOF + +# +# CSEs in derived tables +# + +query T multiline +EXPLAIN SELECT * +FROM (SELECT * FROM t1 WHERE f1 = 1) AS a1 +JOIN (SELECT * FROM t1 WHERE f1 = 1) AS a2 +ON TRUE +---- +%0 = Let l0 = +| Get materialize.public.t1 (u1) +| Filter (#0 = 1) + +%1 = +| Get %0 (l0) +| ArrangeBy () + +%2 = +| Join %1 %0 +| | implementation = Differential %0 %1.() +| | demand = (#0..#3) + +EOF + +query T multiline +EXPLAIN SELECT * +FROM (SELECT * FROM t1 WHERE f1 = 1) AS a1 +, (SELECT * FROM t1 WHERE f1 = 1) AS a2 +WHERE a1.f2 = 2 +AND a2.f2 = 2 +---- +%0 = Let l0 = +| Get materialize.public.t1 (u1) +| Filter (#0 = 1), (#1 = 2) + +%1 = +| Get %0 (l0) +| ArrangeBy () + +%2 = +| Join %1 %0 +| | implementation = Differential %0 %1.() +| | demand = (#0..#3) + +EOF + +# No CSE with only partially-overlapping predicates +query T multiline +EXPLAIN SELECT * +FROM (SELECT * FROM t1 WHERE f1 = 1) AS a1 +, (SELECT * FROM t1 WHERE f1 = 1) AS a2 +WHERE a1.f2 = 2 +AND a2.f2 = 3 +---- +%0 = +| Get materialize.public.t1 (u1) +| Filter (#0 = 1), (#1 = 2) +| ArrangeBy () + +%1 = +| Get materialize.public.t1 (u1) +| Filter (#0 = 1), (#1 = 3) + +%2 = +| Join %0 %1 +| | implementation = Differential %1 %0.() +| | demand = (#0..#3) + +EOF + +# +# CSEs in UNION branches +# + +query T multiline +EXPLAIN SELECT * FROM t1 WHERE f1 = 1 UNION ALL SELECT * FROM t1 WHERE f1 = 1 UNION ALL SELECT * FROM t1 WHERE f1 = 1; +---- +%0 = Let l0 = +| Get materialize.public.t1 (u1) +| Filter (#0 = 1) + +%1 = +| Union %0 %0 %0 + +EOF + +query T multiline +EXPLAIN SELECT * FROM t1 WHERE f1 = 1 UNION ALL SELECT * FROM t1 WHERE f1 = 1 UNION SELECT * FROM t1 WHERE f1 = 1; +---- +%0 = Let l0 = +| Get materialize.public.t1 (u1) +| Filter (#0 = 1) + +%1 = +| Union %0 %0 %0 +| Distinct group=(#1) +| Map 1 +| Project (#1, #0) + +EOF + +# +# CSEs in the SELECT list +# + +query T multiline +EXPLAIN SELECT (SELECT f1 FROM t1 WHERE f1 = 1) , (SELECT f1 FROM t1 WHERE f1 = 1) FROM t1; +---- +%0 = Let l0 = +| Get materialize.public.t1 (u1) +| Filter (#0 = 1) + +%1 = +| Get %0 (l0) +| Project (#0) + +%2 = +| Get %0 (l0) +| Reduce group=() +| | agg count(true) +| Filter (#0 > 1) +| Map (err: more than one record produced in subquery) +| Project (#1) + +%3 = Let l1 = +| Union %1 %2 + +%4 = +| Get %3 (l1) +| Distinct group=() +| Negate + +%5 = +| Constant () + +%6 = +| Union %4 %5 +| Map null + +%7 = Let l2 = +| Union %3 %6 + +%8 = +| Get materialize.public.t1 (u1) +| ArrangeBy () + +%9 = +| Get %7 (l2) +| ArrangeBy () + +%10 = +| Join %8 %9 %7 +| | implementation = Differential %7 %8.() %9.() +| | demand = (#2, #3) +| Project (#2, #3) + +EOF + +query T multiline +EXPLAIN SELECT MIN((SELECT f1 FROM t1 WHERE f1 = 1)), MAX((SELECT f1 FROM t1 WHERE f1 = 1)) FROM t1; +---- +%0 = Let l0 = +| Get materialize.public.t1 (u1) +| Filter (#0 = 1) + +%1 = +| Get %0 (l0) +| Project (#0) + +%2 = +| Get %0 (l0) +| Reduce group=() +| | agg count(true) +| Filter (#0 > 1) +| Map (err: more than one record produced in subquery) +| Project (#1) + +%3 = Let l1 = +| Union %1 %2 + +%4 = +| Get %3 (l1) +| Distinct group=() +| Negate + +%5 = +| Constant () + +%6 = +| Union %4 %5 +| Map null + +%7 = Let l2 = +| Union %3 %6 + +%8 = +| Get materialize.public.t1 (u1) +| ArrangeBy () + +%9 = +| Get %7 (l2) +| ArrangeBy () + +%10 = Let l3 = +| Join %8 %9 %7 +| | implementation = Differential %7 %8.() %9.() +| | demand = (#2, #3) +| Reduce group=() +| | agg min(#2) +| | agg max(#3) + +%11 = +| Get %10 (l3) +| Negate +| Project () + +%12 = +| Constant () + +%13 = +| Union %11 %12 +| Map null, null + +%14 = +| Union %10 %13 + +EOF + +# +# CSEs at two distinct positions within the query +# + +query T multiline +EXPLAIN SELECT (SELECT f1 FROM t1 WHERE f1 = 1) FROM t1 WHERE EXISTS (SELECT f1 FROM t1 WHERE f1 = 1); +---- +%0 = Let l0 = +| Get materialize.public.t1 (u1) +| Filter (#0 = 1) + +%1 = +| Get %0 (l0) +| Project (#0) + +%2 = +| Get %0 (l0) +| Reduce group=() +| | agg count(true) +| Filter (#0 > 1) +| Map (err: more than one record produced in subquery) +| Project (#1) + +%3 = Let l1 = +| Union %1 %2 + +%4 = +| Get materialize.public.t1 (u1) +| ArrangeBy () + +%5 = +| Get %0 (l0) +| Distinct group=() +| ArrangeBy () + +%6 = +| Get %3 (l1) +| Distinct group=() +| Negate + +%7 = +| Constant () + +%8 = +| Union %6 %7 +| Map null + +%9 = +| Union %3 %8 + +%10 = +| Join %4 %5 %9 +| | implementation = Differential %9 %5.() %4.() +| | demand = (#2) +| Project (#2) + +EOF + +query T multiline +EXPLAIN SELECT (SELECT f1 FROM t1 WHERE f1 = 1) FROM t1 +UNION ALL +SELECT f1 FROM t1 WHERE f1 = 1 +---- +%0 = Let l0 = +| Get materialize.public.t1 (u1) +| Filter (#0 = 1) + +%1 = Let l1 = +| Get %0 (l0) +| Project (#0) + +%2 = +| Get %0 (l0) +| Reduce group=() +| | agg count(true) +| Filter (#0 > 1) +| Map (err: more than one record produced in subquery) +| Project (#1) + +%3 = Let l2 = +| Union %1 %2 + +%4 = +| Get materialize.public.t1 (u1) +| ArrangeBy () + +%5 = +| Get %3 (l2) +| Distinct group=() +| Negate + +%6 = +| Constant () + +%7 = +| Union %5 %6 +| Map null + +%8 = +| Union %3 %7 + +%9 = +| Join %4 %8 +| | implementation = Differential %8 %4.() +| | demand = (#2) +| Project (#2) + +%10 = +| Union %9 %1 + +EOF + +## +## Tests around the contents of the CSE itself +## + +# +# CSE containing a join +# + +query T multiline +EXPLAIN +SELECT * FROM t1 AS a1, t1 AS a2 +UNION ALL +SELECT * FROM t1 AS a1, t1 AS a2 +---- +%0 = +| Get materialize.public.t1 (u1) +| ArrangeBy () + +%1 = +| Get materialize.public.t1 (u1) + +%2 = Let l0 = +| Join %0 %1 +| | implementation = Differential %1 %0.() +| | demand = (#0..#3) + +%3 = +| Union %2 %2 + +EOF + +query T multiline +EXPLAIN +SELECT * FROM t1 AS a1 LEFT JOIN t1 AS a2 USING (f1) +UNION ALL +SELECT * FROM t1 AS a1 LEFT JOIN t1 AS a2 USING (f1) +---- +%0 = Let l0 = +| Get materialize.public.t1 (u1) +| Filter !(isnull(#0)) + +%1 = +| Get %0 (l0) +| ArrangeBy (#0) + +%2 = Let l1 = +| Join %1 %0 (= #0 #2) +| | implementation = Differential %0 %1.(#0) +| | demand = (#0, #1, #3) + +%3 = +| Get %2 (l1) +| Project (#0, #1, #0, #3) + +%4 = +| Get %2 (l1) +| Distinct group=(#0, #1) +| Negate + +%5 = +| Get materialize.public.t1 (u1) +| Distinct group=(#0, #1) + +%6 = +| Union %4 %5 + +%7 = +| Get materialize.public.t1 (u1) +| ArrangeBy (#0, #1) + +%8 = +| Join %6 %7 (= #0 #2) (= #1 #3) +| | implementation = Differential %6 %7.(#0, #1) +| | demand = (#0, #1) +| Map null, null +| Project (#0, #1, #4, #5) + +%9 = Let l2 = +| Union %3 %8 +| Map coalesce(#0, #2) + +%10 = +| Union %9 %9 +| Project (#4, #1, #3) + +EOF + +query T multiline +EXPLAIN +SELECT * FROM (SELECT a2.f1 AS f1 FROM t1 AS a1 LEFT JOIN t1 AS a2 USING (f1)) WHERE f1 = 1 +UNION ALL +SELECT * FROM (SELECT a2.f1 AS f1 FROM t1 AS a1 LEFT JOIN t1 AS a2 USING (f1)) WHERE f1 = 2 +---- +%0 = Let l0 = +| Get materialize.public.t1 (u1) +| Filter !(isnull(#0)) + +%1 = +| Get %0 (l0) +| ArrangeBy (#0) + +%2 = Let l1 = +| Join %1 %0 (= #0 #2) +| | implementation = Differential %0 %1.(#0) +| | demand = (#0, #1) + +%3 = +| Get %2 (l1) +| Project (#0, #1, #0, #3) + +%4 = +| Get %2 (l1) +| Distinct group=(#0, #1) +| Negate + +%5 = +| Get materialize.public.t1 (u1) +| Distinct group=(#0, #1) + +%6 = +| Union %4 %5 + +%7 = +| Get materialize.public.t1 (u1) +| ArrangeBy (#0, #1) + +%8 = +| Join %6 %7 (= #0 #2) (= #1 #3) +| | implementation = Differential %6 %7.(#0, #1) +| | demand = (#0) +| Map null, null +| Project (#0, #1, #4, #5) + +%9 = Let l2 = +| Union %3 %8 +| Map coalesce(#0, #2) + +%10 = +| Get %9 (l2) +| Filter (#4 = 1) + +%11 = +| Get %9 (l2) +| Filter (#4 = 2) + +%12 = +| Union %10 %11 +| Project (#4) + +EOF + +query T multiline +EXPLAIN +SELECT * FROM +(SELECT a2.f1 AS f1 FROM t1 AS a1 LEFT JOIN t1 AS a2 USING (f1)) AS s1, +(SELECT a2.f1 AS f1 FROM t1 AS a1 LEFT JOIN t1 AS a2 USING (f1)) AS s2 +WHERE s1.f1 = 1 AND s2.f1 = 1 +---- +%0 = Let l0 = +| Get materialize.public.t1 (u1) +| Filter !(isnull(#0)) + +%1 = +| Get %0 (l0) +| ArrangeBy (#0) + +%2 = Let l1 = +| Join %1 %0 (= #0 #2) +| | implementation = Differential %0 %1.(#0) +| | demand = (#0, #1) + +%3 = +| Get %2 (l1) +| Project (#0, #1, #0, #3) + +%4 = +| Get %2 (l1) +| Distinct group=(#0, #1) +| Negate + +%5 = +| Get materialize.public.t1 (u1) +| Distinct group=(#0, #1) + +%6 = +| Union %4 %5 + +%7 = +| Get materialize.public.t1 (u1) +| ArrangeBy (#0, #1) + +%8 = +| Join %6 %7 (= #0 #2) (= #1 #3) +| | implementation = Differential %6 %7.(#0, #1) +| | demand = (#0) +| Map null, null +| Project (#0, #1, #4, #5) + +%9 = Let l2 = +| Union %3 %8 +| Map coalesce(#0, #2) +| Filter (#4 = 1) + +%10 = +| Get %9 (l2) +| ArrangeBy () + +%11 = +| Join %10 %9 +| | implementation = Differential %9 %10.() +| | demand = (#4, #9) +| Project (#4, #9) + +EOF + +query T multiline +EXPLAIN +SELECT * FROM +(SELECT a2.f1 AS f1 FROM t1 AS a1 LEFT JOIN t1 AS a2 USING (f1)) AS s1, +(SELECT a2.f1 AS f1 FROM t1 AS a1 LEFT JOIN t1 AS a2 USING (f1)) AS s2 +WHERE s1.f1 = 1 AND s2.f1 = 2 +---- +%0 = Let l0 = +| Get materialize.public.t1 (u1) +| Filter !(isnull(#0)) + +%1 = +| Get %0 (l0) +| ArrangeBy (#0) + +%2 = Let l1 = +| Join %1 %0 (= #0 #2) +| | implementation = Differential %0 %1.(#0) +| | demand = (#0, #1) + +%3 = +| Get %2 (l1) +| Project (#0, #1, #0, #3) + +%4 = +| Get %2 (l1) +| Distinct group=(#0, #1) +| Negate + +%5 = +| Get materialize.public.t1 (u1) +| Distinct group=(#0, #1) + +%6 = +| Union %4 %5 + +%7 = +| Get materialize.public.t1 (u1) +| ArrangeBy (#0, #1) + +%8 = +| Join %6 %7 (= #0 #2) (= #1 #3) +| | implementation = Differential %6 %7.(#0, #1) +| | demand = (#0) +| Map null, null +| Project (#0, #1, #4, #5) + +%9 = Let l2 = +| Union %3 %8 +| Map coalesce(#0, #2) + +%10 = +| Get %9 (l2) +| Filter (#4 = 1) +| ArrangeBy () + +%11 = +| Get %9 (l2) +| Filter (#4 = 2) + +%12 = +| Join %10 %11 +| | implementation = Differential %11 %10.() +| | demand = (#4, #9) +| Project (#4, #9) + +EOF + +# +# CSE containing a disjunction (AND) +# + +query T multiline +EXPLAIN +SELECT * FROM t1 WHERE f1 = 1 AND f2 = 2 +UNION ALL +SELECT * FROM t1 WHERE f1 = 1 AND f2 = 2 +---- +%0 = Let l0 = +| Get materialize.public.t1 (u1) +| Filter (#0 = 1), (#1 = 2) + +%1 = +| Union %0 %0 + +EOF + +# +# CSE containing a conjunction (OR) +# + +query T multiline +EXPLAIN +SELECT * FROM t1 WHERE f1 = 1 OR f1 = 2 +UNION ALL +SELECT * FROM t1 WHERE f1 = 1 OR f1 = 2 +---- +%0 = Let l0 = +| Get materialize.public.t1 (u1) +| Filter ((#0 = 1) || (#0 = 2)) + +%1 = +| Union %0 %0 + +EOF + +# +# CSE containing a subquery +# + +query T multiline +EXPLAIN +SELECT * FROM t1 WHERE EXISTS (SELECT * FROM t1 WHERE f1 = 1) +UNION ALL +SELECT * FROM t1 WHERE EXISTS (SELECT * FROM t1 WHERE f1 = 1) +---- +%0 = +| Get materialize.public.t1 (u1) + +%1 = +| Get materialize.public.t1 (u1) +| Filter (#0 = 1) +| Distinct group=() +| ArrangeBy () + +%2 = Let l0 = +| Join %0 %1 +| | implementation = Differential %0 %1.() +| | demand = (#0, #1) + +%3 = +| Union %2 %2 + +EOF + +query T multiline +EXPLAIN +SELECT * FROM t1 WHERE f1 = (SELECT f1 FROM t1 WHERE f1 = 1) +UNION ALL +SELECT * FROM t1 WHERE f1 = (SELECT f1 FROM t1 WHERE f1 = 1) +---- +%0 = Let l0 = +| Get materialize.public.t1 (u1) +| Filter (#0 = 1) + +%1 = +| Get materialize.public.t1 (u1) +| ArrangeBy (#0) + +%2 = +| Get %0 (l0) +| Project (#0) + +%3 = +| Get %0 (l0) +| Reduce group=() +| | agg count(true) +| Filter (err: more than one record produced in subquery), (#0 > 1) +| Map (err: more than one record produced in subquery) +| Project (#1) + +%4 = +| Union %2 %3 + +%5 = Let l1 = +| Join %1 %4 (= #0 #2) +| | implementation = Differential %4 %1.(#0) +| | demand = (#0, #1) + +%6 = +| Union %5 %5 +| Project (#0, #1) + +EOF + +# +# CSE containing a derived table +# + +query T multiline +EXPLAIN +SELECT f1 + 1 FROM (SELECT f1 + 2 AS f1 FROM t1) +UNION ALL +SELECT f1 + 1 FROM (SELECT f1 + 2 AS f1 FROM t1) +---- +%0 = Let l0 = +| Get materialize.public.t1 (u1) +| Map (#0 + 2), (#2 + 1) + +%1 = +| Union %0 %0 +| Project (#3) + +EOF + +# +# CSEs containing an aggregate +# + +query T multiline +EXPLAIN +SELECT MIN(f1) FROM t1 +UNION ALL +SELECT MIN(f1) FROM t1; +---- +%0 = Let l0 = +| Get materialize.public.t1 (u1) +| Reduce group=() +| | agg min(#0) + +%1 = +| Get %0 (l0) +| Negate +| Project () + +%2 = +| Constant () + +%3 = Let l1 = +| Union %1 %2 +| Map null + +%4 = +| Union %0 %3 %0 %3 + +EOF + +query T multiline +EXPLAIN +SELECT DISTINCT f1 FROM t1 +UNION ALL +SELECT DISTINCT f1 FROM t1 +---- +%0 = Let l0 = +| Get materialize.public.t1 (u1) +| Distinct group=(#0) + +%1 = +| Union %0 %0 + +EOF + +query T multiline +EXPLAIN +SELECT f1, COUNT(*) FROM t1 GROUP BY f1 +UNION ALL +SELECT f1, COUNT(*) FROM t1 GROUP BY f1 +---- +%0 = Let l0 = +| Get materialize.public.t1 (u1) +| Reduce group=(#0) +| | agg count(true) + +%1 = +| Union %0 %0 + +EOF + +# +# CSEs containing an expression / function + +query T multiline +EXPLAIN +SELECT f1 + f1 + f1 + f1 FROM t1 +UNION ALL +SELECT f1 + f1 + f1 + f1 FROM t1 +---- +%0 = Let l0 = +| Get materialize.public.t1 (u1) +| Map (((#0 + #0) + #0) + #0) + +%1 = +| Union %0 %0 +| Project (#2) + +EOF + +query T multiline +EXPLAIN +SELECT ABS(f1) FROM t1 +UNION ALL +SELECT ABS(f1) FROM t1 +---- +%0 = Let l0 = +| Get materialize.public.t1 (u1) +| Map abs(#0) + +%1 = +| Union %0 %0 +| Project (#2) + +EOF + +## +## Nested CSEs +## + +query T multiline +EXPLAIN +(SELECT * FROM t1 WHERE f1 = 1 UNION ALL SELECT * FROM t1 WHERE f1 = 1) +UNION ALL +(SELECT * FROM t1 WHERE f1 = 1 UNION ALL SELECT * FROM t1 WHERE f1 = 1) +---- +%0 = Let l0 = +| Get materialize.public.t1 (u1) +| Filter (#0 = 1) + +%1 = +| Union %0 %0 %0 %0 + +EOF + +query T multiline +EXPLAIN +(SELECT * FROM t1 WHERE EXISTS (SELECT * FROM t1 WHERE f1 = 1)) +UNION ALL +(SELECT * FROM t1 WHERE EXISTS (SELECT * FROM t1 WHERE f1 = 1)) +---- +%0 = +| Get materialize.public.t1 (u1) + +%1 = +| Get materialize.public.t1 (u1) +| Filter (#0 = 1) +| Distinct group=() +| ArrangeBy () + +%2 = Let l0 = +| Join %0 %1 +| | implementation = Differential %0 %1.() +| | demand = (#0, #1) + +%3 = +| Union %2 %2 + +EOF + +## +## Deeper-placed CSEs (where a higher-level construct is not a CSE but a lower-level one is) +## + +query T multiline +EXPLAIN +(SELECT f1 FROM t1 WHERE EXISTS (SELECT * FROM t1 WHERE f1 = 1)) +UNION ALL +(SELECT f2 FROM t1 WHERE EXISTS (SELECT * FROM t1 WHERE f1 = 1)) +---- +%0 = +| Get materialize.public.t1 (u1) + +%1 = +| Get materialize.public.t1 (u1) +| Filter (#0 = 1) +| Distinct group=() +| ArrangeBy () + +%2 = Let l0 = +| Join %0 %1 +| | implementation = Differential %0 %1.() +| | demand = (#0, #1) + +%3 = +| Get %2 (l0) +| Project (#0) + +%4 = +| Get %2 (l0) +| Project (#1) + +%5 = +| Union %3 %4 + +EOF + +query T multiline +EXPLAIN +(SELECT * FROM t1 WHERE EXISTS (SELECT * FROM t1 WHERE f1 = 1)) +UNION ALL +(SELECT * FROM t2 WHERE EXISTS (SELECT * FROM t1 WHERE f1 = 1)) +---- +%0 = Let l0 = +| Get materialize.public.t1 (u1) +| Filter (#0 = 1) +| Distinct group=() +| ArrangeBy () + +%1 = +| Get materialize.public.t1 (u1) + +%2 = +| Join %1 %0 +| | implementation = Differential %1 %0.() +| | demand = (#0, #1) + +%3 = +| Get materialize.public.t2 (u4) + +%4 = +| Join %3 %0 +| | implementation = Differential %3 %0.() +| | demand = (#0, #1) + +%5 = +| Union %2 %4 + +EOF + +query T multiline +EXPLAIN SELECT * FROM +(SELECT f1 FROM t2 UNION ALL SELECT f1 FROM t1 WHERE f1 = 1) , +(SELECT f2 FROM t2 UNION ALL SELECT f1 FROM t1 WHERE f1 = 1) +---- +%0 = Let l0 = +| Get materialize.public.t1 (u1) +| Filter (#0 = 1) +| Project (#0) + +%1 = +| Get materialize.public.t2 (u4) +| Project (#0) + +%2 = +| Union %1 %0 +| ArrangeBy () + +%3 = +| Get materialize.public.t2 (u4) +| Project (#1) + +%4 = +| Union %3 %0 + +%5 = +| Join %2 %4 +| | implementation = Differential %4 %2.() +| | demand = (#0, #1) + +EOF + +# Same predicate, different projections +query T multiline +EXPLAIN +SELECT f1 FROM t1 WHERE f1 = 1 +UNION ALL +SELECT f2 FROM t1 WHERE f1 = 1 +---- +%0 = Let l0 = +| Get materialize.public.t1 (u1) +| Filter (#0 = 1) + +%1 = +| Get %0 (l0) +| Project (#0) + +%2 = +| Get %0 (l0) +| Project (#1) + +%3 = +| Union %1 %2 + +EOF + +## +## Negative cases - CSEs should not be identified where they do not exist +## + +# +query T multiline +EXPLAIN SELECT * FROM t1 AS a1, t1 AS a2 WHERE a1.f1 = 1 AND a2.f1 = 2 +---- +%0 = +| Get materialize.public.t1 (u1) +| Filter (#0 = 1) +| ArrangeBy () + +%1 = +| Get materialize.public.t1 (u1) +| Filter (#0 = 2) + +%2 = +| Join %0 %1 +| | implementation = Differential %1 %0.() +| | demand = (#0..#3) + +EOF + +query T multiline +EXPLAIN +SELECT * FROM (SELECT a2.f1 AS f1 FROM t1 AS a1 JOIN t1 AS a2 USING (f1)) WHERE f1 = 1 +UNION ALL +SELECT * FROM (SELECT a2.f1 AS f1 FROM t1 AS a1 JOIN t1 AS a2 USING (f1)) WHERE f1 = 2 +---- +%0 = Let l0 = +| Get materialize.public.t1 (u1) +| Filter (#0 = 1) + +%1 = Let l1 = +| Get materialize.public.t1 (u1) +| Filter (#0 = 2) + +%2 = +| Get %0 (l0) +| ArrangeBy () + +%3 = +| Join %2 %0 +| | implementation = Differential %0 %2.() +| | demand = (#0) + +%4 = +| Get %1 (l1) +| ArrangeBy () + +%5 = +| Join %4 %1 +| | implementation = Differential %1 %4.() +| | demand = (#0) + +%6 = +| Union %3 %5 +| Project (#0) + +EOF + +query T multiline +EXPLAIN +SELECT * FROM t1 WHERE f1 = 1 +UNION ALL +SELECT * FROM t1 WHERE f1 = 2 +---- +%0 = +| Get materialize.public.t1 (u1) +| Filter (#0 = 1) + +%1 = +| Get materialize.public.t1 (u1) +| Filter (#0 = 2) + +%2 = +| Union %0 %1 + +EOF + +query T multiline +EXPLAIN +SELECT MIN(f1) FROM t1 +UNION ALL +SELECT MAX(f1) FROM t1 +---- +%0 = Let l0 = +| Get materialize.public.t1 (u1) +| Reduce group=() +| | agg min(#0) + +%1 = Let l1 = +| Get materialize.public.t1 (u1) +| Reduce group=() +| | agg max(#0) + +%2 = +| Get %0 (l0) +| Negate +| Project () + +%3 = +| Constant () + +%4 = +| Union %2 %3 +| Map null + +%5 = +| Get %1 (l1) +| Negate +| Project () + +%6 = +| Constant () + +%7 = +| Union %5 %6 +| Map null + +%8 = +| Union %0 %4 %1 %7 + +EOF + +query T multiline +EXPLAIN +SELECT MIN(f1) FROM t1 +UNION ALL +SELECT MIN(f2) FROM t1 +---- +%0 = Let l0 = +| Get materialize.public.t1 (u1) +| Reduce group=() +| | agg min(#0) + +%1 = Let l1 = +| Get materialize.public.t1 (u1) +| Reduce group=() +| | agg min(#1) + +%2 = +| Get %0 (l0) +| Negate +| Project () + +%3 = +| Constant () + +%4 = +| Union %2 %3 +| Map null + +%5 = +| Get %1 (l1) +| Negate +| Project () + +%6 = +| Constant () + +%7 = +| Union %5 %6 +| Map null + +%8 = +| Union %0 %4 %1 %7 + +EOF diff --git a/test/sqllogictest/tpch.slt b/test/sqllogictest/tpch.slt index cf729721400d8..c69498eb929fd 100644 --- a/test/sqllogictest/tpch.slt +++ b/test/sqllogictest/tpch.slt @@ -213,75 +213,59 @@ WHERE ORDER BY s_acctbal DESC, n_name, s_name, p_partkey ---- -%0 = -| Get materialize.public.part (u6) -| ArrangeBy (#0) - -%1 = +%0 = Let l0 = | Get materialize.public.supplier (u8) | ArrangeBy (#0) (#3) -%2 = +%1 = Let l1 = | Get materialize.public.partsupp (u11) | ArrangeBy (#0) (#1) -%3 = +%2 = Let l2 = | Get materialize.public.nation (u1) | ArrangeBy (#0) (#2) -%4 = +%3 = Let l3 = | Get materialize.public.region (u4) +| Filter (#1 = "EUROPE") | ArrangeBy (#0) -%5 = Let l0 = -| Join %0 %1 %2 %3 %4 (= #0 #16) (= #9 #17) (= #12 #21) (= #23 #25) +%4 = +| Get materialize.public.part (u6) +| ArrangeBy (#0) + +%5 = Let l4 = +| Join %4 %0 %1 %2 %3 (= #0 #16) (= #9 #17) (= #12 #21) (= #23 #25) | | implementation = DeltaQuery -| | delta %0 %2.(#0) %1.(#0) %3.(#0) %4.(#0) -| | delta %1 %3.(#0) %4.(#0) %2.(#1) %0.(#0) -| | delta %2 %0.(#0) %1.(#0) %3.(#0) %4.(#0) -| | delta %3 %4.(#0) %1.(#3) %2.(#1) %0.(#0) -| | delta %4 %3.(#2) %1.(#3) %2.(#1) %0.(#0) -| | demand = (#0, #2, #4, #5, #10, #11, #13..#15, #19, #22, #26) -| Filter "^.*BRASS$" ~(varchartostr(#4)), (#5 = 15), (#26 = "EUROPE") +| | delta %4 %1.(#0) %0.(#0) %2.(#0) %3.(#0) +| | delta %0 %2.(#0) %3.(#0) %1.(#1) %4.(#0) +| | delta %1 %4.(#0) %0.(#0) %2.(#0) %3.(#0) +| | delta %2 %3.(#0) %0.(#3) %1.(#1) %4.(#0) +| | delta %3 %2.(#2) %0.(#3) %1.(#1) %4.(#0) +| | demand = (#0, #2, #4, #5, #10, #11, #13..#15, #19, #22) +| Filter "^.*BRASS$" ~(varchartostr(#4)), (#5 = 15) %6 = -| Get %5 (l0) +| Get %5 (l4) | Distinct group=(#0) | ArrangeBy (#0) %7 = -| Get materialize.public.partsupp (u11) -| ArrangeBy (#0) (#1) - -%8 = -| Get materialize.public.supplier (u8) -| ArrangeBy (#0) (#3) - -%9 = -| Get materialize.public.nation (u1) -| ArrangeBy (#0) (#2) - -%10 = -| Get materialize.public.region (u4) -| ArrangeBy (#0) - -%11 = -| Join %6 %7 %8 %9 %10 (= #0 #1) (= #2 #6) (= #9 #13) (= #15 #17) +| Join %6 %1 %0 %2 %3 (= #0 #1) (= #2 #6) (= #9 #13) (= #15 #17) | | implementation = DeltaQuery -| | delta %6 %7.(#0) %8.(#0) %9.(#0) %10.(#0) -| | delta %7 %6.(#0) %8.(#0) %9.(#0) %10.(#0) -| | delta %8 %9.(#0) %10.(#0) %7.(#1) %6.(#0) -| | delta %9 %10.(#0) %8.(#3) %7.(#1) %6.(#0) -| | delta %10 %9.(#2) %8.(#3) %7.(#1) %6.(#0) -| | demand = (#0, #4, #18) -| Filter (#18 = "EUROPE") +| | delta %6 %1.(#0) %0.(#0) %2.(#0) %3.(#0) +| | delta %1 %6.(#0) %0.(#0) %2.(#0) %3.(#0) +| | delta %0 %2.(#0) %3.(#0) %1.(#1) %6.(#0) +| | delta %2 %3.(#0) %0.(#3) %1.(#1) %6.(#0) +| | delta %3 %2.(#2) %0.(#3) %1.(#1) %6.(#0) +| | demand = (#0, #4) | Reduce group=(#0) | | agg min(#4) | ArrangeBy (#0, #1) -%12 = -| Join %5 %11 (= #0 #28) (= #19 #29) -| | implementation = Differential %5 %11.(#0, #1) +%8 = +| Join %5 %7 (= #0 #28) (= #19 #29) +| | implementation = Differential %5 %7.(#0, #1) | | demand = (#0, #2, #10, #11, #13..#15, #22) | Project (#14, #10, #22, #0, #2, #11, #13, #15) @@ -544,39 +528,35 @@ ORDER BY cust_nation, l_year ---- -%0 = +%0 = Let l0 = +| Get materialize.public.nation (u1) +| ArrangeBy (#0) + +%1 = | Get materialize.public.supplier (u8) | ArrangeBy (#0) (#3) -%1 = +%2 = | Get materialize.public.lineitem (u21) | ArrangeBy (#0) (#2) -%2 = +%3 = | Get materialize.public.orders (u18) | ArrangeBy (#0) (#1) -%3 = +%4 = | Get materialize.public.customer (u15) | ArrangeBy (#0) (#3) -%4 = -| Get materialize.public.nation (u1) -| ArrangeBy (#0) - %5 = -| Get materialize.public.nation (u1) -| ArrangeBy (#0) - -%6 = -| Join %0 %1 %2 %3 %4 %5 (= #0 #9) (= #3 #40) (= #7 #23) (= #24 #32) (= #35 #44) +| Join %1 %2 %3 %4 %0 %0 (= #0 #9) (= #3 #40) (= #7 #23) (= #24 #32) (= #35 #44) | | implementation = DeltaQuery -| | delta %0 %4.(#0) %1.(#2) %2.(#0) %3.(#0) %5.(#0) -| | delta %1 %0.(#0) %2.(#0) %3.(#0) %4.(#0) %5.(#0) -| | delta %2 %3.(#0) %5.(#0) %1.(#0) %0.(#0) %4.(#0) -| | delta %3 %5.(#0) %2.(#1) %1.(#0) %0.(#0) %4.(#0) -| | delta %4 %0.(#3) %1.(#2) %2.(#0) %3.(#0) %5.(#0) -| | delta %5 %3.(#3) %2.(#1) %1.(#0) %0.(#0) %4.(#0) +| | delta %1 %0.(#0) %2.(#2) %3.(#0) %4.(#0) %0.(#0) +| | delta %2 %1.(#0) %3.(#0) %4.(#0) %0.(#0) %0.(#0) +| | delta %3 %4.(#0) %0.(#0) %2.(#0) %1.(#0) %0.(#0) +| | delta %4 %0.(#0) %3.(#1) %2.(#0) %1.(#0) %0.(#0) +| | delta %0 %1.(#3) %2.(#2) %3.(#0) %4.(#0) %0.(#0) +| | delta %0 %4.(#3) %3.(#1) %2.(#0) %1.(#0) %0.(#0) | | demand = (#12, #13, #17, #41, #45) | Filter (((#41 = "FRANCE") && (#45 = "GERMANY")) || ((#41 = "GERMANY") && (#45 = "FRANCE"))), (#17 <= 1996-12-31), (#17 >= 1995-01-01) | Reduce group=(#41, #45, date_part_year_tstz(datetotstz(#17))) @@ -872,7 +852,7 @@ ORDER BY | Get materialize.public.nation (u1) | ArrangeBy (#0) -%3 = +%3 = Let l0 = | Join %0 %1 %2 (= #1 #5) (= #8 #12) | | implementation = DeltaQuery | | delta %0 %1.(#0) %2.(#0) @@ -880,37 +860,22 @@ ORDER BY | | delta %2 %1.(#3) %0.(#1) | | demand = (#0, #2, #3, #13) | Filter (#13 = "GERMANY") -| Reduce group=(#0) -| | agg sum((#3 * i32tonumeric(#2))) %4 = -| Get materialize.public.partsupp (u11) -| ArrangeBy (#1) +| Get %3 (l0) +| Reduce group=(#0) +| | agg sum((#3 * i32tonumeric(#2))) %5 = -| Get materialize.public.supplier (u8) -| ArrangeBy (#0) (#3) - -%6 = -| Get materialize.public.nation (u1) -| ArrangeBy (#0) - -%7 = -| Join %4 %5 %6 (= #1 #5) (= #8 #12) -| | implementation = DeltaQuery -| | delta %4 %5.(#0) %6.(#0) -| | delta %5 %6.(#0) %4.(#1) -| | delta %6 %5.(#3) %4.(#1) -| | demand = (#2, #3, #13) -| Filter (#13 = "GERMANY") +| Get %3 (l0) | Reduce group=() | | agg sum((#3 * i32tonumeric(#2))) | Map (#0 * 0.0001) | ArrangeBy () -%8 = -| Join %3 %7 -| | implementation = Differential %3 %7.() +%6 = +| Join %4 %5 +| | implementation = Differential %4 %5.() | | demand = (#0, #1, #3) | Filter (#1 > #3) | Project (#0, #1) @@ -1272,7 +1237,7 @@ WHERE l_partkey = p_partkey ) ---- -%0 = +%0 = Let l0 = | Get materialize.public.lineitem (u21) | ArrangeBy (#1) @@ -1280,7 +1245,7 @@ WHERE | Get materialize.public.part (u6) | ArrangeBy (#0) -%2 = Let l0 = +%2 = Let l1 = | Join %0 %1 (= #1 #16) | | implementation = DeltaQuery | | delta %0 %1.(#0) @@ -1289,19 +1254,15 @@ WHERE | Filter (#19 = "Brand#23"), (#22 = "MED BOX") %3 = -| Get %2 (l0) +| Get %2 (l1) | Distinct group=(#1) | ArrangeBy (#0) %4 = -| Get materialize.public.lineitem (u21) -| ArrangeBy (#1) - -%5 = -| Join %3 %4 (= #0 #2) +| Join %3 %0 (= #0 #2) | | implementation = DeltaQuery -| | delta %3 %4.(#1) -| | delta %4 %3.(#0) +| | delta %3 %0.(#1) +| | delta %0 %3.(#0) | | demand = (#0, #5) | Reduce group=(#0) | | agg sum(#5) @@ -1309,28 +1270,28 @@ WHERE | Map (0.2 * (numerictof64(#1) / i64tof64(if (#2 = 0) then {null} else {#2}))) | ArrangeBy (#0) -%6 = Let l1 = -| Join %2 %5 (= #1 #25) -| | implementation = Differential %2 %5.(#0) +%5 = Let l2 = +| Join %2 %4 (= #1 #25) +| | implementation = Differential %2 %4.(#0) | | demand = (#4, #5, #28) | Filter (numerictof64(#4) < #28) | Reduce group=() | | agg sum(#5) -%7 = -| Get %6 (l1) +%6 = +| Get %5 (l2) | Negate | Project () -%8 = +%7 = | Constant () -%9 = -| Union %7 %8 +%8 = +| Union %6 %7 | Map null -%10 = -| Union %6 %9 +%9 = +| Union %5 %8 | Map (#0 / 7) | Project (#1) @@ -1372,48 +1333,44 @@ ORDER BY o_totalprice DESC, o_orderdate ---- -%0 = -| Get materialize.public.customer (u15) +%0 = Let l0 = +| Get materialize.public.lineitem (u21) | ArrangeBy (#0) %1 = -| Get materialize.public.orders (u18) -| ArrangeBy (#0) (#1) +| Get materialize.public.customer (u15) +| ArrangeBy (#0) %2 = -| Get materialize.public.lineitem (u21) -| ArrangeBy (#0) +| Get materialize.public.orders (u18) +| ArrangeBy (#0) (#1) -%3 = Let l0 = -| Join %0 %1 %2 (= #0 #9) (= #8 #17) +%3 = Let l1 = +| Join %1 %2 %0 (= #0 #9) (= #8 #17) | | implementation = DeltaQuery -| | delta %0 %1.(#1) %2.(#0) -| | delta %1 %0.(#0) %2.(#0) +| | delta %1 %2.(#1) %0.(#0) | | delta %2 %1.(#0) %0.(#0) +| | delta %0 %2.(#0) %1.(#0) | | demand = (#0, #1, #8, #11, #12, #21) %4 = -| Get %3 (l0) +| Get %3 (l1) | Distinct group=(#8) | ArrangeBy (#0) %5 = -| Get materialize.public.lineitem (u21) -| ArrangeBy (#0) - -%6 = -| Join %4 %5 (= #0 #1) +| Join %4 %0 (= #0 #1) | | implementation = DeltaQuery -| | delta %4 %5.(#0) -| | delta %5 %4.(#0) +| | delta %4 %0.(#0) +| | delta %0 %4.(#0) | | demand = (#0, #5) | Reduce group=(#0) | | agg sum(#5) | ArrangeBy (#0) -%7 = -| Join %3 %6 (= #8 #33) -| | implementation = Differential %3 %6.(#0) +%6 = +| Join %3 %5 (= #8 #33) +| | implementation = Differential %3 %5.(#0) | | demand = (#0, #1, #8, #11, #12, #21, #34) | Filter (#34 > 300) | Reduce group=(#1, #0, #8, #12, #11) @@ -1658,13 +1615,13 @@ ORDER BY numwait DESC, s_name ---- -%0 = -| Get materialize.public.supplier (u8) -| ArrangeBy (#0) (#3) +%0 = Let l0 = +| Get materialize.public.lineitem (u21) +| Filter (#12 > #11) %1 = -| Get materialize.public.lineitem (u21) -| ArrangeBy (#0) (#2) +| Get materialize.public.supplier (u8) +| ArrangeBy (#0) %2 = | Get materialize.public.orders (u18) @@ -1674,18 +1631,14 @@ ORDER BY | Get materialize.public.nation (u1) | ArrangeBy (#0) -%4 = Let l0 = -| Join %0 %1 %2 %3 (= #0 #9) (= #3 #32) (= #7 #23) -| | implementation = DeltaQuery -| | delta %0 %3.(#0) %1.(#2) %2.(#0) -| | delta %1 %0.(#0) %2.(#0) %3.(#0) -| | delta %2 %1.(#0) %0.(#0) %3.(#0) -| | delta %3 %0.(#3) %1.(#2) %2.(#0) -| | demand = (#0, #1, #7, #18, #19, #25, #33) -| Filter (#25 = "F"), (#33 = "SAUDI ARABIA"), (#19 > #18) +%4 = Let l1 = +| Join %1 %0 %2 %3 (= #0 #9) (= #3 #32) (= #7 #23) +| | implementation = Differential %0 %1.(#0) %2.(#0) %3.(#0) +| | demand = (#0, #1, #7, #25, #33) +| Filter (#25 = "F"), (#33 = "SAUDI ARABIA") %5 = -| Get %4 (l0) +| Get %4 (l1) | Distinct group=(#7, #0) %6 = @@ -1700,28 +1653,28 @@ ORDER BY | Distinct group=(#0, #1) | ArrangeBy (#0, #1) -%8 = Let l1 = +%8 = Let l2 = | Join %4 %7 (= #0 #37) (= #7 #36) | | implementation = Differential %4 %7.(#0, #1) | | demand = (#0, #1, #7) -%9 = Let l2 = -| Get %8 (l1) +%9 = Let l3 = +| Get %8 (l2) | Distinct group=(#7, #0) %10 = -| Get %8 (l1) +| Get %8 (l2) | ArrangeBy (#0, #7) %11 = -| Get materialize.public.lineitem (u21) +| Get %9 (l3) | ArrangeBy (#0) %12 = -| Join %9 %11 (= #0 #2) -| | implementation = Differential %9 %11.(#0) -| | demand = (#0, #1, #4, #13, #14) -| Filter (#1 != #4), (#14 > #13) +| Join %11 %0 (= #0 #2) +| | implementation = Differential %0 %11.(#0) +| | demand = (#0, #1, #4) +| Filter (#1 != #4) | Distinct group=(#0, #1) | Negate @@ -1809,37 +1762,33 @@ ORDER BY | | demand = (#0, #4, #5, #10) | Filter (numerictof64(#5) > #10) -%3 = +%3 = Let l1 = | Get %2 (l0) | ArrangeBy (#0) %4 = -| Get %2 (l0) -| ArrangeBy (#0) - -%5 = | Get materialize.public.orders (u18) | ArrangeBy (#1) -%6 = -| Join %4 %5 (= #0 #12) +%5 = +| Join %3 %4 (= #0 #12) | | implementation = DeltaQuery -| | delta %4 %5.(#1) -| | delta %5 %4.(#0) +| | delta %3 %4.(#1) +| | delta %4 %3.(#0) | | demand = (#0) | Distinct group=(#0) | Negate -%7 = +%6 = | Get %2 (l0) | Project (#0) -%8 = -| Union %6 %7 +%7 = +| Union %5 %6 -%9 = -| Join %3 %8 (= #0 #11) -| | implementation = Differential %8 %3.(#0) +%8 = +| Join %3 %7 (= #0 #11) +| | implementation = Differential %7 %3.(#0) | | demand = (#4, #5) | Reduce group=(substr(chartostr(#4), 1, 2)) | | agg count(true) diff --git a/test/sqllogictest/transform/union.slt b/test/sqllogictest/transform/union.slt index 89a5e7a4ee31a..9884718cc25e7 100644 --- a/test/sqllogictest/transform/union.slt +++ b/test/sqllogictest/transform/union.slt @@ -60,22 +60,18 @@ query II query T multiline EXPLAIN (SELECT * FROM t1 UNION ALL SELECT * FROM t1) EXCEPT ALL (SELECT * FROM t2 UNION ALL SELECT * FROM t2); ---- -%0 = -| Get materialize.public.t1 (u1) +%0 = Let l0 = +| Get materialize.public.t2 (u3) +| Negate %1 = | Get materialize.public.t1 (u1) %2 = -| Get materialize.public.t2 (u3) -| Negate +| Get materialize.public.t1 (u1) %3 = -| Get materialize.public.t2 (u3) -| Negate - -%4 = -| Union %0 %1 %2 %3 +| Union %1 %2 %0 %0 | Threshold EOF