From dfaac7882f73d036ac59f5f47be7043ab1797c17 Mon Sep 17 00:00:00 2001 From: xuyu <11161569@vivo.com> Date: Wed, 6 Nov 2024 20:34:43 +0800 Subject: [PATCH 01/10] [core] Introduce PredicateUtils for Predicate handle --- .../paimon/predicate/PredicateUtils.java | 56 +++++++++++++++++++ .../paimon/table/system/SchemasTable.java | 51 ++++++++--------- .../paimon/table/system/SnapshotsTable.java | 48 +++++++--------- .../apache/paimon/table/system/TagsTable.java | 49 +++++++--------- 4 files changed, 119 insertions(+), 85 deletions(-) create mode 100644 paimon-common/src/main/java/org/apache/paimon/predicate/PredicateUtils.java diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateUtils.java b/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateUtils.java new file mode 100644 index 000000000000..df1c3ab6841e --- /dev/null +++ b/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateUtils.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.predicate; + +import java.util.List; +import java.util.function.Consumer; + +/** A utils to handle {@link Predicate}. */ +public class PredicateUtils { + + /** + * Method for handling with CompoundPredicate. + * + * @param predicate CompoundPredicate to traverse handle + * @param leafName LeafPredicate name + * @param matchConsumer leafName matched handle + * @param unMatchConsumer leafName unmatched handle + */ + public static void traverseCompoundPredicate( + Predicate predicate, + String leafName, + Consumer matchConsumer, + Consumer unMatchConsumer) { + if (!(predicate instanceof CompoundPredicate)) { + throw new RuntimeException( + "PredicateUtils##handleCompoundPredicate should handle with a CompoundPredicate."); + } + CompoundPredicate compoundPredicate = (CompoundPredicate) predicate; + List children = compoundPredicate.children(); + for (Predicate leaf : children) { + if (leaf instanceof LeafPredicate + && (((LeafPredicate) leaf).function() instanceof Equal) + && leaf.visit(LeafPredicateExtractor.INSTANCE).get(leafName) != null) { + matchConsumer.accept(leaf); + } else { + unMatchConsumer.accept(leaf); + } + } + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/SchemasTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/SchemasTable.java index b6150ef7524c..e6fe16ca7870 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/system/SchemasTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/system/SchemasTable.java @@ -26,17 +26,7 @@ import org.apache.paimon.disk.IOManager; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; -import org.apache.paimon.predicate.And; -import org.apache.paimon.predicate.CompoundPredicate; -import org.apache.paimon.predicate.Equal; -import org.apache.paimon.predicate.GreaterOrEqual; -import org.apache.paimon.predicate.GreaterThan; -import org.apache.paimon.predicate.LeafPredicate; -import org.apache.paimon.predicate.LeafPredicateExtractor; -import org.apache.paimon.predicate.LessOrEqual; -import org.apache.paimon.predicate.LessThan; -import org.apache.paimon.predicate.Or; -import org.apache.paimon.predicate.Predicate; +import org.apache.paimon.predicate.*; import org.apache.paimon.reader.RecordReader; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; @@ -205,7 +195,7 @@ private class SchemasRead implements InnerTableRead { private Optional optionalFilterSchemaIdMax = Optional.empty(); private Optional optionalFilterSchemaIdMin = Optional.empty(); - private final List schemaIds = new ArrayList<>(); + private List schemaIds = new ArrayList<>(); public SchemasRead(FileIO fileIO) { this.fileIO = fileIO; @@ -221,26 +211,29 @@ public InnerTableRead withFilter(Predicate predicate) { if (predicate instanceof CompoundPredicate) { CompoundPredicate compoundPredicate = (CompoundPredicate) predicate; if ((compoundPredicate.function()) instanceof And) { - List children = compoundPredicate.children(); - for (Predicate leaf : children) { - handleLeafPredicate(leaf, leafName); - } + PredicateUtils.traverseCompoundPredicate( + predicate, + leafName, + (Predicate p) -> { + handleLeafPredicate(p, leafName); + }, + null); } // optimize for IN filter if ((compoundPredicate.function()) instanceof Or) { - List children = compoundPredicate.children(); - for (Predicate leaf : children) { - if (leaf instanceof LeafPredicate - && (((LeafPredicate) leaf).function() instanceof Equal) - && leaf.visit(LeafPredicateExtractor.INSTANCE).get(leafName) - != null) { - schemaIds.add((Long) ((LeafPredicate) leaf).literals().get(0)); - } else { - schemaIds.clear(); - break; - } - } + PredicateUtils.traverseCompoundPredicate( + predicate, + leafName, + (Predicate p) -> { + if (schemaIds != null) { + schemaIds.add((Long) ((LeafPredicate) p).literals().get(0)); + } + }, + (Predicate p) -> { + schemaIds.clear(); + schemaIds = null; + }); } } else { handleLeafPredicate(predicate, leafName); @@ -299,7 +292,7 @@ public RecordReader createReader(Split split) { SchemaManager manager = new SchemaManager(fileIO, location, branch); Collection tableSchemas; - if (!schemaIds.isEmpty()) { + if (schemaIds != null && !schemaIds.isEmpty()) { tableSchemas = manager.schemasWithId(schemaIds); } else { tableSchemas = diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/SnapshotsTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/SnapshotsTable.java index 8bf4766d580e..cfb8e6456d65 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/system/SnapshotsTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/system/SnapshotsTable.java @@ -27,17 +27,7 @@ import org.apache.paimon.disk.IOManager; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; -import org.apache.paimon.predicate.And; -import org.apache.paimon.predicate.CompoundPredicate; -import org.apache.paimon.predicate.Equal; -import org.apache.paimon.predicate.GreaterOrEqual; -import org.apache.paimon.predicate.GreaterThan; -import org.apache.paimon.predicate.LeafPredicate; -import org.apache.paimon.predicate.LeafPredicateExtractor; -import org.apache.paimon.predicate.LessOrEqual; -import org.apache.paimon.predicate.LessThan; -import org.apache.paimon.predicate.Or; -import org.apache.paimon.predicate.Predicate; +import org.apache.paimon.predicate.*; import org.apache.paimon.reader.RecordReader; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.ReadonlyTable; @@ -208,7 +198,7 @@ private class SnapshotsRead implements InnerTableRead { private RowType readType; private Optional optionalFilterSnapshotIdMax = Optional.empty(); private Optional optionalFilterSnapshotIdMin = Optional.empty(); - private final List snapshotIds = new ArrayList<>(); + private List snapshotIds = new ArrayList<>(); public SnapshotsRead(FileIO fileIO) { this.fileIO = fileIO; @@ -223,26 +213,28 @@ public InnerTableRead withFilter(Predicate predicate) { String leafName = "snapshot_id"; if (predicate instanceof CompoundPredicate) { CompoundPredicate compoundPredicate = (CompoundPredicate) predicate; - List children = compoundPredicate.children(); if ((compoundPredicate.function()) instanceof And) { - for (Predicate leaf : children) { - handleLeafPredicate(leaf, leafName); - } + PredicateUtils.traverseCompoundPredicate( + predicate, + leafName, + (Predicate p) -> handleLeafPredicate(p, leafName), + null); } // optimize for IN filter if ((compoundPredicate.function()) instanceof Or) { - for (Predicate leaf : children) { - if (leaf instanceof LeafPredicate - && (((LeafPredicate) leaf).function() instanceof Equal) - && leaf.visit(LeafPredicateExtractor.INSTANCE).get(leafName) - != null) { - snapshotIds.add((Long) ((LeafPredicate) leaf).literals().get(0)); - } else { - snapshotIds.clear(); - break; - } - } + PredicateUtils.traverseCompoundPredicate( + predicate, + leafName, + (Predicate p) -> { + if (snapshotIds != null) { + snapshotIds.add((Long) ((LeafPredicate) p).literals().get(0)); + } + }, + (Predicate p) -> { + snapshotIds.clear(); + snapshotIds = null; + }); } } else { handleLeafPredicate(predicate, leafName); @@ -304,7 +296,7 @@ public RecordReader createReader(Split split) throws IOException { new SnapshotManager(fileIO, ((SnapshotsSplit) split).location, branch); Iterator snapshots; - if (!snapshotIds.isEmpty()) { + if (snapshotIds != null && !snapshotIds.isEmpty()) { snapshots = snapshotManager.snapshotsWithId(snapshotIds); } else { snapshots = diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/TagsTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/TagsTable.java index f3342e9f2cb3..fa94196afce9 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/system/TagsTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/system/TagsTable.java @@ -26,12 +26,7 @@ import org.apache.paimon.disk.IOManager; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; -import org.apache.paimon.predicate.CompoundPredicate; -import org.apache.paimon.predicate.Equal; -import org.apache.paimon.predicate.LeafPredicate; -import org.apache.paimon.predicate.LeafPredicateExtractor; -import org.apache.paimon.predicate.Or; -import org.apache.paimon.predicate.Predicate; +import org.apache.paimon.predicate.*; import org.apache.paimon.reader.RecordReader; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.ReadonlyTable; @@ -191,6 +186,7 @@ private class TagsRead implements InnerTableRead { private final FileIO fileIO; private RowType readType; + private Map predicateMap = new TreeMap<>(); public TagsRead(FileIO fileIO) { this.fileIO = fileIO; @@ -223,7 +219,7 @@ public RecordReader createReader(Split split) { TagManager tagManager = new TagManager(fileIO, location, branch); Map nameToSnapshot = new TreeMap<>(); - Map predicateMap = new TreeMap<>(); + predicateMap = new TreeMap<>(); if (predicate != null) { if (predicate instanceof LeafPredicate && ((LeafPredicate) predicate).function() instanceof Equal @@ -239,31 +235,28 @@ public RecordReader createReader(Split split) { CompoundPredicate compoundPredicate = (CompoundPredicate) predicate; // optimize for IN filter if ((compoundPredicate.function()) instanceof Or) { - List children = compoundPredicate.children(); - for (Predicate leaf : children) { - if (leaf instanceof LeafPredicate - && (((LeafPredicate) leaf).function() instanceof Equal - && ((LeafPredicate) leaf).literals().get(0) - instanceof BinaryString) - && predicate - .visit(LeafPredicateExtractor.INSTANCE) - .get(TAG_NAME) - != null) { - String equalValue = - ((LeafPredicate) leaf).literals().get(0).toString(); - if (tagManager.tagExists(equalValue)) { - predicateMap.put(equalValue, tagManager.tag(equalValue)); - } - } else { - predicateMap.clear(); - break; - } - } + PredicateUtils.traverseCompoundPredicate( + predicate, + TAG_NAME, + (Predicate p) -> { + String equalValue = + ((LeafPredicate) predicate) + .literals() + .get(0) + .toString(); + if (predicateMap != null && tagManager.tagExists(equalValue)) { + predicateMap.put(equalValue, tagManager.tag(equalValue)); + } + }, + (Predicate p) -> { + predicateMap.clear(); + predicateMap = null; + }); } } } - if (!predicateMap.isEmpty()) { + if (predicateMap != null && !predicateMap.isEmpty()) { nameToSnapshot.putAll(predicateMap); } else { for (Pair tag : tagManager.tagObjects()) { From e79c558d61b3f5c53694896f2288d17ab71da0dc Mon Sep 17 00:00:00 2001 From: xuzifu666 <1206332514@qq.com> Date: Wed, 6 Nov 2024 20:52:14 +0800 Subject: [PATCH 02/10] fix imports --- .../apache/paimon/predicate/PredicateUtils.java | 7 +++++-- .../apache/paimon/table/system/SchemasTable.java | 14 ++++++++++++-- .../paimon/table/system/SnapshotsTable.java | 14 ++++++++++++-- .../org/apache/paimon/table/system/TagsTable.java | 15 ++++++++------- 4 files changed, 37 insertions(+), 13 deletions(-) diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateUtils.java b/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateUtils.java index df1c3ab6841e..2e354e34a4b7 100644 --- a/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateUtils.java +++ b/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateUtils.java @@ -46,10 +46,13 @@ public static void traverseCompoundPredicate( for (Predicate leaf : children) { if (leaf instanceof LeafPredicate && (((LeafPredicate) leaf).function() instanceof Equal) - && leaf.visit(LeafPredicateExtractor.INSTANCE).get(leafName) != null) { + && leaf.visit(LeafPredicateExtractor.INSTANCE).get(leafName) != null + && matchConsumer != null) { matchConsumer.accept(leaf); } else { - unMatchConsumer.accept(leaf); + if (unMatchConsumer != null) { + unMatchConsumer.accept(leaf); + } } } } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/SchemasTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/SchemasTable.java index e6fe16ca7870..7c222c090e76 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/system/SchemasTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/system/SchemasTable.java @@ -26,7 +26,18 @@ import org.apache.paimon.disk.IOManager; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; -import org.apache.paimon.predicate.*; +import org.apache.paimon.predicate.And; +import org.apache.paimon.predicate.CompoundPredicate; +import org.apache.paimon.predicate.Equal; +import org.apache.paimon.predicate.GreaterOrEqual; +import org.apache.paimon.predicate.GreaterThan; +import org.apache.paimon.predicate.LeafPredicate; +import org.apache.paimon.predicate.LeafPredicateExtractor; +import org.apache.paimon.predicate.LessOrEqual; +import org.apache.paimon.predicate.LessThan; +import org.apache.paimon.predicate.Or; +import org.apache.paimon.predicate.Predicate; +import org.apache.paimon.predicate.PredicateUtils; import org.apache.paimon.reader.RecordReader; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; @@ -231,7 +242,6 @@ public InnerTableRead withFilter(Predicate predicate) { } }, (Predicate p) -> { - schemaIds.clear(); schemaIds = null; }); } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/SnapshotsTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/SnapshotsTable.java index cfb8e6456d65..5afb4f7512a3 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/system/SnapshotsTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/system/SnapshotsTable.java @@ -27,7 +27,18 @@ import org.apache.paimon.disk.IOManager; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; -import org.apache.paimon.predicate.*; +import org.apache.paimon.predicate.And; +import org.apache.paimon.predicate.CompoundPredicate; +import org.apache.paimon.predicate.Equal; +import org.apache.paimon.predicate.GreaterOrEqual; +import org.apache.paimon.predicate.GreaterThan; +import org.apache.paimon.predicate.LeafPredicate; +import org.apache.paimon.predicate.LeafPredicateExtractor; +import org.apache.paimon.predicate.LessOrEqual; +import org.apache.paimon.predicate.LessThan; +import org.apache.paimon.predicate.Or; +import org.apache.paimon.predicate.Predicate; +import org.apache.paimon.predicate.PredicateUtils; import org.apache.paimon.reader.RecordReader; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.ReadonlyTable; @@ -232,7 +243,6 @@ public InnerTableRead withFilter(Predicate predicate) { } }, (Predicate p) -> { - snapshotIds.clear(); snapshotIds = null; }); } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/TagsTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/TagsTable.java index fa94196afce9..b27c80483632 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/system/TagsTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/system/TagsTable.java @@ -26,7 +26,13 @@ import org.apache.paimon.disk.IOManager; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; -import org.apache.paimon.predicate.*; +import org.apache.paimon.predicate.CompoundPredicate; +import org.apache.paimon.predicate.Equal; +import org.apache.paimon.predicate.LeafPredicate; +import org.apache.paimon.predicate.LeafPredicateExtractor; +import org.apache.paimon.predicate.Or; +import org.apache.paimon.predicate.Predicate; +import org.apache.paimon.predicate.PredicateUtils; import org.apache.paimon.reader.RecordReader; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.ReadonlyTable; @@ -219,7 +225,6 @@ public RecordReader createReader(Split split) { TagManager tagManager = new TagManager(fileIO, location, branch); Map nameToSnapshot = new TreeMap<>(); - predicateMap = new TreeMap<>(); if (predicate != null) { if (predicate instanceof LeafPredicate && ((LeafPredicate) predicate).function() instanceof Equal @@ -240,16 +245,12 @@ public RecordReader createReader(Split split) { TAG_NAME, (Predicate p) -> { String equalValue = - ((LeafPredicate) predicate) - .literals() - .get(0) - .toString(); + ((LeafPredicate) p).literals().get(0).toString(); if (predicateMap != null && tagManager.tagExists(equalValue)) { predicateMap.put(equalValue, tagManager.tag(equalValue)); } }, (Predicate p) -> { - predicateMap.clear(); predicateMap = null; }); } From b8c80f739be889fa0dedc89762a7651e8e362c62 Mon Sep 17 00:00:00 2001 From: xuyu <11161569@vivo.com> Date: Wed, 6 Nov 2024 21:33:03 +0800 Subject: [PATCH 03/10] add order by for ut avoid Incidental failed --- .../test/java/org/apache/paimon/flink/CatalogTableITCase.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/CatalogTableITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/CatalogTableITCase.java index 9c1a2f4e3918..34c00a3703f5 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/CatalogTableITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/CatalogTableITCase.java @@ -298,7 +298,7 @@ public void testSchemasTable() { result = sql( "SELECT schema_id, fields, partition_keys, " - + "primary_keys, options, `comment` FROM T$schemas where schema_id>0 and schema_id<3"); + + "primary_keys, options, `comment` FROM T$schemas where schema_id>0 and schema_id<3 order by schema_id"); assertThat(result.toString()) .isEqualTo( "[+I[1, [{\"id\":0,\"name\":\"a\",\"type\":\"INT NOT NULL\"}," @@ -312,7 +312,7 @@ public void testSchemasTable() { result = sql( "SELECT schema_id, fields, partition_keys, " - + "primary_keys, options, `comment` FROM T$schemas where schema_id in (1, 3)"); + + "primary_keys, options, `comment` FROM T$schemas where schema_id in (1, 3) order by schema_id"); assertThat(result.toString()) .isEqualTo( "[+I[1, [{\"id\":0,\"name\":\"a\",\"type\":\"INT NOT NULL\"}," From 9ad4e26762195d5fe6b6ac4d48f14273ab506d4a Mon Sep 17 00:00:00 2001 From: xuyu <11161569@vivo.com> Date: Thu, 7 Nov 2024 09:23:56 +0800 Subject: [PATCH 04/10] minor fix --- .../org/apache/paimon/predicate/PredicateUtils.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateUtils.java b/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateUtils.java index 2e354e34a4b7..a6b86f32854d 100644 --- a/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateUtils.java +++ b/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateUtils.java @@ -18,6 +18,8 @@ package org.apache.paimon.predicate; +import org.apache.paimon.utils.Preconditions; + import java.util.List; import java.util.function.Consumer; @@ -37,10 +39,10 @@ public static void traverseCompoundPredicate( String leafName, Consumer matchConsumer, Consumer unMatchConsumer) { - if (!(predicate instanceof CompoundPredicate)) { - throw new RuntimeException( - "PredicateUtils##handleCompoundPredicate should handle with a CompoundPredicate."); - } + Preconditions.checkState( + predicate instanceof CompoundPredicate, + "PredicateUtils##handleCompoundPredicate should handle with a CompoundPredicate."); + CompoundPredicate compoundPredicate = (CompoundPredicate) predicate; List children = compoundPredicate.children(); for (Predicate leaf : children) { From d973c879e8c7740ad7540893d37b0e4f4ea8b188 Mon Sep 17 00:00:00 2001 From: xuyu <11161569@vivo.com> Date: Thu, 7 Nov 2024 10:14:47 +0800 Subject: [PATCH 05/10] address --- .../main/java/org/apache/paimon/predicate/PredicateUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateUtils.java b/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateUtils.java index a6b86f32854d..4503ddd2f3ad 100644 --- a/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateUtils.java +++ b/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateUtils.java @@ -41,7 +41,7 @@ public static void traverseCompoundPredicate( Consumer unMatchConsumer) { Preconditions.checkState( predicate instanceof CompoundPredicate, - "PredicateUtils##handleCompoundPredicate should handle with a CompoundPredicate."); + "PredicateUtils.traverseCompoundPredicate only supports processing Predicates of CompoundPredicate type."); CompoundPredicate compoundPredicate = (CompoundPredicate) predicate; List children = compoundPredicate.children(); From 1698c937982e147ac4a68db3928db426aaf65db3 Mon Sep 17 00:00:00 2001 From: xuyu <11161569@vivo.com> Date: Thu, 7 Nov 2024 11:14:20 +0800 Subject: [PATCH 06/10] minor fix --- .../main/java/org/apache/paimon/table/system/SchemasTable.java | 1 + .../main/java/org/apache/paimon/table/system/SnapshotsTable.java | 1 + .../src/main/java/org/apache/paimon/table/system/TagsTable.java | 1 + 3 files changed, 3 insertions(+) diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/SchemasTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/SchemasTable.java index 7c222c090e76..619d924cc3cd 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/system/SchemasTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/system/SchemasTable.java @@ -242,6 +242,7 @@ public InnerTableRead withFilter(Predicate predicate) { } }, (Predicate p) -> { + schemaIds.clear(); schemaIds = null; }); } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/SnapshotsTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/SnapshotsTable.java index 5afb4f7512a3..6112440ec5ed 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/system/SnapshotsTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/system/SnapshotsTable.java @@ -243,6 +243,7 @@ public InnerTableRead withFilter(Predicate predicate) { } }, (Predicate p) -> { + snapshotIds.clear(); snapshotIds = null; }); } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/TagsTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/TagsTable.java index b27c80483632..d0e98608334c 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/system/TagsTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/system/TagsTable.java @@ -251,6 +251,7 @@ public RecordReader createReader(Split split) { } }, (Predicate p) -> { + predicateMap.clear(); predicateMap = null; }); } From 188bf2cb4af3c9a85a77e55c0ca61774cfc59102 Mon Sep 17 00:00:00 2001 From: xuyu <11161569@vivo.com> Date: Thu, 7 Nov 2024 11:19:30 +0800 Subject: [PATCH 07/10] address --- .../main/java/org/apache/paimon/predicate/PredicateUtils.java | 2 +- .../java/org/apache/paimon/table/system/SchemasTable.java | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateUtils.java b/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateUtils.java index 4503ddd2f3ad..9c33cd5c1dd8 100644 --- a/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateUtils.java +++ b/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateUtils.java @@ -39,7 +39,7 @@ public static void traverseCompoundPredicate( String leafName, Consumer matchConsumer, Consumer unMatchConsumer) { - Preconditions.checkState( + Preconditions.checkArgument( predicate instanceof CompoundPredicate, "PredicateUtils.traverseCompoundPredicate only supports processing Predicates of CompoundPredicate type."); diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/SchemasTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/SchemasTable.java index 619d924cc3cd..11c93ec13559 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/system/SchemasTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/system/SchemasTable.java @@ -225,9 +225,7 @@ public InnerTableRead withFilter(Predicate predicate) { PredicateUtils.traverseCompoundPredicate( predicate, leafName, - (Predicate p) -> { - handleLeafPredicate(p, leafName); - }, + (Predicate p) -> handleLeafPredicate(p, leafName), null); } From 8b4aba9f38da441df120a44ab487c00491c1de88 Mon Sep 17 00:00:00 2001 From: xuyu <11161569@vivo.com> Date: Thu, 7 Nov 2024 11:52:36 +0800 Subject: [PATCH 08/10] address --- .../java/org/apache/paimon/table/system/SchemasTable.java | 7 +++++-- .../org/apache/paimon/table/system/SnapshotsTable.java | 6 ++++-- .../java/org/apache/paimon/table/system/TagsTable.java | 6 ++++-- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/SchemasTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/SchemasTable.java index 11c93ec13559..60993262b8d0 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/system/SchemasTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/system/SchemasTable.java @@ -240,8 +240,11 @@ public InnerTableRead withFilter(Predicate predicate) { } }, (Predicate p) -> { - schemaIds.clear(); - schemaIds = null; + if (schemaIds != null) { + schemaIds.clear(); + schemaIds = null; + } + }); } } else { diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/SnapshotsTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/SnapshotsTable.java index 6112440ec5ed..e296fb40fac9 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/system/SnapshotsTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/system/SnapshotsTable.java @@ -243,8 +243,10 @@ public InnerTableRead withFilter(Predicate predicate) { } }, (Predicate p) -> { - snapshotIds.clear(); - snapshotIds = null; + if (snapshotIds != null) { + snapshotIds.clear(); + snapshotIds = null; + } }); } } else { diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/TagsTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/TagsTable.java index d0e98608334c..c814d1171bfd 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/system/TagsTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/system/TagsTable.java @@ -251,8 +251,10 @@ public RecordReader createReader(Split split) { } }, (Predicate p) -> { - predicateMap.clear(); - predicateMap = null; + if (predicateMap != null) { + predicateMap.clear(); + predicateMap = null; + } }); } } From 3f1b0d1bf4ee5e8745c16dc85617acb37064b388 Mon Sep 17 00:00:00 2001 From: xuyu <11161569@vivo.com> Date: Thu, 7 Nov 2024 13:18:00 +0800 Subject: [PATCH 09/10] fix style --- .../main/java/org/apache/paimon/table/system/SchemasTable.java | 1 - 1 file changed, 1 deletion(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/SchemasTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/SchemasTable.java index 60993262b8d0..29863652c3c3 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/system/SchemasTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/system/SchemasTable.java @@ -244,7 +244,6 @@ public InnerTableRead withFilter(Predicate predicate) { schemaIds.clear(); schemaIds = null; } - }); } } else { From cb36eb21721acc03ad8c31dde35122f73769e9c2 Mon Sep 17 00:00:00 2001 From: xuyu <11161569@vivo.com> Date: Thu, 7 Nov 2024 13:21:14 +0800 Subject: [PATCH 10/10] fix --- .../java/org/apache/paimon/table/system/SchemasTable.java | 7 +------ .../org/apache/paimon/table/system/SnapshotsTable.java | 7 +------ .../java/org/apache/paimon/table/system/TagsTable.java | 7 +------ 3 files changed, 3 insertions(+), 18 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/SchemasTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/SchemasTable.java index 29863652c3c3..05aa5a6735e9 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/system/SchemasTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/system/SchemasTable.java @@ -239,12 +239,7 @@ public InnerTableRead withFilter(Predicate predicate) { schemaIds.add((Long) ((LeafPredicate) p).literals().get(0)); } }, - (Predicate p) -> { - if (schemaIds != null) { - schemaIds.clear(); - schemaIds = null; - } - }); + (Predicate p) -> schemaIds = null); } } else { handleLeafPredicate(predicate, leafName); diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/SnapshotsTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/SnapshotsTable.java index e296fb40fac9..fa9368e6b4d4 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/system/SnapshotsTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/system/SnapshotsTable.java @@ -242,12 +242,7 @@ public InnerTableRead withFilter(Predicate predicate) { snapshotIds.add((Long) ((LeafPredicate) p).literals().get(0)); } }, - (Predicate p) -> { - if (snapshotIds != null) { - snapshotIds.clear(); - snapshotIds = null; - } - }); + (Predicate p) -> snapshotIds = null); } } else { handleLeafPredicate(predicate, leafName); diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/TagsTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/TagsTable.java index c814d1171bfd..13d7bc6615fd 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/system/TagsTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/system/TagsTable.java @@ -250,12 +250,7 @@ public RecordReader createReader(Split split) { predicateMap.put(equalValue, tagManager.tag(equalValue)); } }, - (Predicate p) -> { - if (predicateMap != null) { - predicateMap.clear(); - predicateMap = null; - } - }); + (Predicate p) -> predicateMap = null); } } }