Skip to content

Commit

Permalink
[core] Introduce InPredicateVisitor to refactor Predicate handle (#4486)
Browse files Browse the repository at this point in the history
  • Loading branch information
@xuzifu666
xuzifu666 authored Nov 11, 2024
1 parent fd6786f commit f19a4c9
Show file tree
Hide file tree
Showing 5 changed files with 86 additions and 45 deletions.
53 changes: 53 additions & 0 deletions paimon-common/src/main/java/org/apache/paimon/predicate/InPredicateVisitor.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.paimon.predicate;

import java.util.ArrayList;
import java.util.List;
import java.util.Optional;

/** A utils to handle {@link Predicate}. */
public class InPredicateVisitor {

/**
* Method for handling with In CompoundPredicate.
*
* @param predicate CompoundPredicate to traverse handle
* @param leafName LeafPredicate name
*/
public static Optional<List<Object>> extractInElements(Predicate predicate, String leafName) {
if (!(predicate instanceof CompoundPredicate)) {
return Optional.empty();
}

CompoundPredicate compoundPredicate = (CompoundPredicate) predicate;
List<Object> leafValues = new ArrayList<>();
List<Predicate> children = compoundPredicate.children();
for (Predicate leaf : children) {
if (leaf instanceof LeafPredicate
&& (((LeafPredicate) leaf).function() instanceof Equal)
&& leaf.visit(LeafPredicateExtractor.INSTANCE).get(leafName) != null) {
leafValues.add(((LeafPredicate) leaf).literals().get(0));
} else {
return Optional.empty();
}
}
return Optional.of(leafValues);
}
}
21 changes: 9 additions & 12 deletions paimon-core/src/main/java/org/apache/paimon/table/system/SchemasTable.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import org.apache.paimon.predicate.Equal;
import org.apache.paimon.predicate.GreaterOrEqual;
import org.apache.paimon.predicate.GreaterThan;
import org.apache.paimon.predicate.InPredicateVisitor;
import org.apache.paimon.predicate.LeafPredicate;
import org.apache.paimon.predicate.LeafPredicateExtractor;
import org.apache.paimon.predicate.LessOrEqual;
Expand Down Expand Up @@ -229,18 +230,14 @@ public InnerTableRead withFilter(Predicate predicate) {

// optimize for IN filter
if ((compoundPredicate.function()) instanceof Or) {
List<Predicate> children = compoundPredicate.children();
for (Predicate leaf : children) {
if (leaf instanceof LeafPredicate
&& (((LeafPredicate) leaf).function() instanceof Equal)
&& leaf.visit(LeafPredicateExtractor.INSTANCE).get(leafName)
!= null) {
schemaIds.add((Long) ((LeafPredicate) leaf).literals().get(0));
} else {
schemaIds.clear();
break;
}
}
InPredicateVisitor.extractInElements(predicate, leafName)
.ifPresent(
leafs ->
leafs.forEach(
leaf ->
schemaIds.add(
Long.parseLong(
leaf.toString()))));
}
} else {
handleLeafPredicate(predicate, leafName);
Expand Down
20 changes: 9 additions & 11 deletions paimon-core/src/main/java/org/apache/paimon/table/system/SnapshotsTable.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import org.apache.paimon.predicate.Equal;
import org.apache.paimon.predicate.GreaterOrEqual;
import org.apache.paimon.predicate.GreaterThan;
import org.apache.paimon.predicate.InPredicateVisitor;
import org.apache.paimon.predicate.LeafPredicate;
import org.apache.paimon.predicate.LeafPredicateExtractor;
import org.apache.paimon.predicate.LessOrEqual;
Expand Down Expand Up @@ -232,17 +233,14 @@ public InnerTableRead withFilter(Predicate predicate) {

// optimize for IN filter
if ((compoundPredicate.function()) instanceof Or) {
for (Predicate leaf : children) {
if (leaf instanceof LeafPredicate
&& (((LeafPredicate) leaf).function() instanceof Equal)
&& leaf.visit(LeafPredicateExtractor.INSTANCE).get(leafName)
!= null) {
snapshotIds.add((Long) ((LeafPredicate) leaf).literals().get(0));
} else {
snapshotIds.clear();
break;
}
}
InPredicateVisitor.extractInElements(predicate, leafName)
.ifPresent(
leafs ->
leafs.forEach(
leaf ->
snapshotIds.add(
Long.parseLong(
leaf.toString()))));
}
} else {
handleLeafPredicate(predicate, leafName);
Expand Down
33 changes: 13 additions & 20 deletions paimon-core/src/main/java/org/apache/paimon/table/system/TagsTable.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import org.apache.paimon.fs.Path;
import org.apache.paimon.predicate.CompoundPredicate;
import org.apache.paimon.predicate.Equal;
import org.apache.paimon.predicate.InPredicateVisitor;
import org.apache.paimon.predicate.LeafPredicate;
import org.apache.paimon.predicate.LeafPredicateExtractor;
import org.apache.paimon.predicate.Or;
Expand Down Expand Up @@ -239,26 +240,18 @@ public RecordReader<InternalRow> createReader(Split split) {
CompoundPredicate compoundPredicate = (CompoundPredicate) predicate;
// optimize for IN filter
if ((compoundPredicate.function()) instanceof Or) {
List<Predicate> children = compoundPredicate.children();
for (Predicate leaf : children) {
if (leaf instanceof LeafPredicate
&& (((LeafPredicate) leaf).function() instanceof Equal
&& ((LeafPredicate) leaf).literals().get(0)
instanceof BinaryString)
&& predicate
.visit(LeafPredicateExtractor.INSTANCE)
.get(TAG_NAME)
!= null) {
String equalValue =
((LeafPredicate) leaf).literals().get(0).toString();
if (tagManager.tagExists(equalValue)) {
predicateMap.put(equalValue, tagManager.tag(equalValue));
}
} else {
predicateMap.clear();
break;
}
}
InPredicateVisitor.extractInElements(predicate, TAG_NAME)
.ifPresent(
leafs ->
leafs.forEach(
leaf -> {
String leftName = leaf.toString();
if (tagManager.tagExists(leftName)) {
predicateMap.put(
leftName,
tagManager.tag(leftName));
}
}));
}
}
}
Expand Down
4 changes: 2 additions & 2 deletions ...n-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/CatalogTableITCase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -299,7 +299,7 @@ public void testSchemasTable() {
result =
sql(
"SELECT schema_id, fields, partition_keys, "
+ "primary_keys, options, `comment` FROM T$schemas where schema_id>0 and schema_id<3");
+ "primary_keys, options, `comment` FROM T$schemas where schema_id>0 and schema_id<3 order by schema_id");
assertThat(result.toString())
.isEqualTo(
"[+I[1, [{\"id\":0,\"name\":\"a\",\"type\":\"INT NOT NULL\"},"
Expand All @@ -313,7 +313,7 @@ public void testSchemasTable() {
result =
sql(
"SELECT schema_id, fields, partition_keys, "
+ "primary_keys, options, `comment` FROM T$schemas where schema_id in (1, 3)");
+ "primary_keys, options, `comment` FROM T$schemas where schema_id in (1, 3) order by schema_id");
assertThat(result.toString())
.isEqualTo(
"[+I[1, [{\"id\":0,\"name\":\"a\",\"type\":\"INT NOT NULL\"},"
Expand Down

0 comments on commit f19a4c9

Please sign in to comment.