[core] Fix FilesTable splits too big to distribute (apache#3454)

askwang · May 31, 2024 · ee1d541 · ee1d541
1 parent ebb3d06
commit ee1d541
Show file tree

Hide file tree

Showing 14 changed files with 125 additions and 221 deletions.
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/SingletonSplit.java b/paimon-core/src/main/java/org/apache/paimon/table/source/SingletonSplit.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.table.source;
+
+/** Singleton split use for system table, in which, scan always just produce one split. */
+public abstract class SingletonSplit implements Split {
+
+    @Override
+    public long rowCount() {
+        return 1;
+    }
+}
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/AggregationFieldsTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/AggregationFieldsTable.java
@@ -33,6 +33,7 @@
 import org.apache.paimon.table.source.InnerTableRead;
 import org.apache.paimon.table.source.InnerTableScan;
 import org.apache.paimon.table.source.ReadOnceTableScan;
+import org.apache.paimon.table.source.SingletonSplit;
 import org.apache.paimon.table.source.Split;
 import org.apache.paimon.table.source.TableRead;
 import org.apache.paimon.types.DataField;
@@ -120,32 +121,21 @@ public InnerTableScan withFilter(Predicate predicate) {
 
         @Override
         public Plan innerPlan() {
-            return () ->
-                    Collections.singletonList(
-                            new AggregationSplit(
-                                    new SchemaManager(fileIO, location).listAllIds().size(),
-                                    location));
+            return () -> Collections.singletonList(new AggregationSplit(location));
         }
     }
 
     /** {@link Split} implementation for {@link AggregationFieldsTable}. */
-    private static class AggregationSplit implements Split {
+    private static class AggregationSplit extends SingletonSplit {
 
         private static final long serialVersionUID = 1L;
 
-        private final long rowCount;
         private final Path location;
 
-        private AggregationSplit(long rowCount, Path location) {
-            this.rowCount = rowCount;
+        private AggregationSplit(Path location) {
             this.location = location;
         }
 
-        @Override
-        public long rowCount() {
-            return rowCount;
-        }
-
         @Override
         public boolean equals(Object o) {
             if (this == o) {

diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/AllTableOptionsTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/AllTableOptionsTable.java
@@ -32,6 +32,7 @@
 import org.apache.paimon.table.source.InnerTableRead;
 import org.apache.paimon.table.source.InnerTableScan;
 import org.apache.paimon.table.source.ReadOnceTableScan;
+import org.apache.paimon.table.source.SingletonSplit;
 import org.apache.paimon.table.source.Split;
 import org.apache.paimon.table.source.TableRead;
 import org.apache.paimon.types.DataField;
@@ -42,7 +43,6 @@
 
 import org.apache.paimon.shade.guava30.com.google.common.collect.Iterators;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -121,33 +121,20 @@ public InnerTableScan withFilter(Predicate predicate) {
 
         @Override
         public Plan innerPlan() {
-            return () ->
-                    Collections.singletonList(
-                            new AllTableSplit(
-                                    options(fileIO, allTablePaths).values().stream()
-                                            .flatMap(t -> t.values().stream())
-                                            .reduce(0, (a, b) -> a + b.size(), Integer::sum),
-                                    allTablePaths));
+            return () -> Collections.singletonList(new AllTableSplit(allTablePaths));
         }
     }
 
-    private static class AllTableSplit implements Split {
+    private static class AllTableSplit extends SingletonSplit {
 
         private static final long serialVersionUID = 1L;
 
-        private final long rowCount;
         private final Map<String, Map<String, Path>> allTablePaths;
 
-        private AllTableSplit(long rowCount, Map<String, Map<String, Path>> allTablePaths) {
-            this.rowCount = rowCount;
+        private AllTableSplit(Map<String, Map<String, Path>> allTablePaths) {
             this.allTablePaths = allTablePaths;
         }
 
-        @Override
-        public long rowCount() {
-            return rowCount;
-        }
-
         @Override
         public boolean equals(Object o) {
             if (this == o) {
@@ -192,7 +179,7 @@ public TableRead withIOManager(IOManager ioManager) {
         }
 
         @Override
-        public RecordReader<InternalRow> createReader(Split split) throws IOException {
+        public RecordReader<InternalRow> createReader(Split split) {
             if (!(split instanceof AllTableSplit)) {
                 throw new IllegalArgumentException("Unsupported split: " + split.getClass());
             }

diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/BranchesTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/BranchesTable.java
@@ -35,6 +35,7 @@
 import org.apache.paimon.table.source.InnerTableRead;
 import org.apache.paimon.table.source.InnerTableScan;
 import org.apache.paimon.table.source.ReadOnceTableScan;
+import org.apache.paimon.table.source.SingletonSplit;
 import org.apache.paimon.table.source.Split;
 import org.apache.paimon.table.source.TableRead;
 import org.apache.paimon.types.BigIntType;
@@ -122,28 +123,19 @@ public InnerTableScan withFilter(Predicate predicate) {
 
         @Override
         public Plan innerPlan() {
-            FileStoreTable table = FileStoreTableFactory.create(fileIO, location);
-            long rowCount = table.branchManager().branchCount();
-            return () -> Collections.singletonList(new BranchesSplit(rowCount, location));
+            return () -> Collections.singletonList(new BranchesSplit(location));
         }
     }
 
-    private static class BranchesSplit implements Split {
+    private static class BranchesSplit extends SingletonSplit {
         private static final long serialVersionUID = 1L;
 
-        private final long rowCount;
         private final Path location;
 
-        private BranchesSplit(long rowCount, Path location) {
-            this.rowCount = rowCount;
+        private BranchesSplit(Path location) {
             this.location = location;
         }
 
-        @Override
-        public long rowCount() {
-            return rowCount;
-        }
-
         @Override
         public boolean equals(Object o) {
             if (this == o) {

diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/CatalogOptionsTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/CatalogOptionsTable.java
@@ -30,6 +30,7 @@
 import org.apache.paimon.table.source.InnerTableRead;
 import org.apache.paimon.table.source.InnerTableScan;
 import org.apache.paimon.table.source.ReadOnceTableScan;
+import org.apache.paimon.table.source.SingletonSplit;
 import org.apache.paimon.table.source.Split;
 import org.apache.paimon.table.source.TableRead;
 import org.apache.paimon.types.DataField;
@@ -39,7 +40,6 @@
 
 import org.apache.paimon.shade.guava30.com.google.common.collect.Iterators;
 
-import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.Iterator;
@@ -113,7 +113,7 @@ public Plan innerPlan() {
         }
     }
 
-    private static class CatalogOptionsSplit implements Split {
+    private static class CatalogOptionsSplit extends SingletonSplit {
 
         private static final long serialVersionUID = 1L;
 
@@ -123,11 +123,6 @@ private CatalogOptionsSplit(Options catalogOptions) {
             this.catalogOptions = catalogOptions.toMap();
         }
 
-        @Override
-        public long rowCount() {
-            return catalogOptions.size();
-        }
-
         @Override
         public boolean equals(Object o) {
             if (this == o) {
@@ -168,7 +163,7 @@ public TableRead withIOManager(IOManager ioManager) {
         }
 
         @Override
-        public RecordReader<InternalRow> createReader(Split split) throws IOException {
+        public RecordReader<InternalRow> createReader(Split split) {
             if (!(split instanceof CatalogOptionsTable.CatalogOptionsSplit)) {
                 throw new IllegalArgumentException("Unsupported split: " + split.getClass());
             }

diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/ConsumersTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/ConsumersTable.java
@@ -32,6 +32,7 @@
 import org.apache.paimon.table.source.InnerTableRead;
 import org.apache.paimon.table.source.InnerTableScan;
 import org.apache.paimon.table.source.ReadOnceTableScan;
+import org.apache.paimon.table.source.SingletonSplit;
 import org.apache.paimon.table.source.Split;
 import org.apache.paimon.table.source.TableRead;
 import org.apache.paimon.types.BigIntType;
@@ -114,32 +115,21 @@ public InnerTableScan withFilter(Predicate predicate) {
 
         @Override
         public Plan innerPlan() {
-            return () ->
-                    Collections.singletonList(
-                            new ConsumersTable.ConsumersSplit(
-                                    new ConsumerManager(fileIO, location).listAllIds().size(),
-                                    location));
+            return () -> Collections.singletonList(new ConsumersTable.ConsumersSplit(location));
         }
     }
 
     /** {@link Split} implementation for {@link ConsumersTable}. */
-    private static class ConsumersSplit implements Split {
+    private static class ConsumersSplit extends SingletonSplit {
 
         private static final long serialVersionUID = 1L;
 
-        private final long rowCount;
         private final Path location;
 
-        private ConsumersSplit(long rowCount, Path location) {
-            this.rowCount = rowCount;
+        private ConsumersSplit(Path location) {
             this.location = location;
         }
 
-        @Override
-        public long rowCount() {
-            return rowCount;
-        }
-
         @Override
         public boolean equals(Object o) {
             if (this == o) {