[core] Add Table.indexManifestFileReader (apache#4068)

liming30 · Aug 27, 2024 · d373a86 · d373a86
1 parent db77eec
commit d373a86
Show file tree

Hide file tree

Showing 15 changed files with 87 additions and 3 deletions.
diff --git a/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java b/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java
@@ -141,7 +141,8 @@ protected ManifestList.Factory manifestListFactory(boolean forWrite) {
                 forWrite ? writeManifestCache : readManifestCache);
     }
 
-    protected IndexManifestFile.Factory indexManifestFileFactory() {
+    @Override
+    public IndexManifestFile.Factory indexManifestFileFactory() {
         return new IndexManifestFile.Factory(
                 fileIO,
                 options.manifestFormat(),

diff --git a/paimon-core/src/main/java/org/apache/paimon/FileStore.java b/paimon-core/src/main/java/org/apache/paimon/FileStore.java
@@ -20,6 +20,7 @@
 
 import org.apache.paimon.fs.Path;
 import org.apache.paimon.index.IndexFileHandler;
+import org.apache.paimon.manifest.IndexManifestFile;
 import org.apache.paimon.manifest.ManifestCacheFilter;
 import org.apache.paimon.manifest.ManifestFile;
 import org.apache.paimon.manifest.ManifestList;
@@ -70,6 +71,8 @@ public interface FileStore<T> {
 
     ManifestFile.Factory manifestFileFactory();
 
+    IndexManifestFile.Factory indexManifestFileFactory();
+
     IndexFileHandler newIndexFileHandler();
 
     StatsFileHandler newStatsFileHandler();

diff --git a/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMeta.java b/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMeta.java
@@ -18,6 +18,7 @@
 
 package org.apache.paimon.index;
 
+import org.apache.paimon.annotation.Public;
 import org.apache.paimon.deletionvectors.DeletionVectorsIndexFile;
 import org.apache.paimon.types.ArrayType;
 import org.apache.paimon.types.BigIntType;
@@ -34,7 +35,12 @@
 
 import static org.apache.paimon.utils.SerializationUtils.newStringType;
 
-/** Metadata of index file. */
+/**
+ * Metadata of index file.
+ *
+ * @since 0.9.0
+ */
+@Public
 public class IndexFileMeta {
 
     public static final RowType SCHEMA =

diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntry.java b/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntry.java
@@ -18,6 +18,7 @@
 
 package org.apache.paimon.manifest;
 
+import org.apache.paimon.annotation.Public;
 import org.apache.paimon.data.BinaryRow;
 import org.apache.paimon.index.IndexFileMeta;
 import org.apache.paimon.types.ArrayType;
@@ -34,7 +35,12 @@
 import static org.apache.paimon.utils.SerializationUtils.newBytesType;
 import static org.apache.paimon.utils.SerializationUtils.newStringType;
 
-/** Manifest entry for index file. */
+/**
+ * Manifest entry for index file.
+ *
+ * @since 0.9.0
+ */
+@Public
 public class IndexManifestEntry {
 
     public static final RowType SCHEMA =

diff --git a/paimon-core/src/main/java/org/apache/paimon/privilege/PrivilegedFileStore.java b/paimon-core/src/main/java/org/apache/paimon/privilege/PrivilegedFileStore.java
@@ -23,6 +23,7 @@
 import org.apache.paimon.catalog.Identifier;
 import org.apache.paimon.fs.Path;
 import org.apache.paimon.index.IndexFileHandler;
+import org.apache.paimon.manifest.IndexManifestFile;
 import org.apache.paimon.manifest.ManifestCacheFilter;
 import org.apache.paimon.manifest.ManifestFile;
 import org.apache.paimon.manifest.ManifestList;
@@ -106,6 +107,11 @@ public ManifestFile.Factory manifestFileFactory() {
         return wrapped.manifestFileFactory();
     }
 
+    @Override
+    public IndexManifestFile.Factory indexManifestFileFactory() {
+        return wrapped.indexManifestFileFactory();
+    }
+
     @Override
     public IndexFileHandler newIndexFileHandler() {
         return wrapped.newIndexFileHandler();

diff --git a/paimon-core/src/main/java/org/apache/paimon/privilege/PrivilegedFileStoreTable.java b/paimon-core/src/main/java/org/apache/paimon/privilege/PrivilegedFileStoreTable.java
@@ -20,6 +20,7 @@
 
 import org.apache.paimon.FileStore;
 import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.manifest.IndexManifestEntry;
 import org.apache.paimon.manifest.ManifestCacheFilter;
 import org.apache.paimon.manifest.ManifestEntry;
 import org.apache.paimon.manifest.ManifestFileMeta;
@@ -104,6 +105,11 @@ public SimpleFileReader<ManifestEntry> manifestFileReader() {
         return wrapped.manifestFileReader();
     }
 
+    @Override
+    public SimpleFileReader<IndexManifestEntry> indexManifestFileReader() {
+        return wrapped.indexManifestFileReader();
+    }
+
     @Override
     public FileStoreTable copy(TableSchema newTableSchema) {
         return new PrivilegedFileStoreTable(

diff --git a/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java b/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java
@@ -24,6 +24,7 @@
 import org.apache.paimon.consumer.ConsumerManager;
 import org.apache.paimon.fs.FileIO;
 import org.apache.paimon.fs.Path;
+import org.apache.paimon.manifest.IndexManifestEntry;
 import org.apache.paimon.manifest.ManifestEntry;
 import org.apache.paimon.manifest.ManifestFileMeta;
 import org.apache.paimon.metastore.AddPartitionCommitCallback;
@@ -138,6 +139,11 @@ public SimpleFileReader<ManifestEntry> manifestFileReader() {
         return store().manifestFileFactory().create();
     }
 
+    @Override
+    public SimpleFileReader<IndexManifestEntry> indexManifestFileReader() {
+        return store().indexManifestFileFactory().create();
+    }
+
     @Override
     public String name() {
         return identifier().getObjectName();

diff --git a/paimon-core/src/main/java/org/apache/paimon/table/FallbackReadFileStoreTable.java b/paimon-core/src/main/java/org/apache/paimon/table/FallbackReadFileStoreTable.java
@@ -23,6 +23,7 @@
 import org.apache.paimon.data.BinaryRow;
 import org.apache.paimon.data.InternalRow;
 import org.apache.paimon.disk.IOManager;
+import org.apache.paimon.manifest.IndexManifestEntry;
 import org.apache.paimon.manifest.ManifestEntry;
 import org.apache.paimon.manifest.ManifestFileMeta;
 import org.apache.paimon.manifest.PartitionEntry;
@@ -86,6 +87,11 @@ public SimpleFileReader<ManifestEntry> manifestFileReader() {
         return wrapped.manifestFileReader();
     }
 
+    @Override
+    public SimpleFileReader<IndexManifestEntry> indexManifestFileReader() {
+        return wrapped.indexManifestFileReader();
+    }
+
     @Override
     public FileStoreTable copy(TableSchema newTableSchema) {
         return new FallbackReadFileStoreTable(

diff --git a/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java b/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java
@@ -21,6 +21,7 @@
 import org.apache.paimon.Snapshot;
 import org.apache.paimon.annotation.Public;
 import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.manifest.IndexManifestEntry;
 import org.apache.paimon.manifest.ManifestEntry;
 import org.apache.paimon.manifest.ManifestFileMeta;
 import org.apache.paimon.stats.Statistics;
@@ -235,6 +236,11 @@ default SimpleFileReader<ManifestEntry> manifestFileReader() {
         throw new UnsupportedOperationException();
     }
 
+    @Override
+    default SimpleFileReader<IndexManifestEntry> indexManifestFileReader() {
+        throw new UnsupportedOperationException();
+    }
+
     @Override
     default void rollbackTo(long snapshotId) {
         throw new UnsupportedOperationException();

diff --git a/paimon-core/src/main/java/org/apache/paimon/table/ReadonlyTable.java b/paimon-core/src/main/java/org/apache/paimon/table/ReadonlyTable.java
@@ -19,6 +19,7 @@
 package org.apache.paimon.table;
 
 import org.apache.paimon.Snapshot;
+import org.apache.paimon.manifest.IndexManifestEntry;
 import org.apache.paimon.manifest.ManifestEntry;
 import org.apache.paimon.manifest.ManifestFileMeta;
 import org.apache.paimon.stats.Statistics;
@@ -140,6 +141,14 @@ default SimpleFileReader<ManifestEntry> manifestFileReader() {
                         this.getClass().getSimpleName()));
     }
 
+    @Override
+    default SimpleFileReader<IndexManifestEntry> indexManifestFileReader() {
+        throw new UnsupportedOperationException(
+                String.format(
+                        "Readonly Table %s does not support indexManifestFileReader.",
+                        this.getClass().getSimpleName()));
+    }
+
     @Override
     default void rollbackTo(long snapshotId) {
         throw new UnsupportedOperationException(

diff --git a/paimon-core/src/main/java/org/apache/paimon/table/Table.java b/paimon-core/src/main/java/org/apache/paimon/table/Table.java
@@ -21,6 +21,7 @@
 import org.apache.paimon.Snapshot;
 import org.apache.paimon.annotation.Experimental;
 import org.apache.paimon.annotation.Public;
+import org.apache.paimon.manifest.IndexManifestEntry;
 import org.apache.paimon.manifest.ManifestEntry;
 import org.apache.paimon.manifest.ManifestFileMeta;
 import org.apache.paimon.stats.Statistics;
@@ -94,6 +95,10 @@ default String fullName() {
     @Experimental
     SimpleFileReader<ManifestEntry> manifestFileReader();
 
+    /** Reader to read index manifest entry from index manifest file. */
+    @Experimental
+    SimpleFileReader<IndexManifestEntry> indexManifestFileReader();
+
     /** Rollback table's state to a specific snapshot. */
     @Experimental
     void rollbackTo(long snapshotId);

diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/AuditLogTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/AuditLogTable.java
@@ -27,6 +27,7 @@
 import org.apache.paimon.disk.IOManager;
 import org.apache.paimon.fs.FileIO;
 import org.apache.paimon.fs.Path;
+import org.apache.paimon.manifest.IndexManifestEntry;
 import org.apache.paimon.manifest.ManifestEntry;
 import org.apache.paimon.manifest.ManifestFileMeta;
 import org.apache.paimon.manifest.PartitionEntry;
@@ -124,6 +125,11 @@ public SimpleFileReader<ManifestEntry> manifestFileReader() {
         return wrapped.manifestFileReader();
     }
 
+    @Override
+    public SimpleFileReader<IndexManifestEntry> indexManifestFileReader() {
+        return wrapped.indexManifestFileReader();
+    }
+
     @Override
     public String name() {
         return wrapped.name() + SYSTEM_TABLE_SPLITTER + AUDIT_LOG;

diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/BucketsTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/BucketsTable.java
@@ -28,6 +28,7 @@
 import org.apache.paimon.fs.Path;
 import org.apache.paimon.io.DataFileMeta;
 import org.apache.paimon.io.DataFileMetaSerializer;
+import org.apache.paimon.manifest.IndexManifestEntry;
 import org.apache.paimon.manifest.ManifestEntry;
 import org.apache.paimon.manifest.ManifestFileMeta;
 import org.apache.paimon.predicate.Predicate;
@@ -128,6 +129,11 @@ public SimpleFileReader<ManifestEntry> manifestFileReader() {
         return wrapped.manifestFileReader();
     }
 
+    @Override
+    public SimpleFileReader<IndexManifestEntry> indexManifestFileReader() {
+        return wrapped.indexManifestFileReader();
+    }
+
     @Override
     public Path location() {
         return wrapped.location();

diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/FileMonitorTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/FileMonitorTable.java
@@ -29,6 +29,7 @@
 import org.apache.paimon.fs.Path;
 import org.apache.paimon.io.DataFileMeta;
 import org.apache.paimon.io.DataFileMetaSerializer;
+import org.apache.paimon.manifest.IndexManifestEntry;
 import org.apache.paimon.manifest.ManifestEntry;
 import org.apache.paimon.manifest.ManifestFileMeta;
 import org.apache.paimon.predicate.Predicate;
@@ -115,6 +116,11 @@ public SimpleFileReader<ManifestEntry> manifestFileReader() {
         return wrapped.manifestFileReader();
     }
 
+    @Override
+    public SimpleFileReader<IndexManifestEntry> indexManifestFileReader() {
+        return wrapped.indexManifestFileReader();
+    }
+
     @Override
     public Path location() {
         return wrapped.location();

diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/ReadOptimizedTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/ReadOptimizedTable.java
@@ -22,6 +22,7 @@
 import org.apache.paimon.Snapshot;
 import org.apache.paimon.fs.FileIO;
 import org.apache.paimon.fs.Path;
+import org.apache.paimon.manifest.IndexManifestEntry;
 import org.apache.paimon.manifest.ManifestEntry;
 import org.apache.paimon.manifest.ManifestFileMeta;
 import org.apache.paimon.operation.DefaultValueAssigner;
@@ -85,6 +86,11 @@ public SimpleFileReader<ManifestEntry> manifestFileReader() {
         return wrapped.manifestFileReader();
     }
 
+    @Override
+    public SimpleFileReader<IndexManifestEntry> indexManifestFileReader() {
+        return wrapped.indexManifestFileReader();
+    }
+
     @Override
     public String name() {
         return wrapped.name() + SYSTEM_TABLE_SPLITTER + READ_OPTIMIZED;