diff --git a/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java b/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java index 21fb87562d2d..7c10a1aa7c25 100644 --- a/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java +++ b/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java @@ -141,7 +141,8 @@ protected ManifestList.Factory manifestListFactory(boolean forWrite) { forWrite ? writeManifestCache : readManifestCache); } - protected IndexManifestFile.Factory indexManifestFileFactory() { + @Override + public IndexManifestFile.Factory indexManifestFileFactory() { return new IndexManifestFile.Factory( fileIO, options.manifestFormat(), diff --git a/paimon-core/src/main/java/org/apache/paimon/FileStore.java b/paimon-core/src/main/java/org/apache/paimon/FileStore.java index 715062565683..f9bf4c8440bd 100644 --- a/paimon-core/src/main/java/org/apache/paimon/FileStore.java +++ b/paimon-core/src/main/java/org/apache/paimon/FileStore.java @@ -20,6 +20,7 @@ import org.apache.paimon.fs.Path; import org.apache.paimon.index.IndexFileHandler; +import org.apache.paimon.manifest.IndexManifestFile; import org.apache.paimon.manifest.ManifestCacheFilter; import org.apache.paimon.manifest.ManifestFile; import org.apache.paimon.manifest.ManifestList; @@ -70,6 +71,8 @@ public interface FileStore { ManifestFile.Factory manifestFileFactory(); + IndexManifestFile.Factory indexManifestFileFactory(); + IndexFileHandler newIndexFileHandler(); StatsFileHandler newStatsFileHandler(); diff --git a/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMeta.java b/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMeta.java index 97bb182556ad..7c3b4038205c 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMeta.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMeta.java @@ -18,6 +18,7 @@ package org.apache.paimon.index; +import org.apache.paimon.annotation.Public; import org.apache.paimon.deletionvectors.DeletionVectorsIndexFile; import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.BigIntType; @@ -34,7 +35,12 @@ import static org.apache.paimon.utils.SerializationUtils.newStringType; -/** Metadata of index file. */ +/** + * Metadata of index file. + * + * @since 0.9.0 + */ +@Public public class IndexFileMeta { public static final RowType SCHEMA = diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntry.java b/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntry.java index 41e94e9d7da6..a52d9e8af40f 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntry.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntry.java @@ -18,6 +18,7 @@ package org.apache.paimon.manifest; +import org.apache.paimon.annotation.Public; import org.apache.paimon.data.BinaryRow; import org.apache.paimon.index.IndexFileMeta; import org.apache.paimon.types.ArrayType; @@ -34,7 +35,12 @@ import static org.apache.paimon.utils.SerializationUtils.newBytesType; import static org.apache.paimon.utils.SerializationUtils.newStringType; -/** Manifest entry for index file. */ +/** + * Manifest entry for index file. + * + * @since 0.9.0 + */ +@Public public class IndexManifestEntry { public static final RowType SCHEMA = diff --git a/paimon-core/src/main/java/org/apache/paimon/privilege/PrivilegedFileStore.java b/paimon-core/src/main/java/org/apache/paimon/privilege/PrivilegedFileStore.java index 768e6259e2c1..ec068b25acb3 100644 --- a/paimon-core/src/main/java/org/apache/paimon/privilege/PrivilegedFileStore.java +++ b/paimon-core/src/main/java/org/apache/paimon/privilege/PrivilegedFileStore.java @@ -23,6 +23,7 @@ import org.apache.paimon.catalog.Identifier; import org.apache.paimon.fs.Path; import org.apache.paimon.index.IndexFileHandler; +import org.apache.paimon.manifest.IndexManifestFile; import org.apache.paimon.manifest.ManifestCacheFilter; import org.apache.paimon.manifest.ManifestFile; import org.apache.paimon.manifest.ManifestList; @@ -106,6 +107,11 @@ public ManifestFile.Factory manifestFileFactory() { return wrapped.manifestFileFactory(); } + @Override + public IndexManifestFile.Factory indexManifestFileFactory() { + return wrapped.indexManifestFileFactory(); + } + @Override public IndexFileHandler newIndexFileHandler() { return wrapped.newIndexFileHandler(); diff --git a/paimon-core/src/main/java/org/apache/paimon/privilege/PrivilegedFileStoreTable.java b/paimon-core/src/main/java/org/apache/paimon/privilege/PrivilegedFileStoreTable.java index af48c1337da3..5809efef5856 100644 --- a/paimon-core/src/main/java/org/apache/paimon/privilege/PrivilegedFileStoreTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/privilege/PrivilegedFileStoreTable.java @@ -20,6 +20,7 @@ import org.apache.paimon.FileStore; import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.manifest.IndexManifestEntry; import org.apache.paimon.manifest.ManifestCacheFilter; import org.apache.paimon.manifest.ManifestEntry; import org.apache.paimon.manifest.ManifestFileMeta; @@ -104,6 +105,11 @@ public SimpleFileReader manifestFileReader() { return wrapped.manifestFileReader(); } + @Override + public SimpleFileReader indexManifestFileReader() { + return wrapped.indexManifestFileReader(); + } + @Override public FileStoreTable copy(TableSchema newTableSchema) { return new PrivilegedFileStoreTable( diff --git a/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java b/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java index a7c2ff1a91c7..361442073525 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java @@ -24,6 +24,7 @@ import org.apache.paimon.consumer.ConsumerManager; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; +import org.apache.paimon.manifest.IndexManifestEntry; import org.apache.paimon.manifest.ManifestEntry; import org.apache.paimon.manifest.ManifestFileMeta; import org.apache.paimon.metastore.AddPartitionCommitCallback; @@ -138,6 +139,11 @@ public SimpleFileReader manifestFileReader() { return store().manifestFileFactory().create(); } + @Override + public SimpleFileReader indexManifestFileReader() { + return store().indexManifestFileFactory().create(); + } + @Override public String name() { return identifier().getObjectName(); diff --git a/paimon-core/src/main/java/org/apache/paimon/table/FallbackReadFileStoreTable.java b/paimon-core/src/main/java/org/apache/paimon/table/FallbackReadFileStoreTable.java index 559cafea0ac5..6a06e4d5d0d6 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/FallbackReadFileStoreTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/FallbackReadFileStoreTable.java @@ -23,6 +23,7 @@ import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.InternalRow; import org.apache.paimon.disk.IOManager; +import org.apache.paimon.manifest.IndexManifestEntry; import org.apache.paimon.manifest.ManifestEntry; import org.apache.paimon.manifest.ManifestFileMeta; import org.apache.paimon.manifest.PartitionEntry; @@ -86,6 +87,11 @@ public SimpleFileReader manifestFileReader() { return wrapped.manifestFileReader(); } + @Override + public SimpleFileReader indexManifestFileReader() { + return wrapped.indexManifestFileReader(); + } + @Override public FileStoreTable copy(TableSchema newTableSchema) { return new FallbackReadFileStoreTable( diff --git a/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java b/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java index 2ec38a70d247..efd4a00d0627 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java @@ -21,6 +21,7 @@ import org.apache.paimon.Snapshot; import org.apache.paimon.annotation.Public; import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.manifest.IndexManifestEntry; import org.apache.paimon.manifest.ManifestEntry; import org.apache.paimon.manifest.ManifestFileMeta; import org.apache.paimon.stats.Statistics; @@ -235,6 +236,11 @@ default SimpleFileReader manifestFileReader() { throw new UnsupportedOperationException(); } + @Override + default SimpleFileReader indexManifestFileReader() { + throw new UnsupportedOperationException(); + } + @Override default void rollbackTo(long snapshotId) { throw new UnsupportedOperationException(); diff --git a/paimon-core/src/main/java/org/apache/paimon/table/ReadonlyTable.java b/paimon-core/src/main/java/org/apache/paimon/table/ReadonlyTable.java index 027f26e445fb..82edaa667c2a 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/ReadonlyTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/ReadonlyTable.java @@ -19,6 +19,7 @@ package org.apache.paimon.table; import org.apache.paimon.Snapshot; +import org.apache.paimon.manifest.IndexManifestEntry; import org.apache.paimon.manifest.ManifestEntry; import org.apache.paimon.manifest.ManifestFileMeta; import org.apache.paimon.stats.Statistics; @@ -140,6 +141,14 @@ default SimpleFileReader manifestFileReader() { this.getClass().getSimpleName())); } + @Override + default SimpleFileReader indexManifestFileReader() { + throw new UnsupportedOperationException( + String.format( + "Readonly Table %s does not support indexManifestFileReader.", + this.getClass().getSimpleName())); + } + @Override default void rollbackTo(long snapshotId) { throw new UnsupportedOperationException( diff --git a/paimon-core/src/main/java/org/apache/paimon/table/Table.java b/paimon-core/src/main/java/org/apache/paimon/table/Table.java index ef23040402db..d542732e24c4 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/Table.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/Table.java @@ -21,6 +21,7 @@ import org.apache.paimon.Snapshot; import org.apache.paimon.annotation.Experimental; import org.apache.paimon.annotation.Public; +import org.apache.paimon.manifest.IndexManifestEntry; import org.apache.paimon.manifest.ManifestEntry; import org.apache.paimon.manifest.ManifestFileMeta; import org.apache.paimon.stats.Statistics; @@ -94,6 +95,10 @@ default String fullName() { @Experimental SimpleFileReader manifestFileReader(); + /** Reader to read index manifest entry from index manifest file. */ + @Experimental + SimpleFileReader indexManifestFileReader(); + /** Rollback table's state to a specific snapshot. */ @Experimental void rollbackTo(long snapshotId); diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/AuditLogTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/AuditLogTable.java index 5a2b83234c79..b78444625372 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/system/AuditLogTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/system/AuditLogTable.java @@ -27,6 +27,7 @@ import org.apache.paimon.disk.IOManager; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; +import org.apache.paimon.manifest.IndexManifestEntry; import org.apache.paimon.manifest.ManifestEntry; import org.apache.paimon.manifest.ManifestFileMeta; import org.apache.paimon.manifest.PartitionEntry; @@ -124,6 +125,11 @@ public SimpleFileReader manifestFileReader() { return wrapped.manifestFileReader(); } + @Override + public SimpleFileReader indexManifestFileReader() { + return wrapped.indexManifestFileReader(); + } + @Override public String name() { return wrapped.name() + SYSTEM_TABLE_SPLITTER + AUDIT_LOG; diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/BucketsTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/BucketsTable.java index be3d210f2fb1..ddf38b7b2c4a 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/system/BucketsTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/system/BucketsTable.java @@ -28,6 +28,7 @@ import org.apache.paimon.fs.Path; import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.io.DataFileMetaSerializer; +import org.apache.paimon.manifest.IndexManifestEntry; import org.apache.paimon.manifest.ManifestEntry; import org.apache.paimon.manifest.ManifestFileMeta; import org.apache.paimon.predicate.Predicate; @@ -128,6 +129,11 @@ public SimpleFileReader manifestFileReader() { return wrapped.manifestFileReader(); } + @Override + public SimpleFileReader indexManifestFileReader() { + return wrapped.indexManifestFileReader(); + } + @Override public Path location() { return wrapped.location(); diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/FileMonitorTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/FileMonitorTable.java index 45e8e74785e3..c826c405d0e6 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/system/FileMonitorTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/system/FileMonitorTable.java @@ -29,6 +29,7 @@ import org.apache.paimon.fs.Path; import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.io.DataFileMetaSerializer; +import org.apache.paimon.manifest.IndexManifestEntry; import org.apache.paimon.manifest.ManifestEntry; import org.apache.paimon.manifest.ManifestFileMeta; import org.apache.paimon.predicate.Predicate; @@ -115,6 +116,11 @@ public SimpleFileReader manifestFileReader() { return wrapped.manifestFileReader(); } + @Override + public SimpleFileReader indexManifestFileReader() { + return wrapped.indexManifestFileReader(); + } + @Override public Path location() { return wrapped.location(); diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/ReadOptimizedTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/ReadOptimizedTable.java index cc7e7619ccb5..4b1ad4b7f3a7 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/system/ReadOptimizedTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/system/ReadOptimizedTable.java @@ -22,6 +22,7 @@ import org.apache.paimon.Snapshot; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; +import org.apache.paimon.manifest.IndexManifestEntry; import org.apache.paimon.manifest.ManifestEntry; import org.apache.paimon.manifest.ManifestFileMeta; import org.apache.paimon.operation.DefaultValueAssigner; @@ -85,6 +86,11 @@ public SimpleFileReader manifestFileReader() { return wrapped.manifestFileReader(); } + @Override + public SimpleFileReader indexManifestFileReader() { + return wrapped.indexManifestFileReader(); + } + @Override public String name() { return wrapped.name() + SYSTEM_TABLE_SPLITTER + READ_OPTIMIZED;