[core] Drop stats for deleted data files to reduce memory

apache · Dec 4, 2024 · 9d2a63a · 9d2a63a
1 parent 11cebd4
commit 9d2a63a
Show file tree

Hide file tree

Showing 3 changed files with 6 additions and 2 deletions.
diff --git a/...-core/src/main/java/org/apache/paimon/append/UnawareAppendTableCompactionCoordinator.java b/...-core/src/main/java/org/apache/paimon/append/UnawareAppendTableCompactionCoordinator.java
@@ -380,6 +380,8 @@ public FilesIterator(
             if (filter != null) {
                 snapshotReader.withFilter(filter);
             }
+            // drop stats to reduce memory
+            snapshotReader.dropStats();
             this.streamingMode = isStreaming;
         }
 

diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreWrite.java b/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreWrite.java
@@ -101,7 +101,8 @@ protected AbstractFileStoreWrite(
             int writerNumberMax,
             boolean legacyPartitionName) {
         this.snapshotManager = snapshotManager;
-        this.scan = scan;
+        // Statistic is useless in writer
+        this.scan = scan.dropStats();
         this.indexFactory = indexFactory;
         this.dvMaintainerFactory = dvMaintainerFactory;
         this.totalBuckets = totalBuckets;

diff --git a/...k/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonCommand.scala b/...k/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonCommand.scala
@@ -94,7 +94,8 @@ trait PaimonCommand extends WithFileStoreTable with ExpressionHelper with SQLCon
       condition: Expression,
       output: Seq[Attribute]): Seq[DataSplit] = {
     // low level snapshot reader, it can not be affected by 'scan.mode'
-    val snapshotReader = table.newSnapshotReader()
+    // dropStats after filter push down
+    val snapshotReader = table.newSnapshotReader().dropStats()
     if (condition != TrueLiteral) {
       val filter =
         convertConditionToPaimonPredicate(condition, output, rowType, ignoreFailure = true)