From 1515c7ea36060bef74be5fa0fbe12a0b4cb08dfd Mon Sep 17 00:00:00 2001 From: "jiangqiao.xu" Date: Fri, 20 Dec 2024 19:39:03 +0800 Subject: [PATCH 1/2] [flink] Bugfix expire snapshot don't work when some snapshot between earliest and latest are removed for some reason --- .../java/org/apache/paimon/table/ExpireSnapshotsImpl.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/table/ExpireSnapshotsImpl.java b/paimon-core/src/main/java/org/apache/paimon/table/ExpireSnapshotsImpl.java index dc1c2d6bdbc5..ba95bde5fc09 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/ExpireSnapshotsImpl.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/ExpireSnapshotsImpl.java @@ -115,7 +115,12 @@ public int expire() { // protected by 'snapshot.expire.limit' // (the maximum number of snapshots allowed to expire at a time) - maxExclusive = Math.min(maxExclusive, earliest + maxDeletes); + long maxExclusiveByExpireLimit = earliest + maxDeletes; + while (!snapshotManager.snapshotExists(maxExclusiveByExpireLimit)) { + // Ensure maxExclusive exist, deal some snapshot between earliest and latest is deleted + maxExclusiveByExpireLimit++; + } + maxExclusive = Math.min(maxExclusive, maxExclusiveByExpireLimit); for (long id = min; id < maxExclusive; id++) { // Early exit the loop for 'snapshot.time-retained' From 24f0fd3069e6e28e5838e4d77a89b956a33b27c6 Mon Sep 17 00:00:00 2001 From: "jiangqiao.xu" Date: Fri, 20 Dec 2024 19:58:50 +0800 Subject: [PATCH 2/2] [flink] Bugfix expire snapshot don't work when some snapshot between earliest and latest are removed for some reason --- .../main/java/org/apache/paimon/table/ExpireSnapshotsImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/table/ExpireSnapshotsImpl.java b/paimon-core/src/main/java/org/apache/paimon/table/ExpireSnapshotsImpl.java index ba95bde5fc09..ab89cad26bf1 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/ExpireSnapshotsImpl.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/ExpireSnapshotsImpl.java @@ -120,7 +120,7 @@ public int expire() { // Ensure maxExclusive exist, deal some snapshot between earliest and latest is deleted maxExclusiveByExpireLimit++; } - maxExclusive = Math.min(maxExclusive, maxExclusiveByExpireLimit); + maxExclusive = Math.min(maxExclusive, earliest + maxDeletes); for (long id = min; id < maxExclusive; id++) { // Early exit the loop for 'snapshot.time-retained'