From ac92d1d084921cc8d3d6310140f0a7e776c0af25 Mon Sep 17 00:00:00 2001 From: Jingsong Date: Sun, 11 Aug 2024 21:19:21 +0800 Subject: [PATCH] [core] Rename snapshots to tags in expiration classes --- .../paimon/operation/FileDeletionBase.java | 29 +++++++++---------- .../paimon/table/ExpireChangelogImpl.java | 6 ++-- .../paimon/table/ExpireSnapshotsImpl.java | 6 ++-- 3 files changed, 20 insertions(+), 21 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/FileDeletionBase.java b/paimon-core/src/main/java/org/apache/paimon/operation/FileDeletionBase.java index 978735babc33..0ce94bb9b112 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/FileDeletionBase.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/FileDeletionBase.java @@ -77,12 +77,11 @@ public abstract class FileDeletionBase { protected boolean changelogDecoupled; - /** Used to record which snapshot is cached. */ - private long cachedSnapshotId = 0; + /** Used to record which tag is cached. */ + private long cachedTag = 0; - /** Used to cache data files used by current snapshot. */ - private final Map>> cachedSnapshotDataFiles = - new HashMap<>(); + /** Used to cache data files used by current tag. */ + private final Map>> cachedTagDataFiles = new HashMap<>(); public FileDeletionBase( FileIO fileIO, @@ -328,18 +327,18 @@ protected void cleanUnusedManifests( cleanUnusedStatisticsManifests(snapshot, skippingSet); } - public Predicate dataFileSkipper( - List skippingSnapshots, long expiringSnapshotId) throws Exception { - int index = SnapshotManager.findPreviousSnapshot(skippingSnapshots, expiringSnapshotId); - // refresh snapshot data files + public Predicate createDataFileSkipperForTags( + List taggedSnapshots, long expiringSnapshotId) throws Exception { + int index = SnapshotManager.findPreviousSnapshot(taggedSnapshots, expiringSnapshotId); + // refresh tag data files if (index >= 0) { - Snapshot previousSnapshot = skippingSnapshots.get(index); - if (previousSnapshot.id() != cachedSnapshotId) { - cachedSnapshotId = previousSnapshot.id(); - cachedSnapshotDataFiles.clear(); - addMergedDataFiles(cachedSnapshotDataFiles, previousSnapshot); + Snapshot previousTag = taggedSnapshots.get(index); + if (previousTag.id() != cachedTag) { + cachedTag = previousTag.id(); + cachedTagDataFiles.clear(); + addMergedDataFiles(cachedTagDataFiles, previousTag); } - return entry -> containsDataFile(cachedSnapshotDataFiles, entry); + return entry -> containsDataFile(cachedTagDataFiles, entry); } return entry -> false; } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/ExpireChangelogImpl.java b/paimon-core/src/main/java/org/apache/paimon/table/ExpireChangelogImpl.java index 72f655bbaafc..246a85cbdf03 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/ExpireChangelogImpl.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/ExpireChangelogImpl.java @@ -134,11 +134,11 @@ public int expireUntil(long earliestId, long endExclusiveId) { LOG.debug("Changelog expire range is [" + earliestId + ", " + endExclusiveId + ")"); } - List referencedSnapshots = tagManager.taggedSnapshots(); + List taggedSnapshots = tagManager.taggedSnapshots(); List skippingSnapshots = SnapshotManager.findOverlappedSnapshots( - referencedSnapshots, earliestId, endExclusiveId); + taggedSnapshots, earliestId, endExclusiveId); skippingSnapshots.add(snapshotManager.changelog(endExclusiveId)); Set manifestSkippSet = changelogDeletion.manifestSkippingSet(skippingSnapshots); for (long id = earliestId; id < endExclusiveId; id++) { @@ -148,7 +148,7 @@ public int expireUntil(long earliestId, long endExclusiveId) { Changelog changelog = snapshotManager.longLivedChangelog(id); Predicate skipper; try { - skipper = changelogDeletion.dataFileSkipper(referencedSnapshots, id); + skipper = changelogDeletion.createDataFileSkipperForTags(taggedSnapshots, id); } catch (Exception e) { LOG.info( String.format( diff --git a/paimon-core/src/main/java/org/apache/paimon/table/ExpireSnapshotsImpl.java b/paimon-core/src/main/java/org/apache/paimon/table/ExpireSnapshotsImpl.java index c49db64981d1..71abd90e6b4f 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/ExpireSnapshotsImpl.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/ExpireSnapshotsImpl.java @@ -153,7 +153,7 @@ public int expireUntil(long earliestId, long endExclusiveId) { "Snapshot expire range is [" + beginInclusiveId + ", " + endExclusiveId + ")"); } - List referencedSnapshots = tagManager.taggedSnapshots(); + List taggedSnapshots = tagManager.taggedSnapshots(); // delete merge tree files // deleted merge tree files in a snapshot are not used by the next snapshot, so the range of @@ -166,7 +166,7 @@ public int expireUntil(long earliestId, long endExclusiveId) { // expire merge tree files and collect changed buckets Predicate skipper; try { - skipper = snapshotDeletion.dataFileSkipper(referencedSnapshots, id); + skipper = snapshotDeletion.createDataFileSkipperForTags(taggedSnapshots, id); } catch (Exception e) { LOG.info( String.format( @@ -199,7 +199,7 @@ public int expireUntil(long earliestId, long endExclusiveId) { // delete manifests and indexFiles List skippingSnapshots = SnapshotManager.findOverlappedSnapshots( - referencedSnapshots, beginInclusiveId, endExclusiveId); + taggedSnapshots, beginInclusiveId, endExclusiveId); skippingSnapshots.add(snapshotManager.snapshot(endExclusiveId)); Set skippingSet = snapshotDeletion.manifestSkippingSet(skippingSnapshots); for (long id = beginInclusiveId; id < endExclusiveId; id++) {