From 7fa9598ad51931b174ca6f032b16b17d9c8baf10 Mon Sep 17 00:00:00 2001 From: big face cat <731030576@qq.com> Date: Wed, 7 Feb 2024 15:20:48 +0800 Subject: [PATCH] [AMORO-2412] Automatically delete expired iceberg Table tag (#2526) * Automatically delete expired iceberg Table tag * update config and document * fixed * fixed --------- Co-authored-by: huyuanfeng --- .../AutoCreateIcebergTagAction.java | 7 ++- .../arctic/server/table/TagConfiguration.java | 26 ++++++++++- .../TestAutoCreateIcebergTagAction.java | 45 +++++++++++++++++++ .../netease/arctic/table/TableProperties.java | 3 ++ docs/user-guides/configurations.md | 15 ++++--- 5 files changed, 86 insertions(+), 10 deletions(-) diff --git a/ams/server/src/main/java/com/netease/arctic/server/optimizing/maintainer/AutoCreateIcebergTagAction.java b/ams/server/src/main/java/com/netease/arctic/server/optimizing/maintainer/AutoCreateIcebergTagAction.java index bca5998601..f99aee7c29 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/optimizing/maintainer/AutoCreateIcebergTagAction.java +++ b/ams/server/src/main/java/com/netease/arctic/server/optimizing/maintainer/AutoCreateIcebergTagAction.java @@ -19,6 +19,7 @@ package com.netease.arctic.server.optimizing.maintainer; import com.netease.arctic.server.table.TagConfiguration; +import org.apache.iceberg.ManageSnapshots; import org.apache.iceberg.Snapshot; import org.apache.iceberg.Table; import org.slf4j.Logger; @@ -95,7 +96,11 @@ private boolean createTag() { tagTriggerTimestampMillis); return false; } - table.manageSnapshots().createTag(tagName, snapshot.snapshotId()).commit(); + ManageSnapshots tag = table.manageSnapshots().createTag(tagName, snapshot.snapshotId()); + if (tagConfig.getTagMaxAgeMs() > 0) { + tag.setMaxRefAgeMs(tagName, tagConfig.getTagMaxAgeMs()); + } + tag.commit(); LOG.info( "Created a tag {} for {} on snapshot {} at {}", tagName, diff --git a/ams/server/src/main/java/com/netease/arctic/server/table/TagConfiguration.java b/ams/server/src/main/java/com/netease/arctic/server/table/TagConfiguration.java index 6a3a23bb23..bac4145ebd 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/table/TagConfiguration.java +++ b/ams/server/src/main/java/com/netease/arctic/server/table/TagConfiguration.java @@ -44,6 +44,8 @@ public class TagConfiguration { private int triggerOffsetMinutes; // tag.auto-create.trigger.max-delay.minutes private int maxDelayMinutes; + // tag.auto-create.max-age-ms + private long tagMaxAgeMs; /** The interval for periodically triggering creating tags */ public enum Period { @@ -142,6 +144,11 @@ public static TagConfiguration parse(Map tableProperties) { tableProperties, TableProperties.AUTO_CREATE_TAG_MAX_DELAY_MINUTES, TableProperties.AUTO_CREATE_TAG_MAX_DELAY_MINUTES_DEFAULT)); + tagConfig.setTagMaxAgeMs( + CompatiblePropertyUtil.propertyAsLong( + tableProperties, + TableProperties.AUTO_CREATE_TAG_MAX_AGE_MS, + TableProperties.AUTO_CREATE_TAG_MAX_AGE_MS_DEFAULT)); return tagConfig; } @@ -185,6 +192,14 @@ public void setMaxDelayMinutes(int maxDelayMinutes) { this.maxDelayMinutes = maxDelayMinutes; } + public long getTagMaxAgeMs() { + return tagMaxAgeMs; + } + + public void setTagMaxAgeMs(long tagMaxAgeMs) { + this.tagMaxAgeMs = tagMaxAgeMs; + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -194,13 +209,19 @@ public boolean equals(Object o) { && triggerOffsetMinutes == that.triggerOffsetMinutes && maxDelayMinutes == that.maxDelayMinutes && Objects.equal(tagFormat, that.tagFormat) - && triggerPeriod == that.triggerPeriod; + && triggerPeriod == that.triggerPeriod + && tagMaxAgeMs == that.tagMaxAgeMs; } @Override public int hashCode() { return Objects.hashCode( - autoCreateTag, tagFormat, triggerPeriod, triggerOffsetMinutes, maxDelayMinutes); + autoCreateTag, + tagFormat, + triggerPeriod, + triggerOffsetMinutes, + maxDelayMinutes, + tagMaxAgeMs); } @Override @@ -211,6 +232,7 @@ public String toString() { .add("triggerPeriod", triggerPeriod) .add("triggerOffsetMinutes", triggerOffsetMinutes) .add("maxDelayMinutes", maxDelayMinutes) + .add("tagMaxAgeMs", tagMaxAgeMs) .toString(); } } diff --git a/ams/server/src/test/java/com/netease/arctic/server/optimizing/maintainer/TestAutoCreateIcebergTagAction.java b/ams/server/src/test/java/com/netease/arctic/server/optimizing/maintainer/TestAutoCreateIcebergTagAction.java index 34f3c3ba3f..a539234be8 100644 --- a/ams/server/src/test/java/com/netease/arctic/server/optimizing/maintainer/TestAutoCreateIcebergTagAction.java +++ b/ams/server/src/test/java/com/netease/arctic/server/optimizing/maintainer/TestAutoCreateIcebergTagAction.java @@ -24,6 +24,7 @@ import com.netease.arctic.catalog.TableTestBase; import com.netease.arctic.server.table.TagConfiguration; import com.netease.arctic.table.TableProperties; +import org.apache.iceberg.ExpireSnapshots; import org.apache.iceberg.Snapshot; import org.apache.iceberg.SnapshotRef; import org.apache.iceberg.Table; @@ -272,6 +273,37 @@ public void testTriggerTimePeriod() { testTagTimePeriodDaily("2022-08-09T00:10:00", 30, "2022-08-08T00:00:00"); } + @Test + public void testTagExpiration() { + Table table = getArcticTable().asUnkeyedTable(); + long expireKeepMs = 20; + table + .updateProperties() + .set(TableProperties.ENABLE_AUTO_CREATE_TAG, "true") + .set(TableProperties.AUTO_CREATE_TAG_MAX_DELAY_MINUTES, "0") + .set(TableProperties.AUTO_CREATE_TAG_TRIGGER_PERIOD, "hourly") + .set(TableProperties.AUTO_CREATE_TAG_MAX_AGE_MS, expireKeepMs + "") + .commit(); + + table.newAppend().commit(); + checkSnapshots(table, 1); + checkNoTag(table); + + Snapshot snapshot = table.currentSnapshot(); + LocalDateTime now = fromEpochMillis(snapshot.timestampMillis()); + newAutoCreateIcebergTagAction(table, now).execute(); + checkTagCount(table, 1); + checkTag(table, "tag-" + formatDateTime(now.minusHours(1)), snapshot); + + // should not recreate tag + newAutoCreateIcebergTagAction(table, now).execute(); + checkTagCount(table, 1); + long expirationTime = System.currentTimeMillis() + expireKeepMs; + waitUntilAfter(expirationTime); + expireSnapshots(table); + checkTagCount(table, 0); + } + private void testTagTimePeriodHourly( String checkTimeStr, int offsetMinutes, String expectedResultStr) { LocalDateTime checkTime = LocalDateTime.parse(checkTimeStr); @@ -356,4 +388,17 @@ private void checkTag(Table table, String tagName, Snapshot snapshot) { private void checkSnapshots(Table table, int count) { Assert.assertEquals(Iterables.size(table.snapshots()), count); } + + private long waitUntilAfter(long timestampMillis) { + long current = System.currentTimeMillis(); + while (current <= timestampMillis) { + current = System.currentTimeMillis(); + } + return current; + } + + private void expireSnapshots(Table table) { + ExpireSnapshots expireSnapshots = table.expireSnapshots(); + expireSnapshots.commit(); + } } diff --git a/core/src/main/java/com/netease/arctic/table/TableProperties.java b/core/src/main/java/com/netease/arctic/table/TableProperties.java index 20fe3f6655..9e91cd9441 100644 --- a/core/src/main/java/com/netease/arctic/table/TableProperties.java +++ b/core/src/main/java/com/netease/arctic/table/TableProperties.java @@ -216,6 +216,9 @@ private TableProperties() {} "tag.auto-create.trigger.max-delay.minutes"; public static final int AUTO_CREATE_TAG_MAX_DELAY_MINUTES_DEFAULT = 60; + public static final String AUTO_CREATE_TAG_MAX_AGE_MS = "tag.auto-create.max-age-ms"; + public static final int AUTO_CREATE_TAG_MAX_AGE_MS_DEFAULT = -1; + public static final String AUTO_CREATE_TAG_FORMAT = "tag.auto-create.tag-format"; public static final String AUTO_CREATE_TAG_FORMAT_DAILY_DEFAULT = "'tag-'yyyyMMdd"; public static final String AUTO_CREATE_TAG_FORMAT_HOURLY_DEFAULT = "'tag-'yyyyMMddHH"; diff --git a/docs/user-guides/configurations.md b/docs/user-guides/configurations.md index ade12d5727..8a02c1d276 100644 --- a/docs/user-guides/configurations.md +++ b/docs/user-guides/configurations.md @@ -70,13 +70,14 @@ Data-cleaning configurations are applicable to both Iceberg Format and Mixed str Tags configurations are applicable to Iceberg Format only now, and will be supported in Mixed Format soon. -| Key | Default | Description | -|--------------------------------------------|--------------------------------------------------|-------------------------------------------------------------| -| tag.auto-create.enabled | false | Enables automatically creating tags | -| tag.auto-create.trigger.period | daily | Period of creating tags, support `daily`,`hourly` now | -| tag.auto-create.trigger.offset.minutes | 0 | The minutes by which the tag is created after midnight (00:00) | -| tag.auto-create.trigger.max-delay.minutes | 60 | The maximum delay time for creating a tag | -| tag.auto-create.tag-format | 'tag-'yyyyMMdd for daily and 'tag-'yyyyMMddHH for hourly periods | The format of the name for tag | +| Key | Default | Description | +|-------------------------------------------|------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------| +| tag.auto-create.enabled | false | Enables automatically creating tags | +| tag.auto-create.trigger.period | daily | Period of creating tags, support `daily`,`hourly` now | +| tag.auto-create.trigger.offset.minutes | 0 | The minutes by which the tag is created after midnight (00:00) | +| tag.auto-create.trigger.max-delay.minutes | 60 | The maximum delay time for creating a tag | +| tag.auto-create.tag-format | 'tag-'yyyyMMdd for daily and 'tag-'yyyyMMddHH for hourly periods | The format of the name for tag. Modifying this configuration will not take effect on old tags | +| tag.auto-create.max-age-ms | -1 | Time of automatically created Tag to retain, -1 means keep it forever. Modifying this configuration will not take effect on old tags | ## Mixed Format configurations