[AMORO-2412] Automatically delete expired iceberg Table tag (apache#2526

) * Automatically delete expired iceberg Table tag * update config and document * fixed * fixed --------- Co-authored-by: huyuanfeng <[email protected]>
huyuanfeng2018 · Feb 7, 2024 · 7fa9598 · 7fa9598
1 parent b87ff4a
commit 7fa9598
Show file tree

Hide file tree

Showing 5 changed files with 86 additions and 10 deletions.
diff --git a/...main/java/com/netease/arctic/server/optimizing/maintainer/AutoCreateIcebergTagAction.java b/...main/java/com/netease/arctic/server/optimizing/maintainer/AutoCreateIcebergTagAction.java
@@ -19,6 +19,7 @@
 package com.netease.arctic.server.optimizing.maintainer;
 
 import com.netease.arctic.server.table.TagConfiguration;
+import org.apache.iceberg.ManageSnapshots;
 import org.apache.iceberg.Snapshot;
 import org.apache.iceberg.Table;
 import org.slf4j.Logger;
@@ -95,7 +96,11 @@ private boolean createTag() {
           tagTriggerTimestampMillis);
       return false;
     }
-    table.manageSnapshots().createTag(tagName, snapshot.snapshotId()).commit();
+    ManageSnapshots tag = table.manageSnapshots().createTag(tagName, snapshot.snapshotId());
+    if (tagConfig.getTagMaxAgeMs() > 0) {
+      tag.setMaxRefAgeMs(tagName, tagConfig.getTagMaxAgeMs());
+    }
+    tag.commit();
     LOG.info(
         "Created a tag {} for {} on snapshot {} at {}",
         tagName,

diff --git a/ams/server/src/main/java/com/netease/arctic/server/table/TagConfiguration.java b/ams/server/src/main/java/com/netease/arctic/server/table/TagConfiguration.java
@@ -44,6 +44,8 @@ public class TagConfiguration {
   private int triggerOffsetMinutes;
   // tag.auto-create.trigger.max-delay.minutes
   private int maxDelayMinutes;
+  // tag.auto-create.max-age-ms
+  private long tagMaxAgeMs;
 
   /** The interval for periodically triggering creating tags */
   public enum Period {
@@ -142,6 +144,11 @@ public static TagConfiguration parse(Map<String, String> tableProperties) {
             tableProperties,
             TableProperties.AUTO_CREATE_TAG_MAX_DELAY_MINUTES,
             TableProperties.AUTO_CREATE_TAG_MAX_DELAY_MINUTES_DEFAULT));
+    tagConfig.setTagMaxAgeMs(
+        CompatiblePropertyUtil.propertyAsLong(
+            tableProperties,
+            TableProperties.AUTO_CREATE_TAG_MAX_AGE_MS,
+            TableProperties.AUTO_CREATE_TAG_MAX_AGE_MS_DEFAULT));
     return tagConfig;
   }
 
@@ -185,6 +192,14 @@ public void setMaxDelayMinutes(int maxDelayMinutes) {
     this.maxDelayMinutes = maxDelayMinutes;
   }
 
+  public long getTagMaxAgeMs() {
+    return tagMaxAgeMs;
+  }
+
+  public void setTagMaxAgeMs(long tagMaxAgeMs) {
+    this.tagMaxAgeMs = tagMaxAgeMs;
+  }
+
   @Override
   public boolean equals(Object o) {
     if (this == o) return true;
@@ -194,13 +209,19 @@ public boolean equals(Object o) {
         && triggerOffsetMinutes == that.triggerOffsetMinutes
         && maxDelayMinutes == that.maxDelayMinutes
         && Objects.equal(tagFormat, that.tagFormat)
-        && triggerPeriod == that.triggerPeriod;
+        && triggerPeriod == that.triggerPeriod
+        && tagMaxAgeMs == that.tagMaxAgeMs;
   }
 
   @Override
   public int hashCode() {
     return Objects.hashCode(
-        autoCreateTag, tagFormat, triggerPeriod, triggerOffsetMinutes, maxDelayMinutes);
+        autoCreateTag,
+        tagFormat,
+        triggerPeriod,
+        triggerOffsetMinutes,
+        maxDelayMinutes,
+        tagMaxAgeMs);
   }
 
   @Override
@@ -211,6 +232,7 @@ public String toString() {
         .add("triggerPeriod", triggerPeriod)
         .add("triggerOffsetMinutes", triggerOffsetMinutes)
         .add("maxDelayMinutes", maxDelayMinutes)
+        .add("tagMaxAgeMs", tagMaxAgeMs)
         .toString();
   }
 }
diff --git a/.../java/com/netease/arctic/server/optimizing/maintainer/TestAutoCreateIcebergTagAction.java b/.../java/com/netease/arctic/server/optimizing/maintainer/TestAutoCreateIcebergTagAction.java
@@ -24,6 +24,7 @@
 import com.netease.arctic.catalog.TableTestBase;
 import com.netease.arctic.server.table.TagConfiguration;
 import com.netease.arctic.table.TableProperties;
+import org.apache.iceberg.ExpireSnapshots;
 import org.apache.iceberg.Snapshot;
 import org.apache.iceberg.SnapshotRef;
 import org.apache.iceberg.Table;
@@ -272,6 +273,37 @@ public void testTriggerTimePeriod() {
     testTagTimePeriodDaily("2022-08-09T00:10:00", 30, "2022-08-08T00:00:00");
   }
 
+  @Test
+  public void testTagExpiration() {
+    Table table = getArcticTable().asUnkeyedTable();
+    long expireKeepMs = 20;
+    table
+        .updateProperties()
+        .set(TableProperties.ENABLE_AUTO_CREATE_TAG, "true")
+        .set(TableProperties.AUTO_CREATE_TAG_MAX_DELAY_MINUTES, "0")
+        .set(TableProperties.AUTO_CREATE_TAG_TRIGGER_PERIOD, "hourly")
+        .set(TableProperties.AUTO_CREATE_TAG_MAX_AGE_MS, expireKeepMs + "")
+        .commit();
+
+    table.newAppend().commit();
+    checkSnapshots(table, 1);
+    checkNoTag(table);
+
+    Snapshot snapshot = table.currentSnapshot();
+    LocalDateTime now = fromEpochMillis(snapshot.timestampMillis());
+    newAutoCreateIcebergTagAction(table, now).execute();
+    checkTagCount(table, 1);
+    checkTag(table, "tag-" + formatDateTime(now.minusHours(1)), snapshot);
+
+    // should not recreate tag
+    newAutoCreateIcebergTagAction(table, now).execute();
+    checkTagCount(table, 1);
+    long expirationTime = System.currentTimeMillis() + expireKeepMs;
+    waitUntilAfter(expirationTime);
+    expireSnapshots(table);
+    checkTagCount(table, 0);
+  }
+
   private void testTagTimePeriodHourly(
       String checkTimeStr, int offsetMinutes, String expectedResultStr) {
     LocalDateTime checkTime = LocalDateTime.parse(checkTimeStr);
@@ -356,4 +388,17 @@ private void checkTag(Table table, String tagName, Snapshot snapshot) {
   private void checkSnapshots(Table table, int count) {
     Assert.assertEquals(Iterables.size(table.snapshots()), count);
   }
+
+  private long waitUntilAfter(long timestampMillis) {
+    long current = System.currentTimeMillis();
+    while (current <= timestampMillis) {
+      current = System.currentTimeMillis();
+    }
+    return current;
+  }
+
+  private void expireSnapshots(Table table) {
+    ExpireSnapshots expireSnapshots = table.expireSnapshots();
+    expireSnapshots.commit();
+  }
 }
diff --git a/core/src/main/java/com/netease/arctic/table/TableProperties.java b/core/src/main/java/com/netease/arctic/table/TableProperties.java
@@ -216,6 +216,9 @@ private TableProperties() {}
       "tag.auto-create.trigger.max-delay.minutes";
   public static final int AUTO_CREATE_TAG_MAX_DELAY_MINUTES_DEFAULT = 60;
 
+  public static final String AUTO_CREATE_TAG_MAX_AGE_MS = "tag.auto-create.max-age-ms";
+  public static final int AUTO_CREATE_TAG_MAX_AGE_MS_DEFAULT = -1;
+
   public static final String AUTO_CREATE_TAG_FORMAT = "tag.auto-create.tag-format";
   public static final String AUTO_CREATE_TAG_FORMAT_DAILY_DEFAULT = "'tag-'yyyyMMdd";
   public static final String AUTO_CREATE_TAG_FORMAT_HOURLY_DEFAULT = "'tag-'yyyyMMddHH";

diff --git a/docs/user-guides/configurations.md b/docs/user-guides/configurations.md
@@ -70,13 +70,14 @@ Data-cleaning configurations are applicable to both Iceberg Format and Mixed str
 Tags configurations are applicable to Iceberg Format only now, and will be supported in Mixed Format
 soon.
 
-| Key                                        | Default                                          | Description                                                 |
-|--------------------------------------------|--------------------------------------------------|-------------------------------------------------------------|
-| tag.auto-create.enabled                    | false                                            | Enables automatically creating tags                         |
-| tag.auto-create.trigger.period             | daily                                            | Period of creating tags, support `daily`,`hourly` now       |
-| tag.auto-create.trigger.offset.minutes     | 0                                                | The minutes by which the tag is created after midnight (00:00) |
-| tag.auto-create.trigger.max-delay.minutes  | 60                                               | The maximum delay time for creating a tag                   |
-| tag.auto-create.tag-format            | 'tag-'yyyyMMdd for daily and 'tag-'yyyyMMddHH for hourly periods | The format of the name for tag              |
+| Key                                       | Default                                                          | Description                                                                                                                          |
+|-------------------------------------------|------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------|
+| tag.auto-create.enabled                   | false                                                            | Enables automatically creating tags                                                                                                  |
+| tag.auto-create.trigger.period            | daily                                                            | Period of creating tags, support `daily`,`hourly` now                                                                                |
+| tag.auto-create.trigger.offset.minutes    | 0                                                                | The minutes by which the tag is created after midnight (00:00)                                                                       |
+| tag.auto-create.trigger.max-delay.minutes | 60                                                               | The maximum delay time for creating a tag                                                                                            |
+| tag.auto-create.tag-format                | 'tag-'yyyyMMdd for daily and 'tag-'yyyyMMddHH for hourly periods | The format of the name for tag. Modifying this configuration will not take effect on old tags                                        |
+| tag.auto-create.max-age-ms                | -1                                                               | Time of automatically created Tag to retain, -1 means keep it forever. Modifying this configuration will not take effect on old tags |
 
 ## Mixed Format configurations