[FSTORE-1327] Activity UI doesn't correctly report Delta commit activ…

…ities (#1384)
logicalclocks · Sep 30, 2024 · 7ca8dc9 · 7ca8dc9
1 parent 7558e8f
commit 7ca8dc9
Show file tree

Hide file tree

Showing 8 changed files with 17 additions and 9 deletions.
diff --git a/java/beam/pom.xml b/java/beam/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <artifactId>hsfs-parent</artifactId>
     <groupId>com.logicalclocks</groupId>
-    <version>3.8.0-RC3</version>
+    <version>3.8.0-RC4</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

diff --git a/java/flink/pom.xml b/java/flink/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <artifactId>hsfs-parent</artifactId>
     <groupId>com.logicalclocks</groupId>
-    <version>3.8.0-RC3</version>
+    <version>3.8.0-RC4</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

diff --git a/java/hsfs/pom.xml b/java/hsfs/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <artifactId>hsfs-parent</artifactId>
     <groupId>com.logicalclocks</groupId>
-    <version>3.8.0-RC3</version>
+    <version>3.8.0-RC4</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

diff --git a/java/pom.xml b/java/pom.xml
@@ -7,7 +7,7 @@
     <groupId>com.logicalclocks</groupId>
     <artifactId>hsfs-parent</artifactId>
     <packaging>pom</packaging>
-    <version>3.8.0-RC3</version>
+    <version>3.8.0-RC4</version>
     <modules>
         <module>hsfs</module>
         <module>spark</module>

diff --git a/java/spark/pom.xml b/java/spark/pom.xml
@@ -22,7 +22,7 @@
     <parent>
         <artifactId>hsfs-parent</artifactId>
         <groupId>com.logicalclocks</groupId>
-        <version>3.8.0-RC3</version>
+        <version>3.8.0-RC4</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/python/hsfs/core/delta_engine.py b/python/hsfs/core/delta_engine.py
@@ -172,6 +172,8 @@ def _generate_merge_query(self, source_alias, updates_alias):
     @staticmethod
     def _get_last_commit_metadata(spark_context, base_path):
         fg_source_table = DeltaTable.forPath(spark_context, base_path)
+
+        # Get info about the latest commit
         last_commit = fg_source_table.history(1).first().asDict()
         version = last_commit["version"]
         commit_timestamp = util.convert_event_time_to_timestamp(
@@ -180,6 +182,12 @@ def _get_last_commit_metadata(spark_context, base_path):
         commit_date_string = util.get_hudi_datestr_from_timestamp(commit_timestamp)
         operation_metrics = last_commit["operationMetrics"]
 
+        # Get info about the oldest remaining commit
+        oldest_commit = fg_source_table.history().orderBy("version").first().asDict()
+        oldest_commit_timestamp = util.convert_event_time_to_timestamp(
+            oldest_commit["timestamp"]
+        )
+
         if version == 0:
             fg_commit = feature_group_commit.FeatureGroupCommit(
                 commitid=None,
@@ -188,7 +196,7 @@ def _get_last_commit_metadata(spark_context, base_path):
                 rows_inserted=operation_metrics["numOutputRows"],
                 rows_updated=0,
                 rows_deleted=0,
-                last_active_commit_time=commit_timestamp,
+                last_active_commit_time=oldest_commit_timestamp,
             )
         else:
             fg_commit = feature_group_commit.FeatureGroupCommit(
@@ -198,7 +206,7 @@ def _get_last_commit_metadata(spark_context, base_path):
                 rows_inserted=operation_metrics["numTargetRowsInserted"],
                 rows_updated=operation_metrics["numTargetRowsUpdated"],
                 rows_deleted=operation_metrics["numTargetRowsDeleted"],
-                last_active_commit_time=commit_timestamp,
+                last_active_commit_time=oldest_commit_timestamp,
             )
 
         return fg_commit
diff --git a/python/hsfs/version.py b/python/hsfs/version.py
@@ -14,4 +14,4 @@
 #   limitations under the License.
 #
 
-__version__ = "3.8.0rc3"
+__version__ = "3.8.0rc4"
diff --git a/utils/java/pom.xml b/utils/java/pom.xml
@@ -5,7 +5,7 @@
 
     <groupId>com.logicalclocks</groupId>
     <artifactId>hsfs-utils</artifactId>
-    <version>3.8.0-RC3</version>
+    <version>3.8.0-RC4</version>
 
     <properties>
         <hops.version>3.2.0.0-SNAPSHOT</hops.version>