use as of system time when listing tables to improve performance

Netflix · Jan 3, 2024 · b8d8ebd · b8d8ebd
1 parent 2c0f438
commit b8d8ebd
Show file tree

Hide file tree

Showing 7 changed files with 28 additions and 5 deletions.
diff --git a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/Config.java b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/Config.java
@@ -552,6 +552,14 @@ public interface Config {
      */
     boolean listDatabaseNameByDefaultOnGetCatalog();
 
+    /**
+     * Specifies the relative past timestamps for the snapshot. This is used when listing tableEntities from crdb.
+     * eg: The value "-5s" means it looks at the state of tables 5 seconds ago.
+     * According to https://www.cockroachlabs.com/docs/stable/as-of-system-time, follower_read_timestamp() used
+     * current timestamp - 4.8s to ensure exact staleness follower read so -5s should also be safe
+     */
+    String getTableEntitiesRelativeTimeAgo();
+
     /**
      * Metadata query timeout in seconds.
      *

diff --git a/...-server/src/main/java/com/netflix/metacat/common/server/properties/DefaultConfigImpl.java b/...-server/src/main/java/com/netflix/metacat/common/server/properties/DefaultConfigImpl.java
@@ -639,6 +639,12 @@ public boolean listDatabaseNameByDefaultOnGetCatalog() {
         return this.metacatProperties.getService().isListDatabaseNameByDefaultOnGetCatalog();
     }
 
+    @Override
+    public String getTableEntitiesRelativeTimeAgo() {
+        return this.metacatProperties.getService().getGetTableEntitiesRelativeTimeAgo();
+    }
+
+
     @Override
     public int getMetadataQueryTimeout() {
         return this.metacatProperties.getUsermetadata().getQueryTimeoutInSeconds();

diff --git a/...-server/src/main/java/com/netflix/metacat/common/server/properties/ServiceProperties.java b/...-server/src/main/java/com/netflix/metacat/common/server/properties/ServiceProperties.java
@@ -37,6 +37,7 @@ public class ServiceProperties {
     private Tables tables = new Tables();
     private boolean listTableNamesByDefaultOnGetDatabase = true;
     private boolean listDatabaseNameByDefaultOnGetCatalog = true;
+    private String getTableEntitiesRelativeTimeAgo = "-5s";
 
     /**
      * Max related properties.

diff --git a/...ris/src/main/java/com/netflix/metacat/connector/polaris/PolarisConnectorTableService.java b/...ris/src/main/java/com/netflix/metacat/connector/polaris/PolarisConnectorTableService.java
@@ -338,7 +338,9 @@ public List<TableInfo> list(
         try {
             final String tableFilter = (prefix != null && prefix.isTableDefinition()) ? prefix.getTableName() : "";
             final List<PolarisTableEntity> tbls =
-                polarisStoreService.getTableEntities(name.getDatabaseName(), tableFilter);
+                polarisStoreService.getTableEntities(name.getDatabaseName(),
+                    tableFilter,
+                    connectorContext.getConfig().getTableEntitiesRelativeTimeAgo());
             if (sort != null) {
                 ConnectorUtils.sort(tbls, sort, Comparator.comparing(t -> t.getTblName()));
             }

diff --git a/...aris/src/main/java/com/netflix/metacat/connector/polaris/store/PolarisStoreConnector.java b/...aris/src/main/java/com/netflix/metacat/connector/polaris/store/PolarisStoreConnector.java
@@ -155,15 +155,17 @@ public Optional<PolarisTableEntity> getTable(final String dbName, final String t
      */
     @Override
     @Transactional(propagation = Propagation.SUPPORTS)
-    public List<PolarisTableEntity> getTableEntities(final String databaseName, final String tableNamePrefix) {
+    public List<PolarisTableEntity> getTableEntities(final String databaseName,
+                                                     final String tableNamePrefix,
+                                                     final String relativeTimeAgo) {
         final int pageFetchSize = 1000;
         final List<PolarisTableEntity> retval = new ArrayList<>();
         final String tblPrefix =  tableNamePrefix == null ? "" : tableNamePrefix;
         Pageable page = PageRequest.of(0, pageFetchSize, Sort.by("tblName").ascending());
         Slice<PolarisTableEntity> tbls;
         boolean hasNext;
         do {
-            tbls = tblRepo.findAllTablesByDbNameAndTablePrefix(databaseName, tblPrefix, page);
+            tbls = tblRepo.findAllTablesByDbNameAndTablePrefix(databaseName, tblPrefix, relativeTimeAgo, page);
             retval.addAll(tbls.toList());
             hasNext = tbls.hasNext();
             if (hasNext) {

diff --git a/...olaris/src/main/java/com/netflix/metacat/connector/polaris/store/PolarisStoreService.java b/...olaris/src/main/java/com/netflix/metacat/connector/polaris/store/PolarisStoreService.java
@@ -77,7 +77,9 @@ public interface PolarisStoreService {
      * @param tableNamePrefix table name prefix. can be empty.
      * @return table entities in the database.
      */
-    List<PolarisTableEntity> getTableEntities(final String databaseName, final String tableNamePrefix);
+    List<PolarisTableEntity> getTableEntities(final String databaseName,
+                                              final String tableNamePrefix,
+                                              final String relativeTimeAgo);
 
     /**
      * Updates existing or creates new table entry.

diff --git a/...c/main/java/com/netflix/metacat/connector/polaris/store/repos/PolarisTableRepository.java b/...c/main/java/com/netflix/metacat/connector/polaris/store/repos/PolarisTableRepository.java
@@ -74,10 +74,12 @@ boolean existsByDbNameAndTblName(
      * @param page pageable.
      * @return table entities that belong to the database.
      */
-    @Query("SELECT e FROM PolarisTableEntity e WHERE e.dbName = :dbName AND e.tblName LIKE :tableNamePrefix%")
+
+    @Query(nativeQuery = true, value = "SELECT e FROM PolarisTableEntity e WHERE e.dbName = :dbName AND e.tblName LIKE :tableNamePrefix% AS OF SYSTEM TIME :relativeTimeAgo")
     Slice<PolarisTableEntity> findAllTablesByDbNameAndTablePrefix(
         @Param("dbName") final String dbName,
         @Param("tableNamePrefix") final String tableNamePrefix,
+        @Param("relativeTimeAgo") final String relativeTimeAgo,
         Pageable page);
 
     /**