diff --git a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/Config.java b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/Config.java index b2a3bd180..9cd0c835d 100644 --- a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/Config.java +++ b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/Config.java @@ -552,6 +552,14 @@ public interface Config { */ boolean listDatabaseNameByDefaultOnGetCatalog(); + /** + * Specifies the relative past timestamps for the snapshot. This is used when listing tableEntities from crdb. + * eg: The value "-5s" means it looks at the state of tables 5 seconds ago. + * According to https://www.cockroachlabs.com/docs/stable/as-of-system-time, follower_read_timestamp() used + * current timestamp - 4.8s to ensure exact staleness follower read so -5s should also be safe + */ + String getTableEntitiesRelativeTimeAgo(); + /** * Metadata query timeout in seconds. * diff --git a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/DefaultConfigImpl.java b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/DefaultConfigImpl.java index 2fe336fd0..9fa4dc109 100644 --- a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/DefaultConfigImpl.java +++ b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/DefaultConfigImpl.java @@ -639,6 +639,12 @@ public boolean listDatabaseNameByDefaultOnGetCatalog() { return this.metacatProperties.getService().isListDatabaseNameByDefaultOnGetCatalog(); } + @Override + public String getTableEntitiesRelativeTimeAgo() { + return this.metacatProperties.getService().getGetTableEntitiesRelativeTimeAgo(); + } + + @Override public int getMetadataQueryTimeout() { return this.metacatProperties.getUsermetadata().getQueryTimeoutInSeconds(); diff --git a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/ServiceProperties.java b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/ServiceProperties.java index efbefd37f..782a3d00b 100644 --- a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/ServiceProperties.java +++ b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/ServiceProperties.java @@ -37,6 +37,7 @@ public class ServiceProperties { private Tables tables = new Tables(); private boolean listTableNamesByDefaultOnGetDatabase = true; private boolean listDatabaseNameByDefaultOnGetCatalog = true; + private String getTableEntitiesRelativeTimeAgo = "-5s"; /** * Max related properties. diff --git a/metacat-connector-polaris/src/main/java/com/netflix/metacat/connector/polaris/PolarisConnectorTableService.java b/metacat-connector-polaris/src/main/java/com/netflix/metacat/connector/polaris/PolarisConnectorTableService.java index 31138fc22..9b0d5fbc0 100644 --- a/metacat-connector-polaris/src/main/java/com/netflix/metacat/connector/polaris/PolarisConnectorTableService.java +++ b/metacat-connector-polaris/src/main/java/com/netflix/metacat/connector/polaris/PolarisConnectorTableService.java @@ -338,7 +338,9 @@ public List list( try { final String tableFilter = (prefix != null && prefix.isTableDefinition()) ? prefix.getTableName() : ""; final List tbls = - polarisStoreService.getTableEntities(name.getDatabaseName(), tableFilter); + polarisStoreService.getTableEntities(name.getDatabaseName(), + tableFilter, + connectorContext.getConfig().getTableEntitiesRelativeTimeAgo()); if (sort != null) { ConnectorUtils.sort(tbls, sort, Comparator.comparing(t -> t.getTblName())); } diff --git a/metacat-connector-polaris/src/main/java/com/netflix/metacat/connector/polaris/store/PolarisStoreConnector.java b/metacat-connector-polaris/src/main/java/com/netflix/metacat/connector/polaris/store/PolarisStoreConnector.java index 5c3b203e9..e43d19027 100644 --- a/metacat-connector-polaris/src/main/java/com/netflix/metacat/connector/polaris/store/PolarisStoreConnector.java +++ b/metacat-connector-polaris/src/main/java/com/netflix/metacat/connector/polaris/store/PolarisStoreConnector.java @@ -155,7 +155,9 @@ public Optional getTable(final String dbName, final String t */ @Override @Transactional(propagation = Propagation.SUPPORTS) - public List getTableEntities(final String databaseName, final String tableNamePrefix) { + public List getTableEntities(final String databaseName, + final String tableNamePrefix, + final String relativeTimeAgo) { final int pageFetchSize = 1000; final List retval = new ArrayList<>(); final String tblPrefix = tableNamePrefix == null ? "" : tableNamePrefix; @@ -163,7 +165,7 @@ public List getTableEntities(final String databaseName, fina Slice tbls; boolean hasNext; do { - tbls = tblRepo.findAllTablesByDbNameAndTablePrefix(databaseName, tblPrefix, page); + tbls = tblRepo.findAllTablesByDbNameAndTablePrefix(databaseName, tblPrefix, relativeTimeAgo, page); retval.addAll(tbls.toList()); hasNext = tbls.hasNext(); if (hasNext) { diff --git a/metacat-connector-polaris/src/main/java/com/netflix/metacat/connector/polaris/store/PolarisStoreService.java b/metacat-connector-polaris/src/main/java/com/netflix/metacat/connector/polaris/store/PolarisStoreService.java index b1b32013b..13411936c 100644 --- a/metacat-connector-polaris/src/main/java/com/netflix/metacat/connector/polaris/store/PolarisStoreService.java +++ b/metacat-connector-polaris/src/main/java/com/netflix/metacat/connector/polaris/store/PolarisStoreService.java @@ -77,7 +77,9 @@ public interface PolarisStoreService { * @param tableNamePrefix table name prefix. can be empty. * @return table entities in the database. */ - List getTableEntities(final String databaseName, final String tableNamePrefix); + List getTableEntities(final String databaseName, + final String tableNamePrefix, + final String relativeTimeAgo); /** * Updates existing or creates new table entry. diff --git a/metacat-connector-polaris/src/main/java/com/netflix/metacat/connector/polaris/store/repos/PolarisTableRepository.java b/metacat-connector-polaris/src/main/java/com/netflix/metacat/connector/polaris/store/repos/PolarisTableRepository.java index a707caad8..0db3f59f6 100644 --- a/metacat-connector-polaris/src/main/java/com/netflix/metacat/connector/polaris/store/repos/PolarisTableRepository.java +++ b/metacat-connector-polaris/src/main/java/com/netflix/metacat/connector/polaris/store/repos/PolarisTableRepository.java @@ -74,10 +74,12 @@ boolean existsByDbNameAndTblName( * @param page pageable. * @return table entities that belong to the database. */ - @Query("SELECT e FROM PolarisTableEntity e WHERE e.dbName = :dbName AND e.tblName LIKE :tableNamePrefix%") + + @Query(nativeQuery = true, value = "SELECT e FROM PolarisTableEntity e WHERE e.dbName = :dbName AND e.tblName LIKE :tableNamePrefix% AS OF SYSTEM TIME :relativeTimeAgo") Slice findAllTablesByDbNameAndTablePrefix( @Param("dbName") final String dbName, @Param("tableNamePrefix") final String tableNamePrefix, + @Param("relativeTimeAgo") final String relativeTimeAgo, Pageable page); /**