address comments

Netflix · Jan 9, 2024 · 7aa38e9 · 7aa38e9
1 parent c39b912
commit 7aa38e9
Show file tree

Hide file tree

Showing 6 changed files with 37 additions and 9 deletions.
diff --git a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/Config.java b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/Config.java
@@ -552,6 +552,20 @@ public interface Config {
      */
     boolean listDatabaseNameByDefaultOnGetCatalog();
 
+    /**
+     * Get the page size when listing table entities.
+     *
+     * @return True if it is.
+     */
+    int getListTableEntitiesPageSize();
+
+    /**
+     * Get the page size when listing table names.
+     *
+     * @return True if it is.
+     */
+    int getListTableNamesPageSize();
+
     /**
      * Metadata query timeout in seconds.
      *

diff --git a/...-server/src/main/java/com/netflix/metacat/common/server/properties/DefaultConfigImpl.java b/...-server/src/main/java/com/netflix/metacat/common/server/properties/DefaultConfigImpl.java
@@ -639,6 +639,16 @@ public boolean listDatabaseNameByDefaultOnGetCatalog() {
         return this.metacatProperties.getService().isListDatabaseNameByDefaultOnGetCatalog();
     }
 
+    @Override
+    public int getListTableEntitiesPageSize() {
+        return this.metacatProperties.getService().getListTableEntitiesPageSize();
+    }
+
+    @Override
+    public int getListTableNamesPageSize() {
+        return this.metacatProperties.getService().getListTableNamesPageSize();
+    }
+
     @Override
     public int getMetadataQueryTimeout() {
         return this.metacatProperties.getUsermetadata().getQueryTimeoutInSeconds();

diff --git a/...-server/src/main/java/com/netflix/metacat/common/server/properties/ServiceProperties.java b/...-server/src/main/java/com/netflix/metacat/common/server/properties/ServiceProperties.java
@@ -37,6 +37,8 @@ public class ServiceProperties {
     private Tables tables = new Tables();
     private boolean listTableNamesByDefaultOnGetDatabase = true;
     private boolean listDatabaseNameByDefaultOnGetCatalog = true;
+    private int listTableEntitiesPageSize = 1000;
+    private int listTableNamesPageSize = 10000;
 
     /**
      * Max related properties.

diff --git a/...ris/src/main/java/com/netflix/metacat/connector/polaris/PolarisConnectorTableService.java b/...ris/src/main/java/com/netflix/metacat/connector/polaris/PolarisConnectorTableService.java
@@ -189,7 +189,7 @@ public List<QualifiedName> listNames(
             final String tableFilter = (prefix != null && prefix.isTableDefinition()) ? prefix.getTableName() : "";
             for (String tableName : polarisStoreService.getTables(name.getDatabaseName(),
                 tableFilter,
-                1000)
+                connectorContext.getConfig().getListTableNamesPageSize())
             ) {
                 final QualifiedName qualifiedName =
                     QualifiedName.ofTable(name.getCatalogName(), name.getDatabaseName(), tableName);
@@ -341,7 +341,9 @@ public List<TableInfo> list(
         try {
             final String tableFilter = (prefix != null && prefix.isTableDefinition()) ? prefix.getTableName() : "";
             final List<PolarisTableEntity> tbls =
-                polarisStoreService.getTableEntities(name.getDatabaseName(), tableFilter, 1000);
+                polarisStoreService.getTableEntities(name.getDatabaseName(),
+                    tableFilter,
+                    connectorContext.getConfig().getListTableEntitiesPageSize());
             if (sort != null) {
                 ConnectorUtils.sort(tbls, sort, Comparator.comparing(t -> t.getTblName()));
             }

diff --git a/.../java/com/netflix/metacat/connector/polaris/store/repos/PolarisTableCustomRepository.java b/.../java/com/netflix/metacat/connector/polaris/store/repos/PolarisTableCustomRepository.java
@@ -11,9 +11,9 @@ public interface PolarisTableCustomRepository {
      * @param dbName database name
      * @param tableNamePrefix table name prefix. can be empty.
      * @param pageSize target size for each page
-     * @param selectAll if true return the PolarisEntity else return name of the entity
+     * @param selectAllColumns if true return the PolarisEntity else return name of the entity
      * @return table entities in the database.
      */
     List<?> findAllTablesByDbNameAndTablePrefix(
-        String dbName, String tableNamePrefix, int pageSize, boolean selectAll);
+        String dbName, String tableNamePrefix, int pageSize, boolean selectAllColumns);
 }
diff --git a/...a/com/netflix/metacat/connector/polaris/store/repos/PolarisTableCustomRepositoryImpl.java b/...a/com/netflix/metacat/connector/polaris/store/repos/PolarisTableCustomRepositoryImpl.java
@@ -26,7 +26,7 @@ public class PolarisTableCustomRepositoryImpl implements PolarisTableCustomRepos
     private EntityManager entityManager;
 
     private <T> Slice<T> findAllTablesByDbNameAndTablePrefixForCurrentPage(
-        final String dbName, final String tableNamePrefix, final Pageable page, final boolean selectAll) {
+        final String dbName, final String tableNamePrefix, final Pageable page, final boolean selectAllColumns) {
 
         // Generate ORDER BY clause
         String orderBy = "";
@@ -37,12 +37,12 @@ private <T> Slice<T> findAllTablesByDbNameAndTablePrefixForCurrentPage(
             orderBy = " ORDER BY " + orderBy;
         }
 
-        final String selectClause = selectAll ? "t.*" : "t.tbl_name";
+        final String selectClause = selectAllColumns ? "t.*" : "t.tbl_name";
         final String sql = "SELECT " + selectClause + " FROM TBLS t "
             + "WHERE t.db_name = :dbName AND t.tbl_name LIKE :tableNamePrefix" + orderBy;
 
         Query query;
-        if (selectAll) {
+        if (selectAllColumns) {
             query = entityManager.createNativeQuery(sql, PolarisTableEntity.class);
         } else {
             query = entityManager.createNativeQuery(sql);
@@ -64,7 +64,7 @@ private <T> Slice<T> findAllTablesByDbNameAndTablePrefixForCurrentPage(
     @Override
     @Transactional
     public List<?> findAllTablesByDbNameAndTablePrefix(
-        final String dbName, final String tableNamePrefix, final int pageFetchSize, final boolean selectAll) {
+        final String dbName, final String tableNamePrefix, final int pageFetchSize, final boolean selectAllColumns) {
         Pageable page = PageRequest.of(0, pageFetchSize, Sort.by("tbl_name").ascending());
         entityManager.createNativeQuery("SET TRANSACTION AS OF SYSTEM TIME follower_read_timestamp()")
             .executeUpdate();
@@ -73,7 +73,7 @@ public List<?> findAllTablesByDbNameAndTablePrefix(
         Slice<?> tbls;
         boolean hasNext;
         do {
-            tbls = findAllTablesByDbNameAndTablePrefixForCurrentPage(dbName, tblPrefix, page, selectAll);
+            tbls = findAllTablesByDbNameAndTablePrefixForCurrentPage(dbName, tblPrefix, page, selectAllColumns);
             retval.addAll(tbls.getContent());
             hasNext = tbls.hasNext();
             if (hasNext) {