diff --git a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/Config.java b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/Config.java index b2a3bd180..f2515a725 100644 --- a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/Config.java +++ b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/Config.java @@ -552,6 +552,20 @@ public interface Config { */ boolean listDatabaseNameByDefaultOnGetCatalog(); + /** + * Get the page size when listing table entities. + * + * @return True if it is. + */ + int getListTableEntitiesPageSize(); + + /** + * Get the page size when listing table names. + * + * @return True if it is. + */ + int getListTableNamesPageSize(); + /** * Metadata query timeout in seconds. * diff --git a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/DefaultConfigImpl.java b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/DefaultConfigImpl.java index 2fe336fd0..ad27e0057 100644 --- a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/DefaultConfigImpl.java +++ b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/DefaultConfigImpl.java @@ -639,6 +639,16 @@ public boolean listDatabaseNameByDefaultOnGetCatalog() { return this.metacatProperties.getService().isListDatabaseNameByDefaultOnGetCatalog(); } + @Override + public int getListTableEntitiesPageSize() { + return this.metacatProperties.getService().getListTableEntitiesPageSize(); + } + + @Override + public int getListTableNamesPageSize() { + return this.metacatProperties.getService().getListTableNamesPageSize(); + } + @Override public int getMetadataQueryTimeout() { return this.metacatProperties.getUsermetadata().getQueryTimeoutInSeconds(); diff --git a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/ServiceProperties.java b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/ServiceProperties.java index efbefd37f..cec2f452e 100644 --- a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/ServiceProperties.java +++ b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/ServiceProperties.java @@ -37,6 +37,8 @@ public class ServiceProperties { private Tables tables = new Tables(); private boolean listTableNamesByDefaultOnGetDatabase = true; private boolean listDatabaseNameByDefaultOnGetCatalog = true; + private int listTableEntitiesPageSize = 1000; + private int listTableNamesPageSize = 10000; /** * Max related properties. diff --git a/metacat-connector-polaris/src/main/java/com/netflix/metacat/connector/polaris/PolarisConnectorTableService.java b/metacat-connector-polaris/src/main/java/com/netflix/metacat/connector/polaris/PolarisConnectorTableService.java index 6c12ce3df..a9cfc277a 100644 --- a/metacat-connector-polaris/src/main/java/com/netflix/metacat/connector/polaris/PolarisConnectorTableService.java +++ b/metacat-connector-polaris/src/main/java/com/netflix/metacat/connector/polaris/PolarisConnectorTableService.java @@ -189,7 +189,7 @@ public List listNames( final String tableFilter = (prefix != null && prefix.isTableDefinition()) ? prefix.getTableName() : ""; for (String tableName : polarisStoreService.getTables(name.getDatabaseName(), tableFilter, - 1000) + connectorContext.getConfig().getListTableNamesPageSize()) ) { final QualifiedName qualifiedName = QualifiedName.ofTable(name.getCatalogName(), name.getDatabaseName(), tableName); @@ -341,7 +341,9 @@ public List list( try { final String tableFilter = (prefix != null && prefix.isTableDefinition()) ? prefix.getTableName() : ""; final List tbls = - polarisStoreService.getTableEntities(name.getDatabaseName(), tableFilter, 1000); + polarisStoreService.getTableEntities(name.getDatabaseName(), + tableFilter, + connectorContext.getConfig().getListTableEntitiesPageSize()); if (sort != null) { ConnectorUtils.sort(tbls, sort, Comparator.comparing(t -> t.getTblName())); } diff --git a/metacat-connector-polaris/src/main/java/com/netflix/metacat/connector/polaris/store/repos/PolarisTableCustomRepository.java b/metacat-connector-polaris/src/main/java/com/netflix/metacat/connector/polaris/store/repos/PolarisTableCustomRepository.java index 113cb4b81..fe3572041 100644 --- a/metacat-connector-polaris/src/main/java/com/netflix/metacat/connector/polaris/store/repos/PolarisTableCustomRepository.java +++ b/metacat-connector-polaris/src/main/java/com/netflix/metacat/connector/polaris/store/repos/PolarisTableCustomRepository.java @@ -11,9 +11,9 @@ public interface PolarisTableCustomRepository { * @param dbName database name * @param tableNamePrefix table name prefix. can be empty. * @param pageSize target size for each page - * @param selectAll if true return the PolarisEntity else return name of the entity + * @param selectAllColumns if true return the PolarisEntity else return name of the entity * @return table entities in the database. */ List findAllTablesByDbNameAndTablePrefix( - String dbName, String tableNamePrefix, int pageSize, boolean selectAll); + String dbName, String tableNamePrefix, int pageSize, boolean selectAllColumns); } diff --git a/metacat-connector-polaris/src/main/java/com/netflix/metacat/connector/polaris/store/repos/PolarisTableCustomRepositoryImpl.java b/metacat-connector-polaris/src/main/java/com/netflix/metacat/connector/polaris/store/repos/PolarisTableCustomRepositoryImpl.java index e52f00745..1d9ba4020 100644 --- a/metacat-connector-polaris/src/main/java/com/netflix/metacat/connector/polaris/store/repos/PolarisTableCustomRepositoryImpl.java +++ b/metacat-connector-polaris/src/main/java/com/netflix/metacat/connector/polaris/store/repos/PolarisTableCustomRepositoryImpl.java @@ -26,7 +26,7 @@ public class PolarisTableCustomRepositoryImpl implements PolarisTableCustomRepos private EntityManager entityManager; private Slice findAllTablesByDbNameAndTablePrefixForCurrentPage( - final String dbName, final String tableNamePrefix, final Pageable page, final boolean selectAll) { + final String dbName, final String tableNamePrefix, final Pageable page, final boolean selectAllColumns) { // Generate ORDER BY clause String orderBy = ""; @@ -37,12 +37,12 @@ private Slice findAllTablesByDbNameAndTablePrefixForCurrentPage( orderBy = " ORDER BY " + orderBy; } - final String selectClause = selectAll ? "t.*" : "t.tbl_name"; + final String selectClause = selectAllColumns ? "t.*" : "t.tbl_name"; final String sql = "SELECT " + selectClause + " FROM TBLS t " + "WHERE t.db_name = :dbName AND t.tbl_name LIKE :tableNamePrefix" + orderBy; Query query; - if (selectAll) { + if (selectAllColumns) { query = entityManager.createNativeQuery(sql, PolarisTableEntity.class); } else { query = entityManager.createNativeQuery(sql); @@ -64,7 +64,7 @@ private Slice findAllTablesByDbNameAndTablePrefixForCurrentPage( @Override @Transactional public List findAllTablesByDbNameAndTablePrefix( - final String dbName, final String tableNamePrefix, final int pageFetchSize, final boolean selectAll) { + final String dbName, final String tableNamePrefix, final int pageFetchSize, final boolean selectAllColumns) { Pageable page = PageRequest.of(0, pageFetchSize, Sort.by("tbl_name").ascending()); entityManager.createNativeQuery("SET TRANSACTION AS OF SYSTEM TIME follower_read_timestamp()") .executeUpdate(); @@ -73,7 +73,7 @@ public List findAllTablesByDbNameAndTablePrefix( Slice tbls; boolean hasNext; do { - tbls = findAllTablesByDbNameAndTablePrefixForCurrentPage(dbName, tblPrefix, page, selectAll); + tbls = findAllTablesByDbNameAndTablePrefixForCurrentPage(dbName, tblPrefix, page, selectAllColumns); retval.addAll(tbls.getContent()); hasNext = tbls.hasNext(); if (hasNext) {