diff --git a/metacat-connector-polaris/src/functionalTest/java/com/netflix/metacat/connector/polaris/PolarisConnectorTableServiceFunctionalTest.java b/metacat-connector-polaris/src/functionalTest/java/com/netflix/metacat/connector/polaris/PolarisConnectorTableServiceFunctionalTest.java index c94d04b57..5b647e348 100644 --- a/metacat-connector-polaris/src/functionalTest/java/com/netflix/metacat/connector/polaris/PolarisConnectorTableServiceFunctionalTest.java +++ b/metacat-connector-polaris/src/functionalTest/java/com/netflix/metacat/connector/polaris/PolarisConnectorTableServiceFunctionalTest.java @@ -1,5 +1,6 @@ package com.netflix.metacat.connector.polaris; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.netflix.metacat.common.QualifiedName; @@ -17,6 +18,7 @@ import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.junit.jupiter.SpringExtension; +import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; @@ -31,6 +33,88 @@ @ActiveProfiles(profiles = {"polaris_functional_test"}) @AutoConfigureDataJpa public class PolarisConnectorTableServiceFunctionalTest extends PolarisConnectorTableServiceTest { + /** + * Test get table names. + */ + @Test + public void testGetTableNames() { + final QualifiedName name1 = QualifiedName.ofTable(CATALOG_NAME, DB_NAME, "table1"); + final TableInfo tableInfo1 = TableInfo.builder() + .name(name1) + .metadata(ImmutableMap.of("table_type", "ICEBERG", "metadata_location", "loc1")) + .build(); + getPolarisTableService().create(getRequestContext(), tableInfo1); + final QualifiedName name2 = QualifiedName.ofTable(CATALOG_NAME, DB_NAME, "table2"); + final TableInfo tableInfo2 = TableInfo.builder() + .name(name2) + .metadata(ImmutableMap.of("table_type", "ICEBERG", "metadata_location", "loc2")) + .build(); + getPolarisTableService().create(getRequestContext(), tableInfo2); + final QualifiedName name3 = QualifiedName.ofTable(CATALOG_NAME, DB_NAME, "table3"); + final TableInfo tableInfo3 = TableInfo.builder() + .name(name3) + .metadata(ImmutableMap.of("table_type", "ICEBERG", "metadata_location", "loc3")) + .build(); + getPolarisTableService().create(getRequestContext(), tableInfo3); + + try { + // pause execution for 10000 milliseconds (10 seconds) + Thread.sleep(10000); + } catch (InterruptedException e) { + log.debug("Sleep was interrupted"); + } + + final List tables = getPolarisTableService() + .getTableNames(getRequestContext(), DB_QUALIFIED_NAME, "", -1); + Assert.assertEquals(tables.size(), 3); + Assert.assertEquals(tables, ImmutableList.of(name1, name2, name3)); + } + + /** + * Test empty list tables. + */ + @Test + public void testListTablesEmpty() { + final QualifiedName qualifiedName = QualifiedName.ofTable(CATALOG_NAME, DB_NAME, ""); + + try { + // pause execution for 10000 milliseconds (10 seconds) + Thread.sleep(10000); + } catch (InterruptedException e) { + log.debug("Sleep was interrupted"); + } + + final List names = getPolarisTableService().listNames( + getRequestContext(), DB_QUALIFIED_NAME, qualifiedName, + new Sort(null, SortOrder.ASC), new Pageable(2, 0)); + Assert.assertEquals(names, Arrays.asList()); + } + + /** + * Test table creation then list tables. + */ + @Test + public void testTableCreationAndList() { + final QualifiedName qualifiedName = QualifiedName.ofTable(CATALOG_NAME, DB_NAME, "table1"); + final TableInfo tableInfo = TableInfo.builder() + .name(qualifiedName) + .metadata(ImmutableMap.of("table_type", "ICEBERG", "metadata_location", "loc1")) + .build(); + getPolarisTableService().create(getRequestContext(), tableInfo); + + try { + // pause execution for 10000 milliseconds (10 seconds) + Thread.sleep(10000); + } catch (InterruptedException e) { + log.debug("Sleep was interrupted"); + } + + final List names = getPolarisTableService().listNames( + getRequestContext(), DB_QUALIFIED_NAME, qualifiedName, + new Sort(null, SortOrder.ASC), new Pageable(2, 0)); + Assert.assertEquals(names, Arrays.asList(qualifiedName)); + } + /** * Test table list. */ diff --git a/metacat-connector-polaris/src/functionalTest/java/com/netflix/metacat/connector/polaris/PolarisStoreConnectorFunctionalTest.java b/metacat-connector-polaris/src/functionalTest/java/com/netflix/metacat/connector/polaris/PolarisStoreConnectorFunctionalTest.java index e1c1915e3..2cdf296ff 100644 --- a/metacat-connector-polaris/src/functionalTest/java/com/netflix/metacat/connector/polaris/PolarisStoreConnectorFunctionalTest.java +++ b/metacat-connector-polaris/src/functionalTest/java/com/netflix/metacat/connector/polaris/PolarisStoreConnectorFunctionalTest.java @@ -25,6 +25,37 @@ @AutoConfigureDataJpa public class PolarisStoreConnectorFunctionalTest extends PolarisStoreConnectorTest { + /** + * Test to verify that table names fetch works. + */ + @Test + public void testPaginatedFetch() { + final String dbName = generateDatabaseName(); + createDB(dbName); + List tblNames = getPolarisConnector().getTables(dbName, "", 1000); + Assert.assertEquals(0, tblNames.size()); + + final String tblNameA = "A_" + generateTableName(); + final String tblNameB = "B_" + generateTableName(); + final String tblNameC = "C_" + generateTableName(); + createTable(dbName, tblNameA); + createTable(dbName, tblNameB); + createTable(dbName, tblNameC); + + try { + // pause execution for 10000 milliseconds (10 seconds) + Thread.sleep(10000); + } catch (InterruptedException e) { + log.debug("Sleep was interrupted"); + } + + tblNames = getPolarisConnector().getTables(dbName, "", 1000); + Assert.assertEquals(3, tblNames.size()); + Assert.assertEquals(tblNameA, tblNames.get(0)); + Assert.assertEquals(tblNameB, tblNames.get(1)); + Assert.assertEquals(tblNameC, tblNames.get(2)); + } + /** * Test getTableEntities. */ diff --git a/metacat-connector-polaris/src/main/java/com/netflix/metacat/connector/polaris/PolarisConnectorTableService.java b/metacat-connector-polaris/src/main/java/com/netflix/metacat/connector/polaris/PolarisConnectorTableService.java index 723f4f6aa..6c12ce3df 100644 --- a/metacat-connector-polaris/src/main/java/com/netflix/metacat/connector/polaris/PolarisConnectorTableService.java +++ b/metacat-connector-polaris/src/main/java/com/netflix/metacat/connector/polaris/PolarisConnectorTableService.java @@ -187,7 +187,10 @@ public List listNames( try { final List qualifiedNames = Lists.newArrayList(); final String tableFilter = (prefix != null && prefix.isTableDefinition()) ? prefix.getTableName() : ""; - for (String tableName : polarisStoreService.getTables(name.getDatabaseName(), tableFilter)) { + for (String tableName : polarisStoreService.getTables(name.getDatabaseName(), + tableFilter, + 1000) + ) { final QualifiedName qualifiedName = QualifiedName.ofTable(name.getCatalogName(), name.getDatabaseName(), tableName); if (prefix != null && !qualifiedName.toString().startsWith(prefix.toString())) { @@ -388,7 +391,10 @@ public List getTableNames( final List result = Lists.newArrayList(); for (int i = 0; i < databaseNames.size() && limitSize > 0; i++) { final String databaseName = databaseNames.get(i); - final List tableNames = polarisStoreService.getTables(name.getDatabaseName(), ""); + final List tableNames = polarisStoreService.getTables( + name.getDatabaseName(), + "", + 1000); result.addAll(tableNames.stream() .map(n -> QualifiedName.ofTable(name.getCatalogName(), databaseName, n)) .limit(limitSize) diff --git a/metacat-connector-polaris/src/main/java/com/netflix/metacat/connector/polaris/store/PolarisStoreConnector.java b/metacat-connector-polaris/src/main/java/com/netflix/metacat/connector/polaris/store/PolarisStoreConnector.java index 337f6694a..5c63890d9 100644 --- a/metacat-connector-polaris/src/main/java/com/netflix/metacat/connector/polaris/store/PolarisStoreConnector.java +++ b/metacat-connector-polaris/src/main/java/com/netflix/metacat/connector/polaris/store/PolarisStoreConnector.java @@ -9,7 +9,6 @@ import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Pageable; import org.springframework.data.domain.Slice; -import org.springframework.data.domain.Sort; import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; @@ -158,7 +157,8 @@ public Optional getTable(final String dbName, final String t public List getTableEntities(final String databaseName, final String tableNamePrefix, final int pageFetchSize) { - return tblRepo.findAllTablesByDbNameAndTablePrefix(databaseName, tableNamePrefix, pageFetchSize); + return (List) + tblRepo.findAllTablesByDbNameAndTablePrefix(databaseName, tableNamePrefix, pageFetchSize, true); } /** @@ -205,22 +205,9 @@ boolean tableExistsById(final String tblId) { */ @Override @Transactional(propagation = Propagation.SUPPORTS) - public List getTables(final String databaseName, final String tableNamePrefix) { - final int pageFetchSize = 1000; - final List retval = new ArrayList<>(); - final String tblPrefix = tableNamePrefix == null ? "" : tableNamePrefix; - Pageable page = PageRequest.of(0, pageFetchSize, Sort.by("tblName").ascending()); - Slice tblNames = null; - boolean hasNext = true; - do { - tblNames = tblRepo.findAllByDbNameAndTablePrefix(databaseName, tblPrefix, page); - retval.addAll(tblNames.toList()); - hasNext = tblNames.hasNext(); - if (hasNext) { - page = tblNames.nextPageable(); - } - } while (hasNext); - return retval; + public List getTables(final String databaseName, final String tableNamePrefix, final int pageFetchSize) { + return (List) + tblRepo.findAllTablesByDbNameAndTablePrefix(databaseName, tableNamePrefix, pageFetchSize, false); } /** diff --git a/metacat-connector-polaris/src/main/java/com/netflix/metacat/connector/polaris/store/PolarisStoreService.java b/metacat-connector-polaris/src/main/java/com/netflix/metacat/connector/polaris/store/PolarisStoreService.java index 6cb108410..db358abce 100644 --- a/metacat-connector-polaris/src/main/java/com/netflix/metacat/connector/polaris/store/PolarisStoreService.java +++ b/metacat-connector-polaris/src/main/java/com/netflix/metacat/connector/polaris/store/PolarisStoreService.java @@ -106,9 +106,10 @@ public interface PolarisStoreService { * Gets tables in the database and tableName prefix. * @param databaseName database name * @param tableNamePrefix table name prefix + * @param pageFetchSize size of each page * @return list of table names in the database with the table name prefix. */ - List getTables(String databaseName, String tableNamePrefix); + List getTables(String databaseName, String tableNamePrefix, int pageFetchSize); /** * Do an atomic compare-and-swap to update the table's metadata location. diff --git a/metacat-connector-polaris/src/main/java/com/netflix/metacat/connector/polaris/store/repos/PolarisTableCustomRepository.java b/metacat-connector-polaris/src/main/java/com/netflix/metacat/connector/polaris/store/repos/PolarisTableCustomRepository.java index 34cd2cb2e..113cb4b81 100644 --- a/metacat-connector-polaris/src/main/java/com/netflix/metacat/connector/polaris/store/repos/PolarisTableCustomRepository.java +++ b/metacat-connector-polaris/src/main/java/com/netflix/metacat/connector/polaris/store/repos/PolarisTableCustomRepository.java @@ -1,6 +1,5 @@ package com.netflix.metacat.connector.polaris.store.repos; -import com.netflix.metacat.connector.polaris.store.entities.PolarisTableEntity; import java.util.List; /** @@ -12,8 +11,9 @@ public interface PolarisTableCustomRepository { * @param dbName database name * @param tableNamePrefix table name prefix. can be empty. * @param pageSize target size for each page + * @param selectAll if true return the PolarisEntity else return name of the entity * @return table entities in the database. */ - List findAllTablesByDbNameAndTablePrefix( - String dbName, String tableNamePrefix, int pageSize); + List findAllTablesByDbNameAndTablePrefix( + String dbName, String tableNamePrefix, int pageSize, boolean selectAll); } diff --git a/metacat-connector-polaris/src/main/java/com/netflix/metacat/connector/polaris/store/repos/PolarisTableCustomRepositoryImpl.java b/metacat-connector-polaris/src/main/java/com/netflix/metacat/connector/polaris/store/repos/PolarisTableCustomRepositoryImpl.java index 007961763..e52f00745 100644 --- a/metacat-connector-polaris/src/main/java/com/netflix/metacat/connector/polaris/store/repos/PolarisTableCustomRepositoryImpl.java +++ b/metacat-connector-polaris/src/main/java/com/netflix/metacat/connector/polaris/store/repos/PolarisTableCustomRepositoryImpl.java @@ -25,8 +25,8 @@ public class PolarisTableCustomRepositoryImpl implements PolarisTableCustomRepos @PersistenceContext private EntityManager entityManager; - private Slice findAllTablesByDbNameAndTablePrefixForCurrentPage( - final String dbName, final String tableNamePrefix, final Pageable page) { + private Slice findAllTablesByDbNameAndTablePrefixForCurrentPage( + final String dbName, final String tableNamePrefix, final Pageable page, final boolean selectAll) { // Generate ORDER BY clause String orderBy = ""; @@ -37,14 +37,21 @@ private Slice findAllTablesByDbNameAndTablePrefixForCurrentP orderBy = " ORDER BY " + orderBy; } - final String sql = "SELECT t.* FROM TBLS t " + final String selectClause = selectAll ? "t.*" : "t.tbl_name"; + final String sql = "SELECT " + selectClause + " FROM TBLS t " + "WHERE t.db_name = :dbName AND t.tbl_name LIKE :tableNamePrefix" + orderBy; - final Query query = entityManager.createNativeQuery(sql, PolarisTableEntity.class); + + Query query; + if (selectAll) { + query = entityManager.createNativeQuery(sql, PolarisTableEntity.class); + } else { + query = entityManager.createNativeQuery(sql); + } query.setParameter("dbName", dbName); query.setParameter("tableNamePrefix", tableNamePrefix + "%"); query.setFirstResult(page.getPageNumber() * page.getPageSize()); query.setMaxResults(page.getPageSize() + 1); // Fetch one extra result to determine if there is a next page - final List resultList = query.getResultList(); + final List resultList = query.getResultList(); // Check if there is a next page final boolean hasNext = resultList.size() > page.getPageSize(); // If there is a next page, remove the last item from the list @@ -56,18 +63,18 @@ private Slice findAllTablesByDbNameAndTablePrefixForCurrentP @Override @Transactional - public List findAllTablesByDbNameAndTablePrefix( - final String dbName, final String tableNamePrefix, final int pageFetchSize) { + public List findAllTablesByDbNameAndTablePrefix( + final String dbName, final String tableNamePrefix, final int pageFetchSize, final boolean selectAll) { Pageable page = PageRequest.of(0, pageFetchSize, Sort.by("tbl_name").ascending()); entityManager.createNativeQuery("SET TRANSACTION AS OF SYSTEM TIME follower_read_timestamp()") .executeUpdate(); - final List retval = new ArrayList<>(); + final List retval = new ArrayList<>(); final String tblPrefix = tableNamePrefix == null ? "" : tableNamePrefix; - Slice tbls; + Slice tbls; boolean hasNext; do { - tbls = findAllTablesByDbNameAndTablePrefixForCurrentPage(dbName, tblPrefix, page); - retval.addAll(tbls.toList()); + tbls = findAllTablesByDbNameAndTablePrefixForCurrentPage(dbName, tblPrefix, page, selectAll); + retval.addAll(tbls.getContent()); hasNext = tbls.hasNext(); if (hasNext) { page = tbls.nextPageable(); diff --git a/metacat-connector-polaris/src/test/java/com/netflix/metacat/connector/polaris/PolarisConnectorTableServiceTest.java b/metacat-connector-polaris/src/test/java/com/netflix/metacat/connector/polaris/PolarisConnectorTableServiceTest.java index 1a470dcba..22f95fdfd 100644 --- a/metacat-connector-polaris/src/test/java/com/netflix/metacat/connector/polaris/PolarisConnectorTableServiceTest.java +++ b/metacat-connector-polaris/src/test/java/com/netflix/metacat/connector/polaris/PolarisConnectorTableServiceTest.java @@ -1,13 +1,9 @@ package com.netflix.metacat.connector.polaris; -import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import com.netflix.metacat.common.QualifiedName; -import com.netflix.metacat.common.dto.Pageable; -import com.netflix.metacat.common.dto.Sort; -import com.netflix.metacat.common.dto.SortOrder; import com.netflix.metacat.common.server.connectors.ConnectorContext; import com.netflix.metacat.common.server.connectors.ConnectorRequestContext; import com.netflix.metacat.common.server.connectors.exception.InvalidMetaException; @@ -42,8 +38,6 @@ import org.springframework.test.context.junit.jupiter.SpringExtension; import spock.lang.Shared; -import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -104,17 +98,6 @@ public void init() { connectorContext); } - /** - * Test empty list tables. - */ - @Test - public void testListTablesEmpty() { - final QualifiedName qualifiedName = QualifiedName.ofTable(CATALOG_NAME, DB_NAME, ""); - final List names = polarisTableService.listNames( - requestContext, DB_QUALIFIED_NAME, qualifiedName, new Sort(null, SortOrder.ASC), new Pageable(2, 0)); - Assert.assertEquals(names, Arrays.asList()); - } - /** * Test table exists. */ @@ -132,43 +115,6 @@ public void testTableExists() { Assert.assertTrue(exists); } - /** - * Test table creation then list tables. - */ - @Test - public void testTableCreationAndList() { - final QualifiedName qualifiedName = QualifiedName.ofTable(CATALOG_NAME, DB_NAME, "table1"); - final TableInfo tableInfo = TableInfo.builder() - .name(qualifiedName) - .metadata(ImmutableMap.of("table_type", "ICEBERG", "metadata_location", "loc1")) - .build(); - polarisTableService.create(requestContext, tableInfo); - final List names = polarisTableService.listNames( - requestContext, DB_QUALIFIED_NAME, qualifiedName, new Sort(null, SortOrder.ASC), new Pageable(2, 0)); - Assert.assertEquals(names, Arrays.asList(qualifiedName)); - } - - /** - * Test multiple table creation then list tables. - */ - @Test - public void testMultipleTableCreationAndList() { - final List createdTables = new ArrayList<>(); - for (int i = 0; i < 10; i++) { - final QualifiedName qualifiedName = QualifiedName.ofTable(CATALOG_NAME, DB_NAME, "table" + i); - final TableInfo tableInfo = TableInfo.builder() - .name(qualifiedName) - .metadata(ImmutableMap.of("table_type", "ICEBERG", "metadata_location", "loc" + i)) - .build(); - polarisTableService.create(requestContext, tableInfo); - createdTables.add(qualifiedName); - } - final QualifiedName qualifiedName = QualifiedName.ofTable(CATALOG_NAME, DB_NAME, ""); - final List names = polarisTableService.listNames( - requestContext, DB_QUALIFIED_NAME, qualifiedName, new Sort(null, SortOrder.ASC), new Pageable(20, 0)); - Assert.assertEquals(names, createdTables); - } - /** * Test table rename. */ @@ -210,34 +156,6 @@ public void testDeleteTable() { Assert.assertFalse(exists); } - /** - * Test get table names. - */ - @Test - public void testGetTableNames() { - final QualifiedName name1 = QualifiedName.ofTable(CATALOG_NAME, DB_NAME, "table1"); - final TableInfo tableInfo1 = TableInfo.builder() - .name(name1) - .metadata(ImmutableMap.of("table_type", "ICEBERG", "metadata_location", "loc1")) - .build(); - polarisTableService.create(requestContext, tableInfo1); - final QualifiedName name2 = QualifiedName.ofTable(CATALOG_NAME, DB_NAME, "table2"); - final TableInfo tableInfo2 = TableInfo.builder() - .name(name2) - .metadata(ImmutableMap.of("table_type", "ICEBERG", "metadata_location", "loc2")) - .build(); - polarisTableService.create(requestContext, tableInfo2); - final QualifiedName name3 = QualifiedName.ofTable(CATALOG_NAME, DB_NAME, "table3"); - final TableInfo tableInfo3 = TableInfo.builder() - .name(name3) - .metadata(ImmutableMap.of("table_type", "ICEBERG", "metadata_location", "loc3")) - .build(); - polarisTableService.create(requestContext, tableInfo3); - final List tables = polarisTableService.getTableNames(requestContext, DB_QUALIFIED_NAME, "", -1); - Assert.assertEquals(tables.size(), 3); - Assert.assertEquals(tables, ImmutableList.of(name1, name2, name3)); - } - /** * Test get table using metadata json resource file. */ diff --git a/metacat-connector-polaris/src/test/java/com/netflix/metacat/connector/polaris/store/PolarisStoreConnectorTest.java b/metacat-connector-polaris/src/test/java/com/netflix/metacat/connector/polaris/store/PolarisStoreConnectorTest.java index 6dd04ccc2..9479c6e14 100644 --- a/metacat-connector-polaris/src/test/java/com/netflix/metacat/connector/polaris/store/PolarisStoreConnectorTest.java +++ b/metacat-connector-polaris/src/test/java/com/netflix/metacat/connector/polaris/store/PolarisStoreConnectorTest.java @@ -20,7 +20,6 @@ import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.junit.jupiter.SpringExtension; -import java.util.List; import java.util.Optional; import java.util.Random; @@ -128,30 +127,6 @@ public void testTableCreationAndDeletion() { Assert.assertFalse(polarisConnector.tableExistsById(tblEntity.getTblId())); } - /** - * Test to verify that table names fetch works. - */ - @Test - public void testPaginatedFetch() { - final String dbName = generateDatabaseName(); - final PolarisDatabaseEntity dbEntity = createDB(dbName); - List tblNames = polarisConnector.getTables(dbName, ""); - Assert.assertEquals(0, tblNames.size()); - - final String tblNameA = "A_" + generateTableName(); - final String tblNameB = "B_" + generateTableName(); - final String tblNameC = "C_" + generateTableName(); - createTable(dbName, tblNameA); - createTable(dbName, tblNameB); - createTable(dbName, tblNameC); - - tblNames = polarisConnector.getTables(dbName, ""); - Assert.assertEquals(3, tblNames.size()); - Assert.assertEquals(tblNameA, tblNames.get(0)); - Assert.assertEquals(tblNameB, tblNames.get(1)); - Assert.assertEquals(tblNameC, tblNames.get(2)); - } - /** * Test to verify that table name can be updated. */