From 89cf227422a63c8c273b46856b94a8c105fabf62 Mon Sep 17 00:00:00 2001 From: Vinish Reddy Date: Sun, 29 Dec 2024 23:55:02 -0800 Subject: [PATCH] Fix test deps and Add assertions --- .../apache/xtable/testutil/ITTestUtils.java | 40 ++++++++++++++--- xtable-utilities/pom.xml | 14 ++++++ .../xtable/utilities/ITRunCatalogSync.java | 45 +++++++++++-------- 3 files changed, 75 insertions(+), 24 deletions(-) diff --git a/xtable-core/src/test/java/org/apache/xtable/testutil/ITTestUtils.java b/xtable-core/src/test/java/org/apache/xtable/testutil/ITTestUtils.java index d194558ac..409043807 100644 --- a/xtable-core/src/test/java/org/apache/xtable/testutil/ITTestUtils.java +++ b/xtable-core/src/test/java/org/apache/xtable/testutil/ITTestUtils.java @@ -18,7 +18,9 @@ package org.apache.xtable.testutil; +import java.util.HashMap; import java.util.List; +import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.junit.jupiter.api.Assertions; @@ -52,48 +54,74 @@ public static void validateTable( } public static class TestCatalogSyncImpl implements CatalogSyncClient { + private static final Map FUNCTION_CALLS = new HashMap<>(); public TestCatalogSyncImpl( ExternalCatalogConfig catalogConfig, String tableFormat, Configuration hadoopConf) {} @Override public String getCatalogId() { + trackFunctionCall(); return null; } @Override public String getStorageLocation(Object o) { + trackFunctionCall(); return null; } @Override public boolean hasDatabase(CatalogTableIdentifier tableIdentifier) { + trackFunctionCall(); return false; } @Override - public void createDatabase(CatalogTableIdentifier tableIdentifier) {} + public void createDatabase(CatalogTableIdentifier tableIdentifier) { + trackFunctionCall(); + } @Override public Object getTable(CatalogTableIdentifier tableIdentifier) { + trackFunctionCall(); return null; } @Override - public void createTable(InternalTable table, CatalogTableIdentifier tableIdentifier) {} + public void createTable(InternalTable table, CatalogTableIdentifier tableIdentifier) { + trackFunctionCall(); + } @Override public void refreshTable( - InternalTable table, Object catalogTable, CatalogTableIdentifier tableIdentifier) {} + InternalTable table, Object catalogTable, CatalogTableIdentifier tableIdentifier) { + trackFunctionCall(); + } @Override - public void createOrReplaceTable(InternalTable table, CatalogTableIdentifier tableIdentifier) {} + public void createOrReplaceTable(InternalTable table, CatalogTableIdentifier tableIdentifier) { + trackFunctionCall(); + } @Override - public void dropTable(InternalTable table, CatalogTableIdentifier tableIdentifier) {} + public void dropTable(InternalTable table, CatalogTableIdentifier tableIdentifier) { + trackFunctionCall(); + } @Override - public void close() throws Exception {} + public void close() throws Exception { + trackFunctionCall(); + } + + private void trackFunctionCall() { + String methodName = Thread.currentThread().getStackTrace()[2].getMethodName(); + FUNCTION_CALLS.put(methodName, FUNCTION_CALLS.getOrDefault(methodName, 0) + 1); + } + + public static Map getFunctionCalls() { + return FUNCTION_CALLS; + } } public static class TestCatalogConversionSourceImpl implements CatalogConversionSource { diff --git a/xtable-utilities/pom.xml b/xtable-utilities/pom.xml index 25d559730..a48eeec48 100644 --- a/xtable-utilities/pom.xml +++ b/xtable-utilities/pom.xml @@ -134,6 +134,20 @@ junit-jupiter-engine test + + org.apache.xtable + xtable-core_${scala.binary.version} + ${project.version} + tests + test-jar + test + + + + org.apache.hudi + hudi-spark${spark.version.prefix}-bundle_${scala.binary.version} + test + diff --git a/xtable-utilities/src/test/java/org/apache/xtable/utilities/ITRunCatalogSync.java b/xtable-utilities/src/test/java/org/apache/xtable/utilities/ITRunCatalogSync.java index 1de5a6ad3..52cff85ad 100644 --- a/xtable-utilities/src/test/java/org/apache/xtable/utilities/ITRunCatalogSync.java +++ b/xtable-utilities/src/test/java/org/apache/xtable/utilities/ITRunCatalogSync.java @@ -20,14 +20,18 @@ import java.io.File; import java.io.IOException; +import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Arrays; import java.util.Collections; -import java.util.concurrent.TimeUnit; +import java.util.List; +import java.util.Map; import lombok.SneakyThrows; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -46,6 +50,15 @@ public class ITRunCatalogSync { + private static final List EXPECTED_FUNCTION_CALLS = + Arrays.asList( + "hasDatabase", + "createDatabase", + "getTable", + "getStorageLocation", + "createTable", + "getCatalogId"); + @Test void testCatalogSync(@TempDir Path tempDir) throws Exception { String tableName = "test-table"; @@ -56,6 +69,10 @@ void testCatalogSync(@TempDir Path tempDir) throws Exception { File configFile = writeConfigFile(tempDir, table, tableName); String[] args = new String[] {"-catalogConfig", configFile.getPath()}; RunCatalogSync.main(args); + validateTargetMetadataIsPresent(table.getBasePath()); + Map functionCalls = ITTestUtils.TestCatalogSyncImpl.getFunctionCalls(); + EXPECTED_FUNCTION_CALLS.forEach( + (function -> Assertions.assertEquals(2, functionCalls.get(function)))); } } @@ -112,22 +129,14 @@ private static File writeConfigFile(Path tempDir, GenericTable table, String tab } @SneakyThrows - private static void waitForNumIcebergCommits(Path metadataPath, int count) { - long start = System.currentTimeMillis(); - while (System.currentTimeMillis() - start < TimeUnit.MINUTES.toMillis(5)) { - if (numIcebergMetadataJsonFiles(metadataPath) == count) { - break; - } - Thread.sleep(5000); - } - } - - @SneakyThrows - private static long numIcebergMetadataJsonFiles(Path path) { - long count = 0; - if (Files.exists(path)) { - count = Files.list(path).filter(p -> p.toString().endsWith("metadata.json")).count(); - } - return count; + private void validateTargetMetadataIsPresent(String basePath) { + Path icebergMetadataPath = Paths.get(URI.create(basePath + "/metadata")); + long icebergMetadataFiles = + Files.list(icebergMetadataPath).filter(p -> p.toString().endsWith("metadata.json")).count(); + Assertions.assertEquals(2, icebergMetadataFiles); + Path deltaMetadataPath = Paths.get(URI.create(basePath + "/_delta_log")); + long deltaMetadataFiles = + Files.list(deltaMetadataPath).filter(p -> p.toString().endsWith(".json")).count(); + Assertions.assertEquals(1, deltaMetadataFiles); } }