Rebase with HierarchicalTableIdentifier changes

apache · Dec 24, 2024 · 893b1a6 · 893b1a6
1 parent c2bf7bb
commit 893b1a6
Show file tree

Hide file tree

Showing 5 changed files with 41 additions and 32 deletions.
diff --git a/xtable-core/src/test/java/org/apache/xtable/catalog/TestCatalogConversionFactory.java b/xtable-core/src/test/java/org/apache/xtable/catalog/TestCatalogConversionFactory.java
@@ -27,7 +27,7 @@
 
 import org.apache.xtable.conversion.ExternalCatalogConfig;
 import org.apache.xtable.conversion.TargetCatalogConfig;
-import org.apache.xtable.model.catalog.CatalogTableIdentifier;
+import org.apache.xtable.model.catalog.HierarchicalTableIdentifier;
 import org.apache.xtable.spi.extractor.CatalogConversionSource;
 import org.apache.xtable.spi.sync.CatalogSyncClient;
 import org.apache.xtable.testutil.ITTestUtils.TestCatalogConversionSourceImpl;
@@ -61,10 +61,7 @@ void createForCatalog() {
                     .catalogProperties(Collections.emptyMap())
                     .build())
             .catalogTableIdentifier(
-                CatalogTableIdentifier.builder()
-                    .databaseName("target-database")
-                    .tableName("target-tableName")
-                    .build())
+                new HierarchicalTableIdentifier("target-database", "target-tableName"))
             .build();
     CatalogSyncClient catalogSyncClient =
         CatalogConversionFactory.getInstance()

diff --git a/xtable-core/src/test/java/org/apache/xtable/conversion/TestConversionController.java b/xtable-core/src/test/java/org/apache/xtable/conversion/TestConversionController.java
@@ -57,7 +57,7 @@
 import org.apache.xtable.model.InternalSnapshot;
 import org.apache.xtable.model.InternalTable;
 import org.apache.xtable.model.TableChange;
-import org.apache.xtable.model.catalog.CatalogTableIdentifier;
+import org.apache.xtable.model.catalog.HierarchicalTableIdentifier;
 import org.apache.xtable.model.metadata.TableSyncMetadata;
 import org.apache.xtable.model.storage.TableFormat;
 import org.apache.xtable.model.sync.SyncMode;
@@ -594,10 +594,8 @@ private TargetCatalogConfig getTargetCatalog(String suffix) {
                 .catalogProperties(Collections.emptyMap())
                 .build())
         .catalogTableIdentifier(
-            CatalogTableIdentifier.builder()
-                .databaseName("target-database" + suffix)
-                .tableName("target-tableName" + suffix)
-                .build())
+            new HierarchicalTableIdentifier(
+                "target-database" + suffix, "target-tableName" + suffix))
         .build();
   }
 

diff --git a/xtable-core/src/test/java/org/apache/xtable/testutil/ITTestUtils.java b/xtable-core/src/test/java/org/apache/xtable/testutil/ITTestUtils.java
@@ -67,12 +67,12 @@ public String getStorageLocation(Object o) {
     }
 
     @Override
-    public boolean hasDatabase(String databaseName) {
+    public boolean hasDatabase(CatalogTableIdentifier tableIdentifier) {
       return false;
     }
 
     @Override
-    public void createDatabase(String databaseName) {}
+    public void createDatabase(CatalogTableIdentifier tableIdentifier) {}
 
     @Override
     public Object getTable(CatalogTableIdentifier tableIdentifier) {

diff --git a/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunCatalogSync.java b/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunCatalogSync.java
@@ -57,10 +57,12 @@
 import org.apache.xtable.conversion.TargetCatalogConfig;
 import org.apache.xtable.conversion.TargetTable;
 import org.apache.xtable.model.catalog.CatalogTableIdentifier;
+import org.apache.xtable.model.catalog.HierarchicalTableIdentifier;
 import org.apache.xtable.model.sync.SyncMode;
 import org.apache.xtable.reflection.ReflectionUtils;
 import org.apache.xtable.spi.extractor.CatalogConversionSource;
 import org.apache.xtable.utilities.RunCatalogSync.DatasetConfig.StorageIdentifier;
+import org.apache.xtable.utilities.RunCatalogSync.DatasetConfig.TableIdentifier;
 import org.apache.xtable.utilities.RunCatalogSync.DatasetConfig.TargetTableIdentifier;
 
 /**
@@ -156,7 +158,8 @@ public static void main(String[] args) throws Exception {
       } else {
         sourceTable =
             catalogConversionSource.getSourceTable(
-                dataset.getSourceCatalogTableIdentifier().getCatalogTableIdentifier());
+                getCatalogTableIdentifier(
+                    dataset.getSourceCatalogTableIdentifier().getTableIdentifier()));
       }
       List<TargetTable> targetTables = new ArrayList<>();
       Map<TargetTable, List<TargetCatalogConfig>> targetCatalogs = new HashMap<>();
@@ -178,7 +181,8 @@ public static void main(String[] args) throws Exception {
             .add(
                 TargetCatalogConfig.builder()
                     .catalogTableIdentifier(
-                        targetCatalogTableIdentifier.getCatalogTableIdentifier())
+                        getCatalogTableIdentifier(
+                            targetCatalogTableIdentifier.getTableIdentifier()))
                     .catalogConfig(catalogsById.get(targetCatalogTableIdentifier.getCatalogId()))
                     .build());
       }
@@ -229,6 +233,13 @@ static Map<String, ConversionSourceProvider> getConversionSourceProviders(
     return CONVERSION_SOURCE_PROVIDERS;
   }
 
+  static CatalogTableIdentifier getCatalogTableIdentifier(TableIdentifier tableIdentifier) {
+    if (tableIdentifier.getHierarchicalId() != null) {
+      return new HierarchicalTableIdentifier(tableIdentifier.getHierarchicalId());
+    }
+    throw new IllegalArgumentException("Invalid tableIdentifier configuration provided");
+  }
+
   @Data
   public static class DatasetConfig {
     /**
@@ -258,7 +269,7 @@ public static class Dataset {
     @Data
     public static class SourceTableIdentifier {
       /** Specifies the table identifier in the source catalog. */
-      CatalogTableIdentifier catalogTableIdentifier;
+      TableIdentifier tableIdentifier;
       /**
        * (Optional) Provides direct storage details such as a table’s base path (like an S3
        * location) and the partition specification. This allows reading from a source even if it is
@@ -280,7 +291,16 @@ public static class TargetTableIdentifier {
        */
       String tableFormat;
       /** Specifies the table identifier in the target catalog. */
-      CatalogTableIdentifier catalogTableIdentifier;
+      TableIdentifier tableIdentifier;
+    }
+
+    @Data
+    public static class TableIdentifier {
+      /**
+       * Specifics the three level hierarchical table identifier for {@link
+       * HierarchicalTableIdentifier}
+       */
+      String hierarchicalId;
     }
 
     /**

diff --git a/xtable-utilities/src/test/resources/catalogConfig.yaml b/xtable-utilities/src/test/resources/catalogConfig.yaml
@@ -43,20 +43,17 @@ targetCatalogs:
       key33: "value3"
 datasets:
   - sourceCatalogTableIdentifier:
-      catalogTableIdentifier:
-        databaseName: "source-database-1"
-        tableName: "source-1"
+      tableIdentifier:
+        hierarchicalId: "source-database-1.source-1"
     targetCatalogTableIdentifiers:
       - catalogId: "target-1"
         tableFormat: "DELTA"
-        catalogTableIdentifier:
-          databaseName: "target-database-1"
-          tableName: "target-tableName-1"
+        tableIdentifier:
+          hierarchicalId: "target-database-1.target-tableName-1"
       - catalogId: "target-2"
         tableFormat: "HUDI"
-        catalogTableIdentifier:
-          databaseName: "target-database-2"
-          tableName: "target-tableName-2-delta"
+        tableIdentifier:
+          hierarchicalId: "target-database-2.target-tableName-2-delta"
   - sourceCatalogTableIdentifier:
       storageIdentifier:
         tableBasePath: s3://tpc-ds-datasets/1GB/hudi/catalog_sales
@@ -66,12 +63,9 @@ datasets:
     targetCatalogTableIdentifiers:
       - catalogId: "target-2"
         tableFormat: "ICEBERG"
-        catalogTableIdentifier:
-          databaseName: "target-database-2"
-          tableName: "target-tableName-2"
+        tableIdentifier:
+          hierarchicalId: "target-database-2.target-tableName-2"
       - catalogId: "target-3"
         tableFormat: "HUDI"
-        catalogTableIdentifier:
-          catalogName: "default-catalog-2"
-          databaseName: "target-database-3"
-          tableName: "target-tableName-3"
+        tableIdentifier:
+          hierarchicalId: "default-catalog-2.target-database-3.target-tableName-3"