diff --git a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/model/BaseRelEntityInfo.java b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/model/BaseRelEntityInfo.java new file mode 100644 index 000000000..319065397 --- /dev/null +++ b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/model/BaseRelEntityInfo.java @@ -0,0 +1,32 @@ +package com.netflix.metacat.common.server.model; + +import lombok.Data; + +/** + * ChildInfo. + */ +@Data +public abstract class BaseRelEntityInfo { + private String name; + private String relationType; + private String uuid; + + /** + Empty Constructor. + */ + public BaseRelEntityInfo() { + + } + + /** + Constructor with all params. + @param name name of the entity + @param relationType type of the relation + @param uuid uuid of the entity + */ + public BaseRelEntityInfo(final String name, final String relationType, final String uuid) { + this.name = name; + this.relationType = relationType; + this.uuid = uuid; + } +} diff --git a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/model/ChildInfo.java b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/model/ChildInfo.java new file mode 100644 index 000000000..7deb0945d --- /dev/null +++ b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/model/ChildInfo.java @@ -0,0 +1,23 @@ +package com.netflix.metacat.common.server.model; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.EqualsAndHashCode; + +/** + * ChildInfo. + */ +@EqualsAndHashCode(callSuper = true) +@AllArgsConstructor +@Data +public class ChildInfo extends BaseRelEntityInfo { + /** + Constructor with all params. + @param name name of the entity + @param relationType type of the relation + @param uuid uuid of the entity + */ + public ChildInfo(final String name, final String relationType, final String uuid) { + super(name, relationType, uuid); + } +} diff --git a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/model/ParentInfo.java b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/model/ParentInfo.java new file mode 100644 index 000000000..ff0cca8b4 --- /dev/null +++ b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/model/ParentInfo.java @@ -0,0 +1,29 @@ +package com.netflix.metacat.common.server.model; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +/** + * ParentInfo. + */ +@EqualsAndHashCode(callSuper = true) +@Data +public class ParentInfo extends BaseRelEntityInfo { + + /** + Empty Constructor. + */ + public ParentInfo() { + + } + + /** + Constructor with all params. + @param name name of the entity + @param relationType type of the relation + @param uuid uuid of the entity + */ + public ParentInfo(final String name, final String relationType, final String uuid) { + super(name, relationType, uuid); + } +} diff --git a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/usermetadata/DefaultTableUUIDProvider.java b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/usermetadata/DefaultTableUUIDProvider.java new file mode 100644 index 000000000..1598ef6c2 --- /dev/null +++ b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/usermetadata/DefaultTableUUIDProvider.java @@ -0,0 +1,17 @@ +package com.netflix.metacat.common.server.usermetadata; + +import com.netflix.metacat.common.dto.TableDto; + +import java.util.Optional; + +/** + * Default Table UUID Provider. + * + * @author yingjianw + */ +public class DefaultTableUUIDProvider implements TableUUIDProvider { + @Override + public Optional getUUID(final TableDto tableDto) { + return Optional.empty(); + } +} diff --git a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/usermetadata/ParentChildRelMetadataConstants.java b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/usermetadata/ParentChildRelMetadataConstants.java new file mode 100644 index 000000000..bb1304ccc --- /dev/null +++ b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/usermetadata/ParentChildRelMetadataConstants.java @@ -0,0 +1,46 @@ +package com.netflix.metacat.common.server.usermetadata; + +/** + * ParentChildRelMetadataConstants. + * + * @author yingjianw + */ +public final class ParentChildRelMetadataConstants { + /** + * Key specified in DefinitionMetadata that indicate the parent table name. + */ + public static final String PARENTNAME = "root_table_name"; + /** + * During create, the key specified in DefinitionMetadata that indicates the parent table uuid. + */ + public static final String PARENTUUID = "root_table_uuid"; + /** + * During create, the key specified in DefinitionMetadata that indicates the child table uuid. + */ + public static final String CHILDUUID = "child_table_uuid"; + + /** + * During create, the key specified in DefinitionMetadata that indicates relationType. + */ + public static final String RELATIONTYPE = "relationType"; + + /** + * During get, the key specified in DefinitionMetadata that indicates the parent child infos. + */ + public static final String PARENTCHILDRELINFO = "parentChildRelationInfo"; + + /** + * During get, the key specified in DefinitionMetadata[PARENTCHILDRELINFO] that indicates parent infos. + */ + public static final String PARENTINFOS = "parentInfos"; + + /** + * During get, the key specified in DefinitionMetadata[PARENTCHILDRELINFO] that indicates child infos. + */ + public static final String CHILDINFOS = "childInfos"; + + private ParentChildRelMetadataConstants() { + + } + +} diff --git a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/usermetadata/ParentChildRelMetadataService.java b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/usermetadata/ParentChildRelMetadataService.java new file mode 100644 index 000000000..e9b60d13e --- /dev/null +++ b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/usermetadata/ParentChildRelMetadataService.java @@ -0,0 +1,110 @@ +package com.netflix.metacat.common.server.usermetadata; +import com.netflix.metacat.common.QualifiedName; +import com.netflix.metacat.common.server.model.ChildInfo; +import com.netflix.metacat.common.server.model.ParentInfo; + +import java.util.Optional; +import java.util.Set; + +/** + * Parent-Child Relationship Metadata Service API. + * + * @author yingjianw + */ +public interface ParentChildRelMetadataService { + /** + * Establishes a parent-child relationship with a specified relation type. + * Currently, exceptions are thrown in the following cases: + * 1. Attempting to create a child table as the parent of another child table. + * 2. A child table having more than one parent. + * + * @param parentName the name of the parent entity + * @param parentUUID the uuid of the parent + * @param childName the name of the child entity + * @param childUUID the uuid of the child + * @param relationType the type of the relationship + */ + void createParentChildRelation( + QualifiedName parentName, + String parentUUID, + QualifiedName childName, + String childUUID, + String relationType + ); + + /** + * Deletes a parent-child relationship with a specified relation type. + * + * @param parentName the name of the parent entity + * @param parentUUID the uuid of the parent + * @param childName the name of the child entity + * @param childUUID the uuid of the child + * @param type the type of the relationship + */ + void deleteParentChildRelation( + QualifiedName parentName, + String parentUUID, + QualifiedName childName, + String childUUID, + String type + ); + + /** + * Renames `oldName` to `newName` in the parentChildRelationship store. + * This involves two steps: + * 1. Rename all records where the child is `oldName` to `newName` + * and if the uuid is present also include it in the sql search string. + * 2. Rename all records where the parent is `oldName` to `newName` + * and if the uuid is present also include it in the sql search string. + * For now, since a child cannot be a parent of another entity, only one of these actions will need to be performed. + * + * @param oldName the current name to be renamed + * @param newName the new name to rename to + * @param uuid the uuid of the table + */ + void rename( + QualifiedName oldName, + QualifiedName newName, + Optional uuid + ); + + /** + * Removes the entity from the parentChildRelationship store. + * An exception is thrown if an attempt is made to drop a parent table that still has existing child tables. + * Note that only dropping a child table will result in the physical deletion of records. + * When all children of a parent table are dropped, no records remain. + * Consequently, at then time when dropping the parent table, there should not have any database records + * with a parent reference to the dropping parent table. + * + * @param name the name of the entity to drop + * @param uuid the uuid of the entity + */ + void drop( + QualifiedName name, + Optional uuid + ); + + /** + * get the set of parent for the input name. + * + * @param name name + * @param uuid the uuid of the entity + * @return parentInfo + */ + Set getParents( + QualifiedName name, + Optional uuid + ); + + /** + * get the set of children for the input name. + * + * @param name name + * @param uuid the uuid of the entity + * @return a set of ChildInfo + */ + Set getChildren( + QualifiedName name, + Optional uuid + ); +} diff --git a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/usermetadata/TableUUIDProvider.java b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/usermetadata/TableUUIDProvider.java new file mode 100644 index 000000000..d7c9de63e --- /dev/null +++ b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/usermetadata/TableUUIDProvider.java @@ -0,0 +1,18 @@ +package com.netflix.metacat.common.server.usermetadata; + +import com.netflix.metacat.common.dto.TableDto; + +import java.util.Optional; + +/** + * Interface for TableUUIDPorvider. + * @author yingjianw + */ +public interface TableUUIDProvider { + /** + * Given a tableDto, get the corresponding table uuid if applicable. + * @param tableDto dto for table + * @return the uuid of the table + **/ + Optional getUUID(TableDto tableDto); +} diff --git a/metacat-functional-tests/metacat-test-cluster/datastores/mysql/docker-entrypoint-initdb.d/metacat.sql b/metacat-functional-tests/metacat-test-cluster/datastores/mysql/docker-entrypoint-initdb.d/metacat.sql index 189700dc6..21316cb68 100644 --- a/metacat-functional-tests/metacat-test-cluster/datastores/mysql/docker-entrypoint-initdb.d/metacat.sql +++ b/metacat-functional-tests/metacat-test-cluster/datastores/mysql/docker-entrypoint-initdb.d/metacat.sql @@ -44,6 +44,21 @@ CREATE TABLE data_metadata_delete ( KEY date_created (date_created) ) DEFAULT CHARSET=latin1; +-- +-- Table structure for table `parent_child_relation` +-- +DROP TABLE IF EXISTS `parent_child_relation`; +CREATE TABLE `parent_child_relation` ( + `parent` varchar(255) NOT NULL, + `parent_uuid` varchar(255) NOT NULL, + `child` varchar(255) NOT NULL, + `child_uuid` varchar(255) NOT NULL, + `relation_type` varchar(255) NOT NULL, + PRIMARY KEY (`parent`, `child`, `parent_uuid`, `child_uuid`, `relation_type`), + INDEX `idx_child` (`child`) +) ENGINE=InnoDB AUTO_INCREMENT=10078235 DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + -- -- Table structure for table `definition_metadata` -- diff --git a/metacat-functional-tests/src/functionalTest/groovy/com/netflix/metacat/MetacatSmokeSpec.groovy b/metacat-functional-tests/src/functionalTest/groovy/com/netflix/metacat/MetacatSmokeSpec.groovy index 697d8d647..353936926 100644 --- a/metacat-functional-tests/src/functionalTest/groovy/com/netflix/metacat/MetacatSmokeSpec.groovy +++ b/metacat-functional-tests/src/functionalTest/groovy/com/netflix/metacat/MetacatSmokeSpec.groovy @@ -1962,4 +1962,313 @@ class MetacatSmokeSpec extends Specification { api.deleteTable(catalogName, databaseName, "table2") api.deleteTable(catalogName, databaseName, "table3") } + + def 'testCloneTableE2E'() { + given: + def catalogName = 'embedded-fast-hive-metastore' + def databaseName = 'iceberg_db' + + // First parent child connected component + def parent1 = "parent1" + def parent1UUID = "p1_uuid" + def renameParent1 = "rename_parent1" + def parent1Uri = isLocalEnv ? String.format('file:/tmp/%s/%s', databaseName, parent1) : null + def parent1FullName = catalogName + "." + databaseName + "." + parent1 + + def child11 = "child11" + def child11UUID = "c11_uuid" + def renameChild11 = "rename_child11" + def child11Uri = isLocalEnv ? String.format('file:/tmp/%s/%s', databaseName, child11) : null + def child11FullName = catalogName + "." + databaseName + "." + child11 + + def child12 = "child12" + def child12UUID = "c12_uuid" + def child12Uri = isLocalEnv ? String.format('file:/tmp/%s/%s', databaseName, child12) : null + + def grandChild121 = "grandchild121" + def grandChild121UUID= "gc121_uuid" + + + // Second parent child connected component + def parent2 = "parent2" + def parent2UUID = "p2_uuid" + def parent2Uri = isLocalEnv ? String.format('file:/tmp/%s/%s', databaseName, parent2) : null + def parent2FullName = catalogName + "." + databaseName + "." + parent2 + def child21 = "child21" + def child21UUID = "c21_uuid" + def child21Uri = isLocalEnv ? String.format('file:/tmp/%s/%s', databaseName, child21) : null + + try { + api.createDatabase(catalogName, databaseName, new DatabaseCreateRequestDto()) + } catch (Exception ignored) { + } + + /* + Step 1: Create one Parent (parent1) and one Child (child11) + */ + when: + // Create Parent1 + def parent1TableDto = PigDataDtoProvider.getTable(catalogName, databaseName, parent1, 'amajumdar', parent1Uri) + api.createTable(catalogName, databaseName, parent1, parent1TableDto) + + // Create child11 Table with parent = parent1 + def child11TableDto = PigDataDtoProvider.getTable(catalogName, databaseName, child11, 'amajumdar', child11Uri) + child11TableDto.definitionMetadata.put("root_table_name", parent1FullName) + child11TableDto.definitionMetadata.put("root_table_uuid", parent1UUID) + child11TableDto.definitionMetadata.put("child_table_uuid", child11UUID) + api.createTable(catalogName, databaseName, child11, child11TableDto) + + def parent1Table = api.getTable(catalogName, databaseName, parent1, true, true, false) + def parent1ParentChildRelationInfo = parent1Table.definitionMetadata.get("parentChildRelationInfo") + def child11Table = api.getTable(catalogName, databaseName, child11, true, true, false) + def child11ParentChildRelationInfo = child11Table.definitionMetadata.get("parentChildRelationInfo") + then: + // Test Parent 1 parentChildInfo + assert !parent1ParentChildRelationInfo.has("parentInfo") + JSONAssert.assertEquals(parent1ParentChildRelationInfo.toString(), + '{"childInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/child11","relationType":"CLONE", "uuid":"c11_uuid"}]}', + false) + + // Test Child11 parentChildInfo + JSONAssert.assertEquals(child11ParentChildRelationInfo.toString(), + '{"parentInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/parent1","relationType":"CLONE", "uuid":"p1_uuid"}]}', + false) + assert !child11ParentChildRelationInfo.has("childInfos") + + /* + Step 2: Create a second child (child12) pointing to parent = parent1 + */ + when: + // Create Child2 Table + def child12TableDto = PigDataDtoProvider.getTable(catalogName, databaseName, child12, 'amajumdar', child12Uri) + child12TableDto.definitionMetadata.put("root_table_name", parent1FullName) + child12TableDto.definitionMetadata.put("root_table_uuid", parent1UUID) + child12TableDto.definitionMetadata.put("child_table_uuid", child12UUID) + api.createTable(catalogName, databaseName, child12, child12TableDto) + parent1Table = api.getTable(catalogName, databaseName, parent1, true, true, false) + parent1ParentChildRelationInfo = parent1Table.definitionMetadata.get("parentChildRelationInfo") + def child12Table = api.getTable(catalogName, databaseName, child12, true, true, false) + def child12ParentChildRelationInfo = child12Table.definitionMetadata.get("parentChildRelationInfo") + + then: + // Test Parent 1 parentChildInfo + assert !parent1ParentChildRelationInfo.has("parentInfo") + JSONAssert.assertEquals(parent1ParentChildRelationInfo.toString(), + '{"childInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/child11","relationType":"CLONE","uuid":"c11_uuid"}, {"name":"embedded-fast-hive-metastore/iceberg_db/child12","relationType":"CLONE","uuid":"c12_uuid"}]}', + false) + + // Test Child12 parentChildInfo + JSONAssert.assertEquals(child12ParentChildRelationInfo.toString(), + '{"parentInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/parent1","relationType":"CLONE","uuid":"p1_uuid"}]}', + false) + assert !child12ParentChildRelationInfo.has("childInfos") + + /* + Step 3: Create one grandChild As a Parent of A child table should fail + */ + when: + def grandchild121TableDto = PigDataDtoProvider.getTable(catalogName, databaseName, grandChild121, 'amajumdar', null) + grandchild121TableDto.definitionMetadata.put("root_table_name", child11FullName) + grandchild121TableDto.definitionMetadata.put("root_table_uuid", child11UUID) + grandchild121TableDto.definitionMetadata.put("child_table_uuid", grandChild121UUID) + api.createTable(catalogName, databaseName, grandChild121, grandchild121TableDto) + + then: + thrown(Exception) + + /* + Step 4: Create another parent child that is disconnected with the above + */ + when: + // Create Parent2 + def parent2TableDto = PigDataDtoProvider.getTable(catalogName, databaseName, parent2, 'amajumdar', parent2Uri) + api.createTable(catalogName, databaseName, parent2, parent2TableDto) + + // Create child21 Table with parent = parent2 + def child21TableDto = PigDataDtoProvider.getTable(catalogName, databaseName, child21, 'amajumdar', child21Uri) + child21TableDto.definitionMetadata.put("root_table_name", parent2FullName) + child21TableDto.definitionMetadata.put("root_table_uuid", parent2UUID) + child21TableDto.definitionMetadata.put("child_table_uuid", child21UUID) + api.createTable(catalogName, databaseName, child21, child21TableDto) + def parent2Table = api.getTable(catalogName, databaseName, parent2, true, true, false) + def parent2ParentChildRelationInfo = parent2Table.definitionMetadata.get("parentChildRelationInfo") + def child21Table = api.getTable(catalogName, databaseName, child21, true, true, false) + def child21ParentChildRelationInfo = child21Table.definitionMetadata.get("parentChildRelationInfo") + + then: + // Test Parent 2 parentChildInfo + assert !parent2ParentChildRelationInfo.has("parentInfo") + JSONAssert.assertEquals(parent2ParentChildRelationInfo.toString(), + '{"childInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/child21","relationType":"CLONE","uuid":"c21_uuid"}]}', + false) + + // Test Child21 parentChildInfo + JSONAssert.assertEquals(child21ParentChildRelationInfo.toString(), + '{"parentInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/parent2","relationType":"CLONE","uuid":"p2_uuid"}]}', + false) + assert !child21ParentChildRelationInfo.has("childInfos") + + /* + Step 5: Rename parent1 to newParent1 + */ + when: + api.renameTable(catalogName, databaseName, parent1, renameParent1) + parent1Table = api.getTable(catalogName, databaseName, renameParent1, true, true, false) + parent1ParentChildRelationInfo = parent1Table.definitionMetadata.get("parentChildRelationInfo") + child11Table = api.getTable(catalogName, databaseName, child11, true, true, false) + child11ParentChildRelationInfo = child11Table.definitionMetadata.get("parentChildRelationInfo") + child12Table = api.getTable(catalogName, databaseName, child12, true, true, false) + child12ParentChildRelationInfo = child12Table.definitionMetadata.get("parentChildRelationInfo") + + then: + // Test Parent 1 parentChildInfo newName + assert !parent1ParentChildRelationInfo.has("parentInfo") + JSONAssert.assertEquals(parent1ParentChildRelationInfo.toString(), + '{"childInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/child11","relationType":"CLONE","uuid":"c11_uuid"}, {"name":"embedded-fast-hive-metastore/iceberg_db/child12","relationType":"CLONE","uuid":"c12_uuid"}]}', + false) + + // Test Child11 parentChildInfo + JSONAssert.assertEquals(child11ParentChildRelationInfo.toString(), + '{"parentInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/rename_parent1","relationType":"CLONE","uuid":"p1_uuid"}]}', + false) + assert !child11ParentChildRelationInfo.has("childInfos") + + // Test Child12 parentChildInfo + JSONAssert.assertEquals(child12ParentChildRelationInfo.toString(), + '{"parentInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/rename_parent1","relationType":"CLONE","uuid":"p1_uuid"}]}', + false) + assert !child12ParentChildRelationInfo.has("childInfos") + + /* + Step 6: Rename child11 to renameChild11 + */ + when: + api.renameTable(catalogName, databaseName, child11, renameChild11) + parent1Table = api.getTable(catalogName, databaseName, renameParent1, true, true, false) + parent1ParentChildRelationInfo = parent1Table.definitionMetadata.get("parentChildRelationInfo") + child11Table = api.getTable(catalogName, databaseName, renameChild11, true, true, false) + child11ParentChildRelationInfo = child11Table.definitionMetadata.get("parentChildRelationInfo") + then: + assert !parent1ParentChildRelationInfo.has("parentInfo") + JSONAssert.assertEquals(parent1ParentChildRelationInfo.toString(), + '{"childInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/rename_child11","relationType":"CLONE","uuid":"c11_uuid"}, {"name":"embedded-fast-hive-metastore/iceberg_db/child12","relationType":"CLONE","uuid":"c12_uuid"}]}', + false) + + // Test Child11 parentChildInfo with newName + JSONAssert.assertEquals(child11ParentChildRelationInfo.toString(), + '{"parentInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/rename_parent1","relationType":"CLONE","uuid":"p1_uuid"}]}', + false) + assert !child12ParentChildRelationInfo.has("childInfos") + + /* + Step 7: Drop parent renameParent1 + */ + when: + api.deleteTable(catalogName, databaseName, renameParent1) + + then: + def e = thrown(Exception) + e.message.contains("because it still has child table") + + /* + Step 8: Drop renameChild11 + */ + when: + api.deleteTable(catalogName, databaseName, renameChild11) + parent1Table = api.getTable(catalogName, databaseName, renameParent1, true, true, false) + parent1ParentChildRelationInfo = parent1Table.definitionMetadata.get("parentChildRelationInfo") + + then: + // Test parent1 Table + assert !parent1ParentChildRelationInfo.has("parentInfo") + JSONAssert.assertEquals(parent1ParentChildRelationInfo.toString(), + '{"childInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/child12","relationType":"CLONE","uuid":"c12_uuid"}]}', + false) + + /* + Step 9: Drop child12 + */ + when: + api.deleteTable(catalogName, databaseName, child12) + parent1Table = api.getTable(catalogName, databaseName, renameParent1, true, true, false) + then: + assert !parent1Table.definitionMetadata.has("parentChildRelationInfo") + + /* + Step 10: Drop renameParent1 + */ + when: + api.deleteTable(catalogName, databaseName, renameParent1) + parent2Table = api.getTable(catalogName, databaseName, parent2, true, true, false) + parent2ParentChildRelationInfo = parent2Table.definitionMetadata.get("parentChildRelationInfo") + child21Table = api.getTable(catalogName, databaseName, child21, true, true, false) + child21ParentChildRelationInfo = child21Table.definitionMetadata.get("parentChildRelationInfo") + + then: + // Since all the operations above are on the first connected relationship, the second connected relationship + // should remain the same + // Test Parent 2 parentChildInfo + assert !parent2ParentChildRelationInfo.has("parentInfo") + JSONAssert.assertEquals(parent2ParentChildRelationInfo.toString(), + '{"childInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/child21","relationType":"CLONE","uuid":"c21_uuid"}]}', + false) + + // Test Child21 parentChildInfo + JSONAssert.assertEquals(child21ParentChildRelationInfo.toString(), + '{"parentInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/parent2","relationType":"CLONE","uuid":"p2_uuid"}]}', + false) + assert !child21ParentChildRelationInfo.has("childInfos") + + /* + Step 11: update parent2 with random parentChildRelationInfo + */ + when: + def updateParent2Dto = parent2Table + updateParent2Dto.definitionMetadata.put("parent", "CLONE") + updateParent2Dto.definitionMetadata.put("childInfos", "CLONE") + api.updateTable(catalogName, databaseName, parent2, updateParent2Dto) + + parent2Table = api.getTable(catalogName, databaseName, parent2, true, true, false) + parent2ParentChildRelationInfo = parent2Table.definitionMetadata.get("parentChildRelationInfo") + child21Table = api.getTable(catalogName, databaseName, child21, true, true, false) + child21ParentChildRelationInfo = child21Table.definitionMetadata.get("parentChildRelationInfo") + then: + // Test Parent 2 parentChildInfo + assert !parent2ParentChildRelationInfo.has("parentInfo") + JSONAssert.assertEquals(parent2ParentChildRelationInfo.toString(), + '{"childInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/child21","relationType":"CLONE","uuid":"c21_uuid"}]}', + false) + + // Test Child21 parentChildInfo + JSONAssert.assertEquals(child21ParentChildRelationInfo.toString(), + '{"parentInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/parent2","relationType":"CLONE","uuid":"p2_uuid"}]}', + false) + assert !child21ParentChildRelationInfo.has("childInfos") + + /* + Step 12: update child21 with random parentChildRelationInfo + */ + when: + def updateChild21Dto = child21Table + updateChild21Dto.definitionMetadata.put("parent", "CLONE") + updateChild21Dto.definitionMetadata.put("childInfos", "CLONE") + api.updateTable(catalogName, databaseName, child21, updateChild21Dto) + + parent2Table = api.getTable(catalogName, databaseName, parent2, true, true, false) + parent2ParentChildRelationInfo = parent2Table.definitionMetadata.get("parentChildRelationInfo") + child21Table = api.getTable(catalogName, databaseName, child21, true, true, false) + child21ParentChildRelationInfo = child21Table.definitionMetadata.get("parentChildRelationInfo") + then: + // Test Parent 2 parentChildInfo + assert !parent2ParentChildRelationInfo.has("parentInfo") + JSONAssert.assertEquals(parent2ParentChildRelationInfo.toString(), + '{"childInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/child21","relationType":"CLONE","uuid":"c21_uuid"}]}', + false) + + // Test Child21 parentChildInfo + JSONAssert.assertEquals(child21ParentChildRelationInfo.toString(), + '{"parentInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/parent2","relationType":"CLONE","uuid":"p2_uuid"}]}', + false) + assert !child21ParentChildRelationInfo.has("childInfos") + } } diff --git a/metacat-functional-tests/src/functionalTest/groovy/com/netflix/metacat/ParentChildRelMetadataServiceSpec.groovy b/metacat-functional-tests/src/functionalTest/groovy/com/netflix/metacat/ParentChildRelMetadataServiceSpec.groovy new file mode 100644 index 000000000..7f7994920 --- /dev/null +++ b/metacat-functional-tests/src/functionalTest/groovy/com/netflix/metacat/ParentChildRelMetadataServiceSpec.groovy @@ -0,0 +1,744 @@ +package com.netflix.metacat + +import com.netflix.metacat.common.QualifiedName +import com.netflix.metacat.common.server.model.ChildInfo +import com.netflix.metacat.common.server.model.ParentInfo +import com.netflix.metacat.common.server.usermetadata.ParentChildRelMetadataService +import com.netflix.metacat.metadata.mysql.MySqlParentChildRelMetaDataService +import org.springframework.jdbc.core.JdbcTemplate +import org.springframework.jdbc.datasource.DriverManagerDataSource +import spock.lang.Shared +import spock.lang.Specification + +import java.sql.Connection +import java.sql.PreparedStatement + +class ParentChildRelMetadataServiceSpec extends Specification{ + @Shared + private ParentChildRelMetadataService service; + @Shared + private JdbcTemplate jdbcTemplate; + + @Shared + private String catalog = "prodhive" + @Shared + private String database = "testpc" + + @Shared + private static final String SQL_CREATE_PARENT_CHILD_RELATIONS = + "INSERT INTO parent_child_relation (parent, parent_uuid, child, child_uuid, relation_type) " + + "VALUES (?, ?, ?, ?, ?)" + + def setupSpec() { + String jdbcUrl = "jdbc:mysql://localhost:3306/metacat" + String username = "metacat_user" + String password = "metacat_user_password" + + DriverManagerDataSource dataSource = new DriverManagerDataSource() + dataSource.setDriverClassName("com.mysql.cj.jdbc.Driver") + dataSource.setUrl(jdbcUrl) + dataSource.setUsername(username) + dataSource.setPassword(password) + + jdbcTemplate = new JdbcTemplate(dataSource) + service = new MySqlParentChildRelMetaDataService(jdbcTemplate) + } + + def cleanup() { + jdbcTemplate.update("DELETE FROM parent_child_relation") + } + + void createParentChildRelation(String parentName, String parentUUID, String childName, String childUUID, String type) { + jdbcTemplate.update { Connection connection -> + PreparedStatement ps = connection.prepareStatement(SQL_CREATE_PARENT_CHILD_RELATIONS) + ps.setString(1, parentName.toString()) + ps.setString(2, parentUUID) + ps.setString(3, childName.toString()) + ps.setString(4, childUUID) + ps.setString(5, type) + return ps + } + } + + def "Test CreateThenDelete - OneChildOneParent"() { + setup: + def parent = QualifiedName.ofTable(catalog, database, "p") + def child = QualifiedName.ofTable(catalog, database, "c") + def parentUUID = "p_uuid"; + def childUUID = "c_uuid"; + def type = "clone"; + def child_parent_expected = [new ParentInfo(parent.toString(), type, parentUUID)] as Set + def parent_children_expected = [new ChildInfo(child.toString(), type, childUUID)] as Set + + when: + service.createParentChildRelation(parent, parentUUID, child, childUUID, type) + + then: + // Test Parent + assert service.getParents(parent, Optional.of(parentUUID)).isEmpty() + assert service.getParents(parent, Optional.empty()).isEmpty() + + assert service.getChildren(parent, Optional.of(parentUUID)) == parent_children_expected + assert service.getChildren(parent, Optional.empty()) == parent_children_expected + + // Test Child + assert service.getParents(child, Optional.of(childUUID)) == child_parent_expected + assert service.getParents(child, Optional.empty()) == child_parent_expected + + assert service.getChildren(child, Optional.of(childUUID)).isEmpty() + assert service.getParents(child, Optional.empty()) == child_parent_expected + + when: + service.deleteParentChildRelation(parent, parentUUID, child, childUUID, type) + + then: + // Test Parent + assert service.getParents(parent, Optional.of(parentUUID)).isEmpty() + assert service.getParents(parent, Optional.empty()).isEmpty() + assert service.getChildren(parent, Optional.of(parentUUID)).isEmpty() + assert service.getChildren(parent, Optional.empty()).isEmpty() + + // Test Child + assert service.getParents(child, Optional.of(childUUID)).isEmpty() + assert service.getParents(child, Optional.empty()).isEmpty() + assert service.getChildren(child, Optional.of(childUUID)).isEmpty() + assert service.getParents(child, Optional.empty()).isEmpty() + + } + + def "Test Create - oneParentMultiChildren"() { + setup: + def parent = QualifiedName.ofTable(catalog, database, "p") + def parentUUID = "p_uuid"; + def child1 = QualifiedName.ofTable(catalog, database, "c1") + def child1UUID = "c1_uuid"; + def child2 = QualifiedName.ofTable(catalog, database, "c2") + def child2UUID = "c2_uuid"; + def type = "clone"; + def parent_children_expected = [ + new ChildInfo(child1.toString(), type, child1UUID), + new ChildInfo(child2.toString(), type, child2UUID), + ] as Set + def child_parent_expected = [new ParentInfo(parent.toString(), type, parentUUID)] as Set + + when: + service.createParentChildRelation(parent, parentUUID, child1, child1UUID, type) + service.createParentChildRelation(parent, parentUUID, child2, child2UUID, type) + + then: + // Test Parent + assert service.getParents(parent, Optional.of(parentUUID)).isEmpty() + assert parent_children_expected == service.getChildren(parent, Optional.of(parentUUID)) + assert parent_children_expected == service.getChildren(parent, Optional.empty()) + + // Test Children + // Test Child 1 + assert child_parent_expected == service.getParents(child1, Optional.of(child1UUID)) + assert child_parent_expected == service.getParents(child1, Optional.empty()) + assert service.getChildren(child1, Optional.of(child1UUID)).isEmpty() + assert service.getChildren(child1, Optional.empty()).isEmpty() + + assert child_parent_expected == service.getParents(child2, Optional.of(child2UUID)) + assert child_parent_expected == service.getParents(child2, Optional.empty()) + assert service.getChildren(child2, Optional.of(child2UUID)).isEmpty() + assert service.getChildren(child2, Optional.empty()).isEmpty() + } + + def "Test Create - oneChildMultiParentException"() { + setup: + def parent1 = QualifiedName.ofTable(catalog, database, "p1") + def parent1UUID = "p1_uuid"; + def parent2 = QualifiedName.ofTable(catalog, database, "p2") + def parent2UUID = "p2_uuid"; + def child = QualifiedName.ofTable(catalog, database, "c") + def childUUID = "c_uuid" + def type = "clone" + service.createParentChildRelation(parent1, parent1UUID, child, childUUID, type) + + when: + service.createParentChildRelation(parent2, parent2UUID, child, childUUID, type) + + then: + def e = thrown(RuntimeException) + assert e.message.contains("Cannot have a child table having more than one parent") + + // Test Child + def child_parent_expected = [new ParentInfo(parent1.toString(), type, parent1UUID)] as Set + assert child_parent_expected == service.getParents(child, Optional.of(childUUID)) + assert child_parent_expected == service.getParents(child, Optional.empty()) + + assert service.getChildren(child, Optional.of(childUUID)).isEmpty() + assert service.getChildren(child, Optional.empty()).isEmpty() + } + + def "Test Create - oneChildAsParentOfAnotherException"() { + setup: + def parent = QualifiedName.ofTable(catalog, database, "p") + def parentUUID = "p_uuid" + def child = QualifiedName.ofTable(catalog, database, "c") + def childUUID = "c_uuid" + def grandChild = QualifiedName.ofTable(catalog, database, "gc") + def grandChildUUID = "gc_uuid" + def type = "clone" + service.createParentChildRelation(parent, parentUUID, child, childUUID, type) + + when: + service.createParentChildRelation(child, childUUID, grandChild, grandChildUUID, type) + + then: + def e = thrown(RuntimeException) + assert e.message.contains("Cannot create a child table as parent") + + // Test Child + def child_parent_expected = [new ParentInfo(parent.toString(), type, parentUUID)] as Set + assert service.getParents(child, Optional.of(childUUID)) == child_parent_expected + assert service.getParents(child, Optional.empty()) == child_parent_expected + + assert service.getChildren(child, Optional.of(childUUID)).isEmpty() + assert service.getChildren(child, Optional.empty()).isEmpty() + } + + def "Test Create - oneParentAsChildOfAnother"() { + setup: + def parent = QualifiedName.ofTable(catalog, database, "p") + def parentUUID = "p_uuid" + def child = QualifiedName.ofTable(catalog, database, "c") + def childUUID = "c_uuid" + def grandChild = QualifiedName.ofTable(catalog, database, "gc") + def grandChildUUID = "gc_uuid" + def type = "clone" + service.createParentChildRelation(child, childUUID, grandChild, grandChildUUID, type) + + when: + service.createParentChildRelation(parent, parentUUID, child, childUUID, type) + + then: + def e = thrown(RuntimeException) + assert e.message.contains("Cannot create a parent table on top of another parent") + } + + def "Test Rename Parent - One Child"() { + setup: + def parent = QualifiedName.ofTable(catalog, database, "p") + def parentUUID = "p_uuid" + def child = QualifiedName.ofTable(catalog, database, "c") + def childUUID = "c_uuid" + def type = "clone"; + def newParent = QualifiedName.ofTable(catalog, database, "np") + service.createParentChildRelation(parent, parentUUID, child, childUUID, type) + + when: + service.rename(parent, newParent, Optional.of(parentUUID)) + + then: + // Test Old Parent Name + assert service.getParents(parent, Optional.of(parentUUID)).isEmpty() + assert service.getParents(parent, Optional.empty()).isEmpty() + assert service.getChildren(parent, Optional.of(parentUUID)).isEmpty() + assert service.getChildren(parent, Optional.empty()).isEmpty() + + // Test New Parent Name + assert service.getParents(newParent, Optional.of(parentUUID)).isEmpty() + assert service.getParents(newParent, Optional.empty()).isEmpty() + def newParent_children_expected = [new ChildInfo(child.toString(), type, childUUID)] as Set + assert service.getChildren(newParent, Optional.of(parentUUID)) == newParent_children_expected + assert service.getChildren(newParent, Optional.empty()) == newParent_children_expected + + // Test Child + def child_parent_expected = [new ParentInfo(newParent.toString(), type, parentUUID)] as Set + assert child_parent_expected == service.getParents(child, Optional.of(childUUID)) + assert child_parent_expected == service.getParents(child, Optional.empty()) + assert service.getChildren(child, Optional.of(childUUID)).isEmpty() + assert service.getChildren(child, Optional.empty()).isEmpty() + + // rename back + when: + service.rename(newParent, parent, Optional.of(parentUUID)) + child_parent_expected = [new ParentInfo(parent.toString(), type, parentUUID)] as Set + + then: + // Test new Parent Name + assert service.getParents(newParent, Optional.of(parentUUID)).isEmpty() + assert service.getParents(newParent, Optional.empty()).isEmpty() + assert service.getChildren(newParent, Optional.of(parentUUID)).isEmpty() + assert service.getChildren(newParent, Optional.empty()).isEmpty() + + // Test old Parent Name + assert service.getParents(parent, Optional.of(parentUUID)).isEmpty() + assert service.getParents(parent, Optional.empty()).isEmpty() + assert service.getChildren(parent, Optional.of(parentUUID)) == newParent_children_expected + assert service.getChildren(parent, Optional.empty()) == [new ChildInfo(child.toString(), type, childUUID)] as Set + + // Test Child + assert child_parent_expected == service.getParents(child, Optional.of(childUUID)) + assert child_parent_expected == service.getParents(child, Optional.empty()) + assert service.getChildren(child, Optional.of(childUUID)).isEmpty() + assert service.getChildren(child, Optional.empty()).isEmpty() + } + + def "Test Rename Parent - Multi Child"() { + setup: + def parent = QualifiedName.ofTable(catalog, database, "p") + def parentUUID = "p_uuid" + def child1 = QualifiedName.ofTable(catalog, database, "c1") + def child1UUID = "c1_uuid" + def child2 = QualifiedName.ofTable(catalog, database, "c2") + def child2UUID = "c2_uuid" + def type = "clone"; + service.createParentChildRelation(parent, parentUUID, child1, child1UUID, type) + service.createParentChildRelation(parent, parentUUID, child2, child2UUID, type) + def newParent = QualifiedName.ofTable(catalog, database, "np") + def child_parent_expected = [new ParentInfo(newParent.toString(), type, parentUUID)] as Set + + when: + service.rename(parent, newParent, Optional.of(parentUUID)) + + then: + // Test Child1 + assert service.getParents(child1, Optional.of(child1UUID)) == child_parent_expected + assert service.getParents(child1, Optional.empty()) == child_parent_expected + //Test Child2 + assert service.getParents(child2, Optional.of(child2UUID)) == child_parent_expected + assert service.getParents(child2, Optional.empty()) == child_parent_expected + } + + def "Test Rename Child"() { + setup: + def parent = QualifiedName.ofTable(catalog, database, "p") + def parentUUID = "p_uuid" + def child = QualifiedName.ofTable(catalog, database, "c") + def childUUID = "c_uuid" + def type = "clone" + service.createParentChildRelation(parent, parentUUID, child, childUUID, type) + def newChild = QualifiedName.ofTable(catalog, database, "nc") + + when: + service.rename(child, newChild, Optional.of(childUUID)) + then: + // Test Parent + assert service.getParents(parent, Optional.of(parentUUID)).isEmpty() + def parent_children_expected = [new ChildInfo(newChild.toString(), type, childUUID)] as Set + assert parent_children_expected == service.getChildren(parent, Optional.of(parentUUID)) + assert parent_children_expected == service.getChildren(parent, Optional.empty()) + + // Test Child + assert service.getParents(child, Optional.of(childUUID)).isEmpty() + assert service.getParents(child, Optional.empty()).isEmpty() + assert service.getChildren(child, Optional.of(childUUID)).isEmpty() + assert service.getChildren(child, Optional.empty()).isEmpty() + + // Test New Child + def child_parent_expected = [new ParentInfo(parent.toString(), type, parentUUID)] as Set + assert child_parent_expected == service.getParents(newChild, Optional.of(childUUID)) + assert child_parent_expected == service.getParents(newChild, Optional.empty()) + assert service.getChildren(child, Optional.of(childUUID)).isEmpty() + assert service.getChildren(child, Optional.empty()).isEmpty() + + // rename back + when: + service.rename(newChild, child, Optional.of(childUUID)) + parent_children_expected = [new ChildInfo(child.toString(), type, childUUID)] as Set + + then: + // Test Parent + assert service.getParents(parent, Optional.of(parentUUID)).isEmpty() + assert parent_children_expected == service.getChildren(parent, Optional.of(parentUUID)) + assert parent_children_expected == service.getChildren(parent, Optional.empty()) + + // Test New Child + assert service.getParents(newChild, Optional.of(childUUID)).isEmpty() + assert service.getParents(newChild, Optional.empty()).isEmpty() + assert service.getChildren(newChild, Optional.of(childUUID)).isEmpty() + assert service.getChildren(newChild, Optional.empty()).isEmpty() + + // Test Child + assert child_parent_expected == service.getParents(child, Optional.of(childUUID)) + assert child_parent_expected == service.getParents(child, Optional.empty()) + assert service.getChildren(child, Optional.of(childUUID)).isEmpty() + assert service.getChildren(child, Optional.empty()).isEmpty() + } + + + def "Test Drop Child"() { + setup: + def parent = QualifiedName.ofTable(catalog, database, "p") + def parentUUID = "p_uuid" + def child = QualifiedName.ofTable(catalog, database, "c") + def childUUID = "c_uuid" + def type = "clone"; + service.createParentChildRelation(parent, parentUUID, child, childUUID, type) + when: + service.drop(child, Optional.of(childUUID)) + + then: + // Test Parent + assert service.getParents(parent, Optional.of(parentUUID)).isEmpty() + assert service.getParents(parent, Optional.empty()).isEmpty() + assert service.getChildren(parent, Optional.of(parentUUID)).isEmpty() + assert service.getChildren(parent, Optional.empty()).isEmpty() + + // Test Child + assert service.getParents(child, Optional.of(childUUID)).isEmpty() + assert service.getParents(child, Optional.empty()).isEmpty() + assert service.getChildren(child, Optional.of(childUUID)).isEmpty() + assert service.getChildren(child, Optional.empty()).isEmpty() + } + + def "Test CreateThenRenameThenDropWithUUIDSpecified- two same parent name but different id"() { + setup: + def parent = QualifiedName.ofTable(catalog, database, "p") + def newParent = QualifiedName.ofTable(catalog, database, "np") + def parentUUID = "p_uuid"; + def parentUUID2 = "p_uuid_2" + def child = QualifiedName.ofTable(catalog, database, "c") + def childUUID = "c_uuid"; + def type = "clone"; + + def child_parent_expected = [new ParentInfo(parent.toString(), type, parentUUID), new ParentInfo(parent.toString(), type, parentUUID2)] as Set + def parent_children_expected = [new ChildInfo(child.toString(), type, childUUID)] as Set + + when: + createParentChildRelation(parent.toString(), parentUUID, child.toString(), childUUID, type) + createParentChildRelation(parent.toString(), parentUUID2, child.toString(), childUUID, type) + + then: + // Test Parent with UUID1 + assert service.getParents(parent, Optional.of(parentUUID)).isEmpty() + assert service.getParents(parent, Optional.empty()).isEmpty() + + assert service.getChildren(parent, Optional.of(parentUUID)) == parent_children_expected + assert service.getChildren(parent, Optional.empty()) == parent_children_expected + + // Test Parent with UUID2 + assert service.getParents(parent, Optional.of(parentUUID2)).isEmpty() + assert service.getParents(parent, Optional.empty()).isEmpty() + + assert service.getChildren(parent, Optional.of(parentUUID2)) == parent_children_expected + assert service.getChildren(parent, Optional.empty()) == parent_children_expected + + // Test Child + assert service.getParents(child, Optional.of(childUUID)) == child_parent_expected + assert service.getParents(child, Optional.empty()) == child_parent_expected + assert service.getChildren(child, Optional.of(childUUID)).isEmpty() + assert service.getChildren(child, Optional.empty()).isEmpty() + + when: + service.rename(parent, newParent, Optional.of(parentUUID)) + // We only do rename on the specific id = parentUUID so the same parent name with uuid2 should not be changed + child_parent_expected = [new ParentInfo(newParent.toString(), type, parentUUID), new ParentInfo(parent.toString(), type, parentUUID2)] as Set + + then: + // Test Parent with UUID1 + assert service.getParents(newParent, Optional.of(parentUUID)).isEmpty() + assert service.getParents(newParent, Optional.empty()).isEmpty() + + assert service.getChildren(newParent, Optional.of(parentUUID)) == parent_children_expected + assert service.getChildren(newParent, Optional.empty()) == parent_children_expected + + // Test Parent with UUID2 + assert service.getParents(parent, Optional.of(parentUUID2)).isEmpty() + assert service.getParents(parent, Optional.empty()).isEmpty() + + assert service.getChildren(parent, Optional.of(parentUUID2)) == parent_children_expected + assert service.getChildren(parent, Optional.empty()) == parent_children_expected + + // Test Child + assert service.getParents(child, Optional.of(childUUID)) == child_parent_expected + assert service.getParents(child, Optional.empty()) == child_parent_expected + assert service.getChildren(child, Optional.of(childUUID)).isEmpty() + assert service.getChildren(child, Optional.empty()).isEmpty() + + // delete the record with parentUUID2 + when: + service.drop(parent, Optional.of(parentUUID2)) + child_parent_expected = [new ParentInfo(newParent.toString(), type, parentUUID)] as Set + + then: + // Test Parent with uuid1 + assert service.getParents(newParent, Optional.of(parentUUID)).isEmpty() + assert service.getParents(newParent, Optional.empty()).isEmpty() + assert service.getChildren(newParent, Optional.of(parentUUID)) == parent_children_expected + assert service.getChildren(newParent, Optional.empty()) == parent_children_expected + + // Test Parent with UUID2 + assert service.getParents(parent, Optional.of(parentUUID2)).isEmpty() + assert service.getParents(parent, Optional.empty()).isEmpty() + assert service.getChildren(parent, Optional.of(parentUUID2)).isEmpty() + assert service.getChildren(parent, Optional.empty()).isEmpty() + + // Test Child + assert service.getParents(child, Optional.of(childUUID)) == child_parent_expected + assert service.getParents(child, Optional.empty()) == child_parent_expected + assert service.getChildren(child, Optional.of(childUUID)).isEmpty() + assert service.getChildren(child, Optional.empty()).isEmpty() + } + + def "Test CreateThenRenameThenDeleteWithNoIDSpecified- two same parent name but different id"() { + setup: + def parent = QualifiedName.ofTable(catalog, database, "p") + def newParent = QualifiedName.ofTable(catalog, database, "np") + def parentUUID = "p_uuid"; + def parentUUID2 = "p_uuid_2" + def child = QualifiedName.ofTable(catalog, database, "c") + def childUUID = "c_uuid"; + def type = "clone"; + + def child_parent_expected = [new ParentInfo(parent.toString(), type, parentUUID), new ParentInfo(parent.toString(), type, parentUUID2)] as Set + def parent_children_expected = [new ChildInfo(child.toString(), type, childUUID)] as Set + + when: + createParentChildRelation(parent.toString(), parentUUID, child.toString(), childUUID, type) + createParentChildRelation(parent.toString(), parentUUID2, child.toString(), childUUID, type) + + then: + // Test Parent with UUID1 + assert service.getParents(parent, Optional.of(parentUUID)).isEmpty() + assert service.getParents(parent, Optional.empty()).isEmpty() + + assert service.getChildren(parent, Optional.of(parentUUID)) == parent_children_expected + assert service.getChildren(parent, Optional.empty()) == parent_children_expected + + // Test Parent with UUID2 + assert service.getParents(parent, Optional.of(parentUUID2)).isEmpty() + assert service.getParents(parent, Optional.empty()).isEmpty() + + assert service.getChildren(parent, Optional.of(parentUUID2)) == parent_children_expected + assert service.getChildren(parent, Optional.empty()) == parent_children_expected + + // Test Child + assert service.getParents(child, Optional.of(childUUID)) == child_parent_expected + assert service.getParents(child, Optional.empty()) == child_parent_expected + assert service.getChildren(child, Optional.of(childUUID)).isEmpty() + assert service.getChildren(child, Optional.empty()).isEmpty() + + when: + service.rename(parent, newParent, Optional.empty()) + // We only do rename on the specific id = parentUUID so the same parent name with uuid2 should not be changed + child_parent_expected = [new ParentInfo(newParent.toString(), type, parentUUID), new ParentInfo(newParent.toString(), type, parentUUID2)] as Set + + then: + // Test Parent with UUID1 + assert service.getParents(newParent, Optional.of(parentUUID)).isEmpty() + assert service.getParents(newParent, Optional.empty()).isEmpty() + + assert service.getChildren(newParent, Optional.of(parentUUID)) == parent_children_expected + assert service.getChildren(newParent, Optional.empty()) == parent_children_expected + + // Test Parent with UUID2 + assert service.getParents(newParent, Optional.of(parentUUID2)).isEmpty() + assert service.getParents(newParent, Optional.empty()).isEmpty() + + assert service.getChildren(newParent, Optional.of(parentUUID2)) == parent_children_expected + assert service.getChildren(newParent, Optional.empty()) == parent_children_expected + + // Test Child + assert service.getParents(child, Optional.of(childUUID)) == child_parent_expected + assert service.getParents(child, Optional.empty()) == child_parent_expected + assert service.getChildren(child, Optional.of(childUUID)).isEmpty() + assert service.getChildren(child, Optional.empty()).isEmpty() + + // drop based only on the parentName + when: + service.drop(newParent, Optional.empty()) + + then: + // Test Parent with uuid1 + assert service.getParents(newParent, Optional.of(parentUUID)).isEmpty() + assert service.getParents(newParent, Optional.empty()).isEmpty() + assert service.getChildren(newParent, Optional.of(parentUUID)).isEmpty() + assert service.getChildren(newParent, Optional.empty()).isEmpty() + + // Test Parent with UUID2 + assert service.getParents(newParent, Optional.of(parentUUID2)).isEmpty() + assert service.getParents(newParent, Optional.empty()).isEmpty() + assert service.getChildren(newParent, Optional.of(parentUUID2)).isEmpty() + assert service.getChildren(newParent, Optional.empty()).isEmpty() + + // Test Child + assert service.getParents(child, Optional.of(childUUID)).isEmpty() + assert service.getParents(child, Optional.empty()).isEmpty() + assert service.getChildren(child, Optional.of(childUUID)).isEmpty() + assert service.getChildren(child, Optional.empty()).isEmpty() + } + + def "Test CreateThenRenameThenDropWithIDSpecified - two same child name but different child id"() { + setup: + def parent = QualifiedName.ofTable(catalog, database, "p") + def child = QualifiedName.ofTable(catalog, database, "c") + def newChild = QualifiedName.ofTable(catalog, database, "nc") + def parentUUID = "p_uuid"; + def childUUID = "c_uuid"; + def childUUID2 = "c_uuid_2" + def type = "clone"; + def child_parent_expected = [new ParentInfo(parent.toString(), type, parentUUID)] as Set + def parent_children_expected = [new ChildInfo(child.toString(), type, childUUID), new ChildInfo(child.toString(), type, childUUID2)] as Set + // fake a record in the db with the same child name but different child uuid + createParentChildRelation(parent.toString(), parentUUID, child.toString(), childUUID2, type) + + when: + service.createParentChildRelation(parent, parentUUID, child, childUUID, type) + then: + // Test Parent + assert service.getParents(parent, Optional.of(parentUUID)).isEmpty() + assert service.getParents(parent, Optional.empty()).isEmpty() + + assert service.getChildren(parent, Optional.of(parentUUID)) == parent_children_expected + assert service.getChildren(parent, Optional.empty()) == parent_children_expected + + // Test Child + assert service.getParents(child, Optional.of(childUUID)) == child_parent_expected + assert service.getParents(child, Optional.empty()) == child_parent_expected + + assert service.getChildren(child, Optional.of(childUUID)).isEmpty() + assert service.getParents(child, Optional.empty()) == child_parent_expected + + // Test Child2 + assert service.getParents(child, Optional.of(childUUID2)) == child_parent_expected + assert service.getParents(child, Optional.empty()) == child_parent_expected + + assert service.getChildren(child, Optional.of(childUUID2)).isEmpty() + assert service.getParents(child, Optional.empty()) == child_parent_expected + + when: + service.rename(child, newChild, Optional.of(childUUID)) + parent_children_expected = [new ChildInfo(newChild.toString(), type, childUUID), new ChildInfo(child.toString(), type, childUUID2)] as Set + + then: + // Test Parent + assert service.getParents(parent, Optional.of(parentUUID)).isEmpty() + assert service.getParents(parent, Optional.empty()).isEmpty() + + assert service.getChildren(parent, Optional.of(parentUUID)) == parent_children_expected + assert service.getChildren(parent, Optional.empty()) == parent_children_expected + + // Test Child + assert service.getParents(newChild, Optional.of(childUUID)) == child_parent_expected + assert service.getParents(newChild, Optional.empty()) == child_parent_expected + + assert service.getChildren(newChild, Optional.of(childUUID)).isEmpty() + assert service.getParents(newChild, Optional.empty()) == child_parent_expected + + // Test Child2 + assert service.getParents(child, Optional.of(childUUID2)) == child_parent_expected + assert service.getParents(child, Optional.empty()) == child_parent_expected + + assert service.getChildren(child, Optional.of(childUUID2)).isEmpty() + assert service.getParents(child, Optional.empty()) == child_parent_expected + + when: + service.drop(child, Optional.of(childUUID2)) + parent_children_expected = [new ChildInfo(newChild.toString(), type, childUUID)] as Set + + then: + // Test Parent + assert service.getParents(parent, Optional.of(parentUUID)).isEmpty() + assert service.getParents(parent, Optional.empty()).isEmpty() + + assert service.getChildren(parent, Optional.of(parentUUID)) == parent_children_expected + assert service.getChildren(parent, Optional.empty()) == parent_children_expected + + // Test Child + assert service.getParents(newChild, Optional.of(childUUID)) == child_parent_expected + assert service.getParents(newChild, Optional.empty()) == child_parent_expected + + assert service.getChildren(newChild, Optional.of(childUUID)).isEmpty() + assert service.getParents(newChild, Optional.empty()) == child_parent_expected + + // Test Child2 + assert service.getParents(child, Optional.of(childUUID2)).isEmpty() + assert service.getParents(child, Optional.empty()).isEmpty() + + assert service.getChildren(child, Optional.of(childUUID2)).isEmpty() + assert service.getParents(child, Optional.empty()).isEmpty() + } + + def "Test CreateThenRenameThenDropWithNoIDSpecified - two same child name but different child id"() { + setup: + def parent = QualifiedName.ofTable(catalog, database, "p") + def child = QualifiedName.ofTable(catalog, database, "c") + def newChild = QualifiedName.ofTable(catalog, database, "nc") + def parentUUID = "p_uuid"; + def childUUID = "c_uuid"; + def childUUID2 = "c_uuid_2" + def type = "clone"; + def child_parent_expected = [new ParentInfo(parent.toString(), type, parentUUID)] as Set + def parent_children_expected = [new ChildInfo(child.toString(), type, childUUID), new ChildInfo(child.toString(), type, childUUID2)] as Set + // fake a record in the db with the same child name but different child uuid + createParentChildRelation(parent.toString(), parentUUID, child.toString(), childUUID2, type) + + when: + service.createParentChildRelation(parent, parentUUID, child, childUUID, type) + then: + // Test Parent + assert service.getParents(parent, Optional.of(parentUUID)).isEmpty() + assert service.getParents(parent, Optional.empty()).isEmpty() + + assert service.getChildren(parent, Optional.of(parentUUID)) == parent_children_expected + assert service.getChildren(parent, Optional.empty()) == parent_children_expected + + // Test Child + assert service.getParents(child, Optional.of(childUUID)) == child_parent_expected + assert service.getParents(child, Optional.empty()) == child_parent_expected + + assert service.getChildren(child, Optional.of(childUUID)).isEmpty() + assert service.getParents(child, Optional.empty()) == child_parent_expected + + // Test Child2 + assert service.getParents(child, Optional.of(childUUID2)) == child_parent_expected + assert service.getParents(child, Optional.empty()) == child_parent_expected + + assert service.getChildren(child, Optional.of(childUUID2)).isEmpty() + assert service.getParents(child, Optional.empty()) == child_parent_expected + + when: + service.rename(child, newChild, Optional.empty()) + parent_children_expected = [new ChildInfo(newChild.toString(), type, childUUID), new ChildInfo(newChild.toString(), type, childUUID2)] as Set + + then: + // Test Parent + assert service.getParents(parent, Optional.of(parentUUID)).isEmpty() + assert service.getParents(parent, Optional.empty()).isEmpty() + + assert service.getChildren(parent, Optional.of(parentUUID)) == parent_children_expected + assert service.getChildren(parent, Optional.empty()) == parent_children_expected + + // Test Child + assert service.getParents(newChild, Optional.of(childUUID)) == child_parent_expected + assert service.getParents(newChild, Optional.empty()) == child_parent_expected + + assert service.getChildren(newChild, Optional.of(childUUID)).isEmpty() + assert service.getParents(newChild, Optional.empty()) == child_parent_expected + + // Test Child2 + assert service.getParents(newChild, Optional.of(childUUID2)) == child_parent_expected + assert service.getParents(newChild, Optional.empty()) == child_parent_expected + + assert service.getChildren(newChild, Optional.of(childUUID2)).isEmpty() + assert service.getParents(newChild, Optional.empty()) == child_parent_expected + + when: + service.drop(newChild, Optional.empty()) + + then: + // Test Parent + assert service.getParents(parent, Optional.of(parentUUID)).isEmpty() + assert service.getParents(parent, Optional.empty()).isEmpty() + + assert service.getChildren(parent, Optional.of(parentUUID)).isEmpty() + assert service.getChildren(parent, Optional.empty()).isEmpty() + + // Test Child + assert service.getParents(newChild, Optional.of(childUUID)).isEmpty() + assert service.getParents(newChild, Optional.empty()).isEmpty() + + assert service.getChildren(newChild, Optional.of(childUUID)).isEmpty() + assert service.getParents(newChild, Optional.empty()).isEmpty() + + // Test Child2 + assert service.getParents(newChild, Optional.of(childUUID2)).isEmpty() + assert service.getParents(newChild, Optional.empty()).isEmpty() + + assert service.getChildren(newChild, Optional.of(childUUID2)).isEmpty() + assert service.getParents(newChild, Optional.empty()).isEmpty() + } +} diff --git a/metacat-main/src/main/java/com/netflix/metacat/main/configs/ServicesConfig.java b/metacat-main/src/main/java/com/netflix/metacat/main/configs/ServicesConfig.java index 02dd8ac9a..8e491dd8d 100644 --- a/metacat-main/src/main/java/com/netflix/metacat/main/configs/ServicesConfig.java +++ b/metacat-main/src/main/java/com/netflix/metacat/main/configs/ServicesConfig.java @@ -35,6 +35,8 @@ import com.netflix.metacat.common.server.usermetadata.LookupService; import com.netflix.metacat.common.server.usermetadata.TagService; import com.netflix.metacat.common.server.usermetadata.UserMetadataService; +import com.netflix.metacat.common.server.usermetadata.ParentChildRelMetadataService; +import com.netflix.metacat.common.server.usermetadata.TableUUIDProvider; import com.netflix.metacat.common.server.util.ThreadServiceManager; import com.netflix.metacat.main.manager.CatalogManager; import com.netflix.metacat.main.manager.ConnectorManager; @@ -228,6 +230,8 @@ public DatabaseService databaseService( * @param converterUtil converter utilities * @param authorizationService authorization Service * @param ownerValidationService owner validation service + * @param parentChildRelMetadataService parentChildRelMetadataService + * @param tableUUIDProvider provider that provides the uuid for a table * * @return The table service bean */ @@ -244,7 +248,9 @@ public TableService tableService( final Config config, final ConverterUtil converterUtil, final AuthorizationService authorizationService, - final OwnerValidationService ownerValidationService) { + final OwnerValidationService ownerValidationService, + final ParentChildRelMetadataService parentChildRelMetadataService, + final TableUUIDProvider tableUUIDProvider) { return new TableServiceImpl( connectorManager, connectorTableServiceProxy, @@ -257,7 +263,9 @@ public TableService tableService( config, converterUtil, authorizationService, - ownerValidationService + ownerValidationService, + parentChildRelMetadataService, + tableUUIDProvider ); } diff --git a/metacat-main/src/main/java/com/netflix/metacat/main/services/impl/TableServiceImpl.java b/metacat-main/src/main/java/com/netflix/metacat/main/services/impl/TableServiceImpl.java index 3ea1c41a2..97f65e5f0 100644 --- a/metacat-main/src/main/java/com/netflix/metacat/main/services/impl/TableServiceImpl.java +++ b/metacat-main/src/main/java/com/netflix/metacat/main/services/impl/TableServiceImpl.java @@ -13,6 +13,8 @@ package com.netflix.metacat.main.services.impl; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -42,14 +44,19 @@ import com.netflix.metacat.common.server.events.MetacatUpdateIcebergTablePostEvent; import com.netflix.metacat.common.server.events.MetacatUpdateTablePostEvent; import com.netflix.metacat.common.server.events.MetacatUpdateTablePreEvent; +import com.netflix.metacat.common.server.model.ChildInfo; +import com.netflix.metacat.common.server.model.ParentInfo; import com.netflix.metacat.common.server.monitoring.Metrics; import com.netflix.metacat.common.server.properties.Config; import com.netflix.metacat.common.server.spi.MetacatCatalogConfig; import com.netflix.metacat.common.server.usermetadata.AuthorizationService; import com.netflix.metacat.common.server.usermetadata.GetMetadataInterceptorParameters; +import com.netflix.metacat.common.server.usermetadata.ParentChildRelMetadataConstants; import com.netflix.metacat.common.server.usermetadata.MetacatOperation; import com.netflix.metacat.common.server.usermetadata.TagService; import com.netflix.metacat.common.server.usermetadata.UserMetadataService; +import com.netflix.metacat.common.server.usermetadata.ParentChildRelMetadataService; +import com.netflix.metacat.common.server.usermetadata.TableUUIDProvider; import com.netflix.metacat.common.server.util.MetacatContextManager; import com.netflix.metacat.common.server.util.MetacatUtils; import com.netflix.metacat.main.manager.ConnectorManager; @@ -62,7 +69,6 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; - import javax.annotation.Nullable; import java.util.List; import java.util.Map; @@ -71,6 +77,8 @@ import java.util.Set; import java.util.concurrent.TimeUnit; + + /** * Table service implementation. */ @@ -89,6 +97,8 @@ public class TableServiceImpl implements TableService { private final ConverterUtil converterUtil; private final AuthorizationService authorizationService; private final OwnerValidationService ownerValidationService; + private final ParentChildRelMetadataService parentChildRelMetadataService; + private final TableUUIDProvider tableUUIDProvider; /** * {@inheritDoc} @@ -106,7 +116,13 @@ public TableDto create(final QualifiedName name, final TableDto tableDto) { log.info("Creating table {}", name); eventBus.post(new MetacatCreateTablePreEvent(name, metacatRequestContext, this, tableDto)); - connectorTableServiceProxy.create(name, converterUtil.fromTableDto(tableDto)); + final Optional unSaveOpOpt = saveParentChildRelationship(name, tableDto); //suceed + try { + connectorTableServiceProxy.create(name, converterUtil.fromTableDto(tableDto)); //failed + } catch (Exception e) { + unSaveOpOpt.ifPresent(Runnable::run); + throw e; + } if (tableDto.getDataMetadata() != null || tableDto.getDefinitionMetadata() != null) { log.info("Saving user metadata for table {}", name); @@ -139,6 +155,98 @@ public TableDto create(final QualifiedName name, final TableDto tableDto) { return dto; } + private ObjectNode createParentChildObjectNode(@Nullable final Set parentInfos, + @Nullable final Set childInfos) { + final ObjectMapper objectMapper = new ObjectMapper(); + final ObjectNode rootNode = objectMapper.createObjectNode(); + + if (parentInfos != null && !parentInfos.isEmpty()) { + final ArrayNode parentArrayNode = objectMapper.createArrayNode(); + for (ParentInfo parentInfo : parentInfos) { + final ObjectNode parentNode = objectMapper.createObjectNode(); + parentNode.put("name", parentInfo.getName()); + parentNode.put("relationType", parentInfo.getRelationType()); + parentNode.put("uuid", parentInfo.getUuid()); + parentArrayNode.add(parentNode); + } + rootNode.set(ParentChildRelMetadataConstants.PARENTINFOS, parentArrayNode); + } + + if (childInfos != null && !childInfos.isEmpty()) { + final ArrayNode childrenArrayNode = objectMapper.createArrayNode(); + for (ChildInfo childInfo : childInfos) { + final ObjectNode childNode = objectMapper.createObjectNode(); + childNode.put("name", childInfo.getName()); + childNode.put("relationType", childInfo.getRelationType()); + childNode.put("uuid", childInfo.getUuid()); + childrenArrayNode.add(childNode); + } + rootNode.set(ParentChildRelMetadataConstants.CHILDINFOS, childrenArrayNode); + } + return rootNode; + } + + // Return the unSaveOperation if applicable + private Optional saveParentChildRelationship(final QualifiedName child, final TableDto tableDto) { + if (tableDto.getDefinitionMetadata() != null) { + final ObjectNode definitionMetadata = tableDto.getDefinitionMetadata(); + if (definitionMetadata.has(ParentChildRelMetadataConstants.PARENTNAME)) { + // fetch parent name + final String parentFullName = definitionMetadata.path(ParentChildRelMetadataConstants.PARENTNAME) + .asText(); + String parentUUID; + String childUUID; + + if (!definitionMetadata.has(ParentChildRelMetadataConstants.PARENTUUID)) { + throw new RuntimeException("root_table_uuid is not specified for parent table=" + parentFullName); + } + + if (!definitionMetadata.has(ParentChildRelMetadataConstants.CHILDUUID)) { + throw new RuntimeException("child_table_uuid is not specified for child table=" + child); + } + parentUUID = definitionMetadata.path(ParentChildRelMetadataConstants.PARENTUUID).asText(); + childUUID = definitionMetadata.path(ParentChildRelMetadataConstants.CHILDUUID).asText(); + + final String[] splits = parentFullName.split("\\."); + if (splits.length != 3) { + throw new RuntimeException("Parent table identifier should pass in the following format " + + "{catalog}.{db}.{parentTable}"); + } + final QualifiedName parent = QualifiedName.ofTable( + splits[0], + splits[1], + splits[2] + ); + validate(parent); + + // fetch relationshipType + String relationType; + if (definitionMetadata.has(ParentChildRelMetadataConstants.RELATIONTYPE)) { + relationType = definitionMetadata.path(ParentChildRelMetadataConstants.RELATIONTYPE).asText(); + } else { + relationType = "CLONE"; + } + parentChildRelMetadataService.createParentChildRelation(parent, parentUUID, + child, childUUID, relationType); + + // Return a Runnable for deleting the relationship + return Optional.of(() -> { + try { + parentChildRelMetadataService.deleteParentChildRelation(parent, + parentUUID, child, childUUID, relationType); + } catch (Exception e) { + log.error("parentChildRelMetadataService: Fail to delete parent child relationship " + + "after failing to create the table={}" + + "with the following parameters: " + + "parent={}, parentUUID={}, child={}, childUUID={}, relationType={}", + child, parent, parentUUID, child, childUUID, relationType, e); + } + }); + } + } + return Optional.empty(); + } + private void setDefaultAttributes(final TableDto tableDto) { setDefaultSerdeIfNull(tableDto); setDefaultDefinitionMetadataIfNull(tableDto); @@ -267,10 +375,13 @@ public TableDto deleteAndReturn(final QualifiedName name, final boolean isMView) } } - // Try to delete the table even if get above fails + final Optional uuid = tableUUIDProvider.getUUID(tableDto); + final Set childInfos = parentChildRelMetadataService.getChildren(name, uuid); + if (childInfos != null && !childInfos.isEmpty()) { + throw new RuntimeException("Fail to drop " + name + " because it still has child table"); + } try { connectorTableServiceProxy.delete(name); - // If this is a common view, the storage_table if present // should also be deleted. if (MetacatUtils.isCommonView(tableDto.getMetadata()) @@ -283,11 +394,16 @@ public TableDto deleteAndReturn(final QualifiedName name, final boolean isMView) deleteCommonViewStorageTable(name, qualifiedStorageTableName); } } - } catch (NotFoundException ignored) { log.debug("NotFoundException ignored for table {}", name); } + try { + parentChildRelMetadataService.drop(name, uuid); + } catch (Exception e) { + log.error("parentChildRelMetadataService: Fail to drop relation for table={} and uuid={}", name, uuid, e); + } + if (canDeleteMetadata(name)) { // Delete the metadata. Type doesn't matter since we discard the result log.info("Deleting user metadata for table {}", name); @@ -373,11 +489,26 @@ public Optional get(final QualifiedName name, final GetTableServicePar } if (getTableServiceParameters.isIncludeDefinitionMetadata()) { - final Optional definitionMetadata = + Optional definitionMetadata = (getTableServiceParameters.isDisableOnReadMetadataIntercetor()) ? userMetadataService.getDefinitionMetadata(name) : userMetadataService.getDefinitionMetadataWithInterceptor(name, GetMetadataInterceptorParameters.builder().hasMetadata(tableInternal).build()); + // Always get the source of truth for parent child relation from the parentChildRelMetadataService + final Optional uuid = tableUUIDProvider.getUUID(table); + final Set parentInfo = parentChildRelMetadataService.getParents(name, uuid); + final Set childInfos = parentChildRelMetadataService.getChildren(name, uuid); + final ObjectNode parentChildRelObjectNode = createParentChildObjectNode(parentInfo, childInfos); + if (definitionMetadata.isPresent()) { + if (!parentChildRelObjectNode.isEmpty()) { + definitionMetadata.get().set(ParentChildRelMetadataConstants.PARENTCHILDRELINFO, + parentChildRelObjectNode); + } + } else { + if (!parentChildRelObjectNode.isEmpty()) { + definitionMetadata = Optional.of(parentChildRelObjectNode); + } + } definitionMetadata.ifPresent(table::setDefinitionMetadata); } @@ -437,7 +568,25 @@ public void rename( if (oldTable != null) { //Ignore if the operation is not supported, so that we can at least go ahead and save the user metadata eventBus.post(new MetacatRenameTablePreEvent(oldName, metacatRequestContext, this, newName)); - connectorTableServiceProxy.rename(oldName, newName, isMView); + + // Before rename, first rename its parent child relation + final Optional uuid = tableUUIDProvider.getUUID(oldTable); + parentChildRelMetadataService.rename(oldName, newName, uuid); + + try { + connectorTableServiceProxy.rename(oldName, newName, isMView); + } catch (Exception e) { + try { + // if rename operation fail, rename back the parent child relation + parentChildRelMetadataService.rename(newName, oldName, uuid); + } catch (Exception renameException) { + log.error("parentChildRelMetadataService: Fail to rename parent child relation " + + "after table fail to rename from {} to {} " + + "with the following parameters table={} and uuid={} to newName={}", + oldName, newName, oldName, uuid, newName, renameException); + } + throw e; + } userMetadataService.renameDefinitionMetadataKey(oldName, newName); tagService.renameTableTags(oldName, newName.getTableName()); diff --git a/metacat-main/src/test/groovy/com/netflix/metacat/main/services/impl/TableServiceImplSpec.groovy b/metacat-main/src/test/groovy/com/netflix/metacat/main/services/impl/TableServiceImplSpec.groovy index 286c78696..350c24e35 100644 --- a/metacat-main/src/test/groovy/com/netflix/metacat/main/services/impl/TableServiceImplSpec.groovy +++ b/metacat-main/src/test/groovy/com/netflix/metacat/main/services/impl/TableServiceImplSpec.groovy @@ -35,9 +35,12 @@ import com.netflix.metacat.common.server.converter.ConverterUtil import com.netflix.metacat.common.server.events.MetacatEventBus import com.netflix.metacat.common.server.events.MetacatUpdateTablePostEvent import com.netflix.metacat.common.server.events.MetacatUpdateTablePreEvent +import com.netflix.metacat.common.server.model.ChildInfo +import com.netflix.metacat.common.server.model.ParentInfo import com.netflix.metacat.common.server.properties.Config import com.netflix.metacat.common.server.spi.MetacatCatalogConfig import com.netflix.metacat.common.server.usermetadata.DefaultAuthorizationService +import com.netflix.metacat.common.server.usermetadata.TableUUIDProvider import com.netflix.metacat.common.server.usermetadata.TagService import com.netflix.metacat.common.server.usermetadata.UserMetadataService import com.netflix.metacat.common.server.util.MetacatContextManager @@ -50,6 +53,7 @@ import com.netflix.spectator.api.DefaultRegistry import com.netflix.spectator.api.NoopRegistry import spock.lang.Specification import spock.lang.Unroll +import com.netflix.metacat.common.server.usermetadata.ParentChildRelMetadataService; import javax.annotation.meta.When @@ -80,6 +84,8 @@ class TableServiceImplSpec extends Specification { def connectorTableServiceProxy def authorizationService def ownerValidationService + def parentChildRelSvc + def tableUUIDProvider def service def setup() { @@ -93,13 +99,16 @@ class TableServiceImplSpec extends Specification { usermetadataService.getDefinitionMetadata(_) >> Optional.empty() usermetadataService.getDataMetadata(_) >> Optional.empty() usermetadataService.getDefinitionMetadataWithInterceptor(_,_) >> Optional.empty() - connectorTableServiceProxy = new ConnectorTableServiceProxy(connectorManager, converterUtil) + connectorTableServiceProxy = Spy(new ConnectorTableServiceProxy(connectorManager, converterUtil)) authorizationService = new DefaultAuthorizationService(config) ownerValidationService = Mock(OwnerValidationService) + parentChildRelSvc = Mock(ParentChildRelMetadataService) + tableUUIDProvider = Mock(TableUUIDProvider) service = new TableServiceImpl(connectorManager, connectorTableServiceProxy, databaseService, tagService, usermetadataService, new MetacatJsonLocator(), - eventBus, registry, config, converterUtil, authorizationService, ownerValidationService) + eventBus, registry, config, converterUtil, authorizationService, ownerValidationService, parentChildRelSvc, + tableUUIDProvider) } def testTableGet() { @@ -270,6 +279,70 @@ class TableServiceImplSpec extends Specification { 0 * ownerValidationService.enforceOwnerValidation(_, _, _) } + def "Test Create - Clone Table Fail to create table"() { + given: + def childTableName = QualifiedName.ofTable("clone", "clone", "c") + def parentTableName = QualifiedName.ofTable("clone", "clone", "p") + def createTableDto = new TableDto( + name: childTableName, + definitionMetadata: toObjectNode('{\"root_table_name\":\"clone.clone.p\", \"root_table_uuid\":\"p_uuid\", \"child_table_uuid\":\"child_uuid\"}'), + serde: new StorageDto(uri: 's3:/clone/clone/c') + ) + when: + service.create(childTableName, createTableDto) + then: + 1 * ownerValidationService.extractPotentialOwners(_) >> ["cloneClient"] + 1 * ownerValidationService.isUserValid(_) >> true + 1 * ownerValidationService.extractPotentialOwnerGroups(_) >> ["cloneClientGroup"] + 1 * ownerValidationService.isGroupValid(_) >> true + + 1 * parentChildRelSvc.createParentChildRelation(parentTableName, "p_uuid", childTableName, "child_uuid", "CLONE") + 1 * connectorTableServiceProxy.create(_, _) >> {throw new RuntimeException("Fail to create")} + 1 * parentChildRelSvc.deleteParentChildRelation(parentTableName, "p_uuid", childTableName, "child_uuid", "CLONE") + thrown(RuntimeException) + } + + def "Test Rename - Clone Table Fail to update parent child relation"() { + given: + def oldName = QualifiedName.ofTable("clone", "clone", "oldChild") + def newName = QualifiedName.ofTable("clone", "clone", "newChild") + when: + service.rename(oldName, newName, false) + + then: + 1 * config.getNoTableRenameOnTags() >> [] + 2 * tableUUIDProvider.getUUID(_) >> { Optional.of("uuid")} + 1 * parentChildRelSvc.rename(oldName, newName, Optional.of("uuid")) + 1 * connectorTableServiceProxy.rename(oldName, newName, _) >> {throw new RuntimeException("Fail to rename")} + 1 * parentChildRelSvc.rename(newName, oldName, Optional.of("uuid")) + thrown(RuntimeException) + } + + def "Test Drop - Clone Table Fail to drop parent child relation"() { + given: + def name = QualifiedName.ofTable("clone", "clone", "child") + + when: + service.delete(name) + then: + 2 * tableUUIDProvider.getUUID(_) >> { Optional.of("uuid")} + 1 * parentChildRelSvc.getParents(name, Optional.of("uuid")) >> {[new ParentInfo("parent", "clone", "parent_uuid")] as Set} + 2 * parentChildRelSvc.getChildren(name, Optional.of("uuid")) >> {[new ChildInfo("child", "clone", "child_uuid")] as Set} + 1 * config.getNoTableDeleteOnTags() >> [] + thrown(RuntimeException) + + when: + service.delete(name) + then: + 2 * tableUUIDProvider.getUUID(_) >> { Optional.of("uuid")} + 1 * parentChildRelSvc.getParents(name, Optional.of("uuid")) + 2 * parentChildRelSvc.getChildren(name, Optional.of("uuid")) + 1 * config.getNoTableDeleteOnTags() >> [] + 1 * connectorTableServiceProxy.delete(_) >> {throw new RuntimeException("Fail to drop")} + 0 * parentChildRelSvc.drop(_, _) + thrown(RuntimeException) + } + def "Will not throw on Successful Table Update with Failed Get"() { given: def updatedTableDto = new TableDto(name: name, serde: new StorageDto(uri: 's3:/a/b/c')) @@ -311,7 +384,7 @@ class TableServiceImplSpec extends Specification { connectorManager, connectorTableServiceProxy, databaseService, tagService, usermetadataService, new MetacatJsonLocator(), eventBus, new DefaultRegistry(), config, converterUtil, authorizationService, - new DefaultOwnerValidationService(new NoopRegistry())) + new DefaultOwnerValidationService(new NoopRegistry()), parentChildRelSvc, tableUUIDProvider) def initialDefinitionMetadataJson = toObjectNode(initialDefinitionMetadata) tableDto = new TableDto( diff --git a/metacat-metadata-mysql/build.gradle b/metacat-metadata-mysql/build.gradle index db158f794..c4291fa8e 100644 --- a/metacat-metadata-mysql/build.gradle +++ b/metacat-metadata-mysql/build.gradle @@ -32,6 +32,7 @@ dependencies { api("com.google.guava:guava") api("org.apache.tomcat:tomcat-jdbc") api("org.slf4j:slf4j-api") + api('org.springframework.retry:spring-retry') /******************************* * Provided Dependencies *******************************/ diff --git a/metacat-metadata-mysql/src/main/java/com/netflix/metacat/metadata/mysql/MySqlParentChildRelMetaDataService.java b/metacat-metadata-mysql/src/main/java/com/netflix/metacat/metadata/mysql/MySqlParentChildRelMetaDataService.java new file mode 100644 index 000000000..5ae01d139 --- /dev/null +++ b/metacat-metadata-mysql/src/main/java/com/netflix/metacat/metadata/mysql/MySqlParentChildRelMetaDataService.java @@ -0,0 +1,243 @@ +package com.netflix.metacat.metadata.mysql; + +import com.netflix.metacat.common.QualifiedName; +import com.netflix.metacat.common.server.model.ChildInfo; +import com.netflix.metacat.common.server.model.ParentInfo; +import com.netflix.metacat.common.server.usermetadata.ParentChildRelMetadataService; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.transaction.annotation.Transactional; + +import java.sql.PreparedStatement; +import java.util.Set; +import java.util.Optional; +import java.util.List; +import java.util.ArrayList; +import java.util.HashSet; + +/** + * Parent Child Relationship Metadata Service. + * This stores the parent child relationship of two entities as first class citizen in Metacat. + */ +@Slf4j +@SuppressFBWarnings +@Transactional("metadataTxManager") +public class MySqlParentChildRelMetaDataService implements ParentChildRelMetadataService { + static final String SQL_CREATE_PARENT_CHILD_RELATIONS = + "INSERT INTO parent_child_relation (parent, parent_uuid, child, child_uuid, relation_type) " + + "VALUES (?, ?, ?, ?, ?)"; + + static final String SQL_DELETE_PARENT_CHILD_RELATIONS = + "DELETE FROM parent_child_relation " + + "WHERE parent = ? AND parent_uuid = ? AND child = ? AND child_uuid = ? AND relation_type = ?"; + + static final String SQL_RENAME_PARENT_ENTITY = "UPDATE parent_child_relation " + + "SET parent = ? WHERE parent = ?"; + static final String SQL_RENAME_CHILD_ENTITY = "UPDATE parent_child_relation " + + "SET child = ? WHERE child = ?"; + + static final String SQL_DROP_CHILD = "DELETE FROM parent_child_relation " + + "WHERE child = ? "; + static final String SQL_DROP_PARENT = "DELETE FROM parent_child_relation " + + "WHERE parent = ? "; + + static final String SQL_GET_PARENTS = "SELECT parent, parent_uuid, relation_type " + + "FROM parent_child_relation WHERE child = ?"; + + static final String SQL_GET_CHILDREN = "SELECT child, child_uuid, relation_type " + + "FROM parent_child_relation WHERE parent = ?"; + + private JdbcTemplate jdbcTemplate; + + /** + * Constructor. + * + * @param jdbcTemplate jdbc template + */ + @Autowired + public MySqlParentChildRelMetaDataService(final JdbcTemplate jdbcTemplate) { + this.jdbcTemplate = jdbcTemplate; + } + + @Override + public void createParentChildRelation(final QualifiedName parentName, + final String parentUUID, + final QualifiedName childName, + final String childUUID, + final String type) { + // Validation to prevent having a child have two parents + final Set childParents = getParents(childName, Optional.of(childUUID)); + if (!childParents.isEmpty()) { + throw new RuntimeException("Cannot have a child table having more than one parent " + + "- Child Table: " + childName + + " already have a parent Table=" + childParents.stream().findFirst().get()); + } + + // Validation to prevent creating a child table as a parent of another child table + final Set parentParents = getParents(parentName, Optional.of(parentUUID)); + if (!parentParents.isEmpty()) { + throw new RuntimeException("Cannot create a child table as parent " + + "- parent table: " + parentName + + " already have a parent table = " + parentParents.stream().findFirst().get()); + } + + // Validation to prevent creating a parent on top of a table that have children + final Set childChildren = getChildren(childName, Optional.of(childUUID)); + if (!childChildren.isEmpty()) { + throw new RuntimeException("Cannot create a parent table on top of another parent " + + "- child table: " + childName + " already have child"); + } + + jdbcTemplate.update(connection -> { + final PreparedStatement ps = connection.prepareStatement(SQL_CREATE_PARENT_CHILD_RELATIONS); + ps.setString(1, parentName.toString()); + ps.setString(2, parentUUID); + ps.setString(3, childName.toString()); + ps.setString(4, childUUID); + ps.setString(5, type); + return ps; + }); + log.info("Successfully create parent child relationship with parent={}, parentUUID={}, " + + "child={}, childUUID={}, relationType = {}", + parentName, parentUUID, childName, childUUID, type + ); + } + + @Override + public void deleteParentChildRelation(final QualifiedName parentName, + final String parentUUID, + final QualifiedName childName, + final String childUUID, + final String type) { + log.info("Deleting parent child relationship with parent={}, parentUUID={}, " + + "child={}, childUUID={}, relationType = {}", + parentName, parentUUID, childName, childUUID, type + ); + jdbcTemplate.update(connection -> { + final PreparedStatement ps = connection.prepareStatement(SQL_DELETE_PARENT_CHILD_RELATIONS); + ps.setString(1, parentName.toString()); + ps.setString(2, parentUUID); + ps.setString(3, childName.toString()); + ps.setString(4, childUUID); + ps.setString(5, type); + return ps; + }); + log.info("Successfully delete parent child relationship with parent={}, parentUUID={}, " + + "child={}, childUUID={}, relationType = {}", + parentName, parentUUID, childName, childUUID, type + ); + } + + @Override + public void rename(final QualifiedName oldName, final QualifiedName newName, final Optional uuid) { + try { + renameParent(oldName, newName, uuid); + renameChild(oldName, newName, uuid); + log.info("Successfully rename parent child relationship for oldName={}, newName={}, uuid={}", + oldName, newName, uuid.orElse("") + ); + } catch (RuntimeException e) { + log.error("Failed to rename entity", e); + throw e; + } + } + + private void renameParent(final QualifiedName oldName, final QualifiedName newName, final Optional uuid) { + final String query = uuid.isPresent() ? SQL_RENAME_PARENT_ENTITY + " AND parent_uuid = ?" + : SQL_RENAME_PARENT_ENTITY; + jdbcTemplate.update(connection -> { + final PreparedStatement ps = connection.prepareStatement(query); + ps.setString(1, newName.toString()); + ps.setString(2, oldName.toString()); + if (uuid.isPresent()) { + ps.setString(3, uuid.get()); + } + return ps; + }); + } + + private void renameChild(final QualifiedName oldName, final QualifiedName newName, final Optional uuid) { + final String query = uuid.isPresent() ? SQL_RENAME_CHILD_ENTITY + " AND child_uuid = ?" + : SQL_RENAME_CHILD_ENTITY; + jdbcTemplate.update(connection -> { + final PreparedStatement ps = connection.prepareStatement(query); + ps.setString(1, newName.toString()); + ps.setString(2, oldName.toString()); + if (uuid.isPresent()) { + ps.setString(3, uuid.get()); + } + return ps; + }); + } + + @Override + public void drop(final QualifiedName name, final Optional uuid) { + dropParent(name, uuid); + dropChild(name, uuid); + log.info("Successfully drop parent child relationship for name={}, uuid={}", name, uuid.orElse("")); + } + + private void dropParent(final QualifiedName name, final Optional uuid) { + final String query = uuid.isPresent() ? SQL_DROP_PARENT + " AND parent_uuid = ?" : SQL_DROP_PARENT; + jdbcTemplate.update(connection -> { + final PreparedStatement ps = connection.prepareStatement(query); + ps.setString(1, name.toString()); + if (uuid.isPresent()) { + ps.setString(2, uuid.get()); + } + return ps; + }); + } + + private void dropChild(final QualifiedName name, final Optional uuid) { + final String query = uuid.isPresent() ? SQL_DROP_CHILD + " AND child_uuid = ?" : SQL_DROP_CHILD; + jdbcTemplate.update(connection -> { + final PreparedStatement ps = connection.prepareStatement(query); + ps.setString(1, name.toString()); + if (uuid.isPresent()) { + ps.setString(2, uuid.get()); + } + return ps; + }); + } + + @Override + public Set getParents(final QualifiedName name, final Optional uuid) { + final String query = uuid.isPresent() ? SQL_GET_PARENTS + " AND child_uuid = ?" : SQL_GET_PARENTS; + final List params = new ArrayList<>(); + params.add(name.toString()); + if (uuid.isPresent()) { + params.add(uuid.get().toString()); + } + final List parents = jdbcTemplate.query( + query, params.toArray(), (rs, rowNum) -> { + final ParentInfo parentInfo = new ParentInfo(); + parentInfo.setName(rs.getString("parent")); + parentInfo.setRelationType(rs.getString("relation_type")); + parentInfo.setUuid(rs.getString("parent_uuid")); + return parentInfo; + }); + return new HashSet<>(parents); + } + + @Override + public Set getChildren(final QualifiedName name, final Optional uuid) { + final List params = new ArrayList<>(); + params.add(name.toString()); + final String query = uuid.isPresent() ? SQL_GET_CHILDREN + " AND parent_uuid = ?" : SQL_GET_CHILDREN; + if (uuid.isPresent()) { + params.add(uuid.get().toString()); + } + final List children = jdbcTemplate.query( + query, params.toArray(), (rs, rowNum) -> { + final ChildInfo childInfo = new ChildInfo(); + childInfo.setName(rs.getString("child")); + childInfo.setRelationType(rs.getString("relation_type")); + childInfo.setUuid(rs.getString("child_uuid")); + return childInfo; + }); + return new HashSet<>(children); + } +} diff --git a/metacat-metadata-mysql/src/main/java/com/netflix/metacat/metadata/mysql/MySqlUserMetadataConfig.java b/metacat-metadata-mysql/src/main/java/com/netflix/metacat/metadata/mysql/MySqlUserMetadataConfig.java index e5df7761d..879a25376 100644 --- a/metacat-metadata-mysql/src/main/java/com/netflix/metacat/metadata/mysql/MySqlUserMetadataConfig.java +++ b/metacat-metadata-mysql/src/main/java/com/netflix/metacat/metadata/mysql/MySqlUserMetadataConfig.java @@ -16,11 +16,14 @@ import com.netflix.metacat.common.json.MetacatJson; import com.netflix.metacat.common.server.properties.Config; import com.netflix.metacat.common.server.properties.MetacatProperties; -import com.netflix.metacat.common.server.usermetadata.MetadataInterceptor; +import com.netflix.metacat.common.server.usermetadata.TableUUIDProvider; +import com.netflix.metacat.common.server.usermetadata.DefaultTableUUIDProvider; +import com.netflix.metacat.common.server.usermetadata.UserMetadataService; import com.netflix.metacat.common.server.usermetadata.LookupService; -import com.netflix.metacat.common.server.usermetadata.MetadataInterceptorImpl; import com.netflix.metacat.common.server.usermetadata.TagService; -import com.netflix.metacat.common.server.usermetadata.UserMetadataService; +import com.netflix.metacat.common.server.usermetadata.ParentChildRelMetadataService; +import com.netflix.metacat.common.server.usermetadata.MetadataInterceptor; +import com.netflix.metacat.common.server.usermetadata.MetadataInterceptorImpl; import com.netflix.metacat.common.server.util.DataSourceManager; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; @@ -41,7 +44,6 @@ @Configuration @ConditionalOnProperty(value = "metacat.mysqlmetadataservice.enabled", havingValue = "true") public class MySqlUserMetadataConfig { - /** * business Metadata Manager. * @return business Metadata Manager @@ -53,6 +55,18 @@ public MetadataInterceptor businessMetadataManager( return new MetadataInterceptorImpl(); } + /** + * The parentChildRelMetadataService to use. + * + * @return The TableUUIDProvider implementation + */ + @Bean + @ConditionalOnMissingBean(TableUUIDProvider.class) + public TableUUIDProvider tableUUIDProvider( + ) { + return new DefaultTableUUIDProvider(); + } + /** * User Metadata service. * @@ -108,6 +122,19 @@ public TagService tagService( return new MySqlTagService(config, jdbcTemplate, lookupService, metacatJson, userMetadataService); } + /** + * The parentChildRelMetadataService to use. + * + * @param jdbcTemplate JDBC template + * @return The parentChildRelMetadataService implementation backed by MySQL + */ + @Bean + ParentChildRelMetadataService parentChildRelMetadataService( + @Qualifier("metadataJdbcTemplate") final JdbcTemplate jdbcTemplate + ) { + return new MySqlParentChildRelMetaDataService(jdbcTemplate); + } + /** * mySql DataSource. *