diff --git a/metacat-client/src/main/java/com/netflix/metacat/client/Client.java b/metacat-client/src/main/java/com/netflix/metacat/client/Client.java index fd224844a..fecdac692 100644 --- a/metacat-client/src/main/java/com/netflix/metacat/client/Client.java +++ b/metacat-client/src/main/java/com/netflix/metacat/client/Client.java @@ -21,15 +21,16 @@ import com.fasterxml.jackson.datatype.guava.GuavaModule; import com.fasterxml.jackson.module.jaxb.JaxbAnnotationModule; import com.google.common.base.Preconditions; +import com.netflix.metacat.client.api.MetacatV1; +import com.netflix.metacat.client.api.MetadataV1; +import com.netflix.metacat.client.api.ParentChildRelV1; +import com.netflix.metacat.client.api.PartitionV1; +import com.netflix.metacat.client.api.ResolverV1; import com.netflix.metacat.client.api.TagV1; import com.netflix.metacat.client.module.JacksonDecoder; import com.netflix.metacat.client.module.JacksonEncoder; import com.netflix.metacat.client.module.MetacatErrorDecoder; import com.netflix.metacat.common.MetacatRequestContext; -import com.netflix.metacat.client.api.MetacatV1; -import com.netflix.metacat.client.api.MetadataV1; -import com.netflix.metacat.client.api.PartitionV1; -import com.netflix.metacat.client.api.ResolverV1; import com.netflix.metacat.common.json.MetacatJsonLocator; import feign.Feign; import feign.Request; @@ -58,6 +59,8 @@ public final class Client { private final ResolverV1 resolverApi; private final TagV1 tagApi; + private final ParentChildRelV1 parentChildRelApi; + private Client( final String host, final feign.Client client, @@ -93,6 +96,7 @@ private Client( metadataApi = getApiClient(MetadataV1.class); resolverApi = getApiClient(ResolverV1.class); tagApi = getApiClient(TagV1.class); + parentChildRelApi = getApiClient(ParentChildRelV1.class); } /** @@ -163,6 +167,15 @@ public TagV1 getTagApi() { return tagApi; } + /** + * Return an API instance that can be used to interact with + * the metacat server for parent child relationship metadata. + * @return An instance api conforming to ParentChildRelV1 interface + */ + public ParentChildRelV1 getParentChildRelApi() { + return parentChildRelApi; + } + /** * Builder class to build the metacat client. */ diff --git a/metacat-client/src/main/java/com/netflix/metacat/client/api/ParentChildRelV1.java b/metacat-client/src/main/java/com/netflix/metacat/client/api/ParentChildRelV1.java new file mode 100644 index 000000000..546030f69 --- /dev/null +++ b/metacat-client/src/main/java/com/netflix/metacat/client/api/ParentChildRelV1.java @@ -0,0 +1,43 @@ +package com.netflix.metacat.client.api; + +import javax.ws.rs.Path; +import javax.ws.rs.Consumes; +import javax.ws.rs.Produces; +import javax.ws.rs.GET; +import javax.ws.rs.PathParam; + +import javax.ws.rs.core.MediaType; +import com.netflix.metacat.common.dto.notifications.ChildInfoDto; + +import java.util.Set; + +/** + * Metacat API for managing parent child relation. + * + * @author Yingjianw + */ + +@Path("/mds/v1/parentChildRel") +@Consumes(MediaType.APPLICATION_JSON) +@Produces(MediaType.APPLICATION_JSON) +public interface ParentChildRelV1 { + /** + * Return the list of children. + * @param catalogName catalogName + * @param databaseName databaseName + * @param tableName tableName + * @return list of childInfos + */ + @GET + @Path("children/catalog/{catalog-name}/database/{database-name}/table/{table-name}") + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + Set getChildren( + @PathParam("catalog-name") + String catalogName, + @PathParam("database-name") + String databaseName, + @PathParam("table-name") + String tableName + ); +} diff --git a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/converter/ConverterUtil.java b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/converter/ConverterUtil.java index 05690c312..5ad54ace8 100644 --- a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/converter/ConverterUtil.java +++ b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/converter/ConverterUtil.java @@ -33,6 +33,7 @@ import com.netflix.metacat.common.dto.Sort; import com.netflix.metacat.common.dto.StorageDto; import com.netflix.metacat.common.dto.TableDto; +import com.netflix.metacat.common.dto.notifications.ChildInfoDto; import com.netflix.metacat.common.server.connectors.ConnectorRequestContext; import com.netflix.metacat.common.server.connectors.model.AuditInfo; import com.netflix.metacat.common.server.connectors.model.CatalogInfo; @@ -46,6 +47,7 @@ import com.netflix.metacat.common.server.connectors.model.PartitionsSaveResponse; import com.netflix.metacat.common.server.connectors.model.StorageInfo; import com.netflix.metacat.common.server.connectors.model.TableInfo; +import com.netflix.metacat.common.server.model.ChildInfo; import lombok.NonNull; import org.dozer.CustomConverter; import org.dozer.DozerBeanMapper; @@ -276,5 +278,13 @@ public PartitionsSaveResponseDto toPartitionsSaveResponseDto(final PartitionsSav return mapper.map(partitionsSaveResponse, PartitionsSaveResponseDto.class); } - + /** + * Convert ChildInfo to ChildInfoDto. + * + * @param childInfo childInfo + * @return childInfo dto + */ + public ChildInfoDto toChildInfoDto(final ChildInfo childInfo) { + return mapper.map(childInfo, ChildInfoDto.class); + } } diff --git a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/model/BaseRelEntityInfo.java b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/model/BaseRelEntityInfo.java index 8807439bb..b070d7bfc 100644 --- a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/model/BaseRelEntityInfo.java +++ b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/model/BaseRelEntityInfo.java @@ -2,11 +2,14 @@ import lombok.Data; +import java.io.Serializable; + /** * Base class to represent relation entity. */ @Data -public abstract class BaseRelEntityInfo { +public abstract class BaseRelEntityInfo implements Serializable { + private static final long serialVersionUID = 9121109874202888889L; private String name; private String relationType; private String uuid; diff --git a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/usermetadata/ParentChildRelMetadataService.java b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/usermetadata/ParentChildRelMetadataService.java index 255a78d43..b8e6220a2 100644 --- a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/usermetadata/ParentChildRelMetadataService.java +++ b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/usermetadata/ParentChildRelMetadataService.java @@ -1,5 +1,6 @@ package com.netflix.metacat.common.server.usermetadata; import com.netflix.metacat.common.QualifiedName; +import com.netflix.metacat.common.dto.notifications.ChildInfoDto; import com.netflix.metacat.common.server.model.ChildInfo; import com.netflix.metacat.common.server.model.ParentInfo; @@ -93,4 +94,13 @@ Set getParents( Set getChildren( QualifiedName name ); + + /** + * get the set of children dto for the input name. + * @param name name + * @return a set of ChildInfo + */ + Set getChildrenDto( + QualifiedName name + ); } diff --git a/metacat-common/src/main/java/com/netflix/metacat/common/dto/notifications/ChildInfoDto.java b/metacat-common/src/main/java/com/netflix/metacat/common/dto/notifications/ChildInfoDto.java new file mode 100644 index 000000000..bf150c4af --- /dev/null +++ b/metacat-common/src/main/java/com/netflix/metacat/common/dto/notifications/ChildInfoDto.java @@ -0,0 +1,31 @@ +package com.netflix.metacat.common.dto.notifications; + +import com.netflix.metacat.common.dto.BaseDto; +import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; + +/** + * ChildInfo information. + */ +@SuppressWarnings("unused") +@Data +@EqualsAndHashCode(callSuper = false) +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class ChildInfoDto extends BaseDto { + private static final long serialVersionUID = 9121109874202088789L; + /* Name of the child */ + @ApiModelProperty(value = "name of the child") + private String name; + /* Type of the relation */ + @ApiModelProperty(value = "type of the relation") + private String relationType; + /* uuid of the table */ + @ApiModelProperty(value = "uuid of the table") + private String uuid; +} diff --git a/metacat-functional-tests/src/functionalTest/groovy/com/netflix/metacat/MetacatSmokeSpec.groovy b/metacat-functional-tests/src/functionalTest/groovy/com/netflix/metacat/MetacatSmokeSpec.groovy index 353936926..60ec49f09 100644 --- a/metacat-functional-tests/src/functionalTest/groovy/com/netflix/metacat/MetacatSmokeSpec.groovy +++ b/metacat-functional-tests/src/functionalTest/groovy/com/netflix/metacat/MetacatSmokeSpec.groovy @@ -18,10 +18,12 @@ package com.netflix.metacat import com.netflix.metacat.client.Client import com.netflix.metacat.client.api.MetacatV1 import com.netflix.metacat.client.api.MetadataV1 +import com.netflix.metacat.client.api.ParentChildRelV1 import com.netflix.metacat.client.api.PartitionV1 import com.netflix.metacat.client.api.TagV1 import com.netflix.metacat.common.QualifiedName import com.netflix.metacat.common.dto.* +import com.netflix.metacat.common.dto.notifications.ChildInfoDto import com.netflix.metacat.common.exception.MetacatAlreadyExistsException import com.netflix.metacat.common.exception.MetacatBadRequestException import com.netflix.metacat.common.exception.MetacatNotFoundException @@ -59,6 +61,7 @@ class MetacatSmokeSpec extends Specification { public static PartitionV1 partitionApi public static MetadataV1 metadataApi public static TagV1 tagApi + public static ParentChildRelV1 parentChildRelV1 public static MetacatJson metacatJson = new MetacatJsonLocator() def setupSpec() { @@ -85,6 +88,7 @@ class MetacatSmokeSpec extends Specification { partitionApi = client.partitionApi tagApi = client.tagApi metadataApi = client.metadataApi + parentChildRelV1 = client.parentChildRelApi } @Shared @@ -2019,21 +2023,18 @@ class MetacatSmokeSpec extends Specification { api.createTable(catalogName, databaseName, child11, child11TableDto) def parent1Table = api.getTable(catalogName, databaseName, parent1, true, true, false) - def parent1ParentChildRelationInfo = parent1Table.definitionMetadata.get("parentChildRelationInfo") def child11Table = api.getTable(catalogName, databaseName, child11, true, true, false) def child11ParentChildRelationInfo = child11Table.definitionMetadata.get("parentChildRelationInfo") then: // Test Parent 1 parentChildInfo - assert !parent1ParentChildRelationInfo.has("parentInfo") - JSONAssert.assertEquals(parent1ParentChildRelationInfo.toString(), - '{"childInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/child11","relationType":"CLONE", "uuid":"c11_uuid"}]}', - false) + assert !parent1Table.definitionMetadata.has("parentChildRelationInfo") + assert parentChildRelV1.getChildren(catalogName, databaseName, parent1) == [new ChildInfoDto("embedded-fast-hive-metastore/iceberg_db/child11", "CLONE", "c11_uuid")] as Set // Test Child11 parentChildInfo JSONAssert.assertEquals(child11ParentChildRelationInfo.toString(), '{"parentInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/parent1","relationType":"CLONE", "uuid":"p1_uuid"}]}', false) - assert !child11ParentChildRelationInfo.has("childInfos") + assert parentChildRelV1.getChildren(catalogName, databaseName, child11).isEmpty() /* Step 2: Create a second child (child12) pointing to parent = parent1 @@ -2046,22 +2047,22 @@ class MetacatSmokeSpec extends Specification { child12TableDto.definitionMetadata.put("child_table_uuid", child12UUID) api.createTable(catalogName, databaseName, child12, child12TableDto) parent1Table = api.getTable(catalogName, databaseName, parent1, true, true, false) - parent1ParentChildRelationInfo = parent1Table.definitionMetadata.get("parentChildRelationInfo") def child12Table = api.getTable(catalogName, databaseName, child12, true, true, false) def child12ParentChildRelationInfo = child12Table.definitionMetadata.get("parentChildRelationInfo") then: // Test Parent 1 parentChildInfo - assert !parent1ParentChildRelationInfo.has("parentInfo") - JSONAssert.assertEquals(parent1ParentChildRelationInfo.toString(), - '{"childInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/child11","relationType":"CLONE","uuid":"c11_uuid"}, {"name":"embedded-fast-hive-metastore/iceberg_db/child12","relationType":"CLONE","uuid":"c12_uuid"}]}', - false) + assert !parent1Table.definitionMetadata.get("parentChildRelationInfo") + assert parentChildRelV1.getChildren(catalogName, databaseName, parent1) == [ + new ChildInfoDto("embedded-fast-hive-metastore/iceberg_db/child11", "CLONE", "c11_uuid"), + new ChildInfoDto("embedded-fast-hive-metastore/iceberg_db/child12", "CLONE", "c12_uuid") + ] as Set // Test Child12 parentChildInfo JSONAssert.assertEquals(child12ParentChildRelationInfo.toString(), '{"parentInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/parent1","relationType":"CLONE","uuid":"p1_uuid"}]}', false) - assert !child12ParentChildRelationInfo.has("childInfos") + assert parentChildRelV1.getChildren(catalogName, databaseName, child12).isEmpty() /* Step 3: Create one grandChild As a Parent of A child table should fail @@ -2072,6 +2073,7 @@ class MetacatSmokeSpec extends Specification { grandchild121TableDto.definitionMetadata.put("root_table_uuid", child11UUID) grandchild121TableDto.definitionMetadata.put("child_table_uuid", grandChild121UUID) api.createTable(catalogName, databaseName, grandChild121, grandchild121TableDto) + assert parentChildRelV1.getChildren(catalogName, databaseName, grandChild121).isEmpty() then: thrown(Exception) @@ -2091,22 +2093,21 @@ class MetacatSmokeSpec extends Specification { child21TableDto.definitionMetadata.put("child_table_uuid", child21UUID) api.createTable(catalogName, databaseName, child21, child21TableDto) def parent2Table = api.getTable(catalogName, databaseName, parent2, true, true, false) - def parent2ParentChildRelationInfo = parent2Table.definitionMetadata.get("parentChildRelationInfo") def child21Table = api.getTable(catalogName, databaseName, child21, true, true, false) def child21ParentChildRelationInfo = child21Table.definitionMetadata.get("parentChildRelationInfo") then: // Test Parent 2 parentChildInfo - assert !parent2ParentChildRelationInfo.has("parentInfo") - JSONAssert.assertEquals(parent2ParentChildRelationInfo.toString(), - '{"childInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/child21","relationType":"CLONE","uuid":"c21_uuid"}]}', - false) + assert !parent2Table.definitionMetadata.has("parentChildRelationInfo") + assert parentChildRelV1.getChildren(catalogName, databaseName, parent2) == [ + new ChildInfoDto("embedded-fast-hive-metastore/iceberg_db/child21", "CLONE", "c21_uuid") + ] as Set // Test Child21 parentChildInfo JSONAssert.assertEquals(child21ParentChildRelationInfo.toString(), '{"parentInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/parent2","relationType":"CLONE","uuid":"p2_uuid"}]}', false) - assert !child21ParentChildRelationInfo.has("childInfos") + assert parentChildRelV1.getChildren(catalogName, databaseName, child21).isEmpty() /* Step 5: Rename parent1 to newParent1 @@ -2114,7 +2115,6 @@ class MetacatSmokeSpec extends Specification { when: api.renameTable(catalogName, databaseName, parent1, renameParent1) parent1Table = api.getTable(catalogName, databaseName, renameParent1, true, true, false) - parent1ParentChildRelationInfo = parent1Table.definitionMetadata.get("parentChildRelationInfo") child11Table = api.getTable(catalogName, databaseName, child11, true, true, false) child11ParentChildRelationInfo = child11Table.definitionMetadata.get("parentChildRelationInfo") child12Table = api.getTable(catalogName, databaseName, child12, true, true, false) @@ -2122,22 +2122,23 @@ class MetacatSmokeSpec extends Specification { then: // Test Parent 1 parentChildInfo newName - assert !parent1ParentChildRelationInfo.has("parentInfo") - JSONAssert.assertEquals(parent1ParentChildRelationInfo.toString(), - '{"childInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/child11","relationType":"CLONE","uuid":"c11_uuid"}, {"name":"embedded-fast-hive-metastore/iceberg_db/child12","relationType":"CLONE","uuid":"c12_uuid"}]}', - false) - + assert !parent1Table.definitionMetadata.has("parentChildRelationInfo") + assert parentChildRelV1.getChildren(catalogName, databaseName, renameParent1) == + [ + new ChildInfoDto("embedded-fast-hive-metastore/iceberg_db/child11", "CLONE", "c11_uuid"), + new ChildInfoDto("embedded-fast-hive-metastore/iceberg_db/child12", "CLONE", "c12_uuid") + ] as Set // Test Child11 parentChildInfo JSONAssert.assertEquals(child11ParentChildRelationInfo.toString(), '{"parentInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/rename_parent1","relationType":"CLONE","uuid":"p1_uuid"}]}', false) - assert !child11ParentChildRelationInfo.has("childInfos") + assert parentChildRelV1.getChildren(catalogName, databaseName, child11).isEmpty() // Test Child12 parentChildInfo JSONAssert.assertEquals(child12ParentChildRelationInfo.toString(), '{"parentInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/rename_parent1","relationType":"CLONE","uuid":"p1_uuid"}]}', false) - assert !child12ParentChildRelationInfo.has("childInfos") + assert parentChildRelV1.getChildren(catalogName, databaseName, child12).isEmpty() /* Step 6: Rename child11 to renameChild11 @@ -2145,20 +2146,19 @@ class MetacatSmokeSpec extends Specification { when: api.renameTable(catalogName, databaseName, child11, renameChild11) parent1Table = api.getTable(catalogName, databaseName, renameParent1, true, true, false) - parent1ParentChildRelationInfo = parent1Table.definitionMetadata.get("parentChildRelationInfo") child11Table = api.getTable(catalogName, databaseName, renameChild11, true, true, false) child11ParentChildRelationInfo = child11Table.definitionMetadata.get("parentChildRelationInfo") then: - assert !parent1ParentChildRelationInfo.has("parentInfo") - JSONAssert.assertEquals(parent1ParentChildRelationInfo.toString(), - '{"childInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/rename_child11","relationType":"CLONE","uuid":"c11_uuid"}, {"name":"embedded-fast-hive-metastore/iceberg_db/child12","relationType":"CLONE","uuid":"c12_uuid"}]}', - false) - + assert !parent1Table.definitionMetadata.has("parentChildRelationInfo") + assert parentChildRelV1.getChildren(catalogName, databaseName, renameParent1) == [ + new ChildInfoDto("embedded-fast-hive-metastore/iceberg_db/rename_child11", "CLONE", "c11_uuid"), + new ChildInfoDto("embedded-fast-hive-metastore/iceberg_db/child12", "CLONE", "c12_uuid") + ] as Set // Test Child11 parentChildInfo with newName JSONAssert.assertEquals(child11ParentChildRelationInfo.toString(), '{"parentInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/rename_parent1","relationType":"CLONE","uuid":"p1_uuid"}]}', false) - assert !child12ParentChildRelationInfo.has("childInfos") + assert parentChildRelV1.getChildren(catalogName, databaseName, renameChild11).isEmpty() /* Step 7: Drop parent renameParent1 @@ -2176,15 +2176,13 @@ class MetacatSmokeSpec extends Specification { when: api.deleteTable(catalogName, databaseName, renameChild11) parent1Table = api.getTable(catalogName, databaseName, renameParent1, true, true, false) - parent1ParentChildRelationInfo = parent1Table.definitionMetadata.get("parentChildRelationInfo") then: // Test parent1 Table - assert !parent1ParentChildRelationInfo.has("parentInfo") - JSONAssert.assertEquals(parent1ParentChildRelationInfo.toString(), - '{"childInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/child12","relationType":"CLONE","uuid":"c12_uuid"}]}', - false) - + assert !parent1Table.definitionMetadata.has("parentChildRelationInfo") + assert parentChildRelV1.getChildren(catalogName, databaseName, renameParent1) == [ + new ChildInfoDto("embedded-fast-hive-metastore/iceberg_db/child12", "CLONE", "c12_uuid") + ] as Set /* Step 9: Drop child12 */ @@ -2193,6 +2191,7 @@ class MetacatSmokeSpec extends Specification { parent1Table = api.getTable(catalogName, databaseName, renameParent1, true, true, false) then: assert !parent1Table.definitionMetadata.has("parentChildRelationInfo") + assert parentChildRelV1.getChildren(catalogName, databaseName, renameParent1).isEmpty() /* Step 10: Drop renameParent1 @@ -2200,7 +2199,6 @@ class MetacatSmokeSpec extends Specification { when: api.deleteTable(catalogName, databaseName, renameParent1) parent2Table = api.getTable(catalogName, databaseName, parent2, true, true, false) - parent2ParentChildRelationInfo = parent2Table.definitionMetadata.get("parentChildRelationInfo") child21Table = api.getTable(catalogName, databaseName, child21, true, true, false) child21ParentChildRelationInfo = child21Table.definitionMetadata.get("parentChildRelationInfo") @@ -2208,16 +2206,16 @@ class MetacatSmokeSpec extends Specification { // Since all the operations above are on the first connected relationship, the second connected relationship // should remain the same // Test Parent 2 parentChildInfo - assert !parent2ParentChildRelationInfo.has("parentInfo") - JSONAssert.assertEquals(parent2ParentChildRelationInfo.toString(), - '{"childInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/child21","relationType":"CLONE","uuid":"c21_uuid"}]}', - false) + assert !parent2Table.definitionMetadata.has("parentChildRelationInfo") + assert parentChildRelV1.getChildren(catalogName, databaseName, parent2) == [ + new ChildInfoDto("embedded-fast-hive-metastore/iceberg_db/child21", "CLONE", "c21_uuid") + ] as Set // Test Child21 parentChildInfo JSONAssert.assertEquals(child21ParentChildRelationInfo.toString(), '{"parentInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/parent2","relationType":"CLONE","uuid":"p2_uuid"}]}', false) - assert !child21ParentChildRelationInfo.has("childInfos") + assert parentChildRelV1.getChildren(catalogName, databaseName, child21).isEmpty() /* Step 11: update parent2 with random parentChildRelationInfo @@ -2229,21 +2227,19 @@ class MetacatSmokeSpec extends Specification { api.updateTable(catalogName, databaseName, parent2, updateParent2Dto) parent2Table = api.getTable(catalogName, databaseName, parent2, true, true, false) - parent2ParentChildRelationInfo = parent2Table.definitionMetadata.get("parentChildRelationInfo") child21Table = api.getTable(catalogName, databaseName, child21, true, true, false) child21ParentChildRelationInfo = child21Table.definitionMetadata.get("parentChildRelationInfo") then: - // Test Parent 2 parentChildInfo - assert !parent2ParentChildRelationInfo.has("parentInfo") - JSONAssert.assertEquals(parent2ParentChildRelationInfo.toString(), - '{"childInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/child21","relationType":"CLONE","uuid":"c21_uuid"}]}', - false) + assert !parent2Table.definitionMetadata.has("parentChildRelationInfo") + assert parentChildRelV1.getChildren(catalogName, databaseName, parent2) == [ + new ChildInfoDto("embedded-fast-hive-metastore/iceberg_db/child21", "CLONE", "c21_uuid") + ] as Set // Test Child21 parentChildInfo JSONAssert.assertEquals(child21ParentChildRelationInfo.toString(), '{"parentInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/parent2","relationType":"CLONE","uuid":"p2_uuid"}]}', false) - assert !child21ParentChildRelationInfo.has("childInfos") + assert parentChildRelV1.getChildren(catalogName, databaseName, child21).isEmpty() /* Step 12: update child21 with random parentChildRelationInfo @@ -2255,20 +2251,19 @@ class MetacatSmokeSpec extends Specification { api.updateTable(catalogName, databaseName, child21, updateChild21Dto) parent2Table = api.getTable(catalogName, databaseName, parent2, true, true, false) - parent2ParentChildRelationInfo = parent2Table.definitionMetadata.get("parentChildRelationInfo") child21Table = api.getTable(catalogName, databaseName, child21, true, true, false) child21ParentChildRelationInfo = child21Table.definitionMetadata.get("parentChildRelationInfo") then: // Test Parent 2 parentChildInfo - assert !parent2ParentChildRelationInfo.has("parentInfo") - JSONAssert.assertEquals(parent2ParentChildRelationInfo.toString(), - '{"childInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/child21","relationType":"CLONE","uuid":"c21_uuid"}]}', - false) + assert !parent2Table.definitionMetadata.has("parentChildRelationInfo") + assert parentChildRelV1.getChildren(catalogName, databaseName, parent2) == [ + new ChildInfoDto("embedded-fast-hive-metastore/iceberg_db/child21", "CLONE", "c21_uuid") + ] as Set // Test Child21 parentChildInfo JSONAssert.assertEquals(child21ParentChildRelationInfo.toString(), '{"parentInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/parent2","relationType":"CLONE","uuid":"p2_uuid"}]}', false) - assert !child21ParentChildRelationInfo.has("childInfos") + assert parentChildRelV1.getChildren(catalogName, databaseName, child21).isEmpty() } } diff --git a/metacat-functional-tests/src/functionalTest/groovy/com/netflix/metacat/ParentChildRelMetadataServiceSpec.groovy b/metacat-functional-tests/src/functionalTest/groovy/com/netflix/metacat/ParentChildRelMetadataServiceSpec.groovy index caebc8535..22abe1a69 100644 --- a/metacat-functional-tests/src/functionalTest/groovy/com/netflix/metacat/ParentChildRelMetadataServiceSpec.groovy +++ b/metacat-functional-tests/src/functionalTest/groovy/com/netflix/metacat/ParentChildRelMetadataServiceSpec.groovy @@ -1,6 +1,11 @@ package com.netflix.metacat import com.netflix.metacat.common.QualifiedName +import com.netflix.metacat.common.server.converter.ConverterUtil +import com.netflix.metacat.common.server.converter.DefaultTypeConverter +import com.netflix.metacat.common.server.converter.DozerJsonTypeConverter +import com.netflix.metacat.common.server.converter.DozerTypeConverter +import com.netflix.metacat.common.server.converter.TypeConverterFactory import com.netflix.metacat.common.server.model.ChildInfo import com.netflix.metacat.common.server.model.ParentInfo import com.netflix.metacat.common.server.usermetadata.ParentChildRelMetadataService @@ -10,15 +15,15 @@ import org.springframework.jdbc.datasource.DriverManagerDataSource import spock.lang.Shared import spock.lang.Specification -import java.sql.Connection -import java.sql.PreparedStatement - class ParentChildRelMetadataServiceSpec extends Specification{ @Shared private ParentChildRelMetadataService service; @Shared private JdbcTemplate jdbcTemplate; + @Shared + private ConverterUtil converterUtil; + @Shared private String catalog = "prodhive" @Shared @@ -36,7 +41,9 @@ class ParentChildRelMetadataServiceSpec extends Specification{ dataSource.setPassword(password) jdbcTemplate = new JdbcTemplate(dataSource) - service = new MySqlParentChildRelMetaDataService(jdbcTemplate) + def typeFactory = new TypeConverterFactory(new DefaultTypeConverter()) + def converter = new ConverterUtil(new DozerTypeConverter(typeFactory), new DozerJsonTypeConverter(typeFactory)) + service = new MySqlParentChildRelMetaDataService(jdbcTemplate, converter) } def cleanup() { diff --git a/metacat-main/src/main/java/com/netflix/metacat/main/api/v1/ParentChildRelController.java b/metacat-main/src/main/java/com/netflix/metacat/main/api/v1/ParentChildRelController.java new file mode 100644 index 000000000..65ce6572c --- /dev/null +++ b/metacat-main/src/main/java/com/netflix/metacat/main/api/v1/ParentChildRelController.java @@ -0,0 +1,75 @@ +package com.netflix.metacat.main.api.v1; + +import com.netflix.metacat.common.QualifiedName; +import com.netflix.metacat.common.dto.notifications.ChildInfoDto; +import com.netflix.metacat.common.server.usermetadata.ParentChildRelMetadataService; +import com.netflix.metacat.main.api.RequestWrapper; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; +import lombok.RequiredArgsConstructor; +import org.springframework.context.annotation.DependsOn; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.PathVariable; + +import java.util.Set; + +/** + * Parent Child Relation V1 API implementation. + * + * @author amajumdar + */ + +@RestController +@RequestMapping( + path = "/mds/v1/parentChildRel", + produces = MediaType.APPLICATION_JSON_VALUE +) +@Api( + value = "ParentChildRelV1", + description = "Federated user metadata operations", + produces = MediaType.APPLICATION_JSON_VALUE, + consumes = MediaType.APPLICATION_JSON_VALUE +) +@DependsOn("metacatCoreInitService") +@RequiredArgsConstructor +public class ParentChildRelController { + private final RequestWrapper requestWrapper; + private final ParentChildRelMetadataService parentChildRelMetadataService; + + /** + * Return the list of children for a given table. + * @param catalogName catalogName + * @param databaseName databaseName + * @param tableName tableName + * @return list of childInfoDto + */ + @RequestMapping(method = RequestMethod.GET, + path = "/children/catalog/{catalog-name}/database/{database-name}/table/{table-name}") + @ResponseStatus(HttpStatus.OK) + @ApiOperation( + position = 0, + value = "Returns the children", + notes = "Returns the children" + ) + public Set getChildren( + @ApiParam(value = "The name of the catalog", required = true) + @PathVariable("catalog-name") final String catalogName, + @ApiParam(value = "The name of the database", required = true) + @PathVariable("database-name") final String databaseName, + @ApiParam(value = "The name of the table", required = true) + @PathVariable("table-name") final String tableName + ) { + return this.requestWrapper.processRequest( + "ParentChildRelV1Resource.getChildren", + () -> this.parentChildRelMetadataService.getChildrenDto( + QualifiedName.ofTable(catalogName, databaseName, tableName) + ) + ); + } +} diff --git a/metacat-main/src/main/java/com/netflix/metacat/main/services/impl/TableServiceImpl.java b/metacat-main/src/main/java/com/netflix/metacat/main/services/impl/TableServiceImpl.java index ddeb8fe03..1f1c021ef 100644 --- a/metacat-main/src/main/java/com/netflix/metacat/main/services/impl/TableServiceImpl.java +++ b/metacat-main/src/main/java/com/netflix/metacat/main/services/impl/TableServiceImpl.java @@ -153,11 +153,11 @@ public TableDto create(final QualifiedName name, final TableDto tableDto) { return dto; } - private ObjectNode createParentChildObjectNode(@Nullable final Set parentInfos, - @Nullable final Set childInfos) { + private ObjectNode createParentChildObjectNode(@Nullable final Set parentInfos) { final ObjectMapper objectMapper = new ObjectMapper(); final ObjectNode rootNode = objectMapper.createObjectNode(); + // For now, we only populate parent information for a child table. if (parentInfos != null && !parentInfos.isEmpty()) { final ArrayNode parentArrayNode = objectMapper.createArrayNode(); for (ParentInfo parentInfo : parentInfos) { @@ -169,18 +169,6 @@ private ObjectNode createParentChildObjectNode(@Nullable final Set p } rootNode.set(ParentChildRelMetadataConstants.PARENTINFOS, parentArrayNode); } - - if (childInfos != null && !childInfos.isEmpty()) { - final ArrayNode childrenArrayNode = objectMapper.createArrayNode(); - for (ChildInfo childInfo : childInfos) { - final ObjectNode childNode = objectMapper.createObjectNode(); - childNode.put("name", childInfo.getName()); - childNode.put("relationType", childInfo.getRelationType()); - childNode.put("uuid", childInfo.getUuid()); - childrenArrayNode.add(childNode); - } - rootNode.set(ParentChildRelMetadataConstants.CHILDINFOS, childrenArrayNode); - } return rootNode; } @@ -493,8 +481,7 @@ public Optional get(final QualifiedName name, final GetTableServicePar GetMetadataInterceptorParameters.builder().hasMetadata(tableInternal).build()); // Always get the source of truth for parent child relation from the parentChildRelMetadataService final Set parentInfo = parentChildRelMetadataService.getParents(name); - final Set childInfos = parentChildRelMetadataService.getChildren(name); - final ObjectNode parentChildRelObjectNode = createParentChildObjectNode(parentInfo, childInfos); + final ObjectNode parentChildRelObjectNode = createParentChildObjectNode(parentInfo); if (definitionMetadata.isPresent()) { if (!parentChildRelObjectNode.isEmpty()) { definitionMetadata.get().set(ParentChildRelMetadataConstants.PARENTCHILDRELINFO, diff --git a/metacat-main/src/test/groovy/com/netflix/metacat/main/services/impl/TableServiceImplSpec.groovy b/metacat-main/src/test/groovy/com/netflix/metacat/main/services/impl/TableServiceImplSpec.groovy index 4bd083f5e..340f9890b 100644 --- a/metacat-main/src/test/groovy/com/netflix/metacat/main/services/impl/TableServiceImplSpec.groovy +++ b/metacat-main/src/test/groovy/com/netflix/metacat/main/services/impl/TableServiceImplSpec.groovy @@ -54,8 +54,6 @@ import spock.lang.Specification import spock.lang.Unroll import com.netflix.metacat.common.server.usermetadata.ParentChildRelMetadataService; -import javax.annotation.meta.When - /** * Tests for the TableServiceImpl. * @@ -321,7 +319,7 @@ class TableServiceImplSpec extends Specification { service.delete(name) then: 1 * parentChildRelSvc.getParents(name) >> {[new ParentInfo("parent", "clone", "parent_uuid")] as Set} - 2 * parentChildRelSvc.getChildren(name) >> {[new ChildInfo("child", "clone", "child_uuid")] as Set} + 1 * parentChildRelSvc.getChildren(name) >> {[new ChildInfo("child", "clone", "child_uuid")] as Set} 1 * config.getNoTableDeleteOnTags() >> [] thrown(RuntimeException) @@ -329,7 +327,7 @@ class TableServiceImplSpec extends Specification { service.delete(name) then: 1 * parentChildRelSvc.getParents(name) - 2 * parentChildRelSvc.getChildren(name) + 1 * parentChildRelSvc.getChildren(name) 1 * config.getNoTableDeleteOnTags() >> [] 1 * connectorTableServiceProxy.delete(_) >> {throw new RuntimeException("Fail to drop")} 0 * parentChildRelSvc.drop(_, _) diff --git a/metacat-metadata-mysql/src/main/java/com/netflix/metacat/metadata/mysql/MySqlParentChildRelMetaDataService.java b/metacat-metadata-mysql/src/main/java/com/netflix/metacat/metadata/mysql/MySqlParentChildRelMetaDataService.java index ef8394f7d..73f9cbf35 100644 --- a/metacat-metadata-mysql/src/main/java/com/netflix/metacat/metadata/mysql/MySqlParentChildRelMetaDataService.java +++ b/metacat-metadata-mysql/src/main/java/com/netflix/metacat/metadata/mysql/MySqlParentChildRelMetaDataService.java @@ -1,6 +1,8 @@ package com.netflix.metacat.metadata.mysql; import com.netflix.metacat.common.QualifiedName; +import com.netflix.metacat.common.dto.notifications.ChildInfoDto; +import com.netflix.metacat.common.server.converter.ConverterUtil; import com.netflix.metacat.common.server.model.ChildInfo; import com.netflix.metacat.common.server.model.ParentInfo; import com.netflix.metacat.common.server.usermetadata.ParentChildRelMetadataService; @@ -15,6 +17,7 @@ import java.util.List; import java.util.ArrayList; import java.util.HashSet; +import java.util.stream.Collectors; /** * Parent Child Relationship Metadata Service. @@ -48,16 +51,19 @@ public class MySqlParentChildRelMetaDataService implements ParentChildRelMetadat static final String SQL_GET_CHILDREN = "SELECT child, child_uuid, relation_type " + "FROM parent_child_relation WHERE parent = ?"; - private JdbcTemplate jdbcTemplate; + private final JdbcTemplate jdbcTemplate; + private final ConverterUtil converterUtil; /** * Constructor. * * @param jdbcTemplate jdbc template + * @param converterUtil converterUtil */ @Autowired - public MySqlParentChildRelMetaDataService(final JdbcTemplate jdbcTemplate) { + public MySqlParentChildRelMetaDataService(final JdbcTemplate jdbcTemplate, final ConverterUtil converterUtil) { this.jdbcTemplate = jdbcTemplate; + this.converterUtil = converterUtil; } @Override @@ -214,4 +220,10 @@ public Set getChildren(final QualifiedName name) { }); return new HashSet<>(children); } + + @Override + public Set getChildrenDto(final QualifiedName name) { + return getChildren(name).stream() + .map(converterUtil::toChildInfoDto).collect(Collectors.toSet()); + } } diff --git a/metacat-metadata-mysql/src/main/java/com/netflix/metacat/metadata/mysql/MySqlUserMetadataConfig.java b/metacat-metadata-mysql/src/main/java/com/netflix/metacat/metadata/mysql/MySqlUserMetadataConfig.java index 451b4cf44..a195d403c 100644 --- a/metacat-metadata-mysql/src/main/java/com/netflix/metacat/metadata/mysql/MySqlUserMetadataConfig.java +++ b/metacat-metadata-mysql/src/main/java/com/netflix/metacat/metadata/mysql/MySqlUserMetadataConfig.java @@ -14,6 +14,7 @@ package com.netflix.metacat.metadata.mysql; import com.netflix.metacat.common.json.MetacatJson; +import com.netflix.metacat.common.server.converter.ConverterUtil; import com.netflix.metacat.common.server.properties.Config; import com.netflix.metacat.common.server.properties.MetacatProperties; import com.netflix.metacat.common.server.usermetadata.UserMetadataService; @@ -116,9 +117,10 @@ public TagService tagService( */ @Bean ParentChildRelMetadataService parentChildRelMetadataService( - @Qualifier("metadataJdbcTemplate") final JdbcTemplate jdbcTemplate + @Qualifier("metadataJdbcTemplate") final JdbcTemplate jdbcTemplate, + final ConverterUtil converterUtil ) { - return new MySqlParentChildRelMetaDataService(jdbcTemplate); + return new MySqlParentChildRelMetaDataService(jdbcTemplate, converterUtil); } /**